amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package host
2
3import (
4 "bytes"
5 "context"
6 "crypto/ed25519"
7 "encoding/base64"
8 "encoding/json"
9 "errors"
10 "io"
11 "net/http"
12 "net/http/httptest"
13 "net/url"
14 "os"
15 "path/filepath"
16 "reflect"
17 "slices"
18 "strings"
19 "testing"
20 "time"
21
22 "github.com/carlosframework/platform/internal/activator"
23 "github.com/carlosframework/platform/internal/appconfig"
24 "github.com/carlosframework/platform/internal/console"
25 "github.com/carlosframework/platform/internal/provision"
26 "github.com/carlosframework/platform/internal/registry"
27 "github.com/carlosframework/platform/internal/release"
28 "github.com/carlosframework/platform/internal/rollout"
29 "github.com/carlosframework/platform/internal/status"
30)
31
32// writeFailureBook writes recs to dd's failure book through the book itself,
33// then retries each host in retried, as an armed run would have left them.
34func writeFailureBook(t *testing.T, dd string, recs []rollout.FailureRecord, retried map[string][]string) {
35 t.Helper()
36 b, err := rollout.OpenFailureBook(dd)
37 if err != nil {
38 t.Fatal(err)
39 }
40 for _, r := range recs {
41 if _, err := b.Record(r); err != nil {
42 t.Fatal(err)
43 }
44 for _, h := range retried[r.Build] {
45 if _, err := b.RetryHost(r.Account, r.App, r.Build, h); err != nil {
46 t.Fatal(err)
47 }
48 }
49 }
50}
51
52// bookSnapshot is the book file's bytes and modification time, so a test can
53// show a declined box never wrote it.
54type bookSnapshot struct {
55 raw []byte
56 mod time.Time
57}
58
59func snapshotBook(t *testing.T, dd string) bookSnapshot {
60 t.Helper()
61 p := filepath.Join(dd, rollout.FailureFile)
62 raw, err := os.ReadFile(p)
63 if err != nil {
64 t.Fatal(err)
65 }
66 fi, err := os.Stat(p)
67 if err != nil {
68 t.Fatal(err)
69 }
70 return bookSnapshot{raw, fi.ModTime()}
71}
72
73func (s bookSnapshot) unchanged(t *testing.T, dd string) {
74 t.Helper()
75 now := snapshotBook(t, dd)
76 if !bytes.Equal(now.raw, s.raw) || !now.mod.Equal(s.mod) {
77 t.Fatalf("a declined box wrote the failure book:\nbefore %s\nafter %s", s.raw, now.raw)
78 }
79}
80
81// declineWithBook arms svc as runAgent does on a box with no delegated cgroup
82// subtree: swapArming declined the flag, the Service was built without
83// generations, and armRollout runs with CARLOS_SWAP=overlap set. The store
84// must never be opened.
85func declineWithBook(t *testing.T, svc *activator.Service, reg *registry.Registry, dd string) {
86 t.Helper()
87 _, pub, err := release.GenerateKey()
88 if err != nil {
89 t.Fatal(err)
90 }
91 declineRollout(rolloutNoDelegation)
92 opened := false
93 m := armRollout(context.Background(), overlapEnv(pub), svc, reg, dd, nil,
94 func(context.Context) (release.Store, error) { opened = true; return nil, errors.New("unused") })
95 if m != nil || armedRollout() != nil || opened {
96 t.Fatal("a declined box armed the scheduler or opened the store")
97 }
98 if activator.GenerationsEnabled() || svc.GenerationsOn() {
99 t.Fatal("a declined box turned generations on")
100 }
101}
102
103// managedRoute is a managed, hibernating exec instance of bab/blog installed
104// on ver.
105func managedRoute(host, ver string) registry.Route {
106 rt := execRoute(host, "bab", "blog")
107 rt.Hibernate, rt.Version = true, ver
108 return rt
109}
110
111func isTodaysWake(p activator.WakePlan) bool {
112 return p.Refuse == "" && p.Boot.StagedPath == "" && p.Check == nil && !p.FirstWake && p.Fallback == nil
113}
114
115// Final review F6 (c): a declined box with no failure records, the book
116// absent or empty, is exactly today's box: no activator hook, every leg
117// today's, and the status says only why the flag was declined.
118func TestDeclinedWithNoRecordsIsToday(t *testing.T) {
119 for name, body := range map[string]string{
120 "absent book": "",
121 "empty book": `{"schema": 1, "records": []}`,
122 } {
123 t.Run(name, func(t *testing.T) {
124 t.Setenv("CARLOS_OPT_DIR", t.TempDir())
125 svc, reg, dd := rolloutFixture(t, false)
126 if body != "" {
127 if err := os.WriteFile(filepath.Join(dd, rollout.FailureFile), []byte(body), 0o644); err != nil {
128 t.Fatal(err)
129 }
130 }
131 armAlways(svc, reg, dd)
132 declineWithBook(t, svc, reg, dd)
133 if svc.Planner != nil || svc.OnGen != nil || svc.OnDrift != nil || svc.CoalesceRecycle != nil {
134 t.Fatal("a declined box with no records set an activator hook")
135 }
136 if heldHolds() != nil || rolloutUnavailable() != "" {
137 t.Fatal("a declined box with no records holds or refuses something")
138 }
139 if r := rolloutStatusProbe(); r == nil || r.Unavailable != rolloutNoDelegation || r.FailureBook != "" {
140 t.Fatalf("status = %+v, want only the decline code %q", r, rolloutNoDelegation)
141 }
142 legsAreTodays(t, svc, reg)
143 if body == "" {
144 if _, err := os.Stat(filepath.Join(dd, rollout.FailureFile)); !errors.Is(err, os.ErrNotExist) {
145 t.Fatalf("a declined box created the failure book: %v", err)
146 }
147 }
148 })
149 }
150}
151
152// Final review F6 (a): a declined box with records refuses a managed wake
153// onto a build they hold for that instance, with fixed text, and wakes
154// everything else as today. The book is never written.
155func TestDeclinedBoxRefusesAWakeOntoAHeldBuild(t *testing.T) {
156 optDir := t.TempDir()
157 t.Setenv("CARLOS_OPT_DIR", optDir)
158 svc, reg, dd := rolloutFixture(t, false)
159 acct := release.RouteAccount(managedRoute("x", ""))
160 writeFailureBook(t, dd, []rollout.FailureRecord{{
161 Account: acct, App: "blog", Build: "v2", Host: "sib.example.com", Cause: "adopt", Mode: "hold", Reason: "health",
162 HeldDown: []string{"down.example.com", "downgood.example.com"},
163 }}, map[string][]string{"v2": {"retried.example.com"}})
164 before := snapshotBook(t, dd)
165 declineWithBook(t, svc, reg, dd)
166 if svc.Planner == nil {
167 t.Fatal("no Planner: a wake onto a held build runs from the live link")
168 }
169 if svc.OnGen != nil || svc.OnDrift != nil || svc.CoalesceRecycle != nil || rolloutUnavailable() != "" {
170 t.Fatal("honouring records changed more than the wake: the box must otherwise stay on today's paths")
171 }
172
173 good := managedRoute("good.example.com", "v2")
174 good.GoodVersion, good.GoodDigest = "v2", "sha256:good"
175 // A held instance has a good build (v1) to have fallen back to: spec
176 // §8.5's bootstrap rule refuses on a record only then.
177 blocked := managedRoute("a.example.com", "v2")
178 blocked.GoodVersion, blocked.GoodDigest = "v1", "sha256:v1"
179 // No good build, installed build blocked, not held down: boots as today.
180 noGood := managedRoute("nogood.example.com", "v2")
181 // Held down with no good build: still refused.
182 downNoGood := managedRoute("down.example.com", "v2")
183 downGood := managedRoute("downgood.example.com", "v2")
184 downGood.GoodVersion, downGood.GoodDigest = "v2", "sha256:good"
185 unit := managedRoute("unit.example.com", "v2")
186 unit.Unit = "blog.service"
187 // Installed v1, but root has already repointed live/<host> at v2 and
188 // not yet stamped the row: the wake would exec v2.
189 repointed := managedRoute("repointed.example.com", "v1")
190 repointed.GoodVersion, repointed.GoodDigest = "v1", "sha256:v1"
191 // The same repoint on an instance with no good build boots as today.
192 repointedNoGood := managedRoute("repointednogood.example.com", "v1")
193 if err := release.StageArtifact(optDir, acct, "v2", "blog", []byte("v2")); err != nil {
194 t.Fatal(err)
195 }
196 for _, h := range []string{repointed.Host, repointedNoGood.Host} {
197 if err := release.PointLive(optDir, h, acct, "v2", "blog"); err != nil {
198 t.Fatal(err)
199 }
200 }
201 for _, tc := range []struct {
202 rt registry.Route
203 refuse string
204 }{
205 {blocked, refuseNoSwaps},
206 {noGood, ""},
207 {repointedNoGood, ""},
208 {downNoGood, "build v2 failed in hold mode on this instance, so it stays down until you roll back or retry the build"},
209 {downGood, "build v2 failed in hold mode on this instance, so it stays down until you roll back or retry the build"},
210 {repointed, refuseNoSwaps},
211 {good, ""},
212 {managedRoute("retried.example.com", "v2"), ""},
213 {managedRoute("old.example.com", "v1"), ""},
214 {managedRoute("new.example.com", ""), ""},
215 {unit, ""},
216 } {
217 p := svc.Planner(context.Background(), tc.rt)
218 if tc.refuse == "" {
219 if !isTodaysWake(p) {
220 t.Errorf("%s: planned %+v, want today's wake", tc.rt.Host, p)
221 }
222 continue
223 }
224 if p.Refuse != tc.refuse {
225 t.Errorf("%s: refusal %q, want %q", tc.rt.Host, p.Refuse, tc.refuse)
226 }
227 }
228 if r := rolloutStatusProbe(); r == nil || r.Unavailable != rolloutNoDelegation || r.FailureBook != rolloutBookHonoured {
229 t.Fatalf("status = %+v, want the decline code %q and the book %q", r, rolloutNoDelegation, rolloutBookHonoured)
230 }
231 before.unchanged(t, dd)
232}
233
234// Final review F6 (b), the legs handed fakes: the restart leg and config
235// sync's restart leave a held instance running, and every other host gets
236// today's bounce. A held instance's config ack never claims a restart.
237func TestDeclinedLegsLeaveAHeldInstanceRunning(t *testing.T) {
238 t.Setenv("CARLOS_OPT_DIR", t.TempDir())
239 svc, reg, dd := rolloutFixture(t, false)
240 acct := release.RouteAccount(managedRoute("x", ""))
241 writeFailureBook(t, dd, []rollout.FailureRecord{{Account: acct, App: "blog", Build: "v2", Host: "sib.example.com"}}, nil)
242 before := snapshotBook(t, dd)
243 held, sleeper, free := managedRoute("held.example.com", "v2"), managedRoute("sleeper.example.com", "v2"), managedRoute("free.example.com", "v1")
244 // Held and asleep only because they have a good build to have kept; an
245 // instance with none boots its blocked build as today, so it is free.
246 held.GoodVersion, held.GoodDigest = "v1", "sha256:v1"
247 sleeper.GoodVersion, sleeper.GoodDigest = "v1", "sha256:v1"
248 noGood := managedRoute("nogood.example.com", "v2")
249 for _, rt := range []registry.Route{held, sleeper, free, noGood} {
250 if err := reg.Upsert(rt); err != nil {
251 t.Fatal(err)
252 }
253 }
254 declineWithBook(t, svc, reg, dd)
255
256 // The restart leg.
257 ctx, s := genKit(t)
258 var bounced []string
259 bounce := func(h string) bool { bounced = append(bounced, h); return true }
260 mem := newRestartMemory()
261 routes := []registry.Route{held, free, noGood}
262 askRestart(t, ctx, s, "bab", "blog")
263 restartRequestLeg(ctx, routes, s, t.TempDir(), bounce, nil, mem)
264 askRestart(t, ctx, s, "bab", "blog")
265 if !restartRequestLeg(ctx, routes, s, t.TempDir(), bounce, nil, mem) {
266 t.Fatal("the restart leg did not ask for the root pass")
267 }
268 slices.Sort(bounced)
269 if !slices.Equal(bounced, []string{"free.example.com", "nogood.example.com"}) {
270 t.Fatalf("restart bounced %v, want only the instances no record holds", bounced)
271 }
272
273 // Config sync, end to end over the real pass.
274 store := release.NewDirStore(t.TempDir())
275 privText, _, err := appconfig.GenerateKey()
276 if err != nil {
277 t.Fatal(err)
278 }
279 priv, err := appconfig.ParsePrivateKey(privText, "test")
280 if err != nil {
281 t.Fatal(err)
282 }
283 if err := appconfig.SetEnv(ctx, store, "bab", "blog", map[string]string{"LOG_LEVEL": "debug"}); err != nil {
284 t.Fatal(err)
285 }
286 var restarted []string
287 cb, ok := rolloutConfigCallbacks(ctx, store, configLegs{
288 awake: func(h string) bool { return h != sleeper.Host },
289 restart: func(h string) error { restarted = append(restarted, h); return nil },
290 })
291 if !ok {
292 t.Fatal("config sync took today's SyncExec, which would bounce the held instance")
293 }
294 acks := map[string]appconfig.ConfigAck{}
295 cb.Ack = func(a appconfig.ConfigAck) { acks[a.Host] = a }
296 if err := appconfig.SyncExecWithCallbacks(ctx, reg, store, priv, t.TempDir(), "", cb); err != nil {
297 t.Fatal(err)
298 }
299 slices.Sort(restarted)
300 if !slices.Equal(restarted, []string{free.Host, noGood.Host}) {
301 t.Fatalf("config sync restarted %v, want only the instances no record holds", restarted)
302 }
303 if a := acks[held.Host]; a.Restarted || a.Hibernating || a.RestartError != rolloutRestartRefused {
304 t.Fatalf("held awake ack = %+v, want RestartError %q", a, rolloutRestartRefused)
305 }
306 if a := acks[sleeper.Host]; a.Restarted || !a.Hibernating || a.RestartError != "" {
307 t.Fatalf("held sleeper ack = %+v, want delivered on wake", a)
308 }
309 if a := acks[noGood.Host]; !a.Restarted || a.Hibernating || a.RestartError != "" {
310 t.Fatalf("no-good ack = %+v, want today's restarted", a)
311 }
312 if a := acks[free.Host]; !a.Restarted || a.Hibernating || a.RestartError != "" {
313 t.Fatalf("free ack = %+v, want today's restarted", a)
314 }
315 before.unchanged(t, dd)
316}
317
318// Final review F6 (a) and (b) on a real activator, the converge leg and the
319// adopt bounce as fastConvergeLoop and the adopt pass build them. Root
320// installs a build a failure record holds: neither leg stops the instance
321// still serving the old build, and when something else does stop it, the
322// next wake is refused rather than exec the held build from the live link.
323func TestDeclinedBoxKeepsAHeldBuildOffARealActivator(t *testing.T) {
324 prev := activator.GenerationsEnabled()
325 activator.SetGenerations(false)
326 t.Cleanup(func() { activator.SetGenerations(prev) })
327 resetRollout(t)
328 env := newFastFixture(t)
329 t.Setenv("CARLOS_OPT_DIR", env.optDir)
330 host := "warm.test"
331 // GoodVersion v1: a record holds a build only for an instance that has a
332 // good build (the bootstrap rule), and this one has served v1.
333 rt := registry.Route{Host: host, Kind: "instance", App: "fastapp", Account: "bab", Channel: "edge",
334 Status: "ready", Hibernate: true, Version: "v1", GoodVersion: "v1", GoodDigest: "sha256:v1"}
335 if err := env.reg.Upsert(rt); err != nil {
336 t.Fatal(err)
337 }
338 env.stageLive(t, host, "v1")
339 writeFailureBook(t, env.dataDir, []rollout.FailureRecord{{Account: release.RouteAccount(rt), App: "fastapp", Build: "v2", Host: "sibling.test"}}, nil)
340 backend := &versionedBackend{optDir: env.optDir}
341 svc := activator.New(activator.Config{Bucket: "test-bucket", Idle: time.Hour}, backend, env.reg)
342 svc.Routes = env.reg
343 t.Cleanup(func() {
344 sctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
345 defer cancel()
346 svc.Shutdown(sctx)
347 })
348 declineWithBook(t, svc, env.reg, env.dataDir)
349 request := func() int {
350 rec := httptest.NewRecorder()
351 proxy := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) })
352 if !svc.ServeHTTP(rec, httptest.NewRequest("GET", "http://"+host+"/", nil), rt, proxy) {
353 t.Fatal("the activator did not take the request")
354 }
355 return rec.Code
356 }
357 if code := request(); code != http.StatusNoContent {
358 t.Fatalf("wake onto v1 = %d", code)
359 }
360
361 // Root installs v2: the link first, then the stamp.
362 env.stageLive(t, host, "v2")
363 rt.Version = "v2"
364 if err := env.reg.Upsert(rt); err != nil {
365 t.Fatal(err)
366 }
367 leg := convergeLegFor(svc, env.reg)
368 leg()
369 restart := makeRestartRoute(env.reg, func(string) {}, unitOnly(func(u string) error {
370 t.Fatalf("named-unit arm fired for %q", u)
371 return nil
372 }), adoptBounceFn(svc))
373 if err := restart(host); err != nil {
374 t.Fatal(err)
375 }
376 if !svc.Owns(host) {
377 t.Fatal("a leg stopped the instance serving v1 into a wake onto held v2")
378 }
379 if code := request(); code != http.StatusNoContent {
380 t.Fatalf("request to the running instance = %d", code)
381 }
382 if got := backend.started(); !reflect.DeepEqual(got, []string{"v1"}) {
383 t.Fatalf("spawned builds = %v, want only v1", got)
384 }
385
386 // The sweep's own drift cycle asks the launch gate too (Task 9 fix round
387 // 1); whatever else stops the instance (here a bounce), its next wake is
388 // refused.
389 svc.Bounce(host)
390 deadline := time.Now().Add(10 * time.Second)
391 for svc.Owns(host) {
392 if time.Now().After(deadline) {
393 t.Fatal("the bounce never stopped the instance")
394 }
395 time.Sleep(5 * time.Millisecond)
396 }
397 if code := request(); code == http.StatusNoContent {
398 t.Fatal("the wake onto held v2 was served")
399 }
400 if got := backend.started(); !reflect.DeepEqual(got, []string{"v1"}) {
401 t.Fatalf("spawned builds = %v: held v2 was exec'd from the live link", got)
402 }
403}
404
405// Task 5 rule 5: a declined box honours the records reconciled from an
406// intent an armed run left, and saves them; a save that fails is logged and
407// the records are honoured all the same.
408func TestDeclinedBoxHonoursAReconciledIntent(t *testing.T) {
409 for _, canSave := range []bool{true, false} {
410 t.Run(map[bool]string{true: "saved", false: "not saved"}[canSave], func(t *testing.T) {
411 if !canSave && os.Geteuid() == 0 {
412 t.Skip("running as root: a read-only directory does not fail a save")
413 }
414 t.Setenv("CARLOS_OPT_DIR", t.TempDir())
415 svc, reg, dd := rolloutFixture(t, false)
416 acct := release.RouteAccount(managedRoute("x", ""))
417 leaveIntent(t, dd, "down.example.com", rollout.Intent{Account: acct, App: "blog", Build: "v2", Mode: "hold", ID: "a9"})
418 if !canSave {
419 if err := os.Chmod(dd, 0o500); err != nil {
420 t.Fatal(err)
421 }
422 t.Cleanup(func() { os.Chmod(dd, 0o700) })
423 }
424 declineWithBook(t, svc, reg, dd)
425 if heldHolds() == nil || svc.Planner == nil || rolloutUnavailable() != "" {
426 t.Fatalf("the reconciled record was not honoured (unavailable %q)", rolloutUnavailable())
427 }
428 if p := svc.Planner(context.Background(), managedRoute("down.example.com", "v2")); p.Refuse == "" {
429 t.Fatalf("plan %+v: want the held-down instance refused", p)
430 }
431 os.Chmod(dd, 0o700)
432 b, err := rollout.OpenFailureBook(dd)
433 if err != nil {
434 t.Fatal(err)
435 }
436 if saved := b.Reconciled() == 0; saved != canSave {
437 t.Fatalf("book saved %v, want %v", saved, canSave)
438 }
439 })
440 }
441}
442
443// Final review F6 (d): a declined box whose failure book cannot be read
444// fails closed, as refuseManaged does on an armed box: every managed wake is
445// refused with the fixed text, and the bounce legs stand down. The status
446// keeps the decline code and says why starts are refused. The book is not
447// rewritten.
448func TestDeclinedBoxWithAnUnreadableBookFailsClosed(t *testing.T) {
449 for name, body := range map[string]string{
450 "not json": "{not json",
451 "bad schema": `{"schema": 9}`,
452 } {
453 t.Run(name, func(t *testing.T) {
454 t.Setenv("CARLOS_OPT_DIR", t.TempDir())
455 svc, reg, dd := rolloutFixture(t, false)
456 if err := os.WriteFile(filepath.Join(dd, rollout.FailureFile), []byte(body), 0o644); err != nil {
457 t.Fatal(err)
458 }
459 before := snapshotBook(t, dd)
460 declineWithBook(t, svc, reg, dd)
461 if svc.Planner == nil {
462 t.Fatal("no Planner: an unreadable book let managed wakes run from the live link")
463 }
464 if p := svc.Planner(context.Background(), managedRoute("a.example.com", "v1")); p.Refuse != refuseTexts[rolloutBookUnreadable] {
465 t.Fatalf("managed wake planned %+v, want the fixed refusal %q", p, refuseTexts[rolloutBookUnreadable])
466 }
467 unit := managedRoute("u.example.com", "v1")
468 unit.Unit = "blog.service"
469 if p := svc.Planner(context.Background(), unit); !isTodaysWake(p) {
470 t.Fatalf("an unmanaged wake was changed: %+v", p)
471 }
472 if adoptBounceFn(svc) != nil || fastConvergeLeg(svc, reg) != nil {
473 t.Fatal("a bounce leg still stops instances whose next wake is refused")
474 }
475 if svc.OnDrift == nil || svc.CoalesceRecycle == nil || svc.CoalesceRecycle("bot.internal") {
476 t.Fatal("the activator's own moves onto the live link are still armed")
477 }
478 if svc.OnGen != nil || heldHolds() != nil {
479 t.Fatal("an unreadable book was honoured as records")
480 }
481 if r := rolloutStatusProbe(); r == nil || r.Unavailable != rolloutNoDelegation || r.FailureBook != rolloutBookUnreadable {
482 t.Fatalf("status = %+v, want the decline code %q and the book %q", r, rolloutNoDelegation, rolloutBookUnreadable)
483 }
484 before.unchanged(t, dd)
485 })
486 }
487}
488
489// liveLaunchBackend is fakeWakeBackend's lease and restore around a REAL
490// ExecBackend start, so a wake runs the production live-link spawn. before,
491// when set, runs between the plan and the start: root's repoint landing
492// there.
493type liveLaunchBackend struct {
494 fakeWakeBackend
495 eb *activator.ExecBackend
496 before func()
497}
498
499func (b *liveLaunchBackend) Start(ctx context.Context, rt registry.Route) (activator.Proc, string, error) {
500 if b.before != nil {
501 b.before()
502 }
503 return b.eb.Start(ctx, rt)
504}
505
506func (b *liveLaunchBackend) LiveVersion(host string) (string, error) { return b.eb.LiveVersion(host) }
507
508func (b *liveLaunchBackend) SetLiveExecGate(g func(registry.Route, string) error) {
509 b.eb.SetLiveExecGate(g)
510}
511
512// Task 9 (Astra R1 #6, the R2/R3 declined residuals; plan review #6), brief
513// (a): the plan-time refusal is repeated at the real launch. The wake plans
514// with the live link on the good build v1, which no record holds; root
515// repoints the link at v2, which a record holds for this instance, before
516// the start. The start resolves the link, asks the gate about v2, and execs
517// nothing.
518func TestDeclinedBoxRefusesAHeldBuildAtLaunch(t *testing.T) {
519 prev := activator.GenerationsEnabled()
520 activator.SetGenerations(false)
521 t.Cleanup(func() { activator.SetGenerations(prev) })
522 resetRollout(t)
523 env := newFastFixture(t)
524 t.Setenv("CARLOS_OPT_DIR", env.optDir)
525 host := "warm.test"
526 run := t.TempDir()
527 rt := registry.Route{Host: host, Kind: "instance", App: "fastapp", Account: "bab", Channel: "edge",
528 Status: "ready", Hibernate: true, Version: "v1", GoodVersion: "v1", GoodDigest: "sha256:v1",
529 Socket: filepath.Join(run, host+".sock"), DBPath: filepath.Join(env.dataDir, host+".db")}
530 if err := env.reg.Upsert(rt); err != nil {
531 t.Fatal(err)
532 }
533 env.stageLive(t, host, "v1")
534 mark := filepath.Join(t.TempDir(), "v2.ran")
535 if err := release.StageArtifact(env.optDir, "bab", "v2", "fastapp", []byte("#!/bin/sh\necho ran > '"+mark+"'\nexec sleep 30\n")); err != nil {
536 t.Fatal(err)
537 }
538 writeFailureBook(t, env.dataDir, []rollout.FailureRecord{{Account: release.RouteAccount(rt), App: "fastapp", Build: "v2", Host: "sibling.test"}}, nil)
539 eb := &activator.ExecBackend{Cfg: activator.Config{Bucket: "test-bucket", Idle: time.Hour},
540 OptDir: env.optDir, DataDir: t.TempDir(), RunDirOverride: run}
541 backend := &liveLaunchBackend{eb: eb}
542 svc := activator.New(activator.Config{Bucket: "test-bucket", Idle: time.Hour}, backend, env.reg)
543 svc.Routes = env.reg
544 t.Cleanup(func() {
545 sctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
546 defer cancel()
547 svc.Shutdown(sctx)
548 })
549 declineWithBook(t, svc, env.reg, env.dataDir)
550 if eb.LiveExecGate == nil {
551 t.Fatal("no LiveExecGate: the launch execs whatever the live link names by then")
552 }
553 if p := svc.Planner(context.Background(), rt); !isTodaysWake(p) {
554 t.Fatalf("plan with the link on v1 = %+v, want today's wake", p)
555 }
556 backend.before = func() {
557 if err := release.PointLive(env.optDir, host, "bab", "v2", "fastapp"); err != nil {
558 t.Error(err)
559 }
560 }
561 rec := httptest.NewRecorder()
562 proxy := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) })
563 if !svc.ServeHTTP(rec, httptest.NewRequest("GET", "http://"+host+"/", nil), rt, proxy) {
564 t.Fatal("the activator did not take the request")
565 }
566 if rec.Code == http.StatusNoContent {
567 t.Fatal("the wake onto held v2 was served")
568 }
569 time.Sleep(200 * time.Millisecond)
570 if _, err := os.Stat(mark); err == nil {
571 t.Fatal("held v2 was exec'd: the gate at the launch did not see the repoint")
572 }
573}
574
575// Task 9 rule 2 and Correction 3: a declined box honours hold-ran markers
576// as the armed planner does (decideHoldRan, planCheck), refusing where the
577// armed box would refuse. A book holding only a marker is honoured. The
578// gate at the launch answers as the plan does.
579func TestDeclinedBoxHonoursHoldRanMarkers(t *testing.T) {
580 held := rollout.HeldDownRefusal("v2")
581 route := func(host, installed, good string) registry.Route {
582 rt := managedRoute(host, installed)
583 if good != "" {
584 rt.GoodVersion, rt.GoodDigest = good, "sha256:"+good
585 }
586 return rt
587 }
588 acct := release.RouteAccount(managedRoute("x", ""))
589 mark := func(t *testing.T, dd string, marks map[string]string) {
590 t.Helper()
591 b, err := rollout.OpenFailureBook(dd)
592 if err != nil {
593 t.Fatal(err)
594 }
595 for host, build := range marks {
596 app := "blog"
597 if host == "other.example.com" {
598 app = "shop" // another app's marker under the same host name
599 }
600 if err := b.MarkHoldRan(host, rollout.HoldRan{Account: acct, App: app, Build: build}); err != nil {
601 t.Fatal(err)
602 }
603 }
604 }
605 type want struct {
606 rt registry.Route
607 refuse string
608 }
609 check := func(t *testing.T, svc *activator.Service, cases []want) {
610 t.Helper()
611 h := heldHolds()
612 for _, tc := range cases {
613 p := svc.Planner(context.Background(), tc.rt)
614 if tc.refuse == "" && !isTodaysWake(p) {
615 t.Errorf("%s (installed %s, good %s): planned %+v, want today's wake", tc.rt.Host, tc.rt.Version, tc.rt.GoodVersion, p)
616 }
617 if tc.refuse != "" && p.Refuse != tc.refuse {
618 t.Errorf("%s (installed %s, good %s): refusal %q, want %q", tc.rt.Host, tc.rt.Version, tc.rt.GoodVersion, p.Refuse, tc.refuse)
619 }
620 got := ""
621 if err := h.gate(tc.rt, tc.rt.Version); err != nil {
622 got = err.Error()
623 }
624 if got != tc.refuse {
625 t.Errorf("%s (installed %s, good %s): gate %q, want %q", tc.rt.Host, tc.rt.Version, tc.rt.GoodVersion, got, tc.refuse)
626 }
627 }
628 }
629 t.Run("only a marker", func(t *testing.T) {
630 t.Setenv("CARLOS_OPT_DIR", t.TempDir())
631 svc, reg, dd := rolloutFixture(t, false)
632 mark(t, dd, map[string]string{"m.example.com": "v2", "mg.example.com": "v1", "other.example.com": "v2"})
633 before := snapshotBook(t, dd)
634 declineWithBook(t, svc, reg, dd)
635 if heldHolds() == nil || svc.Planner == nil {
636 t.Fatal("a book holding only a marker was not honoured")
637 }
638 if r := rolloutStatusProbe(); r == nil || r.FailureBook != rolloutBookHonoured {
639 t.Fatalf("status = %+v, want the book %q", r, rolloutBookHonoured)
640 }
641 check(t, svc, []want{
642 // The installed good build is refused while a marker of another
643 // build stands: only a signed rollback clears it (Task 10).
644 {route("m.example.com", "v1", "v1"), held},
645 // The marked build starts: no record holds it.
646 {route("m.example.com", "v2", "v1"), ""},
647 // Any other build starts.
648 {route("m.example.com", "v3", "v1"), ""},
649 // With no good build there is nothing to refuse in its favour.
650 {route("m.example.com", "v2", ""), ""},
651 // A marker naming the installed good build itself lets it start.
652 {route("mg.example.com", "v1", "v1"), ""},
653 // Another app's marker under the host's name says nothing here.
654 {route("other.example.com", "v1", "v1"), ""},
655 {route("free.example.com", "v1", "v1"), ""},
656 })
657 before.unchanged(t, dd)
658 })
659 t.Run("a marker and records", func(t *testing.T) {
660 t.Setenv("CARLOS_OPT_DIR", t.TempDir())
661 svc, reg, dd := rolloutFixture(t, false)
662 writeFailureBook(t, dd, []rollout.FailureRecord{
663 {Account: acct, App: "blog", Build: "v1", Host: "sib.example.com"},
664 {Account: acct, App: "blog", Build: "v2", Host: "sib.example.com"},
665 {Account: acct, App: "blog", Build: "v3", Host: "sib.example.com"},
666 {Account: acct, App: "blog", Build: "v4", Host: "sib.example.com", HeldDown: []string{"m.example.com"}},
667 }, nil)
668 mark(t, dd, map[string]string{"m.example.com": "v2", "mg.example.com": "v1"})
669 declineWithBook(t, svc, reg, dd)
670 check(t, svc, []want{
671 // The marked build, blocked by a record while the route has a good
672 // build.
673 {route("m.example.com", "v2", "v1"), held},
674 // The marked build that is the installed good build: served, so a
675 // sibling's record does not hold it.
676 {route("mg.example.com", "v1", "v1"), ""},
677 // With no good build a record alone refuses nothing.
678 {route("m.example.com", "v2", ""), ""},
679 // Any other build a record blocks.
680 {route("m.example.com", "v3", "v1"), held},
681 // A hold-down still comes first, in its own build's words.
682 {route("m.example.com", "v4", "v1"), rollout.HeldDownRefusal("v4")},
683 {route("m.example.com", "v5", "v1"), ""},
684 })
685 })
686}
687
688// Task 9: a claims-backed wake on a declined box is gated against the build
689// its launch snapshot resolved from the live link, before anything is asked
690// of root. A box that honours nothing is today's.
691func TestDeclinedBoxGatesAClaimsBackedWake(t *testing.T) {
692 f := newAdminTokenFixture(t, "box-1")
693 if err := os.WriteFile(f.envPath(), []byte("CARLOS_ADMIN_TOKEN=tok-0123456789abcdef0123456789abcdef\n"), 0o600); err != nil {
694 t.Fatal(err)
695 }
696 rt := f.rt
697 rt.Kind, rt.Status, rt.Channel, rt.Account, rt.App, rt.Hibernate, rt.Version = "instance", "ready", "edge", "bab", "jam", true, "v1"
698 rt.GoodVersion, rt.GoodDigest = "v0", "sha256:v0"
699 launch := claimServingLaunchSource(f.backend, f.tokens)
700 if _, err := launch(t.Context(), rt); err != nil {
701 t.Fatalf("launch with nothing honoured = %v, want today's", err)
702 }
703 svc, reg, dd := rolloutFixture(t, false)
704 writeFailureBook(t, dd, []rollout.FailureRecord{{Account: release.RouteAccount(rt), App: "jam", Build: "v1", Host: "sib.example.com"}}, nil)
705 declineWithBook(t, svc, reg, dd)
706 if _, err := launch(t.Context(), rt); err == nil || err.Error() != refuseNoSwaps {
707 t.Fatalf("launch of held v1 = %v, want %q", err, refuseNoSwaps)
708 }
709 served := rt
710 served.GoodVersion, served.GoodDigest = "v1", "sha256:v1"
711 if _, err := launch(t.Context(), served); err != nil {
712 t.Fatalf("launch of the instance's own good build = %v, want it allowed", err)
713 }
714}
715
716// Task 9 (brief (e)): with no records and no markers, or the flag off,
717// LiveExecGate stays nil, so every spawn is today's.
718func TestNoLiveExecGateUnlessADeclinedBoxHonoursSomething(t *testing.T) {
719 for name, env := range map[string]func(string) string{"flag off": noEnv, "declined, empty book": nil} {
720 t.Run(name, func(t *testing.T) {
721 t.Setenv("CARLOS_OPT_DIR", t.TempDir())
722 eb := &activator.ExecBackend{}
723 svc, reg, dd := rolloutFixtureWith(t, false, &activator.SplitBackend{Exec: eb})
724 if env == nil {
725 declineWithBook(t, svc, reg, dd)
726 } else {
727 armRollout(context.Background(), env, svc, reg, dd, nil, unusedStore)
728 }
729 if eb.LiveExecGate != nil {
730 t.Fatal("LiveExecGate set on a box that honours nothing")
731 }
732 legsAreTodays(t, svc, reg)
733 })
734 }
735 t.Run("declined, a record", func(t *testing.T) {
736 t.Setenv("CARLOS_OPT_DIR", t.TempDir())
737 eb := &activator.ExecBackend{}
738 svc, reg, dd := rolloutFixtureWith(t, false, &activator.SplitBackend{Exec: eb})
739 writeFailureBook(t, dd, []rollout.FailureRecord{{Account: "a", App: "blog", Build: "v2", Host: "sib.example.com"}}, nil)
740 declineWithBook(t, svc, reg, dd)
741 if eb.LiveExecGate == nil {
742 t.Fatal("a declined box honouring a record left LiveExecGate nil")
743 }
744 })
745}
746
747// Task 9 fix round 1 (ruling 4a): the marked-build rule holds whichever of
748// the stamp and the live link names the marked build. The live link names
749// the marked good build v1 while the stamp says v3 (root repointing): that
750// is the build the marker says may run, and the instance served it, so a
751// sibling's record does not hold it either.
752func TestDeclinedBoxLetsTheMarkedGoodBuildStartFromTheLiveLink(t *testing.T) {
753 optDir := t.TempDir()
754 t.Setenv("CARLOS_OPT_DIR", optDir)
755 svc, reg, dd := rolloutFixture(t, false)
756 acct := release.RouteAccount(managedRoute("x", ""))
757 writeFailureBook(t, dd, []rollout.FailureRecord{{Account: acct, App: "blog", Build: "v1", Host: "sib.example.com"}}, nil)
758 b, err := rollout.OpenFailureBook(dd)
759 if err != nil {
760 t.Fatal(err)
761 }
762 if err := b.MarkHoldRan("mg.example.com", rollout.HoldRan{Account: acct, App: "blog", Build: "v1"}); err != nil {
763 t.Fatal(err)
764 }
765 if err := release.StageArtifact(optDir, acct, "v1", "blog", []byte("v1")); err != nil {
766 t.Fatal(err)
767 }
768 if err := release.PointLive(optDir, "mg.example.com", acct, "v1", "blog"); err != nil {
769 t.Fatal(err)
770 }
771 declineWithBook(t, svc, reg, dd)
772 rt := managedRoute("mg.example.com", "v3")
773 rt.GoodVersion, rt.GoodDigest = "v1", "sha256:v1"
774 if p := svc.Planner(context.Background(), rt); !isTodaysWake(p) {
775 t.Fatalf("plan with the link on the marked good build = %+v, want today's wake", p)
776 }
777 if err := heldHolds().gate(rt, "v1"); err != nil {
778 t.Fatalf("gate refused the marked good build the link names: %v", err)
779 }
780}
781
782// Task 9 fix round 1 (ruling 4c): a claims move launch (a moves probe or
783// landing) on a declined box is gated as a claims-backed wake is, after its
784// snapshot resolves the link.
785func TestDeclinedBoxGatesAClaimsMoveLaunch(t *testing.T) {
786 f := newAdminTokenFixture(t, "box-1")
787 if err := os.WriteFile(f.envPath(), []byte("CARLOS_ADMIN_TOKEN=tok-0123456789abcdef0123456789abcdef\n"), 0o600); err != nil {
788 t.Fatal(err)
789 }
790 t.Setenv("CARLOS_SECRETS_KEY", base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{7}, 32)))
791 rt := f.rt
792 rt.Kind, rt.Status, rt.Channel, rt.Account, rt.App, rt.Hibernate, rt.Version = "instance", "ready", "edge", "bab", "jam", true, "v1"
793 rt.GoodVersion, rt.GoodDigest = "v0", "sha256:v0"
794 launch := claimMoveLaunchSource(f.backend)
795 if _, _, err := launch(t.Context(), rt); err != nil {
796 t.Fatalf("move launch with nothing honoured = %v, want today's", err)
797 }
798 svc, reg, dd := rolloutFixture(t, false)
799 writeFailureBook(t, dd, []rollout.FailureRecord{{Account: release.RouteAccount(rt), App: "jam", Build: "v1", Host: "sib.example.com"}}, nil)
800 declineWithBook(t, svc, reg, dd)
801 if _, _, err := launch(t.Context(), rt); err == nil || err.Error() != refuseNoSwaps {
802 t.Fatalf("move launch of held v1 = %v, want %q", err, refuseNoSwaps)
803 }
804 served := rt
805 served.GoodVersion, served.GoodDigest = "v1", "sha256:v1"
806 if _, _, err := launch(t.Context(), served); err != nil {
807 t.Fatalf("move launch of the instance's own good build = %v, want it allowed", err)
808 }
809}
810
811// heldRouteIn upserts a managed, hibernating instance of bab/blog into reg,
812// installed on installed with good as its good build.
813func heldRouteIn(t *testing.T, reg *registry.Registry, host, installed, good string) registry.Route {
814 t.Helper()
815 rt := managedRoute(host, installed)
816 if good != "" {
817 rt.GoodVersion, rt.GoodDigest = good, "sha256:"+good
818 }
819 if err := reg.Upsert(rt); err != nil {
820 t.Fatal(err)
821 }
822 return rt
823}
824
825// declinedHostReport is rolloutStatusProbe's entry for host, failing the
826// test when the report is not a declined one.
827func declinedHostReport(t *testing.T, host string) (status.HostRollout, bool) {
828 t.Helper()
829 r := rolloutStatusProbe()
830 if r == nil || r.Unavailable != rolloutNoDelegation || r.FailureBook != rolloutBookHonoured {
831 t.Fatalf("status = %+v, want the decline code %q and the book %q", r, rolloutNoDelegation, rolloutBookHonoured)
832 }
833 hr, ok := r.Hosts[host]
834 return hr, ok
835}
836
837// Task 10 (ruling 194, plan review #5): a declined box publishes its holds,
838// with the build and its restriction's epoch, and processes a verified
839// signed retry against the book it honours. A grant for another build, or
840// for an epoch the restriction has since moved past, clears nothing; the
841// current grant clears the hold-down in memory, where the planner and the
842// launch gate see it at once, and on disk.
843func TestDeclinedBoxProcessesAVerifiedSignedRetry(t *testing.T) {
844 t.Setenv("CARLOS_OPT_DIR", t.TempDir())
845 svc, reg, dd := rolloutFixture(t, false)
846 const host = "h.example.com"
847 rt := heldRouteIn(t, reg, host, "v2", "v1")
848 acct := release.RouteAccount(rt)
849 writeFailureBook(t, dd, []rollout.FailureRecord{{
850 Account: acct, App: "blog", Build: "v2", Host: "sib.example.com", Mode: "hold", Reason: "health",
851 }}, nil)
852 b, err := rollout.OpenFailureBook(dd)
853 if err != nil {
854 t.Fatal(err)
855 }
856 stale := b.RestrictionEpoch(acct, "blog", "v2", host)
857 // The armed run saw v2 fail again, holding this host down, after a
858 // grant for the first restriction could have been signed.
859 if _, err := b.Record(rollout.FailureRecord{Account: acct, App: "blog", Build: "v2", Host: "sib2.example.com", Mode: "hold", Reason: "health", HeldDown: []string{host}}); err != nil {
860 t.Fatal(err)
861 }
862 epoch := b.RestrictionEpoch(acct, "blog", "v2", host)
863 if epoch == 0 || epoch == stale {
864 t.Fatalf("epochs %d then %d: the second failure did not move the restriction", stale, epoch)
865 }
866 declineWithBook(t, svc, reg, dd)
867 ctx := context.Background()
868 held := rollout.HeldDownRefusal("v2")
869 if p := svc.Planner(ctx, rt); p.Refuse != held {
870 t.Fatalf("plan before the retry = %+v, want %q", p, held)
871 }
872 hr, ok := declinedHostReport(t, host)
873 if !ok || hr.Account != acct || hr.App != "blog" || hr.Installed != "v2" || hr.HeldBy != "v2" || hr.HeldEpoch != epoch || hr.Attempt != status.AttemptDown ||
874 hr.HeldBook == "" || hr.HeldBook != heldHolds().book.ID() || hr.RetryV != status.HostRetryV {
875 t.Fatalf("published %+v (%v), want v2 holding %s down at epoch %d of the box's book", hr, ok, host, epoch)
876 }
877
878 mem, priv := retryMemory(t)
879 mem.retryHost = rolloutRetryHost()
880 if mem.retryHost == nil {
881 t.Fatal("a declined box honouring its book has no retry path")
882 }
883 _, s := genKit(t)
884 routes, err := reg.List()
885 if err != nil {
886 t.Fatal(err)
887 }
888 bounce := func(h string) bool { t.Errorf("%s was bounced for a retry", h); return true }
889 restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) // baseline
890 askRetry(t, ctx, s, priv, acct, "blog", host, "v9", hr.HeldBook, epoch)
891 askRetry(t, ctx, s, priv, acct, "blog", host, "v2", hr.HeldBook, stale)
892 restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem)
893 if p := svc.Planner(ctx, rt); p.Refuse != held {
894 t.Fatalf("plan after a grant for another build and a stale grant = %+v, want %q still", p, held)
895 }
896 askRetry(t, ctx, s, priv, acct, "blog", host, "v2", hr.HeldBook, epoch)
897 restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem)
898 if p := svc.Planner(ctx, rt); !isTodaysWake(p) {
899 t.Fatalf("plan after the signed retry = %+v, want today's wake onto v2", p)
900 }
901 if err := heldHolds().gate(rt, "v2"); err != nil {
902 t.Fatalf("the launch gate still refuses v2 after the retry: %v", err)
903 }
904 if hr, ok := declinedHostReport(t, host); ok {
905 t.Fatalf("published %+v after the retry, want no hold", hr)
906 }
907 b, err = rollout.OpenFailureBook(dd)
908 if err != nil {
909 t.Fatal(err)
910 }
911 if b.HeldDown(acct, "blog", "v2", host) || b.Blocks(acct, "blog", "v2", host, false) {
912 t.Fatal("the retry was not saved: the book on disk still holds the host")
913 }
914}
915
916// writePreRuleBook writes dd's failure book as an agent before the
917// crash-epoch hard limit left it (readiness round 2, Task 1): schema 2 with
918// a book identity and per-record epochs, no generations and no
919// durable_epochs. v2's record holds host down.
920func writePreRuleBook(t *testing.T, dd, acct, host string) {
921 t.Helper()
922 raw, err := json.Marshal(map[string]any{
923 "schema": 2,
924 "book": "old",
925 "seq": 10,
926 "records": []map[string]any{{"account": acct, "app": "blog", "build": "v2", "host": "sib.example.com",
927 "mode": "hold", "reason": "health", "held_down": []string{host}, "epochs": map[string]uint64{"": 11, host: 11}}},
928 })
929 if err != nil {
930 t.Fatal(err)
931 }
932 if err := os.WriteFile(filepath.Join(dd, rollout.FailureFile), raw, 0o644); err != nil {
933 t.Fatal(err)
934 }
935}
936
937// Readiness round 2, Task 1 (plan 10): a declined box saves a book that
938// owes a save at startup, whatever reconciliation found (here a book an
939// earlier agent wrote, whose identity the open rotates), and publishes its
940// generation at once. When that save fails it publishes the hold with no
941// generation, and the next periodic pass, with no pointer or grant
942// arriving, saves the book once the disk recovers and publishes it.
943func TestDeclinedBoxPublishesAGenerationOnceItsBookIsSaved(t *testing.T) {
944 for _, diskFails := range []bool{false, true} {
945 t.Run(map[bool]string{false: "saved at startup", true: "saved by the periodic pass"}[diskFails], func(t *testing.T) {
946 if diskFails && os.Geteuid() == 0 {
947 t.Skip("running as root: a read-only directory does not fail a save")
948 }
949 t.Setenv("CARLOS_OPT_DIR", t.TempDir())
950 svc, reg, dd := rolloutFixture(t, false)
951 const host = "h.example.com"
952 rt := heldRouteIn(t, reg, host, "v2", "v1")
953 acct := release.RouteAccount(rt)
954 writePreRuleBook(t, dd, acct, host)
955 if diskFails {
956 if err := os.Chmod(dd, 0o500); err != nil {
957 t.Fatal(err)
958 }
959 t.Cleanup(func() { os.Chmod(dd, 0o700) })
960 }
961 declineWithBook(t, svc, reg, dd)
962 hr, ok := declinedHostReport(t, host)
963 if !ok || hr.HeldBy != "v2" || hr.Attempt != status.AttemptDown || hr.HeldBook == "old" || hr.HeldBook == "" {
964 t.Fatalf("published %+v (%v), want v2 holding %s down under a new book identity", hr, ok, host)
965 }
966 if diskFails {
967 if hr.HeldEpoch != 0 {
968 t.Fatalf("published generation %d while the book owes a save, want 0", hr.HeldEpoch)
969 }
970 if err := os.Chmod(dd, 0o700); err != nil {
971 t.Fatal(err)
972 }
973 if hr, _ := declinedHostReport(t, host); hr.HeldEpoch != 0 {
974 t.Fatalf("published generation %d before any save landed, want 0", hr.HeldEpoch)
975 }
976 p := &pointerReads{ptrs: map[string]*release.ChannelPointer{}, errs: map[string]error{"blog/stable": errors.New("no pointer")}}
977 heldHolds().pass(context.Background(), p.read)
978 hr, _ = declinedHostReport(t, host)
979 }
980 if hr.HeldEpoch == 0 {
981 t.Fatalf("published %+v: want a generation once the book is saved", hr)
982 }
983 b, err := rollout.OpenFailureBook(dd)
984 if err != nil {
985 t.Fatal(err)
986 }
987 if b.ID() != hr.HeldBook || b.PublishedEpoch(acct, "blog", "v2", host) != hr.HeldEpoch {
988 t.Fatalf("the book on disk is %q at %d, want the published %q at %d",
989 b.ID(), b.PublishedEpoch(acct, "blog", "v2", host), hr.HeldBook, hr.HeldEpoch)
990 }
991 })
992 }
993}
994
995// pointerReads is a fake signed pointer read keyed "<app>/<channel>", which
996// counts the reads it is asked for.
997type pointerReads struct {
998 ptrs map[string]*release.ChannelPointer
999 errs map[string]error
1000 reads []string
1001}
1002
1003func (p *pointerReads) read(_ context.Context, _, app, channel string) (*release.ChannelPointer, error) {
1004 k := app + "/" + channel
1005 p.reads = append(p.reads, k)
1006 if err := p.errs[k]; err != nil {
1007 return nil, err
1008 }
1009 return p.ptrs[k], nil
1010}
1011
1012// Task 10 rule 2 (`carlos promote --retry`): the declined box's pointer
1013// pass clears a record when its channel's signed pointer names the record's
1014// build with a newer PromotedAt. The pointer the build failed under, a
1015// pointer naming another build and a failed read clear nothing; an app with
1016// nothing to clear costs no read.
1017func TestDeclinedBoxPointerPassClearsARecordOnPromoteRetry(t *testing.T) {
1018 t.Setenv("CARLOS_OPT_DIR", t.TempDir())
1019 svc, reg, dd := rolloutFixture(t, false)
1020 rt := heldRouteIn(t, reg, "h.example.com", "v2", "v1")
1021 other := execRoute("s.example.com", "bab", "shop")
1022 other.Hibernate, other.Version, other.GoodVersion, other.GoodDigest = true, "z2", "z1", "sha256:z1"
1023 if err := reg.Upsert(other); err != nil {
1024 t.Fatal(err)
1025 }
1026 acct := release.RouteAccount(rt)
1027 failedUnder := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC)
1028 writeFailureBook(t, dd, []rollout.FailureRecord{{
1029 Account: acct, App: "blog", Build: "v2", Host: "sib.example.com", Reason: "health", PromotedAt: failedUnder,
1030 }}, nil)
1031 declineWithBook(t, svc, reg, dd)
1032 ctx := context.Background()
1033 routes, err := reg.List()
1034 if err != nil {
1035 t.Fatal(err)
1036 }
1037 h := heldHolds()
1038 stillHeld := func(when string) {
1039 t.Helper()
1040 if p := svc.Planner(ctx, rt); p.Refuse != refuseNoSwaps {
1041 t.Fatalf("%s: plan = %+v, want %q", when, p, refuseNoSwaps)
1042 }
1043 }
1044 stillHeld("before any pointer")
1045 if hr, ok := declinedHostReport(t, rt.Host); !ok || hr.HeldBy != "v2" || hr.Attempt != status.AttemptFailed || hr.HeldEpoch == 0 {
1046 t.Fatalf("published %+v (%v), want v2 holding it as failed, with an epoch", hr, ok)
1047 }
1048
1049 p := &pointerReads{ptrs: map[string]*release.ChannelPointer{}, errs: map[string]error{}}
1050 p.errs["blog/stable"] = errors.New("bucket blip")
1051 h.applyPointers(ctx, p.read, routes)
1052 stillHeld("after a failed read")
1053 delete(p.errs, "blog/stable")
1054 p.ptrs["blog/stable"] = &release.ChannelPointer{App: "blog", Channel: "stable", Version: "v2", PromotedAt: failedUnder}
1055 h.applyPointers(ctx, p.read, routes)
1056 stillHeld("after the pointer v2 failed under")
1057 p.ptrs["blog/stable"] = &release.ChannelPointer{App: "blog", Channel: "stable", Version: "v3", PromotedAt: failedUnder.Add(time.Hour)}
1058 h.applyPointers(ctx, p.read, routes)
1059 stillHeld("after a pointer naming another build")
1060 p.ptrs["blog/stable"] = &release.ChannelPointer{App: "blog", Channel: "stable", Version: "v2", PromotedAt: failedUnder.Add(time.Minute)}
1061 h.applyPointers(ctx, p.read, routes)
1062 if pl := svc.Planner(ctx, rt); !isTodaysWake(pl) {
1063 t.Fatalf("plan after promote --retry = %+v, want today's wake onto v2", pl)
1064 }
1065 for _, r := range p.reads {
1066 if strings.HasPrefix(r, "shop/") {
1067 t.Fatalf("reads %v: an app with nothing to clear was read", p.reads)
1068 }
1069 }
1070 b, err := rollout.OpenFailureBook(dd)
1071 if err != nil {
1072 t.Fatal(err)
1073 }
1074 if _, ok := b.Lookup(acct, "blog", "v2"); ok {
1075 t.Fatal("the cleared record was not saved")
1076 }
1077}
1078
1079// Task 10 Correction 3 (Task 3's rule, ruling 205): the pointer pass also
1080// clears a hold-ran marker on a signed rollback, for an app with markers and
1081// no records: the route installs its good build and the channel's pointer
1082// names that build with a PromotedAt after the marker's At. The planner then
1083// lets the good build start. An older pointer, and a route not installing
1084// its good build, clear nothing.
1085func TestDeclinedBoxPointerPassClearsAMarkerOnASignedRollback(t *testing.T) {
1086 t.Setenv("CARLOS_OPT_DIR", t.TempDir())
1087 svc, reg, dd := rolloutFixture(t, false)
1088 rolled := heldRouteIn(t, reg, "m.example.com", "v1", "v1")
1089 notRolled := heldRouteIn(t, reg, "n.example.com", "v3", "v1")
1090 acct := release.RouteAccount(rolled)
1091 ranAt := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC)
1092 b, err := rollout.OpenFailureBook(dd)
1093 if err != nil {
1094 t.Fatal(err)
1095 }
1096 for _, host := range []string{rolled.Host, notRolled.Host} {
1097 if err := b.MarkHoldRan(host, rollout.HoldRan{Account: acct, App: "blog", Build: "v2", At: ranAt}); err != nil {
1098 t.Fatal(err)
1099 }
1100 }
1101 epoch := b.RestrictionEpoch(acct, "blog", "v2", rolled.Host)
1102 declineWithBook(t, svc, reg, dd)
1103 ctx := context.Background()
1104 routes, err := reg.List()
1105 if err != nil {
1106 t.Fatal(err)
1107 }
1108 held := rollout.HeldDownRefusal("v2")
1109 if p := svc.Planner(ctx, rolled); p.Refuse != held {
1110 t.Fatalf("plan before the rollback = %+v, want %q", p, held)
1111 }
1112 if hr, ok := declinedHostReport(t, rolled.Host); !ok || hr.HeldBy != "v2" || hr.HeldEpoch != epoch || hr.Attempt != status.AttemptDown {
1113 t.Fatalf("published %+v (%v), want the marker's v2 holding the good build down at epoch %d", hr, ok, epoch)
1114 }
1115
1116 h := heldHolds()
1117 p := &pointerReads{ptrs: map[string]*release.ChannelPointer{}}
1118 p.ptrs["blog/stable"] = &release.ChannelPointer{App: "blog", Channel: "stable", Version: "v1", PromotedAt: ranAt.Add(-time.Minute)}
1119 h.applyPointers(ctx, p.read, routes)
1120 if len(p.reads) == 0 {
1121 t.Fatal("an app with only markers was not read")
1122 }
1123 if pl := svc.Planner(ctx, rolled); pl.Refuse != held {
1124 t.Fatalf("plan after a pointer older than the marker = %+v, want %q still", pl, held)
1125 }
1126 p.ptrs["blog/stable"] = &release.ChannelPointer{App: "blog", Channel: "stable", Version: "v1", PromotedAt: ranAt.Add(time.Minute)}
1127 h.applyPointers(ctx, p.read, routes)
1128 if pl := svc.Planner(ctx, rolled); !isTodaysWake(pl) {
1129 t.Fatalf("plan after the signed rollback = %+v, want today's wake onto the good build", pl)
1130 }
1131 if hr, ok := declinedHostReport(t, rolled.Host); ok {
1132 t.Fatalf("published %+v after the rollback, want no hold", hr)
1133 }
1134 b, err = rollout.OpenFailureBook(dd)
1135 if err != nil {
1136 t.Fatal(err)
1137 }
1138 if _, ok := b.HoldRanOn(rolled.Host); ok {
1139 t.Fatal("the cleared marker was not saved")
1140 }
1141 if _, ok := b.HoldRanOn(notRolled.Host); !ok {
1142 t.Fatal("a marker on a route not installing its good build was cleared")
1143 }
1144}
1145
1146// retryConsole is a console with a root key over store, its ops account
1147// owning app, with a CLI token for its owner: the shape the root package's
1148// e2e tests log in through (e2eDeviceLogin).
1149func retryConsole(t *testing.T, store release.Store, app, host string) (base, token string, pub ed25519.PublicKey) {
1150 t.Helper()
1151 seed, pubB64, err := release.GenerateKey()
1152 if err != nil {
1153 t.Fatal(err)
1154 }
1155 priv, err := release.ParseSigningKey(seed, "test")
1156 if err != nil {
1157 t.Fatal(err)
1158 }
1159 if pub, err = release.ParsePublicKey(pubB64, "test"); err != nil {
1160 t.Fatal(err)
1161 }
1162 dir := t.TempDir()
1163 cst, err := console.Open(filepath.Join(dir, "console.db"))
1164 if err != nil {
1165 t.Fatal(err)
1166 }
1167 t.Cleanup(func() { cst.Close() })
1168 var srv *console.Server
1169 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv.ServeHTTP(w, r) }))
1170 t.Cleanup(ts.Close)
1171 if srv, err = console.NewServer(console.Config{Origin: ts.URL, Keymail: "http://keymail.invalid",
1172 Registry: filepath.Join(dir, "registry.db"), Releases: store, ReleasePub: pub, Key: priv, AppScheme: "http"}, cst); err != nil {
1173 t.Fatal(err)
1174 }
1175 const addr = "paul@keymail.dev"
1176 now := time.Now()
1177 if err := cst.AddMember(console.OpsAccountID, addr, console.RoleOwner, now); err != nil {
1178 t.Fatal(err)
1179 }
1180 if _, err := cst.CreateAppIn(console.OpsAccountID, app, addr, now); err != nil {
1181 t.Fatal(err)
1182 }
1183 if err := provision.CreateRecord(context.Background(), store, provision.Record{Schema: 1, Host: host, Slug: strings.SplitN(host, ".", 2)[0],
1184 App: app, Account: release.OpsAccount, Region: "ie", Channel: "stable", CreatedAt: now.UTC(), CreatedBy: addr}); err != nil {
1185 t.Fatal(err)
1186 }
1187 post := func(u string, body any, cookie string, form url.Values) map[string]any {
1188 t.Helper()
1189 var req *http.Request
1190 if form != nil {
1191 req, _ = http.NewRequest(http.MethodPost, u, strings.NewReader(form.Encode()))
1192 req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
1193 req.Header.Set("Cookie", "carlos_console_session="+cookie)
1194 } else {
1195 raw, _ := json.Marshal(body)
1196 req, _ = http.NewRequest(http.MethodPost, u, bytes.NewReader(raw))
1197 req.Header.Set("Content-Type", "application/json")
1198 }
1199 res, err := http.DefaultClient.Do(req)
1200 if err != nil {
1201 t.Fatal(err)
1202 }
1203 defer res.Body.Close()
1204 raw, _ := io.ReadAll(res.Body)
1205 if res.StatusCode != http.StatusOK {
1206 t.Fatalf("POST %s = %d %s", u, res.StatusCode, raw)
1207 }
1208 out := map[string]any{}
1209 _ = json.Unmarshal(raw, &out)
1210 return out
1211 }
1212 start := post(ts.URL+"/auth/cli/start", map[string]string{"label": "test@harness"}, "", nil)
1213 session, err := cst.CreateSession(addr, "", now)
1214 if err != nil {
1215 t.Fatal(err)
1216 }
1217 if err := cst.RecordStepUp(session, addr, now, now); err != nil {
1218 t.Fatal(err)
1219 }
1220 post(ts.URL+"/auth/cli/approve", nil, session, url.Values{"code": {start["user_code"].(string)}, "action": {"approve"}, "csrf": {session}})
1221 poll := post(ts.URL+"/auth/cli/poll", map[string]string{"device_code": start["device_code"].(string)}, "", nil)
1222 token, _ = poll["token"].(string)
1223 if token == "" {
1224 t.Fatalf("no CLI token: %v", poll)
1225 }
1226 return ts.URL, token, pub
1227}
1228
1229// Task 10 Correction 1 (plan review #5), from the API to the box: the
1230// declined box publishes its held host, the console's retry door signs a
1231// grant for it from that report, and the declined box's restart leg
1232// verifies the grant and clears the hold, so the next wake is today's.
1233func TestDeclinedBoxRetryFromTheConsoleDoorToTheRestartLeg(t *testing.T) {
1234 t.Setenv("CARLOS_OPT_DIR", t.TempDir())
1235 svc, reg, dd := rolloutFixture(t, false)
1236 const host = "h.blog.example.com"
1237 rt := heldRouteIn(t, reg, host, "v2", "v1")
1238 acct := release.RouteAccount(rt)
1239 writeFailureBook(t, dd, []rollout.FailureRecord{{
1240 Account: acct, App: "blog", Build: "v2", Host: "sib.example.com", Mode: "hold", Reason: "health", HeldDown: []string{host},
1241 }}, nil)
1242 declineWithBook(t, svc, reg, dd)
1243 ctx, s := genKit(t)
1244 base, token, pub := retryConsole(t, s, "blog", host)
1245
1246 // The box's restart memory, seeded before the request, as at arming.
1247 mem := newRestartMemory()
1248 mem.pub = pub
1249 mem.retryHost = rolloutRetryHost()
1250 routes, err := reg.List()
1251 if err != nil {
1252 t.Fatal(err)
1253 }
1254 bounce := func(h string) bool { t.Errorf("%s was bounced for a retry", h); return true }
1255 restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem)
1256
1257 // The box's status, as the status loop publishes it.
1258 box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now())
1259 box.Metrics = &status.BoxMetrics{RouteVersions: map[string]string{host: rt.Version}, Rollout: rolloutStatusProbe()}
1260 if err := status.WriteBox(ctx, s, box); err != nil {
1261 t.Fatal(err)
1262 }
1263 req, _ := http.NewRequest(http.MethodPost, base+"/api/cli/apps/"+release.OpsAccount+"/blog/instances/"+host+"/retry", strings.NewReader(`{}`))
1264 req.Header.Set("Content-Type", "application/json")
1265 req.Header.Set("Authorization", "Bearer "+token)
1266 res, err := http.DefaultClient.Do(req)
1267 if err != nil {
1268 t.Fatal(err)
1269 }
1270 raw, _ := io.ReadAll(res.Body)
1271 res.Body.Close()
1272 if res.StatusCode != http.StatusOK {
1273 t.Fatalf("the console's retry door = %d %s, want a signed grant for the declined box's hold", res.StatusCode, raw)
1274 }
1275
1276 restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem)
1277 if p := svc.Planner(ctx, rt); !isTodaysWake(p) {
1278 t.Fatalf("plan after the console's retry = %+v, want today's wake onto v2", p)
1279 }
1280 b, err := rollout.OpenFailureBook(dd)
1281 if err != nil {
1282 t.Fatal(err)
1283 }
1284 if b.HeldDown(acct, "blog", "v2", host) {
1285 t.Fatal("the console's retry was not saved")
1286 }
1287}
1288