amadan
zdd-5 final review: a declined box honours failure records; config delivery splits by management (F6, F7)
F6 (replaces PF15). With CARLOS_SWAP=overlap set but arming declined,
generations, containment and the scheduler stay off, but armRollout now
opens the failure book read-only (honourDeclinedFailures). No records,
the file absent or empty: no hook, every leg exactly today's. Records: a
managed wake onto a build they hold for that instance (Blocks with served
only for its good build, or HeldDown; the stamp and the live link both
asked) is refused through the Planner with existing text, and the converge
leg, adopt bounce, restart leg and config sync's restart leave such an
instance running. Unreadable book: refuseManaged, as on an armed box. The
status keeps the decline code and adds RolloutReport.FailureBook
("honoured" or "failure-book-unreadable").
F7. appconfig.Callbacks gains Managed and Unmanaged: with the scheduler
armed or refusing, a socket route it does not manage gets today's restart
(svc.Bounce) and ack instead of a no-op that claimed a restart, and a
managed native sidecar keeps Delivered so the scheduler queues its config
attempt. A refusing box's sleeping managed instance is acked delivered on
wake (appconfig.ErrDeliveredOnWake sets Hibernating), not Restarted. Flag
off, the callbacks are unchanged.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
11 files changed,
+1006
−50
internal/appconfig/claimsync.go+23 −1internal/appconfig/syncexec.go+32 −5internal/appconfig/syncexec_test.go+113 −0internal/host/agent.go+15 −3internal/host/restart.go+6 −1internal/host/rollout.go+76 −30internal/host/rollout_test.go+93 −9internal/host/rolloutdeclined.go+219 −0internal/host/rolloutdeclined_test.go+412 −0internal/rollout/wakeplan.go+9 −1internal/status/snapshot.go+8 −0
diff --git a/internal/appconfig/claimsync.go b/internal/appconfig/claimsync.go| index 44a7d87b9..fecd9456b 100644 |
| --- a/internal/appconfig/claimsync.go |
| +++ b/internal/appconfig/claimsync.go |
| @@ -8,6 +8,7 @@ import ( |
| "fmt" |
| "github.com/carlosframework/platform/internal/hostwrite" |
| + "github.com/carlosframework/platform/internal/registry" |
| ) |
| var errHostWriteOptions = errors.New("appconfig: more than one hostwrite configuration") |
| @@ -52,11 +53,32 @@ type Callbacks struct { |
| // each time this pass writes or removes it, before the restart callback |
| // runs: the zero-downtime scheduler's "delivered" revision (spec §8.1). |
| // It is never called for a shape whose process the platform does not |
| - // restart (the TCP-addr and sidecar shapes). Nil keeps today's behaviour. |
| + // restart (the TCP-addr shape), for a sidecar Managed does not name, or |
| + // for a socket route Managed says no to. Nil keeps today's behaviour. |
| Delivered func(host, rev string) |
| + // Managed, when set, names the routes the release scheduler moves itself |
| + // (rollout.Managed, handed in because this package sits below the |
| + // scheduler). It splits the pass the way the scheduler does (final |
| + // review F7): a socket route it does not manage is restarted by |
| + // Unmanaged, as config sync restarts it with the flag off, and is not |
| + // reported to Delivered; a native sidecar it manages is reported to |
| + // Delivered though config sync never restarts a sidecar, because the |
| + // scheduler's config attempt is that sidecar's restart. Nil: Restart and |
| + // Delivered serve every route, as before. |
| + Managed func(rt registry.Route) bool |
| + // Unmanaged is the restart a socket route outside Managed gets in |
| + // Restart's place. Ignored while Managed is nil. |
| + Unmanaged func(host string) error |
| enabled bool |
| } |
| +// ErrDeliveredOnWake is a restart callback's answer for an instance that is |
| +// asleep and reads the new file when it next wakes: nothing was restarted |
| +// and nothing failed. Its ack carries Hibernating (delivered on wake), the |
| +// marker the unit pass sets for a hibernated unit (sync.go), with Restarted |
| +// false and no RestartError. |
| +var ErrDeliveredOnWake = errors.New("asleep: config delivered on wake") |
| + |
| func (c Callbacks) restart(ctx context.Context, selected hostwrite.Selection) error { |
| if c.enabled { |
| if c.SelectedRestart == nil { |
diff --git a/internal/appconfig/syncexec.go b/internal/appconfig/syncexec.go| index 9f01a2d7f..cca52b4ea 100644 |
| --- a/internal/appconfig/syncexec.go |
| +++ b/internal/appconfig/syncexec.go |
| @@ -613,10 +613,25 @@ func SyncExecWithCallbacks(ctx context.Context, reg *registry.Registry, s releas |
| m = applyInstanceOverlay(k.app, m, inst) |
| } |
| hostCallbacks := callbacks |
| + // With Managed set (the release scheduler is armed or refusing, |
| + // final review F7), the scheduler's own routes and the rest part |
| + // here. A socket route it does not manage is restarted as it is |
| + // with the flag off, by Unmanaged, and its delivery is none of the |
| + // scheduler's business. A native sidecar it manages keeps Delivered |
| + // below though config sync has no restart for it: the scheduler's |
| + // config attempt is how its new file reaches a running worker. |
| + managed := callbacks.Managed != nil && callbacks.Managed(rt) |
| + if callbacks.Managed != nil && hostBounce != nil && !managed { |
| + hostBounce = callbacks.Unmanaged |
| + hostCallbacks.Restart = callbacks.Unmanaged |
| + hostCallbacks.Delivered = nil |
| + } |
| if hostBounce == nil { |
| hostCallbacks.Restart = nil |
| hostCallbacks.SelectedRestart = nil |
| - hostCallbacks.Delivered = nil |
| + if !sidecarShape || !managed { |
| + hostCallbacks.Delivered = nil |
| + } |
| } |
| if err := syncExecHostClaimed(ctx, selected[rt.Host], rt, m, Key(k.acct, k.app), priv, dataDir, hostBounce, header, instETag, ack, hostCallbacks); err != nil && hostWrites.Enabled { |
| return err |
| @@ -1224,7 +1239,10 @@ func syncExecHostClaimed(ctx context.Context, work hostwrite.Work, rt registry.R |
| } |
| if bounce != nil || callbacks.SelectedRestart != nil { |
| if callbacks.enabled { |
| - return callbacks.restart(ctx, hostwrite.Selection{Host: rt.Host, Work: work, Identity: id}) |
| + if err := callbacks.restart(ctx, hostwrite.Selection{Host: rt.Host, Work: work, Identity: id}); !errors.Is(err, ErrDeliveredOnWake) { |
| + return err |
| + } |
| + return nil |
| } |
| bounceLogged(rt.Host, bounce) |
| } |
| @@ -1248,7 +1266,7 @@ func syncExecHostClaimed(ctx context.Context, work hostwrite.Work, rt registry.R |
| if bounce != nil || callbacks.SelectedRestart != nil { |
| if callbacks.enabled { |
| bounceErr = callbacks.restart(ctx, hostwrite.Selection{Host: rt.Host, Work: work, Identity: id}) |
| - if bounceErr != nil { |
| + if bounceErr != nil && !errors.Is(bounceErr, ErrDeliveredOnWake) { |
| return bounceErr |
| } |
| } else { |
| @@ -1269,10 +1287,15 @@ func syncExecHostClaimed(ctx context.Context, work hostwrite.Work, rt registry.R |
| // route's cutover is the ack "since the platform restarted it"), |
| // so the ack has to carry what bounceLogged actually returned |
| // rather than assume success the way the removal branch above may. |
| + // A sleeper's restart answers ErrDeliveredOnWake: nothing ran and |
| + // nothing failed, so the ack carries the unit pass's marker. |
| if bounce != nil { |
| - if bounceErr != nil { |
| + switch { |
| + case errors.Is(bounceErr, ErrDeliveredOnWake): |
| + a.Hibernating = true |
| + case bounceErr != nil: |
| a.RestartError = bounceErr.Error() |
| - } else { |
| + default: |
| a.Restarted = true |
| } |
| } |
| @@ -1402,6 +1425,10 @@ func bounceLogged(host string, bounce func(host string) error) error { |
| return nil |
| } |
| if err := bounce(host); err != nil { |
| + if errors.Is(err, ErrDeliveredOnWake) { |
| + fmt.Printf(" %s: config updated, asleep: delivered on wake\n", host) |
| + return err |
| + } |
| fmt.Printf(" %s: bounce after config change: %v\n", host, err) |
| return err |
| } |
diff --git a/internal/appconfig/syncexec_test.go b/internal/appconfig/syncexec_test.go| index 8a46d6e9b..37440129e 100644 |
| --- a/internal/appconfig/syncexec_test.go |
| +++ b/internal/appconfig/syncexec_test.go |
| @@ -7,6 +7,7 @@ import ( |
| "io" |
| "os" |
| "path/filepath" |
| + "reflect" |
| "slices" |
| "strings" |
| "testing" |
| @@ -2533,3 +2534,115 @@ func TestSyncExecReportsTheDeliveredRevision(t *testing.T) { |
| t.Fatalf("revs = %v, want a new revision for new content", revs) |
| } |
| } |
| + |
| +// Final review F7 (1) and (2): with Managed set, the pass splits as the |
| +// scheduler does. A managed socket route gets Restart and is reported to |
| +// Delivered. A socket route Managed says no to is restarted by Unmanaged, |
| +// as config sync restarts it with the flag off, is not reported, and its |
| +// ack says it was restarted. A managed native sidecar is reported though |
| +// nothing here restarts it (the scheduler's config attempt does); an |
| +// unmanaged one is neither reported nor restarted, as before. |
| +func TestSyncExecSplitsDeliveryByManagement(t *testing.T) { |
| + ctx, s, reg, dd, priv, _ := execKit(t) |
| + pk, err := ParsePrivateKey(priv, "test") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "debug"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + const ( |
| + mInst = "m.example.com" |
| + uInst = "u.example.com" |
| + mBot = "bot.eleven.bab.sidecar.internal" |
| + uBot = "bot2.eleven.bab.sidecar.internal" |
| + ) |
| + for _, rt := range []registry.Route{ |
| + {Host: mInst, Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/m.sock"}, |
| + {Host: uInst, Kind: "instance", App: "eleven", Account: "bab", Status: "ready", Socket: "/run/carlos/u.sock"}, |
| + {Host: mBot, Kind: "sidecar", App: "eleven", Account: "bab", Status: "ready"}, |
| + {Host: uBot, Kind: "sidecar", App: "eleven", Account: "bab", Status: "ready"}, |
| + } { |
| + if err := reg.Upsert(rt); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + calls := map[string][]string{} |
| + acks := map[string]ConfigAck{} |
| + cb := Callbacks{ |
| + Restart: func(h string) error { calls[h] = append(calls[h], "restart"); return nil }, |
| + Unmanaged: func(h string) error { calls[h] = append(calls[h], "unmanaged"); return nil }, |
| + Delivered: func(h, _ string) { calls[h] = append(calls[h], "delivered") }, |
| + Ack: func(a ConfigAck) { acks[a.Host] = a }, |
| + Managed: func(rt registry.Route) bool { return rt.Host == mInst || rt.Host == mBot }, |
| + } |
| + if err := SyncExecWithCallbacks(ctx, reg, s, pk, dd, "", cb); err != nil { |
| + t.Fatal(err) |
| + } |
| + want := map[string][]string{ |
| + mInst: {"delivered", "restart"}, |
| + uInst: {"unmanaged"}, |
| + mBot: {"delivered"}, |
| + } |
| + if !reflect.DeepEqual(calls, want) { |
| + t.Fatalf("calls = %v, want %v", calls, want) |
| + } |
| + if a := acks[uInst]; !a.Restarted || a.RestartError != "" { |
| + t.Fatalf("unmanaged ack = %+v, want today's: restarted", a) |
| + } |
| + if a := acks[mBot]; a.Restarted || a.RestartError != "" { |
| + t.Fatalf("managed sidecar ack = %+v: nothing here restarted it", a) |
| + } |
| + |
| + // Managed nil: every socket route gets Restart and Delivered, and no |
| + // sidecar is reported, exactly as before. |
| + if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "info"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + calls = map[string][]string{} |
| + cb.Managed = nil |
| + if err := SyncExecWithCallbacks(ctx, reg, s, pk, dd, "", cb); err != nil { |
| + t.Fatal(err) |
| + } |
| + want = map[string][]string{ |
| + mInst: {"delivered", "restart"}, |
| + uInst: {"delivered", "restart"}, |
| + } |
| + if !reflect.DeepEqual(calls, want) { |
| + t.Fatalf("with Managed nil, calls = %v, want %v", calls, want) |
| + } |
| +} |
| + |
| +// Final review F7 (3): a restart that answers ErrDeliveredOnWake acks the |
| +// revision as delivered on wake (Hibernating), the unit pass's marker, with |
| +// no restart claimed and no error, on both the write and the removal paths. |
| +func TestSyncExecAcksDeliveredOnWake(t *testing.T) { |
| + ctx, s, reg, dd, priv, _ := execKit(t) |
| + pk, err := ParsePrivateKey(priv, "test") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "debug"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + host := "asleep.example.com" |
| + if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", App: "eleven", Account: "bab", Status: "ready", |
| + Socket: "/run/carlos/asleep.sock", Hibernate: true}); err != nil { |
| + t.Fatal(err) |
| + } |
| + var acks []ConfigAck |
| + restarts := 0 |
| + cb := Callbacks{ |
| + Restart: func(string) error { restarts++; return ErrDeliveredOnWake }, |
| + Ack: func(a ConfigAck) { acks = append(acks, a) }, |
| + } |
| + if err := SyncExecWithCallbacks(ctx, reg, s, pk, dd, "", cb); err != nil { |
| + t.Fatal(err) |
| + } |
| + if restarts != 1 || len(acks) != 1 { |
| + t.Fatalf("restarts %d, acks %+v", restarts, acks) |
| + } |
| + if a := acks[0]; !a.Hibernating || a.Restarted || a.RestartError != "" { |
| + t.Fatalf("ack = %+v, want delivered on wake: Hibernating, not restarted, no error", a) |
| + } |
| +} |
diff --git a/internal/host/agent.go b/internal/host/agent.go| index 2f6217609..160b02cef 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -1128,7 +1128,7 @@ func syncExecSerialized(ctx context.Context, reg *registry.Registry, store relea |
| // CARLOS_SWAP=overlap: the scheduler restarts, paced, and acknowledges a |
| // revision once it is active; refusing: no restart. configSyncMu above |
| // is what makes ConfigDelivered serial per host. |
| - if cb, ok := rolloutConfigCallbacks(ctx, store, managedAwake(svc, reg)); ok { |
| + if cb, ok := rolloutConfigCallbacks(ctx, store, configLegs{awake: managedAwake(svc, reg), restart: bounce(svc)}); ok { |
| return appconfig.SyncExecWithCallbacks(ctx, reg, store, priv, dd, os.Getenv("PLATFORM_APPS_DOMAIN"), cb, opts...) |
| } |
| return appconfig.SyncExec(ctx, reg, store, priv, dd, bounce(svc), os.Getenv("PLATFORM_APPS_DOMAIN"), configAckFunc(ctx, store, "agent"), opts...) |
| @@ -2003,13 +2003,20 @@ const adoptedCyclesPerTick = 1 |
| const adoptedConvergeEvery = 2 * time.Second |
| // convergeLegFor builds the converge leg over the box's activator. nil for a |
| -// box with none armed: there are no exec children to cycle. |
| +// box with none armed: there are no exec children to cycle. On a box that |
| +// declined CARLOS_SWAP=overlap but honours failure records (heldHolds), a |
| +// host whose installed build a record holds for it is passed by: cycling it |
| +// would stop it into a wake its Planner refuses (final review F6). |
| func convergeLegFor(svc *activator.Service, reg *registry.Registry) func() { |
| if svc == nil { |
| return nil |
| } |
| + get := reg.Get |
| + if h := heldHolds(); h != nil { |
| + get = h.skipGet(reg.Get) |
| + } |
| return func() { |
| - convergeAdopted(svc.Drifted, reg.Get, svc.ConvergeAdopted, adoptedCyclesPerTick) |
| + convergeAdopted(svc.Drifted, get, svc.ConvergeAdopted, adoptedCyclesPerTick) |
| } |
| } |
| @@ -3782,6 +3789,11 @@ func adoptBounceFn(svc *activator.Service) func(host string) bool { |
| if rolloutUnavailable() != "" { |
| return nil // no adopt bounce while managed starts are refused |
| } |
| + if h := heldHolds(); h != nil { |
| + // Declined with failure records: a host a record holds keeps |
| + // running rather than stop into a refused wake (final review F6). |
| + return h.skipHost("adopt bounce", svc.BounceAdopted) |
| + } |
| return svc.BounceAdopted |
| } |
diff --git a/internal/host/restart.go b/internal/host/restart.go| index 57fc71f84..09bd5ee70 100644 |
| --- a/internal/host/restart.go |
| +++ b/internal/host/restart.go |
| @@ -878,9 +878,14 @@ func restartRequestLeg(ctx context.Context, routes []registry.Route, store relea |
| // paced and identified by its request (spec §10). While managed starts |
| // are refused (refuseManaged), no exec restart is made: the wake after |
| // the stop would be refused, and the box's status names why. A box that |
| - // declined the flag (swapArming) bounces as today. |
| + // declined the flag (swapArming) bounces as today, except an instance a |
| + // failure record from an armed run holds (final review F6): its wake |
| + // would be refused, so it keeps running. |
| m := armedRollout() |
| legacy := m == nil && rolloutUnavailable() == "" |
| + if h := heldHolds(); legacy && h != nil { |
| + bounce = h.skipRoutes(routes, bounce) |
| + } |
| if m != nil { |
| feedRestarts(routes, moved, scoped, mem.gen, m.Restart) |
| } else if legacy { |
diff --git a/internal/host/rollout.go b/internal/host/rollout.go| index 81a0c3d0d..72ab0b3b7 100644 |
| --- a/internal/host/rollout.go |
| +++ b/internal/host/rollout.go |
| @@ -72,15 +72,19 @@ func nudgeStatus() { |
| } |
| // rolloutDown is why CARLOS_SWAP=overlap is set but the scheduler could not |
| -// arm while generations are on: managed starts are refused and the bounce |
| -// legs stand down (refuseManaged). Nil when it armed, when the flag is off, |
| -// or when the box declined the flag outright (rolloutDeclined). |
| +// arm while generations are on, or why a box that declined the flag cannot |
| +// read the failure records it must honour: managed starts are refused and |
| +// the bounce legs stand down (refuseManaged). Nil when it armed, when the |
| +// flag is off, or when the box declined the flag (rolloutDeclined) with a |
| +// readable failure book. |
| var rolloutDown atomic.Pointer[string] |
| // rolloutDeclined is why CARLOS_SWAP=overlap is set but the box stayed on |
| // today's paths: it cannot contain generations (swapArming). Unlike |
| -// rolloutDown nothing is refused and no leg changes; the reason is only |
| -// published in the box's status. |
| +// rolloutDown nothing is refused and no leg changes, except where failure |
| +// records an earlier armed run left hold a build for an instance |
| +// (honourDeclinedFailures, final review F6). The reason is published in the |
| +// box's status. |
| var rolloutDeclined atomic.Pointer[string] |
| // rolloutUnavailable is rolloutDown's reason, "" when there is none. A leg |
| @@ -132,7 +136,8 @@ func swapDelegation(eb *activator.ExecBackend) string { |
| // fail closed, CODEX-FINDINGS-ZDD-IMPL-3.md:138). With the flag off it is |
| // false and says nothing. With the flag on but no usable containment it is |
| // false too: generations, generation containment and the scheduler all stay |
| -// off, every leg keeps today's paths, and the refusal is logged loudly and |
| +// off, every leg keeps today's paths but for the failure records armRollout |
| +// honours (honourDeclinedFailures), and the refusal is logged loudly and |
| // published as a reason code in the box's status (rolloutStatusProbe). |
| func swapArming(getenv func(string) string, eb *activator.ExecBackend) bool { |
| if !rollout.Enabled(getenv) { |
| @@ -169,7 +174,7 @@ func refuseManaged(svc *activator.Service, reason string) { |
| rolloutDown.Store(&r) |
| text := refuseTexts[reason] |
| if text == "" { |
| - text = "overlap swaps are on but this box cannot run them" |
| + text = refuseNoSwaps |
| } |
| svc.Planner = func(_ context.Context, rt registry.Route) activator.WakePlan { |
| if !rollout.Managed(rt) { |
| @@ -274,7 +279,10 @@ func touchStageMarker(dd string) { |
| // returns nil and touches nothing. With the flag on but svc built without |
| // generations (swapArming declined the flag) it returns nil and touches |
| // nothing either: a manager is never armed over a Service that cannot run |
| -// its attempts. With the flag on but no deploy store, no public key or an |
| +// its attempts. It does honour the failure records an earlier armed run left |
| +// (honourDeclinedFailures, final review F6): an infrastructure prerequisite |
| +// failing is not an operator clearing a failure hold. With the flag on but |
| +// no deploy store, no public key or an |
| // unreadable failure book, it returns nil and refuses every managed start |
| // (refuseManaged): each of those is a fact the scheduler cannot do without |
| // (the canonical demand, the retry signal, the records), and today's paths |
| @@ -289,6 +297,7 @@ func armRollout(ctx context.Context, getenv func(string) string, svc *activator. |
| log.Printf("agent: REFUSING %s=overlap: the activator was built without generations", rollout.SwapEnv) |
| declineRollout(rolloutGenerationsOff) |
| } |
| + honourDeclinedFailures(svc, reg, dd, release.OptDir()) |
| return nil |
| } |
| store, err := openStore(ctx) |
| @@ -355,7 +364,8 @@ func rolloutLegFor(m *rollout.Manager, reg *registry.Registry) func() { |
| // adoptedConvergeEvery: the scheduler's pass when it is armed, none while |
| // managed starts are refused (legacy convergence would stop instances whose |
| // next wake is refused), and today's convergeAdopted otherwise, the flag off |
| -// or declined included. |
| +// or declined included (declined with failure records, it passes by a held |
| +// host: convergeLegFor). |
| func fastConvergeLeg(svc *activator.Service, reg *registry.Registry) func() { |
| if m := armedRollout(); m != nil { |
| return rolloutLegFor(m, reg) |
| @@ -366,35 +376,54 @@ func fastConvergeLeg(svc *activator.Service, reg *registry.Registry) func() { |
| return convergeLegFor(svc, reg) |
| } |
| -// rolloutConfigCallbacks is config sync's callbacks when the scheduler has |
| -// a say, and false when it has none (the flag off or declined), in which case |
| -// syncExecSerialized runs today's SyncExec unchanged. Armed: the scheduler |
| -// restarts exec instances, paced, and a revision is acknowledged only once |
| -// it is active (spec §8.1, §10); appconfig calls Delivered after the file is |
| -// written (syncExecHostClaimed). Refusing: delivered and acknowledged at |
| -// delivery, but no restart, because the wake after it would be refused. An |
| -// AWAKE managed host (awake says which) still runs its old config, so its |
| -// restart answers an error and the ack carries RestartError with Restarted |
| -// false rather than claiming a restart that did not happen (spec §8.1). A |
| -// sleeper, and any unmanaged host, keep the no-op restart and so the ack they |
| -// got before. (The exec pass never sets Hibernating; only the unit pass in |
| -// appconfig/sync.go does, from the route's Hibernate flag.) awake may be nil. |
| -func rolloutConfigCallbacks(ctx context.Context, store release.Store, awake func(host string) bool) (appconfig.Callbacks, bool) { |
| +// rolloutConfigCallbacks is config sync's callbacks when the scheduler, or |
| +// the failure records, have a say, and false when they have none (the flag |
| +// off, or declined with no records), in which case syncExecSerialized runs |
| +// today's SyncExec unchanged. |
| +// |
| +// Armed and refusing, the pass splits as the scheduler does |
| +// (appconfig.Callbacks.Managed, final review F7): an exec route the scheduler |
| +// does not manage gets today's restart (legs.restart, svc.Bounce) and today's |
| +// ack, never a no-op that claims a restart. A managed route: |
| +// |
| +// Armed: the scheduler restarts it, paced, and a revision is acknowledged |
| +// only once it is active (spec §8.1, §10); appconfig calls Delivered after |
| +// the file is written (syncExecHostClaimed), for a managed native sidecar |
| +// too, whose config attempt is the scheduler's. A sleeper's ack is |
| +// Hibernating (configTrack). |
| +// |
| +// Refusing: delivered and acknowledged at delivery, but no restart, because |
| +// the wake after it would be refused. An AWAKE managed host (legs.awake says |
| +// which) still runs its old config, so its restart answers an error and the |
| +// ack carries RestartError with Restarted false rather than claiming a |
| +// restart that did not happen (spec §8.1). A sleeper reads the file on its |
| +// next wake, so its ack says delivered on wake (appconfig.ErrDeliveredOnWake |
| +// sets Hibernating, the unit pass's marker). |
| +// |
| +// Declined with records: today's restart and ack for every host but one a |
| +// record holds (failureHolds.configCallbacks). |
| +func rolloutConfigCallbacks(ctx context.Context, store release.Store, legs configLegs) (appconfig.Callbacks, bool) { |
| if m := armedRollout(); m != nil { |
| return appconfig.Callbacks{ |
| Restart: func(string) error { return nil }, |
| Delivered: m.ConfigDelivered, |
| Ack: m.DeferAck(configAckFunc(ctx, store, "agent")), |
| + Managed: rollout.Managed, |
| + Unmanaged: legs.restart, |
| }, true |
| } |
| if rolloutUnavailable() != "" { |
| restart := func(host string) error { |
| - if awake != nil && awake(host) { |
| + if legs.awake != nil && legs.awake(host) { |
| return errors.New(rolloutRestartRefused) |
| } |
| - return nil |
| + return appconfig.ErrDeliveredOnWake |
| } |
| - return appconfig.Callbacks{Restart: restart, Ack: configAckFunc(ctx, store, "agent")}, true |
| + return appconfig.Callbacks{Restart: restart, Ack: configAckFunc(ctx, store, "agent"), |
| + Managed: rollout.Managed, Unmanaged: legs.restart}, true |
| + } |
| + if h := heldHolds(); h != nil { |
| + return h.configCallbacks(configAckFunc(ctx, store, "agent"), legs), true |
| } |
| return appconfig.Callbacks{}, false |
| } |
| @@ -433,16 +462,33 @@ func feedRestarts(routes []registry.Route, moved [][2]string, scoped []scopedRes |
| return n |
| } |
| +// configLegs is what config sync's callbacks need from the activator. |
| +type configLegs struct { |
| + // awake reports a managed instance the activator has a live entry for |
| + // (managedAwake). Nil: nothing is awake. |
| + awake func(host string) bool |
| + // restart is today's restart, bounce(svc): what config sync does to an |
| + // exec instance with the flag off. Nil: no restart. |
| + restart func(host string) error |
| +} |
| + |
| // rolloutStatusProbe feeds status.Probes.Rollout: nil with the flag off, the |
| // scheduler's report when it is armed, and only the reason code when the |
| -// flag is on but the scheduler is not armed (refusing, or declined). |
| +// flag is on but the scheduler is not armed (refusing, or declined). A |
| +// declined box's report keeps the decline code and says what it does with |
| +// the failure records (FailureBook): honoured, unreadable (every managed |
| +// start refused), or nothing when there are none. |
| func rolloutStatusProbe() *status.RolloutReport { |
| m := armedRollout() |
| if m == nil { |
| - if why := rolloutUnavailable(); why != "" { |
| - return &status.RolloutReport{Unavailable: why} |
| - } |
| if why := rolloutUnarmed(); why != "" { |
| + r := &status.RolloutReport{Unavailable: why, FailureBook: rolloutUnavailable()} |
| + if heldHolds() != nil { |
| + r.FailureBook = rolloutBookHonoured |
| + } |
| + return r |
| + } |
| + if why := rolloutUnavailable(); why != "" { |
| return &status.RolloutReport{Unavailable: why} |
| } |
| return nil |
diff --git a/internal/host/rollout_test.go b/internal/host/rollout_test.go| index c0cfab6de..f8d8acb61 100644 |
| --- a/internal/host/rollout_test.go |
| +++ b/internal/host/rollout_test.go |
| @@ -45,6 +45,7 @@ func resetRollout(t *testing.T) { |
| rolloutMgr.Store(nil) |
| rolloutDown.Store(nil) |
| rolloutDeclined.Store(nil) |
| + declinedHolds.Store(nil) |
| select { |
| case <-statusNudge: |
| default: |
| @@ -76,7 +77,7 @@ func legsAreTodays(t *testing.T, svc *activator.Service, reg *registry.Registry) |
| if b := adoptBounceFn(svc); b == nil || funcPtr(b) != funcPtr(svc.BounceAdopted) { |
| t.Error("the adopt bounce is not BounceAdopted") |
| } |
| - if _, ok := rolloutConfigCallbacks(context.Background(), release.NewDirStore(t.TempDir()), nil); ok { |
| + if _, ok := rolloutConfigCallbacks(context.Background(), release.NewDirStore(t.TempDir()), configLegs{}); ok { |
| t.Error("config sync left today's SyncExec") |
| } |
| ctx, s := genKit(t) |
| @@ -176,7 +177,7 @@ func TestTheFlagOnArmsTheScheduler(t *testing.T) { |
| if fastConvergeLeg(svc, reg) == nil { |
| t.Fatal("no converge leg: the scheduler's pass must run") |
| } |
| - cb, ok := rolloutConfigCallbacks(context.Background(), store, nil) |
| + cb, ok := rolloutConfigCallbacks(context.Background(), store, configLegs{}) |
| if !ok || cb.Delivered == nil || cb.Ack == nil || cb.Restart == nil || cb.Restart("a.example.com") != nil { |
| t.Fatalf("config callbacks = %+v, %v: want the scheduler's delivery and deferred ack, and no restart", cb, ok) |
| } |
| @@ -264,8 +265,10 @@ func TestUnavailableStateRefusesManagedStarts(t *testing.T) { |
| t.Fatal("OnGen set with no scheduler to hear it") |
| } |
| store := release.NewDirStore(t.TempDir()) |
| - cb, ok := rolloutConfigCallbacks(context.Background(), store, nil) |
| - if !ok || cb.Delivered != nil || cb.Restart == nil || cb.Restart("a.example.com") != nil || cb.Ack == nil { |
| + cb, ok := rolloutConfigCallbacks(context.Background(), store, configLegs{}) |
| + // Nothing is awake here: the restart is none, and says the file |
| + // is read on the next wake (final review F7 (3)). |
| + if !ok || cb.Delivered != nil || cb.Restart == nil || !errors.Is(cb.Restart("a.example.com"), appconfig.ErrDeliveredOnWake) || cb.Ack == nil { |
| t.Fatalf("config callbacks = %+v, %v: want delivery and the ack as today, and no restart", cb, ok) |
| } |
| ctx, s := genKit(t) |
| @@ -513,8 +516,9 @@ func TestThePolicyLoaderFailsOnAnyStoreError(t *testing.T) { |
| // restarted: the instance still runs its old config and nothing will restart |
| // it (spec §8.1, an ack only once active). Its ack carries a fixed |
| // RestartError and Restarted stays false. A sleeper reads the file on its |
| -// next wake, which is refused or not on its own terms, so its ack is as it |
| -// is today: no error. |
| +// next wake, so its ack says delivered on wake (Hibernating, final review F7 |
| +// (3)), never Restarted. An exec route the scheduler does not manage is |
| +// restarted as today and acked as restarted (F7 (2)). |
| func TestRefusingConfigSyncDoesNotAckAnAwakeManagedHostAsRestarted(t *testing.T) { |
| ctx := context.Background() |
| svc, reg, dd := rolloutFixture(t, true) |
| @@ -547,7 +551,15 @@ func TestRefusingConfigSyncDoesNotAckAnAwakeManagedHostAsRestarted(t *testing.T) |
| t.Fatal(err) |
| } |
| } |
| - cb, ok := rolloutConfigCallbacks(ctx, store, func(h string) bool { return h == "awake.example.com" }) |
| + // Always on, so not managed (rollout.Managed): today's restart. |
| + if err := reg.Upsert(execRoute("plain.example.com", "bab", "eleven")); err != nil { |
| + t.Fatal(err) |
| + } |
| + var restarted []string |
| + cb, ok := rolloutConfigCallbacks(ctx, store, configLegs{ |
| + awake: func(h string) bool { return h == "awake.example.com" }, |
| + restart: func(h string) error { restarted = append(restarted, h); return nil }, |
| + }) |
| if !ok { |
| t.Fatal("refusing mode has no config callbacks") |
| } |
| @@ -567,8 +579,80 @@ func TestRefusingConfigSyncDoesNotAckAnAwakeManagedHostAsRestarted(t *testing.T) |
| if !ok { |
| t.Fatalf("no ack for the sleeping host: %+v", acks) |
| } |
| - if asleep.RestartError != "" || asleep.Restarted != true || asleep.Hibernating { |
| - t.Fatalf("sleeper ack = %+v, want today's: restarted, no error, no marker (the exec pass never sets Hibernating)", asleep) |
| + if asleep.RestartError != "" || asleep.Restarted || !asleep.Hibernating { |
| + t.Fatalf("sleeper ack = %+v, want delivered on wake: Hibernating, not restarted, no error", asleep) |
| + } |
| + if !slices.Equal(restarted, []string{"plain.example.com"}) { |
| + t.Fatalf("today's restart ran for %v, want only the unmanaged route", restarted) |
| + } |
| + if plain := acks["plain.example.com"]; !plain.Restarted || plain.RestartError != "" || plain.Hibernating { |
| + t.Fatalf("unmanaged ack = %+v, want today's: restarted", plain) |
| + } |
| +} |
| + |
| +// Final review F7 (1) and (2), armed: config sync reports a managed native |
| +// sidecar's delivery to the scheduler, which owns its restart, and restarts |
| +// an exec route the scheduler does not manage as today (svc.Bounce, through |
| +// configLegs.restart), acking it as restarted at once; neither managed route |
| +// is restarted by config sync, and the unmanaged one is not reported. |
| +func TestArmedConfigSyncSplitsManagedFromUnmanaged(t *testing.T) { |
| + ctx, cancel := context.WithCancel(context.Background()) |
| + t.Cleanup(cancel) |
| + svc, reg, dd := rolloutFixture(t, true) |
| + svc.GenerationsFile = release.GenerationsPath(dd) |
| + _, pub, err := release.GenerateKey() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + managed := execRoute("m.example.com", "bab", "eleven") |
| + managed.Hibernate, managed.Version = true, "v1" |
| + bot := registry.Route{Host: "bot.eleven.bab.sidecar.internal", Kind: "sidecar", App: "eleven", Account: "bab", Status: "ready", Version: "v1"} |
| + plain := execRoute("plain.example.com", "bab", "eleven") // always on: not managed |
| + for _, rt := range []registry.Route{managed, bot, plain} { |
| + if err := reg.Upsert(rt); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + if !rollout.Managed(managed) || !rollout.Managed(bot) || rollout.Managed(plain) { |
| + t.Fatal("fixture: the routes are not managed as this test assumes") |
| + } |
| + store := release.NewDirStore(t.TempDir()) |
| + m := armRollout(ctx, overlapEnv(pub), svc, reg, dd, nil, func(context.Context) (release.Store, error) { return store, nil }) |
| + if m == nil { |
| + t.Fatal("not armed") |
| + } |
| + privText, _, err := appconfig.GenerateKey() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + priv, err := appconfig.ParsePrivateKey(privText, "test") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := appconfig.SetEnv(ctx, store, "bab", "eleven", map[string]string{"LOG_LEVEL": "debug"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + var restarted, delivered []string |
| + cb, ok := rolloutConfigCallbacks(ctx, store, configLegs{restart: func(h string) error { restarted = append(restarted, h); return nil }}) |
| + if !ok || cb.Delivered == nil { |
| + t.Fatal("armed mode has no delivery callback") |
| + } |
| + report := cb.Delivered |
| + cb.Delivered = func(h, rev string) { delivered = append(delivered, h); report(h, rev) } |
| + acks := map[string]appconfig.ConfigAck{} |
| + cb.Ack = m.DeferAck(func(a appconfig.ConfigAck) { acks[a.Host] = a }) |
| + if err := appconfig.SyncExecWithCallbacks(ctx, reg, store, priv, t.TempDir(), "", cb); err != nil { |
| + t.Fatal(err) |
| + } |
| + slices.Sort(delivered) |
| + if want := []string{bot.Host, managed.Host}; !slices.Equal(delivered, want) { |
| + t.Fatalf("delivered %v, want %v: the managed sidecar's revision must reach the scheduler, the unmanaged route's must not", delivered, want) |
| + } |
| + if !slices.Equal(restarted, []string{plain.Host}) { |
| + t.Fatalf("today's restart ran for %v, want only the unmanaged route", restarted) |
| + } |
| + if a, ok := acks[plain.Host]; !ok || !a.Restarted || a.RestartError != "" { |
| + t.Fatalf("unmanaged ack = %+v (written %v), want today's: restarted, at once", a, ok) |
| } |
| } |
diff --git a/internal/host/rolloutdeclined.go b/internal/host/rolloutdeclined.go| new file mode 100644 |
| index 000000000..520dd3736 |
| --- /dev/null |
| +++ b/internal/host/rolloutdeclined.go |
| @@ -0,0 +1,219 @@ |
| +package host |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "log" |
| + "sync" |
| + "sync/atomic" |
| + |
| + "github.com/carlosframework/platform/internal/activator" |
| + "github.com/carlosframework/platform/internal/appconfig" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| + "github.com/carlosframework/platform/internal/rollout" |
| +) |
| + |
| +// rolloutBookHonoured is RolloutReport.FailureBook's code for a box that |
| +// declined CARLOS_SWAP=overlap and honours the failure records an earlier |
| +// armed run left (honourDeclinedFailures). A code, not member-facing text. |
| +const rolloutBookHonoured = "honoured" |
| + |
| +// refuseNoSwaps is the wake refusal for a managed instance whose build needs |
| +// the scheduler this box cannot run: refuseManaged's text for a reason it has |
| +// no words for, and a declined box's for a build a failure record holds. |
| +const refuseNoSwaps = "overlap swaps are on but this box cannot run them" |
| + |
| +// failureHolds is the failure book a box that declined CARLOS_SWAP=overlap |
| +// still honours (final review F6, which replaced PF15's "today's paths"). An |
| +// earlier armed run recorded these failures, and losing the delegation the |
| +// scheduler needs is not an operator clearing them: a hold build whose |
| +// migration failed must not run again on the live link. So, with generations |
| +// off and no scheduler, the box keeps today's paths except that a managed |
| +// wake onto a build the book holds for that instance is refused (Planner), |
| +// and the converge leg, the adopt bounce, the restart leg and config sync's |
| +// restart leave such an instance running rather than stop it into a refused |
| +// wake. |
| +// |
| +// The book is read once, at arming, and never written: nothing on a declined |
| +// box records, retries or prunes, so what was read stays the truth until the |
| +// agent restarts. |
| +type failureHolds struct { |
| + book *rollout.FailureBook |
| + // get reads a route by host, for the legs that are handed only a host. |
| + get func(host string) (*registry.Route, error) |
| + // optDir is where the live links are: a wake execs the build |
| + // live/<host> names, which root repoints before it stamps the row. |
| + optDir string |
| + // said is the (leg, host, build) triples already logged, so a leg that |
| + // runs every two seconds says once that it is leaving a host alone. |
| + said sync.Map |
| +} |
| + |
| +// declinedHolds is set by honourDeclinedFailures, before any leg is built, |
| +// and only when the book holds at least one record. Nil: the flag is off, |
| +// armed, refusing, or declined with no records, and every leg is as its own |
| +// branch says. |
| +var declinedHolds atomic.Pointer[failureHolds] |
| + |
| +// heldHolds is declinedHolds' value. |
| +func heldHolds() *failureHolds { return declinedHolds.Load() } |
| + |
| +// honourDeclinedFailures is armRollout's branch for a box that declined the |
| +// flag (spec §§6.4, 8.4; final review F6). An absent book, or one with no |
| +// records, leaves everything exactly as today: no hook, no leg changed. An |
| +// unreadable one fails closed as refuseManaged does for an armed box, since |
| +// any record in it could be a hold-down. A readable one with records is |
| +// honoured, read-only, through the Planner and the legs (failureHolds). |
| +func honourDeclinedFailures(svc *activator.Service, reg *registry.Registry, dd, optDir string) { |
| + book, err := rollout.OpenFailureBook(dd) |
| + if err != nil { |
| + log.Printf("agent: %s=overlap declined, and the failure book is unreadable: %v", rollout.SwapEnv, err) |
| + refuseManaged(svc, rolloutBookUnreadable) |
| + return |
| + } |
| + recs := book.List() |
| + if len(recs) == 0 { |
| + return |
| + } |
| + h := &failureHolds{book: book, get: reg.Get, optDir: optDir} |
| + declinedHolds.Store(h) |
| + svc.Planner = h.plan |
| + log.Printf("agent: %s=overlap declined, but %d failure record(s) from an armed run are honoured: "+ |
| + "no managed instance wakes, or is stopped, onto a build they hold for it", rollout.SwapEnv, len(recs)) |
| +} |
| + |
| +// heldBuild is the build a failure record holds rt off, and whether that |
| +// instance is held down after a failed hold attempt; "" when nothing holds |
| +// it. Only a managed route can be held: the scheduler records nothing else. |
| +// Both the installed build (root's stamp) and the build live/<host> names |
| +// are asked, because a wake execs the live link and root repoints it before |
| +// it stamps the row; an unreadable link is left to the stamp. A build the |
| +// instance has served (its good build) is never held by a record alone, as |
| +// rollout.Blocks says, but a hold-down holds it whatever it served. |
| +func (h *failureHolds) heldBuild(rt registry.Route) (build string, heldDown bool) { |
| + if !rollout.Managed(rt) { |
| + return "", false |
| + } |
| + builds := []string{rt.Version} |
| + if live, err := release.LiveVersion(h.optDir, rt.Host); err == nil && live != "" && live != rt.Version { |
| + builds = append(builds, live) |
| + } |
| + acct := release.RouteAccount(rt) |
| + for _, b := range builds { |
| + if b == "" { |
| + continue |
| + } |
| + if h.book.HeldDown(acct, rt.App, b, rt.Host) { |
| + return b, true |
| + } |
| + served := rt.GoodVersion != "" && b == rt.GoodVersion |
| + if h.book.Blocks(acct, rt.App, b, rt.Host, served) { |
| + return b, false |
| + } |
| + } |
| + return "", false |
| +} |
| + |
| +// plan is the declined box's Planner: a managed wake onto a held build is |
| +// refused with fixed text that names the build at most, never an error; any |
| +// other wake gets the zero plan, which is today's wake from the live link. |
| +// rt is the row doWake has just re-read (Service.Routes). |
| +func (h *failureHolds) plan(_ context.Context, rt registry.Route) activator.WakePlan { |
| + build, heldDown := h.heldBuild(rt) |
| + switch { |
| + case build == "": |
| + return activator.WakePlan{} |
| + case heldDown: |
| + return activator.WakePlan{Refuse: rollout.HeldDownRefusal(build)} |
| + default: |
| + return activator.WakePlan{Refuse: refuseNoSwaps} |
| + } |
| +} |
| + |
| +// heldRoute reports whether rt is held, logging it once per leg and build. |
| +func (h *failureHolds) heldRoute(leg string, rt registry.Route) bool { |
| + build, _ := h.heldBuild(rt) |
| + if build == "" { |
| + return false |
| + } |
| + if _, said := h.said.LoadOrStore(leg+"\x00"+rt.Host+"\x00"+build, true); !said { |
| + log.Printf("agent: %s: leaving %s running: a failure record from an armed run holds build %s for it, and %s=overlap is declined on this box", |
| + leg, rt.Host, build, rollout.SwapEnv) |
| + } |
| + return true |
| +} |
| + |
| +// heldHost is heldRoute for a leg handed only a host. A row that cannot be |
| +// read counts as held: the leg leaves the instance running, which is what |
| +// makeRestartRoute does with such a row anyway. |
| +func (h *failureHolds) heldHost(leg, host string) bool { |
| + rt, err := h.get(host) |
| + if err != nil { |
| + return true |
| + } |
| + return rt != nil && h.heldRoute(leg, *rt) |
| +} |
| + |
| +// skipGet wraps the converge leg's row read: a held row reads as no row, so |
| +// convergeAdopted passes the host by. |
| +func (h *failureHolds) skipGet(get func(string) (*registry.Route, error)) func(string) (*registry.Route, error) { |
| + return func(host string) (*registry.Route, error) { |
| + rt, err := get(host) |
| + if err == nil && rt != nil && h.heldRoute("converge", *rt) { |
| + return nil, nil |
| + } |
| + return rt, err |
| + } |
| +} |
| + |
| +// skipHost wraps a bounce handed only a host (the adopt bounce): a held host |
| +// is not stopped, which its caller reads as nothing running to cycle. |
| +func (h *failureHolds) skipHost(leg string, bounce func(string) bool) func(string) bool { |
| + return func(host string) bool { |
| + if h.heldHost(leg, host) { |
| + return false |
| + } |
| + return bounce(host) |
| + } |
| +} |
| + |
| +// skipRoutes wraps the restart leg's bounce over the routes it was handed, |
| +// so it reads no row of its own. |
| +func (h *failureHolds) skipRoutes(routes []registry.Route, bounce func(string) bool) func(string) bool { |
| + if bounce == nil { |
| + return nil |
| + } |
| + byHost := make(map[string]registry.Route, len(routes)) |
| + for _, rt := range routes { |
| + byHost[rt.Host] = rt |
| + } |
| + return func(host string) bool { |
| + if rt, ok := byHost[host]; ok && h.heldRoute("restart", rt) { |
| + return false |
| + } |
| + return bounce(host) |
| + } |
| +} |
| + |
| +// configCallbacks is config sync's callbacks on a declined box that honours |
| +// records: today's restart and ack for every host but a held one, which is |
| +// not stopped into a refused wake. Its ack must not claim a restart: an |
| +// awake held instance still runs its old config, so the restart answers |
| +// rolloutRestartRefused (the ack's RestartError); a sleeping one reads the |
| +// file when it next wakes, so its ack says delivered on wake. |
| +func (h *failureHolds) configCallbacks(ack func(appconfig.ConfigAck), legs configLegs) appconfig.Callbacks { |
| + restart := func(host string) error { |
| + if h.heldHost("config sync", host) { |
| + if legs.awake != nil && legs.awake(host) { |
| + return errors.New(rolloutRestartRefused) |
| + } |
| + return appconfig.ErrDeliveredOnWake |
| + } |
| + if legs.restart == nil { |
| + return nil |
| + } |
| + return legs.restart(host) |
| + } |
| + return appconfig.Callbacks{Restart: restart, Ack: ack} |
| +} |
diff --git a/internal/host/rolloutdeclined_test.go b/internal/host/rolloutdeclined_test.go| new file mode 100644 |
| index 000000000..242bb1d0b |
| --- /dev/null |
| +++ b/internal/host/rolloutdeclined_test.go |
| @@ -0,0 +1,412 @@ |
| +package host |
| + |
| +import ( |
| + "bytes" |
| + "context" |
| + "errors" |
| + "net/http" |
| + "net/http/httptest" |
| + "os" |
| + "path/filepath" |
| + "reflect" |
| + "slices" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/activator" |
| + "github.com/carlosframework/platform/internal/appconfig" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| + "github.com/carlosframework/platform/internal/rollout" |
| +) |
| + |
| +// writeFailureBook writes recs to dd's failure book through the book itself, |
| +// then retries each host in retried, as an armed run would have left them. |
| +func writeFailureBook(t *testing.T, dd string, recs []rollout.FailureRecord, retried map[string][]string) { |
| + t.Helper() |
| + b, err := rollout.OpenFailureBook(dd) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, r := range recs { |
| + if _, err := b.Record(r); err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, h := range retried[r.Build] { |
| + if _, err := b.RetryHost(r.Account, r.App, r.Build, h); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + } |
| +} |
| + |
| +// bookSnapshot is the book file's bytes and modification time, so a test can |
| +// show a declined box never wrote it. |
| +type bookSnapshot struct { |
| + raw []byte |
| + mod time.Time |
| +} |
| + |
| +func snapshotBook(t *testing.T, dd string) bookSnapshot { |
| + t.Helper() |
| + p := filepath.Join(dd, rollout.FailureFile) |
| + raw, err := os.ReadFile(p) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + fi, err := os.Stat(p) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + return bookSnapshot{raw, fi.ModTime()} |
| +} |
| + |
| +func (s bookSnapshot) unchanged(t *testing.T, dd string) { |
| + t.Helper() |
| + now := snapshotBook(t, dd) |
| + if !bytes.Equal(now.raw, s.raw) || !now.mod.Equal(s.mod) { |
| + t.Fatalf("a declined box wrote the failure book:\nbefore %s\nafter %s", s.raw, now.raw) |
| + } |
| +} |
| + |
| +// declineWithBook arms svc as runAgent does on a box with no delegated cgroup |
| +// subtree: swapArming declined the flag, the Service was built without |
| +// generations, and armRollout runs with CARLOS_SWAP=overlap set. The store |
| +// must never be opened. |
| +func declineWithBook(t *testing.T, svc *activator.Service, reg *registry.Registry, dd string) { |
| + t.Helper() |
| + _, pub, err := release.GenerateKey() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + declineRollout(rolloutNoDelegation) |
| + opened := false |
| + m := armRollout(context.Background(), overlapEnv(pub), svc, reg, dd, nil, |
| + func(context.Context) (release.Store, error) { opened = true; return nil, errors.New("unused") }) |
| + if m != nil || armedRollout() != nil || opened { |
| + t.Fatal("a declined box armed the scheduler or opened the store") |
| + } |
| + if activator.GenerationsEnabled() || svc.GenerationsOn() { |
| + t.Fatal("a declined box turned generations on") |
| + } |
| +} |
| + |
| +// managedRoute is a managed, hibernating exec instance of bab/blog installed |
| +// on ver. |
| +func managedRoute(host, ver string) registry.Route { |
| + rt := execRoute(host, "bab", "blog") |
| + rt.Hibernate, rt.Version = true, ver |
| + return rt |
| +} |
| + |
| +func isTodaysWake(p activator.WakePlan) bool { |
| + return p.Refuse == "" && p.Boot.StagedPath == "" && p.Check == nil && !p.FirstWake && p.Fallback == nil |
| +} |
| + |
| +// Final review F6 (c): a declined box with no failure records, the book |
| +// absent or empty, is exactly today's box: no activator hook, every leg |
| +// today's, and the status says only why the flag was declined. |
| +func TestDeclinedWithNoRecordsIsToday(t *testing.T) { |
| + for name, body := range map[string]string{ |
| + "absent book": "", |
| + "empty book": `{"schema": 1, "records": []}`, |
| + } { |
| + t.Run(name, func(t *testing.T) { |
| + t.Setenv("CARLOS_OPT_DIR", t.TempDir()) |
| + svc, reg, dd := rolloutFixture(t, false) |
| + if body != "" { |
| + if err := os.WriteFile(filepath.Join(dd, rollout.FailureFile), []byte(body), 0o644); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + armAlways(svc, reg, dd) |
| + declineWithBook(t, svc, reg, dd) |
| + if svc.Planner != nil || svc.OnGen != nil || svc.OnDrift != nil || svc.CoalesceRecycle != nil { |
| + t.Fatal("a declined box with no records set an activator hook") |
| + } |
| + if heldHolds() != nil || rolloutUnavailable() != "" { |
| + t.Fatal("a declined box with no records holds or refuses something") |
| + } |
| + if r := rolloutStatusProbe(); r == nil || r.Unavailable != rolloutNoDelegation || r.FailureBook != "" { |
| + t.Fatalf("status = %+v, want only the decline code %q", r, rolloutNoDelegation) |
| + } |
| + legsAreTodays(t, svc, reg) |
| + if body == "" { |
| + if _, err := os.Stat(filepath.Join(dd, rollout.FailureFile)); !errors.Is(err, os.ErrNotExist) { |
| + t.Fatalf("a declined box created the failure book: %v", err) |
| + } |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// Final review F6 (a): a declined box with records refuses a managed wake |
| +// onto a build they hold for that instance, with fixed text, and wakes |
| +// everything else as today. The book is never written. |
| +func TestDeclinedBoxRefusesAWakeOntoAHeldBuild(t *testing.T) { |
| + optDir := t.TempDir() |
| + t.Setenv("CARLOS_OPT_DIR", optDir) |
| + svc, reg, dd := rolloutFixture(t, false) |
| + acct := release.RouteAccount(managedRoute("x", "")) |
| + writeFailureBook(t, dd, []rollout.FailureRecord{{ |
| + Account: acct, App: "blog", Build: "v2", Host: "sib.example.com", Cause: "adopt", Mode: "hold", Reason: "health", |
| + HeldDown: []string{"down.example.com", "downgood.example.com"}, |
| + }}, map[string][]string{"v2": {"retried.example.com"}}) |
| + before := snapshotBook(t, dd) |
| + declineWithBook(t, svc, reg, dd) |
| + if svc.Planner == nil { |
| + t.Fatal("no Planner: a wake onto a held build runs from the live link") |
| + } |
| + if svc.OnGen != nil || svc.OnDrift != nil || svc.CoalesceRecycle != nil || rolloutUnavailable() != "" { |
| + t.Fatal("honouring records changed more than the wake: the box must otherwise stay on today's paths") |
| + } |
| + |
| + good := managedRoute("good.example.com", "v2") |
| + good.GoodVersion, good.GoodDigest = "v2", "sha256:good" |
| + downGood := managedRoute("downgood.example.com", "v2") |
| + downGood.GoodVersion, downGood.GoodDigest = "v2", "sha256:good" |
| + unit := managedRoute("unit.example.com", "v2") |
| + unit.Unit = "blog.service" |
| + // Installed v1, but root has already repointed live/<host> at v2 and |
| + // not yet stamped the row: the wake would exec v2. |
| + repointed := managedRoute("repointed.example.com", "v1") |
| + if err := release.StageArtifact(optDir, acct, "v2", "blog", []byte("v2")); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := release.PointLive(optDir, repointed.Host, acct, "v2", "blog"); err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, tc := range []struct { |
| + rt registry.Route |
| + refuse string |
| + }{ |
| + {managedRoute("a.example.com", "v2"), refuseNoSwaps}, |
| + {managedRoute("down.example.com", "v2"), "build v2 failed in hold mode on this instance; recovery is explicit"}, |
| + {downGood, "build v2 failed in hold mode on this instance; recovery is explicit"}, |
| + {repointed, refuseNoSwaps}, |
| + {good, ""}, |
| + {managedRoute("retried.example.com", "v2"), ""}, |
| + {managedRoute("old.example.com", "v1"), ""}, |
| + {managedRoute("new.example.com", ""), ""}, |
| + {unit, ""}, |
| + } { |
| + p := svc.Planner(context.Background(), tc.rt) |
| + if tc.refuse == "" { |
| + if !isTodaysWake(p) { |
| + t.Errorf("%s: planned %+v, want today's wake", tc.rt.Host, p) |
| + } |
| + continue |
| + } |
| + if p.Refuse != tc.refuse { |
| + t.Errorf("%s: refusal %q, want %q", tc.rt.Host, p.Refuse, tc.refuse) |
| + } |
| + } |
| + if r := rolloutStatusProbe(); r == nil || r.Unavailable != rolloutNoDelegation || r.FailureBook != rolloutBookHonoured { |
| + t.Fatalf("status = %+v, want the decline code %q and the book %q", r, rolloutNoDelegation, rolloutBookHonoured) |
| + } |
| + before.unchanged(t, dd) |
| +} |
| + |
| +// Final review F6 (b), the legs handed fakes: the restart leg and config |
| +// sync's restart leave a held instance running, and every other host gets |
| +// today's bounce. A held instance's config ack never claims a restart. |
| +func TestDeclinedLegsLeaveAHeldInstanceRunning(t *testing.T) { |
| + t.Setenv("CARLOS_OPT_DIR", t.TempDir()) |
| + svc, reg, dd := rolloutFixture(t, false) |
| + acct := release.RouteAccount(managedRoute("x", "")) |
| + writeFailureBook(t, dd, []rollout.FailureRecord{{Account: acct, App: "blog", Build: "v2", Host: "sib.example.com"}}, nil) |
| + before := snapshotBook(t, dd) |
| + held, sleeper, free := managedRoute("held.example.com", "v2"), managedRoute("sleeper.example.com", "v2"), managedRoute("free.example.com", "v1") |
| + for _, rt := range []registry.Route{held, sleeper, free} { |
| + if err := reg.Upsert(rt); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + declineWithBook(t, svc, reg, dd) |
| + |
| + // The restart leg. |
| + ctx, s := genKit(t) |
| + var bounced []string |
| + bounce := func(h string) bool { bounced = append(bounced, h); return true } |
| + mem := newRestartMemory() |
| + routes := []registry.Route{held, free} |
| + askRestart(t, ctx, s, "bab", "blog") |
| + restartRequestLeg(ctx, routes, s, t.TempDir(), bounce, nil, mem) |
| + askRestart(t, ctx, s, "bab", "blog") |
| + if !restartRequestLeg(ctx, routes, s, t.TempDir(), bounce, nil, mem) { |
| + t.Fatal("the restart leg did not ask for the root pass") |
| + } |
| + if !slices.Equal(bounced, []string{"free.example.com"}) { |
| + t.Fatalf("restart bounced %v, want only the instance no record holds", bounced) |
| + } |
| + |
| + // Config sync, end to end over the real pass. |
| + store := release.NewDirStore(t.TempDir()) |
| + privText, _, err := appconfig.GenerateKey() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + priv, err := appconfig.ParsePrivateKey(privText, "test") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := appconfig.SetEnv(ctx, store, "bab", "blog", map[string]string{"LOG_LEVEL": "debug"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + var restarted []string |
| + cb, ok := rolloutConfigCallbacks(ctx, store, configLegs{ |
| + awake: func(h string) bool { return h != sleeper.Host }, |
| + restart: func(h string) error { restarted = append(restarted, h); return nil }, |
| + }) |
| + if !ok { |
| + t.Fatal("config sync took today's SyncExec, which would bounce the held instance") |
| + } |
| + acks := map[string]appconfig.ConfigAck{} |
| + cb.Ack = func(a appconfig.ConfigAck) { acks[a.Host] = a } |
| + if err := appconfig.SyncExecWithCallbacks(ctx, reg, store, priv, t.TempDir(), "", cb); err != nil { |
| + t.Fatal(err) |
| + } |
| + if !slices.Equal(restarted, []string{free.Host}) { |
| + t.Fatalf("config sync restarted %v, want only the instance no record holds", restarted) |
| + } |
| + if a := acks[held.Host]; a.Restarted || a.Hibernating || a.RestartError != rolloutRestartRefused { |
| + t.Fatalf("held awake ack = %+v, want RestartError %q", a, rolloutRestartRefused) |
| + } |
| + if a := acks[sleeper.Host]; a.Restarted || !a.Hibernating || a.RestartError != "" { |
| + t.Fatalf("held sleeper ack = %+v, want delivered on wake", a) |
| + } |
| + if a := acks[free.Host]; !a.Restarted || a.Hibernating || a.RestartError != "" { |
| + t.Fatalf("free ack = %+v, want today's restarted", a) |
| + } |
| + before.unchanged(t, dd) |
| +} |
| + |
| +// Final review F6 (a) and (b) on a real activator, the converge leg and the |
| +// adopt bounce as fastConvergeLoop and the adopt pass build them. Root |
| +// installs a build a failure record holds: neither leg stops the instance |
| +// still serving the old build, and when something else does stop it, the |
| +// next wake is refused rather than exec the held build from the live link. |
| +func TestDeclinedBoxKeepsAHeldBuildOffARealActivator(t *testing.T) { |
| + prev := activator.GenerationsEnabled() |
| + activator.SetGenerations(false) |
| + t.Cleanup(func() { activator.SetGenerations(prev) }) |
| + resetRollout(t) |
| + env := newFastFixture(t) |
| + t.Setenv("CARLOS_OPT_DIR", env.optDir) |
| + host := "warm.test" |
| + rt := registry.Route{Host: host, Kind: "instance", App: "fastapp", Account: "bab", Channel: "edge", |
| + Status: "ready", Hibernate: true, Version: "v1"} |
| + if err := env.reg.Upsert(rt); err != nil { |
| + t.Fatal(err) |
| + } |
| + env.stageLive(t, host, "v1") |
| + writeFailureBook(t, env.dataDir, []rollout.FailureRecord{{Account: release.RouteAccount(rt), App: "fastapp", Build: "v2", Host: "sibling.test"}}, nil) |
| + backend := &versionedBackend{optDir: env.optDir} |
| + svc := activator.New(activator.Config{Bucket: "test-bucket", Idle: time.Hour}, backend, env.reg) |
| + svc.Routes = env.reg |
| + t.Cleanup(func() { |
| + sctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
| + defer cancel() |
| + svc.Shutdown(sctx) |
| + }) |
| + declineWithBook(t, svc, env.reg, env.dataDir) |
| + request := func() int { |
| + rec := httptest.NewRecorder() |
| + proxy := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }) |
| + if !svc.ServeHTTP(rec, httptest.NewRequest("GET", "http://"+host+"/", nil), rt, proxy) { |
| + t.Fatal("the activator did not take the request") |
| + } |
| + return rec.Code |
| + } |
| + if code := request(); code != http.StatusNoContent { |
| + t.Fatalf("wake onto v1 = %d", code) |
| + } |
| + |
| + // Root installs v2: the link first, then the stamp. |
| + env.stageLive(t, host, "v2") |
| + rt.Version = "v2" |
| + if err := env.reg.Upsert(rt); err != nil { |
| + t.Fatal(err) |
| + } |
| + leg := convergeLegFor(svc, env.reg) |
| + leg() |
| + restart := makeRestartRoute(env.reg, func(string) {}, unitOnly(func(u string) error { |
| + t.Fatalf("named-unit arm fired for %q", u) |
| + return nil |
| + }), adoptBounceFn(svc)) |
| + if err := restart(host); err != nil { |
| + t.Fatal(err) |
| + } |
| + if !svc.Owns(host) { |
| + t.Fatal("a leg stopped the instance serving v1 into a wake onto held v2") |
| + } |
| + if code := request(); code != http.StatusNoContent { |
| + t.Fatalf("request to the running instance = %d", code) |
| + } |
| + if got := backend.started(); !reflect.DeepEqual(got, []string{"v1"}) { |
| + t.Fatalf("spawned builds = %v, want only v1", got) |
| + } |
| + |
| + // The sweep's own convergence is the activator's, unchanged on a |
| + // declined box; whatever stops the instance, its next wake is refused. |
| + svc.Bounce(host) |
| + deadline := time.Now().Add(10 * time.Second) |
| + for svc.Owns(host) { |
| + if time.Now().After(deadline) { |
| + t.Fatal("the bounce never stopped the instance") |
| + } |
| + time.Sleep(5 * time.Millisecond) |
| + } |
| + if code := request(); code == http.StatusNoContent { |
| + t.Fatal("the wake onto held v2 was served") |
| + } |
| + if got := backend.started(); !reflect.DeepEqual(got, []string{"v1"}) { |
| + t.Fatalf("spawned builds = %v: held v2 was exec'd from the live link", got) |
| + } |
| +} |
| + |
| +// Final review F6 (d): a declined box whose failure book cannot be read |
| +// fails closed, as refuseManaged does on an armed box: every managed wake is |
| +// refused with the fixed text, and the bounce legs stand down. The status |
| +// keeps the decline code and says why starts are refused. The book is not |
| +// rewritten. |
| +func TestDeclinedBoxWithAnUnreadableBookFailsClosed(t *testing.T) { |
| + for name, body := range map[string]string{ |
| + "not json": "{not json", |
| + "bad schema": `{"schema": 9}`, |
| + } { |
| + t.Run(name, func(t *testing.T) { |
| + t.Setenv("CARLOS_OPT_DIR", t.TempDir()) |
| + svc, reg, dd := rolloutFixture(t, false) |
| + if err := os.WriteFile(filepath.Join(dd, rollout.FailureFile), []byte(body), 0o644); err != nil { |
| + t.Fatal(err) |
| + } |
| + before := snapshotBook(t, dd) |
| + declineWithBook(t, svc, reg, dd) |
| + if svc.Planner == nil { |
| + t.Fatal("no Planner: an unreadable book let managed wakes run from the live link") |
| + } |
| + if p := svc.Planner(context.Background(), managedRoute("a.example.com", "v1")); p.Refuse != refuseTexts[rolloutBookUnreadable] { |
| + t.Fatalf("managed wake planned %+v, want the fixed refusal %q", p, refuseTexts[rolloutBookUnreadable]) |
| + } |
| + unit := managedRoute("u.example.com", "v1") |
| + unit.Unit = "blog.service" |
| + if p := svc.Planner(context.Background(), unit); !isTodaysWake(p) { |
| + t.Fatalf("an unmanaged wake was changed: %+v", p) |
| + } |
| + if adoptBounceFn(svc) != nil || fastConvergeLeg(svc, reg) != nil { |
| + t.Fatal("a bounce leg still stops instances whose next wake is refused") |
| + } |
| + if svc.OnDrift == nil || svc.CoalesceRecycle == nil || svc.CoalesceRecycle("bot.internal") { |
| + t.Fatal("the activator's own moves onto the live link are still armed") |
| + } |
| + if svc.OnGen != nil || heldHolds() != nil { |
| + t.Fatal("an unreadable book was honoured as records") |
| + } |
| + if r := rolloutStatusProbe(); r == nil || r.Unavailable != rolloutNoDelegation || r.FailureBook != rolloutBookUnreadable { |
| + t.Fatalf("status = %+v, want the decline code %q and the book %q", r, rolloutNoDelegation, rolloutBookUnreadable) |
| + } |
| + before.unchanged(t, dd) |
| + }) |
| + } |
| +} |
diff --git a/internal/rollout/wakeplan.go b/internal/rollout/wakeplan.go| index 54cbbe283..50d315379 100644 |
| --- a/internal/rollout/wakeplan.go |
| +++ b/internal/rollout/wakeplan.go |
| @@ -26,6 +26,14 @@ type wakeInputs struct { |
| Demand func(version, stagedPath string) (demand, digest string, err error) |
| } |
| +// HeldDownRefusal is the wake refusal for an instance where build failed in |
| +// hold mode (spec §6.4, §8.3). It reaches the member's "wake failed" event, |
| +// so it names the build only. A box that declined CARLOS_SWAP=overlap but |
| +// honours the failure records refuses with the same words (internal/host). |
| +func HeldDownRefusal(build string) string { |
| + return "build " + build + " failed in hold mode on this instance; recovery is explicit" |
| +} |
| + |
| // decideWake is spec §8.2's "when the wake picks a build", as a pure |
| // function. The installed build (root's stamp) boots unless a failure record |
| // holds it for this instance, in which case the good build boots from its |
| @@ -50,7 +58,7 @@ func decideWake(in wakeInputs) (activator.WakePlan, error) { |
| Digest: rt.GoodDigest, Demand: rt.GoodDemand} |
| recovery := activator.WakePlan{Boot: good, Mode: RecoveryMode(in.Policy, good.Demand)} |
| if in.Book.HeldDown(acct, app, installed, host) { |
| - return activator.WakePlan{Refuse: "build " + installed + " failed in hold mode on this instance; recovery is explicit"}, nil |
| + return activator.WakePlan{Refuse: HeldDownRefusal(installed)}, nil |
| } |
| if hasGood && installed == rt.GoodVersion { |
| return recovery, nil |
diff --git a/internal/status/snapshot.go b/internal/status/snapshot.go| index 22ca73b49..45788606c 100644 |
| --- a/internal/status/snapshot.go |
| +++ b/internal/status/snapshot.go |
| @@ -265,6 +265,14 @@ type RolloutReport struct { |
| // is refused meanwhile (Task 14, internal/host's swapArming and |
| // refuseManaged). The rest of the report is then empty. Plan 6 words it. |
| Unavailable string `json:"unavailable,omitempty"` |
| + // FailureBook is set only beside one of the first three codes: what a |
| + // box that cannot contain generations does with the failure records an |
| + // earlier armed run left (final review F6). "honoured": a managed wake, |
| + // or a bounce, onto a build they hold for that instance is refused. |
| + // "failure-book-unreadable": every managed start is refused and the |
| + // bounce legs stand down. Empty: there are no records, and the box is on |
| + // today's paths. A code, like Unavailable. |
| + FailureBook string `json:"failure_book,omitempty"` |
| } |
| // HostRollout is one managed instance. Attempt and Reason are the closed |