amadan
zdd-readiness: declined boxes process signed retries and promote --retry pointers
A box that declined CARLOS_SWAP=overlap but honours its failure book (Astra R2 ruling 194, plan review #5, readiness finding 5) now publishes each held host with HeldBy, HeldEpoch and an attempt word beside its decline code; the console folds those hosts and its retry doors sign a grant for a decline-coded report with a HeldBy, still refusing every other Unavailable. The box hands each verified grant to failureHolds.retryHost (RetryHostAt, epoch-bound, the same h.book the planner and launch gate read), and fastConvergeLoop runs a pointer pass every rollout.PointerReadEvery: promote --retry clears a record, a signed rollback clears a marker. The pointer rules are factored out of Manager.applyPointers (pointerWork, rollout.ApplyPointers) so both boxes apply the same ones.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
10 files changed,
+829
−98
internal/console/apiinstances.go+17 −3internal/console/apiretry.go+8 −3internal/console/apiretry_test.go+80 −5internal/host/agent.go+15 −4internal/host/restart.go+7 −2internal/host/restartscope.go+7 −6internal/host/rollout.go+6 −2internal/host/rolloutdeclined.go+165 −23internal/host/rolloutdeclined_test.go+406 −0internal/rollout/manager.go+118 −50
diff --git a/internal/console/apiinstances.go b/internal/console/apiinstances.go| index e4242a5dd..3cebf0275 100644 |
| --- a/internal/console/apiinstances.go |
| +++ b/internal/console/apiinstances.go |
| @@ -361,7 +361,10 @@ type rolloutView struct { |
| SchedulerIdle bool `json:"scheduler_idle"` |
| // Unavailable is the reason code a box gives for being unable to arm its |
| // rollout scheduler (RolloutReport.Unavailable). When set, the view is |
| - // only Box, Installed and this: the box reports no per-host state. |
| + // only Box, Installed and this: the box reports no per-host state. The |
| + // one exception is a host a box that declined the flag reports held by |
| + // the failure book it honours (Task 10), whose view also carries Good, |
| + // the attempt, HeldBy and its epoch. |
| Unavailable string `json:"unavailable,omitempty"` |
| account, app string |
| // heldEpoch is the box's epoch for HeldBy's restriction |
| @@ -568,11 +571,22 @@ func (j *boxJoin) foldRollout(b status.Box, override bool) { |
| j.rollouts[host] = v |
| } |
| if ro.Unavailable != "" { |
| - // The scheduler could not arm, so Hosts is empty. Say so for every |
| - // host this box serves, from the version it reports for it. |
| + // The scheduler could not arm. Say so for every host this box |
| + // serves, from the version it reports for it. A box that declined |
| + // the flag but honours its failure book also reports the hosts the |
| + // book holds (Task 10, plan review #5): those keep the code, and |
| + // carry the hold and its epoch so a retry can be signed for them |
| + // (retryHostBuild), under the report's own account and app. |
| for host, v := range b.Metrics.RouteVersions { |
| + if _, held := ro.Hosts[host]; held { |
| + continue |
| + } |
| put(host, rolloutView{Box: b.Label, Installed: v, Unavailable: ro.Unavailable, refused: true}) |
| } |
| + for host, hr := range ro.Hosts { |
| + put(host, rolloutView{Box: b.Label, Installed: hr.Installed, Good: hr.Good, Attempt: hr.Attempt, AttemptBuild: hr.AttemptBuild, |
| + HeldBy: hr.HeldBy, Reason: hr.Reason, Unavailable: ro.Unavailable, account: hr.Account, app: hr.App, heldEpoch: hr.HeldEpoch}) |
| + } |
| return |
| } |
| // Plan 5 marks every host a failure record holds as failed. The |
diff --git a/internal/console/apiretry.go b/internal/console/apiretry.go| index e7fb218de..084edeb28 100644 |
| --- a/internal/console/apiretry.go |
| +++ b/internal/console/apiretry.go |
| @@ -25,6 +25,7 @@ import ( |
| "github.com/carlosframework/platform/internal/provision" |
| "github.com/carlosframework/platform/internal/release" |
| + "github.com/carlosframework/platform/internal/rolloutwords" |
| ) |
| const ( |
| @@ -47,8 +48,12 @@ var ( |
| // must exist in this account and app (the instance family's 404), and the |
| // report must be this app's own (boxJoin.rollout), read through the same |
| // loader as the list and the instance page (loadRolloutJoin). A box that |
| -// could not arm its scheduler reports no hold, so its host is not held. A |
| -// box older than the epoch reports 0, which is signed as it is and which no |
| +// could not arm its scheduler is not held, except one that declined the |
| +// flag (a decline code, rolloutwords.UnavailableDeclines) and reports the |
| +// host held by the failure book it still honours: that box verifies and |
| +// applies a signed retry too (Task 10, plan review #5). A refusing box, or |
| +// one whose code this console does not know, is never retried here. A box |
| +// older than the epoch reports 0, which is signed as it is and which no |
| // box accepts. |
| func (s *Server) retryHostBuild(ctx context.Context, acct *Account, app, host string) (string, uint64, error) { |
| if _, err := provision.GetRecord(ctx, s.cfg.Releases, acct.Sqid, app, host); err != nil { |
| @@ -59,7 +64,7 @@ func (s *Server) retryHostBuild(ctx context.Context, acct *Account, app, host st |
| } |
| join, _ := s.loadRolloutJoin(ctx, acct.Sqid, time.Now(), nil) |
| ro := join.rollout(host, acct.Sqid, app) |
| - if ro == nil || ro.Unavailable != "" || ro.HeldBy == "" { |
| + if ro == nil || ro.HeldBy == "" || (ro.Unavailable != "" && !rolloutwords.UnavailableDeclines(ro.Unavailable)) { |
| return "", 0, errRetryNotHeld |
| } |
| return ro.HeldBy, ro.heldEpoch, nil |
diff --git a/internal/console/apiretry_test.go b/internal/console/apiretry_test.go| index 3e6d25f90..dd118a30e 100644 |
| --- a/internal/console/apiretry_test.go |
| +++ b/internal/console/apiretry_test.go |
| @@ -87,11 +87,10 @@ func TestAPIInstanceRetryIgnoresAnotherAppsReport(t *testing.T) { |
| } |
| } |
| -// A box that could not arm its scheduler reports no per-host state: its view is |
| -// built with Unavailable and no HeldBy, so the host is not held. This pins that |
| -// whole path to the 409. It cannot tell retryHostBuild's own Unavailable check |
| -// from the HeldBy check, because loadBoxJoin never builds a view carrying both; |
| -// the check is a guard for a join that someday does. |
| +// A box that refuses every managed start reports no per-host state: its view |
| +// is built with Unavailable and no HeldBy, so the host is not held. This pins |
| +// that whole path to the 409. TestAPIInstanceRetryOfADeclinedBoxsHold covers |
| +// a view carrying both. |
| func TestAPIInstanceRetryRefusesAnUnavailableBox(t *testing.T) { |
| fx := newConsoleFixture(t) |
| const host = "web.kass.carlosapp.dev" |
| @@ -112,6 +111,82 @@ func TestAPIInstanceRetryRefusesAnUnavailableBox(t *testing.T) { |
| } |
| } |
| +// Task 10 Correction 1 (plan review #5): a box that declined |
| +// CARLOS_SWAP=overlap but honours the failure book an armed run left |
| +// publishes its held hosts beside its decline code, and the door signs a |
| +// retry for one from that report. Any other Unavailable code is still |
| +// refused, whatever its report carries, and a declined box's host it does |
| +// not report as held is not held. |
| +func TestAPIInstanceRetryOfADeclinedBoxsHold(t *testing.T) { |
| + const host = "web.kass.carlosapp.dev" |
| + write := func(t *testing.T, fx *consoleFixture, code string, held bool) { |
| + t.Helper() |
| + ro := &status.RolloutReport{Unavailable: code, FailureBook: "honoured"} |
| + if held { |
| + ro.Hosts = map[string]status.HostRollout{host: {Account: fx.sqid, App: "kass", Installed: "v2", Good: "v1", |
| + Attempt: status.AttemptDown, AttemptBuild: "v2", HeldBy: "v2", HeldEpoch: heldEpoch}} |
| + } |
| + box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) |
| + box.Metrics = &status.BoxMetrics{RouteVersions: map[string]string{host: "v2"}, Rollout: ro} |
| + if err := status.WriteBox(context.Background(), fx.rel, box); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + for _, tc := range []struct { |
| + name string |
| + code string |
| + held bool |
| + ok bool |
| + }{ |
| + {"declined, held", "no-cgroup-delegation", true, true}, |
| + {"claims on, held", "host-claims-on", true, true}, |
| + {"declined, not held", "generations-off", false, false}, |
| + {"refusing, with hosts", "no-deploy-store", true, false}, |
| + {"unknown code, with hosts", "a-newer-code", true, false}, |
| + } { |
| + t.Run(tc.name, func(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + sibling(t, fx, "kass", host, "stable") |
| + write(t, fx, tc.code, tc.held) |
| + token := cliToken(t, fx, "paul@keymail.dev", "code-host-retry-d") |
| + u := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/kass/instances/" + host + "/retry" |
| + code, _, body := rawReq(t, http.MethodPost, u, token, `{}`) |
| + if !tc.ok { |
| + if code != http.StatusConflict || !strings.Contains(body, fmt.Sprintf(apiRetryNotHeld, host)) { |
| + t.Fatalf("retry = %d %s, want 409 naming %s as not held", code, body, host) |
| + } |
| + if g, found, _ := release.LoadRestartScopes(context.Background(), fx.rel, fx.sqid, "kass"); found && len(g.Requests) != 0 { |
| + t.Fatalf("a retry was written: %+v", g.Requests) |
| + } |
| + return |
| + } |
| + if code != http.StatusOK { |
| + t.Fatalf("retry = %d %s, want the declined box's hold retried", code, body) |
| + } |
| + r := lastScoped(t, fx, "kass") |
| + g, err := release.VerifyHostRetry(fx.pub, r.Scope.Signed, fx.sqid, "kass", host, time.Now()) |
| + if err != nil || g.Build != "v2" || g.Epoch != heldEpoch { |
| + t.Fatalf("signed grant = %+v, %v; want v2 at epoch %d", g, err, heldEpoch) |
| + } |
| + }) |
| + } |
| + // Another app's report under the host's name is not this app's hold. |
| + fx := newConsoleFixture(t) |
| + sibling(t, fx, "kass", host, "stable") |
| + box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) |
| + box.Metrics = &status.BoxMetrics{RouteVersions: map[string]string{host: "v2"}, Rollout: &status.RolloutReport{ |
| + Unavailable: "no-cgroup-delegation", FailureBook: "honoured", |
| + Hosts: map[string]status.HostRollout{host: {Account: fx.sqid, App: "otherapp", Installed: "v2", HeldBy: "v2", HeldEpoch: heldEpoch}}}} |
| + if err := status.WriteBox(context.Background(), fx.rel, box); err != nil { |
| + t.Fatal(err) |
| + } |
| + token := cliToken(t, fx, "paul@keymail.dev", "code-host-retry-o") |
| + u := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/kass/instances/" + host + "/retry" |
| + if code, _, body := rawReq(t, http.MethodPost, u, token, `{}`); code != http.StatusConflict { |
| + t.Fatalf("another app's declined hold = %d %s, want 409", code, body) |
| + } |
| +} |
| + |
| // A box that reports the host and holds nothing back: there is nothing to |
| // retry, and no request may be written. |
| func TestAPIInstanceRetryRefusesAHostThatIsNotHeld(t *testing.T) { |
diff --git a/internal/host/agent.go b/internal/host/agent.go| index b4dd61dfc..f4916d876 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -2257,10 +2257,12 @@ func fastConvergeLoop(reg *registry.Registry, svc *activator.Service, sd *schedu |
| ctx := context.Background() |
| store, err := openDeployStore(ctx) |
| if err != nil { |
| + declinedHoldsStay("no deploy store") |
| return // adoptLoop already logged this |
| } |
| pub, err := release.ParsePublicKey(os.Getenv("CARLOS_RELEASE_PUBKEY"), "CARLOS_RELEASE_PUBKEY") |
| if err != nil { |
| + declinedHoldsStay("no release key") |
| return // ditto |
| } |
| var priv *ecdh.PrivateKey |
| @@ -2299,11 +2301,20 @@ func fastConvergeLoop(reg *registry.Registry, svc *activator.Service, sd *schedu |
| // first request to establish itself. |
| restartMem := newRestartMemory() |
| // `carlos instances retry` (zero-downtime spec §8.4) reaches the rollout |
| - // manager through the narrowed-restart log; nil while it is not armed. |
| - // Each entry's grant is verified against the root key first (spec §11). |
| + // manager through the narrowed-restart log, or, on a box that declined |
| + // the flag but honours its failure book, that book (Task 10); nil |
| + // otherwise. Each entry's grant is verified against the root key first |
| + // (spec §11). |
| restartMem.pub = pub |
| - if m := armedRollout(); m != nil { |
| - restartMem.retryHost = m.RetryHost |
| + restartMem.retryHost = rolloutRetryHost() |
| + // The declined box's pointer pass (Task 10, ruling 194): `carlos promote |
| + // --retry` clears a record, a signed rollback a marker, read through |
| + // the same verified pointer read the armed scheduler uses, at most |
| + // every rollout.PointerReadEvery. |
| + if h := heldHolds(); h != nil { |
| + go h.pointerLoop(ctx, func(ctx context.Context, acct, app, channel string) (*release.ChannelPointer, error) { |
| + return release.Channel(ctx, store, pub, acct, app, channel) |
| + }) |
| } |
| restartSeed(ctx, reg, store, dd, restartMem) |
| restartBounce := bounceFn(svc) |
diff --git a/internal/host/restart.go b/internal/host/restart.go| index 3950f4f92..9ad6a94f5 100644 |
| --- a/internal/host/restart.go |
| +++ b/internal/host/restart.go |
| @@ -853,7 +853,8 @@ func restartRequestLeg(ctx context.Context, routes []registry.Route, store relea |
| // entry (spec §11, Astra impl-6 finding 5): every box can write the log, |
| // so an unsigned or foreign entry is dropped here, before anything is |
| // cleared. The grant's epoch is checked against the book where the |
| - // clear is (rollout.Manager.RetryHost). |
| + // clear is (rollout.Manager.RetryHost, or failureHolds.retryHost on a |
| + // box that declined the flag but honours its book). |
| for _, rq := range retryRequests(routes, scoped) { |
| if mem.retryHost == nil { |
| log.Printf("agent: retry requested for %s, dropped: rollout manager not armed on this box, so nothing is held to clear", rq.host) |
| @@ -865,7 +866,11 @@ func restartRequestLeg(ctx context.Context, routes []registry.Route, store relea |
| continue |
| } |
| if mem.retryHost(rq.host, g.Build, g.Epoch) { |
| - log.Printf("agent: retry requested for %s: its failure hold on %s is cleared and its next attempt runs as a canary", rq.host, g.Build) |
| + next := "its next attempt runs as a canary" |
| + if armedRollout() == nil { |
| + next = "it starts the build as today" // a declined box runs no canary |
| + } |
| + log.Printf("agent: retry requested for %s: its failure hold on %s is cleared and %s", rq.host, g.Build, next) |
| } else { |
| log.Printf("agent: retry for %s: nothing held to clear for %s at epoch %d (cleared already, or a newer failure since the grant)", rq.host, g.Build, g.Epoch) |
| } |
diff --git a/internal/host/restartscope.go b/internal/host/restartscope.go| index 63730468f..63913b1fb 100644 |
| --- a/internal/host/restartscope.go |
| +++ b/internal/host/restartscope.go |
| @@ -53,12 +53,13 @@ type restartMemory struct { |
| gen map[[2]string]string // app -> restart-gen.json etag, or genAbsent |
| scope map[[2]string]string // app -> last applied request id, or release.RestartScopeNone |
| - // retryHost clears one host's failure hold (rollout.Manager.RetryHost), |
| - // for a RestartScopeRetry request whose root-signed grant verified: the |
| - // grant's build and the restriction's epoch it was issued for. nil while |
| - // the rollout manager is not armed: there is no hold to clear, and the |
| - // request is dropped (a declined box's holds get retries in a later |
| - // task, through this same signature). |
| + // retryHost clears one host's failure hold, for a RestartScopeRetry |
| + // request whose root-signed grant verified: the grant's build and the |
| + // restriction's epoch it was issued for (rolloutRetryHost). It is the |
| + // rollout manager's (rollout.Manager.RetryHost) when armed, and a |
| + // declined box's that honours its failure book (failureHolds.retryHost, |
| + // Task 10). nil otherwise: there is no hold to clear, and the request is |
| + // dropped. |
| retryHost func(host, build string, epoch uint64) bool |
| // pub is the deployment root's public key (CARLOS_RELEASE_PUBKEY), which |
| // every retry's grant is verified against (spec §11). Nil drops every |
diff --git a/internal/host/rollout.go b/internal/host/rollout.go| index b4d4ccd1e..56130e037 100644 |
| --- a/internal/host/rollout.go |
| +++ b/internal/host/rollout.go |
| @@ -505,14 +505,18 @@ type configLegs struct { |
| // flag is on but the scheduler is not armed (refusing, or declined). A |
| // declined box's report keeps the decline code and says what it does with |
| // the failure records (FailureBook): honoured, unreadable (every managed |
| -// start refused), or nothing when there are none. |
| +// start refused), or nothing when there are none. Honoured, it also names |
| +// each instance the book holds, with the build holding it and that |
| +// restriction's epoch (failureHolds.report; Task 10, plan review #5), so |
| +// the console can sign a retry for it. |
| func rolloutStatusProbe() *status.RolloutReport { |
| m := armedRollout() |
| if m == nil { |
| if why := rolloutUnarmed(); why != "" { |
| r := &status.RolloutReport{Unavailable: why, FailureBook: rolloutUnavailable()} |
| - if heldHolds() != nil { |
| + if h := heldHolds(); h != nil { |
| r.FailureBook = rolloutBookHonoured |
| + r.Hosts = h.report() |
| } |
| return r |
| } |
diff --git a/internal/host/rolloutdeclined.go b/internal/host/rolloutdeclined.go| index 5f9ccb984..3d10ebe55 100644 |
| --- a/internal/host/rolloutdeclined.go |
| +++ b/internal/host/rolloutdeclined.go |
| @@ -6,12 +6,14 @@ import ( |
| "log" |
| "sync" |
| "sync/atomic" |
| + "time" |
| "github.com/carlosframework/platform/internal/activator" |
| "github.com/carlosframework/platform/internal/appconfig" |
| "github.com/carlosframework/platform/internal/registry" |
| "github.com/carlosframework/platform/internal/release" |
| "github.com/carlosframework/platform/internal/rollout" |
| + "github.com/carlosframework/platform/internal/status" |
| ) |
| // rolloutBookHonoured is RolloutReport.FailureBook's code for a box that |
| @@ -36,14 +38,21 @@ const refuseNoSwaps = "this box cannot run two builds side by side, which this i |
| // config sync's restart leave such an instance running rather than stop it |
| // into a refused wake. The book holds records and hold-ran markers. |
| // |
| -// The book is read once, at arming, and written at most once then: when an |
| -// armed run left intents in it, the records reconciled from them are saved |
| -// (honourDeclinedFailures). Nothing on a declined box records, retries or |
| -// prunes, so what was read stays the truth until the agent restarts. |
| +// The book is read once, at arming, and nothing on a declined box records a |
| +// failure or a marker, or prunes (Astra R2 ruling 194, "startup-only |
| +// records"). But it is cleared while the agent runs, as an armed box's is |
| +// (Task 10): a verified signed `carlos instances retry` (retryHost), a |
| +// `carlos promote --retry` pointer and a signed rollback pointer |
| +// (applyPointers) each clear in this one book and save it. The planner, the |
| +// launch gate and the legs read it live, so a clear lets the instance start |
| +// at once, with no agent restart. It is also written once at arming when an |
| +// armed run left intents in it (honourDeclinedFailures). |
| type failureHolds struct { |
| book *rollout.FailureBook |
| // get reads a route by host, for the legs that are handed only a host. |
| get func(host string) (*registry.Route, error) |
| + // list reads every route, for the box's status report. |
| + list func() ([]registry.Route, error) |
| // optDir is where the live links are: a wake execs the build |
| // live/<host> names, which root repoints before it stamps the row. |
| optDir string |
| @@ -91,7 +100,7 @@ func honourDeclinedFailures(svc *activator.Service, reg *registry.Registry, dd, |
| if recs == 0 && marks == 0 { |
| return |
| } |
| - h := &failureHolds{book: book, get: reg.Get, optDir: optDir} |
| + h := &failureHolds{book: book, get: reg.Get, list: reg.List, optDir: optDir} |
| declinedHolds.Store(h) |
| svc.Planner = h.plan |
| // And at the real launch, with the build actually started (Task 9): the |
| @@ -134,52 +143,71 @@ func honourDeclinedFailures(svc *activator.Service, reg *registry.Registry, dd, |
| // sleeping sibling. It reads only memory, so the activator may ask it under |
| // its own lock (gate). |
| func (h *failureHolds) refusal(rt registry.Route, build string) string { |
| + return h.holdOf(rt, build).refuse |
| +} |
| + |
| +// hold is why the book holds an instance off one build: the wake refusal, |
| +// the build whose restriction refuses it (by), which is what a retry of |
| +// the instance must name (RetryHostAt), and whether the instance stays down |
| +// for it (a hold-down, or a hold-ran marker) rather than on what it runs. |
| +type hold struct { |
| + refuse, by string |
| + down bool |
| +} |
| + |
| +// holdOf is refusal with the build it holds by: the hold-down's build, the |
| +// marker's build when the marker refuses the marked build or the good |
| +// build, and otherwise the build a record holds, which a marker may refuse |
| +// in its own words while only that record's retry lifts it. |
| +func (h *failureHolds) holdOf(rt registry.Route, build string) hold { |
| if !rollout.Managed(rt) || build == "" { |
| - return "" |
| + return hold{} |
| } |
| acct, app, host := release.RouteAccount(rt), rt.App, rt.Host |
| hasGood := rt.GoodVersion != "" |
| if h.book.HeldDown(acct, app, build, host) { |
| - return rollout.HeldDownRefusal(build) |
| + return hold{rollout.HeldDownRefusal(build), build, true} |
| } |
| if mark, ok := h.book.HoldRanOn(host); ok && mark.Account == acct && mark.App == app { |
| isGood := hasGood && build == rt.GoodVersion |
| switch { |
| case build == mark.Build: |
| if hasGood && h.book.Blocks(acct, app, build, host, isGood) { |
| - return rollout.HeldDownRefusal(mark.Build) |
| + return hold{rollout.HeldDownRefusal(mark.Build), mark.Build, true} |
| } |
| - return "" |
| - case isGood, hasGood && h.book.Blocks(acct, app, build, host, false): |
| - return rollout.HeldDownRefusal(mark.Build) |
| + return hold{} |
| + case isGood: |
| + return hold{rollout.HeldDownRefusal(mark.Build), mark.Build, true} |
| + case hasGood && h.book.Blocks(acct, app, build, host, false): |
| + return hold{rollout.HeldDownRefusal(mark.Build), build, true} |
| } |
| - return "" |
| + return hold{} |
| } |
| if hasGood && h.book.Blocks(acct, app, build, host, build == rt.GoodVersion) { |
| - return refuseNoSwaps |
| + return hold{refuseNoSwaps, build, false} |
| } |
| - return "" |
| + return hold{} |
| } |
| -// heldBuild is the build the book holds rt off and its refusal (refusal), |
| -// "" when nothing holds it. Both the installed build (root's stamp) and the |
| +// heldBuild is the build the book holds rt off and its hold (holdOf), "" |
| +// when nothing holds it. Both the installed build (root's stamp) and the |
| // build live/<host> names are asked, because a wake execs the live link and |
| // root repoints it before it stamps the row; an unreadable link is left to |
| // the stamp. |
| -func (h *failureHolds) heldBuild(rt registry.Route) (build, refuse string) { |
| +func (h *failureHolds) heldBuild(rt registry.Route) (string, hold) { |
| if !rollout.Managed(rt) { |
| - return "", "" |
| + return "", hold{} |
| } |
| builds := []string{rt.Version} |
| if live, err := release.LiveVersion(h.optDir, rt.Host); err == nil && live != "" && live != rt.Version { |
| builds = append(builds, live) |
| } |
| for _, b := range builds { |
| - if r := h.refusal(rt, b); r != "" { |
| - return b, r |
| + if hd := h.holdOf(rt, b); hd.refuse != "" { |
| + return b, hd |
| } |
| } |
| - return "", "" |
| + return "", hold{} |
| } |
| // plan is the declined box's Planner: a managed wake onto a held build is |
| @@ -187,8 +215,109 @@ func (h *failureHolds) heldBuild(rt registry.Route) (build, refuse string) { |
| // other wake gets the zero plan, which is today's wake from the live link. |
| // rt is the row doWake has just re-read (Service.Routes). |
| func (h *failureHolds) plan(_ context.Context, rt registry.Route) activator.WakePlan { |
| - _, refuse := h.heldBuild(rt) |
| - return activator.WakePlan{Refuse: refuse} |
| + _, hd := h.heldBuild(rt) |
| + return activator.WakePlan{Refuse: hd.refuse} |
| +} |
| + |
| +// report is the declined box's per-host rollout status (Task 10, plan |
| +// review #5): each managed instance the book holds, with the build that |
| +// holds it (HeldBy) and that restriction's epoch (HeldEpoch), which the |
| +// console's retry doors sign into their grant, attempt down when it stays |
| +// down for the hold and failed otherwise. Nil when nothing is held or the |
| +// routes cannot be read. A declined box runs no scheduler, so it reports no |
| +// serving build and no attempt in progress. |
| +func (h *failureHolds) report() map[string]status.HostRollout { |
| + routes, err := h.list() |
| + if err != nil { |
| + return nil |
| + } |
| + var out map[string]status.HostRollout |
| + for _, rt := range routes { |
| + build, hd := h.heldBuild(rt) |
| + if build == "" { |
| + continue |
| + } |
| + acct := release.RouteAccount(rt) |
| + attempt := status.AttemptFailed |
| + if hd.down { |
| + attempt = status.AttemptDown |
| + } |
| + if out == nil { |
| + out = map[string]status.HostRollout{} |
| + } |
| + out[rt.Host] = status.HostRollout{Account: acct, App: rt.App, Installed: rt.Version, Good: rt.GoodVersion, |
| + Attempt: attempt, AttemptBuild: hd.by, HeldBy: hd.by, HeldEpoch: h.book.RestrictionEpoch(acct, rt.App, hd.by, rt.Host)} |
| + } |
| + return out |
| +} |
| + |
| +// retryHost is restartMemory.retryHost on a declined box (Task 10, ruling |
| +// 194): a `carlos instances retry` whose root-signed grant the restart leg |
| +// has verified, applied to this box's book exactly as the armed box applies |
| +// it (RetryHostAt): while build's restriction of host is still at epoch, it |
| +// clears build's record for host and host's marker when the marker names |
| +// build, and saves. A grant for another build, or from before the |
| +// restriction last changed, clears nothing. There is no scheduler to run a |
| +// canary: the next wake, or the converge leg, starts the build as today. |
| +func (h *failureHolds) retryHost(host, build string, epoch uint64) bool { |
| + rt, err := h.get(host) |
| + if err != nil || rt == nil || !rollout.Managed(*rt) || build == "" { |
| + return false |
| + } |
| + acct := release.RouteAccount(*rt) |
| + rec, mark, now, err := h.book.RetryHostAt(acct, rt.App, build, host, epoch) |
| + if err != nil { |
| + log.Printf("agent: retry %s on %s: save the failure book: %v (cleared in memory)", build, host, err) |
| + } |
| + if !rec && !mark && now != epoch { |
| + log.Printf("agent: retry %s on %s: the grant is for epoch %d, the restriction's is %d, so nothing was cleared", build, host, epoch, now) |
| + } |
| + if rec || mark { |
| + nudgeStatus() |
| + } |
| + return rec || mark |
| +} |
| + |
| +// applyPointers is the declined box's pointer pass (Task 10, ruling 194; |
| +// Correction 3): the armed box's two pointer rules (rollout.ApplyPointers) |
| +// on this box's book, for every app with a record or a marker a signed |
| +// rollback could clear. read is release.Channel with the agent's store and |
| +// root key. The caller paces it (pointerLoop). |
| +func (h *failureHolds) applyPointers(ctx context.Context, read rollout.PointerRead, routes []registry.Route) { |
| + if rollout.ApplyPointers(ctx, h.book, read, routes, log.Printf) { |
| + nudgeStatus() |
| + } |
| +} |
| + |
| +// pointerLoop runs applyPointers over the registry's routes, at most every |
| +// rollout.PointerReadEvery, for the life of the agent. An app with nothing |
| +// the pass could clear costs no read. |
| +func (h *failureHolds) pointerLoop(ctx context.Context, read rollout.PointerRead) { |
| + for { |
| + if routes, err := h.list(); err != nil { |
| + log.Printf("agent: declined pointer pass: list routes: %v", err) |
| + } else { |
| + h.applyPointers(ctx, read, routes) |
| + } |
| + select { |
| + case <-ctx.Done(): |
| + return |
| + case <-time.After(rollout.PointerReadEvery): |
| + } |
| + } |
| +} |
| + |
| +// rolloutRetryHost is restartMemory.retryHost for this box: the armed |
| +// manager's, a declined box's that honours its book (failureHolds.retryHost, |
| +// Task 10), or nil, which drops every retry because nothing is held. |
| +func rolloutRetryHost() func(host, build string, epoch uint64) bool { |
| + if m := armedRollout(); m != nil { |
| + return m.RetryHost |
| + } |
| + if h := heldHolds(); h != nil { |
| + return h.retryHost |
| + } |
| + return nil |
| } |
| // gate is the declined box's LiveExecGate (Task 9; Astra R1 #6, the R2/R3 |
| @@ -298,3 +427,16 @@ func (h *failureHolds) configCallbacks(ack func(appconfig.ConfigAck), legs confi |
| } |
| return appconfig.Callbacks{Restart: restart, Ack: ack} |
| } |
| + |
| +// declinedHoldsStay is fastConvergeLoop's one line, at arming, when a box |
| +// that honours its failure book has no deploy store or no release key (Task |
| +// 10 rule 2): with neither a pointer can be read nor a retry's grant |
| +// verified, so no pointer pass runs and no retry is read, and what the book |
| +// holds stays held until the agent restarts with both. |
| +func declinedHoldsStay(why string) { |
| + if heldHolds() == nil { |
| + return |
| + } |
| + log.Printf("agent: %s=overlap declined: %s, so no signed retry or channel pointer can clear the failure records and hold-ran markers honoured here", |
| + rollout.SwapEnv, why) |
| +} |
diff --git a/internal/host/rolloutdeclined_test.go b/internal/host/rolloutdeclined_test.go| index f16e97da7..b78156fe4 100644 |
| --- a/internal/host/rolloutdeclined_test.go |
| +++ b/internal/host/rolloutdeclined_test.go |
| @@ -3,22 +3,30 @@ package host |
| import ( |
| "bytes" |
| "context" |
| + "crypto/ed25519" |
| "encoding/base64" |
| + "encoding/json" |
| "errors" |
| + "io" |
| "net/http" |
| "net/http/httptest" |
| + "net/url" |
| "os" |
| "path/filepath" |
| "reflect" |
| "slices" |
| + "strings" |
| "testing" |
| "time" |
| "github.com/carlosframework/platform/internal/activator" |
| "github.com/carlosframework/platform/internal/appconfig" |
| + "github.com/carlosframework/platform/internal/console" |
| + "github.com/carlosframework/platform/internal/provision" |
| "github.com/carlosframework/platform/internal/registry" |
| "github.com/carlosframework/platform/internal/release" |
| "github.com/carlosframework/platform/internal/rollout" |
| + "github.com/carlosframework/platform/internal/status" |
| ) |
| // writeFailureBook writes recs to dd's failure book through the book itself, |
| @@ -799,3 +807,401 @@ func TestDeclinedBoxGatesAClaimsMoveLaunch(t *testing.T) { |
| t.Fatalf("move launch of the instance's own good build = %v, want it allowed", err) |
| } |
| } |
| + |
| +// heldRouteIn upserts a managed, hibernating instance of bab/blog into reg, |
| +// installed on installed with good as its good build. |
| +func heldRouteIn(t *testing.T, reg *registry.Registry, host, installed, good string) registry.Route { |
| + t.Helper() |
| + rt := managedRoute(host, installed) |
| + if good != "" { |
| + rt.GoodVersion, rt.GoodDigest = good, "sha256:"+good |
| + } |
| + if err := reg.Upsert(rt); err != nil { |
| + t.Fatal(err) |
| + } |
| + return rt |
| +} |
| + |
| +// declinedHostReport is rolloutStatusProbe's entry for host, failing the |
| +// test when the report is not a declined one. |
| +func declinedHostReport(t *testing.T, host string) (status.HostRollout, bool) { |
| + t.Helper() |
| + r := rolloutStatusProbe() |
| + if r == nil || r.Unavailable != rolloutNoDelegation || r.FailureBook != rolloutBookHonoured { |
| + t.Fatalf("status = %+v, want the decline code %q and the book %q", r, rolloutNoDelegation, rolloutBookHonoured) |
| + } |
| + hr, ok := r.Hosts[host] |
| + return hr, ok |
| +} |
| + |
| +// Task 10 (ruling 194, plan review #5): a declined box publishes its holds, |
| +// with the build and its restriction's epoch, and processes a verified |
| +// signed retry against the book it honours. A grant for another build, or |
| +// for an epoch the restriction has since moved past, clears nothing; the |
| +// current grant clears the hold-down in memory, where the planner and the |
| +// launch gate see it at once, and on disk. |
| +func TestDeclinedBoxProcessesAVerifiedSignedRetry(t *testing.T) { |
| + t.Setenv("CARLOS_OPT_DIR", t.TempDir()) |
| + svc, reg, dd := rolloutFixture(t, false) |
| + const host = "h.example.com" |
| + rt := heldRouteIn(t, reg, host, "v2", "v1") |
| + acct := release.RouteAccount(rt) |
| + writeFailureBook(t, dd, []rollout.FailureRecord{{ |
| + Account: acct, App: "blog", Build: "v2", Host: "sib.example.com", Mode: "hold", Reason: "health", |
| + }}, nil) |
| + b, err := rollout.OpenFailureBook(dd) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + stale := b.RestrictionEpoch(acct, "blog", "v2", host) |
| + // The armed run saw v2 fail again, holding this host down, after a |
| + // grant for the first restriction could have been signed. |
| + if _, err := b.Record(rollout.FailureRecord{Account: acct, App: "blog", Build: "v2", Host: "sib2.example.com", Mode: "hold", Reason: "health", HeldDown: []string{host}}); err != nil { |
| + t.Fatal(err) |
| + } |
| + epoch := b.RestrictionEpoch(acct, "blog", "v2", host) |
| + if epoch == 0 || epoch == stale { |
| + t.Fatalf("epochs %d then %d: the second failure did not move the restriction", stale, epoch) |
| + } |
| + declineWithBook(t, svc, reg, dd) |
| + ctx := context.Background() |
| + held := rollout.HeldDownRefusal("v2") |
| + if p := svc.Planner(ctx, rt); p.Refuse != held { |
| + t.Fatalf("plan before the retry = %+v, want %q", p, held) |
| + } |
| + hr, ok := declinedHostReport(t, host) |
| + if !ok || hr.Account != acct || hr.App != "blog" || hr.Installed != "v2" || hr.HeldBy != "v2" || hr.HeldEpoch != epoch || hr.Attempt != status.AttemptDown { |
| + t.Fatalf("published %+v (%v), want v2 holding %s down at epoch %d", hr, ok, host, epoch) |
| + } |
| + |
| + mem, priv := retryMemory(t) |
| + mem.retryHost = rolloutRetryHost() |
| + if mem.retryHost == nil { |
| + t.Fatal("a declined box honouring its book has no retry path") |
| + } |
| + _, s := genKit(t) |
| + routes, err := reg.List() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + bounce := func(h string) bool { t.Errorf("%s was bounced for a retry", h); return true } |
| + restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) // baseline |
| + askRetry(t, ctx, s, priv, acct, "blog", host, "v9", epoch) |
| + askRetry(t, ctx, s, priv, acct, "blog", host, "v2", stale) |
| + restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) |
| + if p := svc.Planner(ctx, rt); p.Refuse != held { |
| + t.Fatalf("plan after a grant for another build and a stale grant = %+v, want %q still", p, held) |
| + } |
| + askRetry(t, ctx, s, priv, acct, "blog", host, "v2", epoch) |
| + restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) |
| + if p := svc.Planner(ctx, rt); !isTodaysWake(p) { |
| + t.Fatalf("plan after the signed retry = %+v, want today's wake onto v2", p) |
| + } |
| + if err := heldHolds().gate(rt, "v2"); err != nil { |
| + t.Fatalf("the launch gate still refuses v2 after the retry: %v", err) |
| + } |
| + if hr, ok := declinedHostReport(t, host); ok { |
| + t.Fatalf("published %+v after the retry, want no hold", hr) |
| + } |
| + b, err = rollout.OpenFailureBook(dd) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if b.HeldDown(acct, "blog", "v2", host) || b.Blocks(acct, "blog", "v2", host, false) { |
| + t.Fatal("the retry was not saved: the book on disk still holds the host") |
| + } |
| +} |
| + |
| +// pointerReads is a fake signed pointer read keyed "<app>/<channel>", which |
| +// counts the reads it is asked for. |
| +type pointerReads struct { |
| + ptrs map[string]*release.ChannelPointer |
| + errs map[string]error |
| + reads []string |
| +} |
| + |
| +func (p *pointerReads) read(_ context.Context, _, app, channel string) (*release.ChannelPointer, error) { |
| + k := app + "/" + channel |
| + p.reads = append(p.reads, k) |
| + if err := p.errs[k]; err != nil { |
| + return nil, err |
| + } |
| + return p.ptrs[k], nil |
| +} |
| + |
| +// Task 10 rule 2 (`carlos promote --retry`): the declined box's pointer |
| +// pass clears a record when its channel's signed pointer names the record's |
| +// build with a newer PromotedAt. The pointer the build failed under, a |
| +// pointer naming another build and a failed read clear nothing; an app with |
| +// nothing to clear costs no read. |
| +func TestDeclinedBoxPointerPassClearsARecordOnPromoteRetry(t *testing.T) { |
| + t.Setenv("CARLOS_OPT_DIR", t.TempDir()) |
| + svc, reg, dd := rolloutFixture(t, false) |
| + rt := heldRouteIn(t, reg, "h.example.com", "v2", "v1") |
| + other := execRoute("s.example.com", "bab", "shop") |
| + other.Hibernate, other.Version, other.GoodVersion, other.GoodDigest = true, "z2", "z1", "sha256:z1" |
| + if err := reg.Upsert(other); err != nil { |
| + t.Fatal(err) |
| + } |
| + acct := release.RouteAccount(rt) |
| + failedUnder := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) |
| + writeFailureBook(t, dd, []rollout.FailureRecord{{ |
| + Account: acct, App: "blog", Build: "v2", Host: "sib.example.com", Reason: "health", PromotedAt: failedUnder, |
| + }}, nil) |
| + declineWithBook(t, svc, reg, dd) |
| + ctx := context.Background() |
| + routes, err := reg.List() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + h := heldHolds() |
| + stillHeld := func(when string) { |
| + t.Helper() |
| + if p := svc.Planner(ctx, rt); p.Refuse != refuseNoSwaps { |
| + t.Fatalf("%s: plan = %+v, want %q", when, p, refuseNoSwaps) |
| + } |
| + } |
| + stillHeld("before any pointer") |
| + if hr, ok := declinedHostReport(t, rt.Host); !ok || hr.HeldBy != "v2" || hr.Attempt != status.AttemptFailed || hr.HeldEpoch == 0 { |
| + t.Fatalf("published %+v (%v), want v2 holding it as failed, with an epoch", hr, ok) |
| + } |
| + |
| + p := &pointerReads{ptrs: map[string]*release.ChannelPointer{}, errs: map[string]error{}} |
| + p.errs["blog/stable"] = errors.New("bucket blip") |
| + h.applyPointers(ctx, p.read, routes) |
| + stillHeld("after a failed read") |
| + delete(p.errs, "blog/stable") |
| + p.ptrs["blog/stable"] = &release.ChannelPointer{App: "blog", Channel: "stable", Version: "v2", PromotedAt: failedUnder} |
| + h.applyPointers(ctx, p.read, routes) |
| + stillHeld("after the pointer v2 failed under") |
| + p.ptrs["blog/stable"] = &release.ChannelPointer{App: "blog", Channel: "stable", Version: "v3", PromotedAt: failedUnder.Add(time.Hour)} |
| + h.applyPointers(ctx, p.read, routes) |
| + stillHeld("after a pointer naming another build") |
| + p.ptrs["blog/stable"] = &release.ChannelPointer{App: "blog", Channel: "stable", Version: "v2", PromotedAt: failedUnder.Add(time.Minute)} |
| + h.applyPointers(ctx, p.read, routes) |
| + if pl := svc.Planner(ctx, rt); !isTodaysWake(pl) { |
| + t.Fatalf("plan after promote --retry = %+v, want today's wake onto v2", pl) |
| + } |
| + for _, r := range p.reads { |
| + if strings.HasPrefix(r, "shop/") { |
| + t.Fatalf("reads %v: an app with nothing to clear was read", p.reads) |
| + } |
| + } |
| + b, err := rollout.OpenFailureBook(dd) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, ok := b.Lookup(acct, "blog", "v2"); ok { |
| + t.Fatal("the cleared record was not saved") |
| + } |
| +} |
| + |
| +// Task 10 Correction 3 (Task 3's rule, ruling 205): the pointer pass also |
| +// clears a hold-ran marker on a signed rollback, for an app with markers and |
| +// no records: the route installs its good build and the channel's pointer |
| +// names that build with a PromotedAt after the marker's At. The planner then |
| +// lets the good build start. An older pointer, and a route not installing |
| +// its good build, clear nothing. |
| +func TestDeclinedBoxPointerPassClearsAMarkerOnASignedRollback(t *testing.T) { |
| + t.Setenv("CARLOS_OPT_DIR", t.TempDir()) |
| + svc, reg, dd := rolloutFixture(t, false) |
| + rolled := heldRouteIn(t, reg, "m.example.com", "v1", "v1") |
| + notRolled := heldRouteIn(t, reg, "n.example.com", "v3", "v1") |
| + acct := release.RouteAccount(rolled) |
| + ranAt := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) |
| + b, err := rollout.OpenFailureBook(dd) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, host := range []string{rolled.Host, notRolled.Host} { |
| + if err := b.MarkHoldRan(host, rollout.HoldRan{Account: acct, App: "blog", Build: "v2", At: ranAt}); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + epoch := b.RestrictionEpoch(acct, "blog", "v2", rolled.Host) |
| + declineWithBook(t, svc, reg, dd) |
| + ctx := context.Background() |
| + routes, err := reg.List() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + held := rollout.HeldDownRefusal("v2") |
| + if p := svc.Planner(ctx, rolled); p.Refuse != held { |
| + t.Fatalf("plan before the rollback = %+v, want %q", p, held) |
| + } |
| + if hr, ok := declinedHostReport(t, rolled.Host); !ok || hr.HeldBy != "v2" || hr.HeldEpoch != epoch || hr.Attempt != status.AttemptDown { |
| + t.Fatalf("published %+v (%v), want the marker's v2 holding the good build down at epoch %d", hr, ok, epoch) |
| + } |
| + |
| + h := heldHolds() |
| + p := &pointerReads{ptrs: map[string]*release.ChannelPointer{}} |
| + p.ptrs["blog/stable"] = &release.ChannelPointer{App: "blog", Channel: "stable", Version: "v1", PromotedAt: ranAt.Add(-time.Minute)} |
| + h.applyPointers(ctx, p.read, routes) |
| + if len(p.reads) == 0 { |
| + t.Fatal("an app with only markers was not read") |
| + } |
| + if pl := svc.Planner(ctx, rolled); pl.Refuse != held { |
| + t.Fatalf("plan after a pointer older than the marker = %+v, want %q still", pl, held) |
| + } |
| + p.ptrs["blog/stable"] = &release.ChannelPointer{App: "blog", Channel: "stable", Version: "v1", PromotedAt: ranAt.Add(time.Minute)} |
| + h.applyPointers(ctx, p.read, routes) |
| + if pl := svc.Planner(ctx, rolled); !isTodaysWake(pl) { |
| + t.Fatalf("plan after the signed rollback = %+v, want today's wake onto the good build", pl) |
| + } |
| + if hr, ok := declinedHostReport(t, rolled.Host); ok { |
| + t.Fatalf("published %+v after the rollback, want no hold", hr) |
| + } |
| + b, err = rollout.OpenFailureBook(dd) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, ok := b.HoldRanOn(rolled.Host); ok { |
| + t.Fatal("the cleared marker was not saved") |
| + } |
| + if _, ok := b.HoldRanOn(notRolled.Host); !ok { |
| + t.Fatal("a marker on a route not installing its good build was cleared") |
| + } |
| +} |
| + |
| +// retryConsole is a console with a root key over store, its ops account |
| +// owning app, with a CLI token for its owner: the shape the root package's |
| +// e2e tests log in through (e2eDeviceLogin). |
| +func retryConsole(t *testing.T, store release.Store, app, host string) (base, token string, pub ed25519.PublicKey) { |
| + t.Helper() |
| + seed, pubB64, err := release.GenerateKey() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + priv, err := release.ParseSigningKey(seed, "test") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if pub, err = release.ParsePublicKey(pubB64, "test"); err != nil { |
| + t.Fatal(err) |
| + } |
| + dir := t.TempDir() |
| + cst, err := console.Open(filepath.Join(dir, "console.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { cst.Close() }) |
| + var srv *console.Server |
| + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv.ServeHTTP(w, r) })) |
| + t.Cleanup(ts.Close) |
| + if srv, err = console.NewServer(console.Config{Origin: ts.URL, Keymail: "http://keymail.invalid", |
| + Registry: filepath.Join(dir, "registry.db"), Releases: store, ReleasePub: pub, Key: priv, AppScheme: "http"}, cst); err != nil { |
| + t.Fatal(err) |
| + } |
| + const addr = "paul@keymail.dev" |
| + now := time.Now() |
| + if err := cst.AddMember(console.OpsAccountID, addr, console.RoleOwner, now); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := cst.CreateAppIn(console.OpsAccountID, app, addr, now); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := provision.CreateRecord(context.Background(), store, provision.Record{Schema: 1, Host: host, Slug: strings.SplitN(host, ".", 2)[0], |
| + App: app, Account: release.OpsAccount, Region: "ie", Channel: "stable", CreatedAt: now.UTC(), CreatedBy: addr}); err != nil { |
| + t.Fatal(err) |
| + } |
| + post := func(u string, body any, cookie string, form url.Values) map[string]any { |
| + t.Helper() |
| + var req *http.Request |
| + if form != nil { |
| + req, _ = http.NewRequest(http.MethodPost, u, strings.NewReader(form.Encode())) |
| + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") |
| + req.Header.Set("Cookie", "carlos_console_session="+cookie) |
| + } else { |
| + raw, _ := json.Marshal(body) |
| + req, _ = http.NewRequest(http.MethodPost, u, bytes.NewReader(raw)) |
| + req.Header.Set("Content-Type", "application/json") |
| + } |
| + res, err := http.DefaultClient.Do(req) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + defer res.Body.Close() |
| + raw, _ := io.ReadAll(res.Body) |
| + if res.StatusCode != http.StatusOK { |
| + t.Fatalf("POST %s = %d %s", u, res.StatusCode, raw) |
| + } |
| + out := map[string]any{} |
| + _ = json.Unmarshal(raw, &out) |
| + return out |
| + } |
| + start := post(ts.URL+"/auth/cli/start", map[string]string{"label": "test@harness"}, "", nil) |
| + session, err := cst.CreateSession(addr, "", now) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := cst.RecordStepUp(session, addr, now, now); err != nil { |
| + t.Fatal(err) |
| + } |
| + post(ts.URL+"/auth/cli/approve", nil, session, url.Values{"code": {start["user_code"].(string)}, "action": {"approve"}, "csrf": {session}}) |
| + poll := post(ts.URL+"/auth/cli/poll", map[string]string{"device_code": start["device_code"].(string)}, "", nil) |
| + token, _ = poll["token"].(string) |
| + if token == "" { |
| + t.Fatalf("no CLI token: %v", poll) |
| + } |
| + return ts.URL, token, pub |
| +} |
| + |
| +// Task 10 Correction 1 (plan review #5), from the API to the box: the |
| +// declined box publishes its held host, the console's retry door signs a |
| +// grant for it from that report, and the declined box's restart leg |
| +// verifies the grant and clears the hold, so the next wake is today's. |
| +func TestDeclinedBoxRetryFromTheConsoleDoorToTheRestartLeg(t *testing.T) { |
| + t.Setenv("CARLOS_OPT_DIR", t.TempDir()) |
| + svc, reg, dd := rolloutFixture(t, false) |
| + const host = "h.blog.example.com" |
| + rt := heldRouteIn(t, reg, host, "v2", "v1") |
| + acct := release.RouteAccount(rt) |
| + writeFailureBook(t, dd, []rollout.FailureRecord{{ |
| + Account: acct, App: "blog", Build: "v2", Host: "sib.example.com", Mode: "hold", Reason: "health", HeldDown: []string{host}, |
| + }}, nil) |
| + declineWithBook(t, svc, reg, dd) |
| + ctx, s := genKit(t) |
| + base, token, pub := retryConsole(t, s, "blog", host) |
| + |
| + // The box's restart memory, seeded before the request, as at arming. |
| + mem := newRestartMemory() |
| + mem.pub = pub |
| + mem.retryHost = rolloutRetryHost() |
| + routes, err := reg.List() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + bounce := func(h string) bool { t.Errorf("%s was bounced for a retry", h); return true } |
| + restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) |
| + |
| + // The box's status, as the status loop publishes it. |
| + box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) |
| + box.Metrics = &status.BoxMetrics{RouteVersions: map[string]string{host: rt.Version}, Rollout: rolloutStatusProbe()} |
| + if err := status.WriteBox(ctx, s, box); err != nil { |
| + t.Fatal(err) |
| + } |
| + req, _ := http.NewRequest(http.MethodPost, base+"/api/cli/apps/"+release.OpsAccount+"/blog/instances/"+host+"/retry", strings.NewReader(`{}`)) |
| + req.Header.Set("Content-Type", "application/json") |
| + req.Header.Set("Authorization", "Bearer "+token) |
| + res, err := http.DefaultClient.Do(req) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + raw, _ := io.ReadAll(res.Body) |
| + res.Body.Close() |
| + if res.StatusCode != http.StatusOK { |
| + t.Fatalf("the console's retry door = %d %s, want a signed grant for the declined box's hold", res.StatusCode, raw) |
| + } |
| + |
| + restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) |
| + if p := svc.Planner(ctx, rt); !isTodaysWake(p) { |
| + t.Fatalf("plan after the console's retry = %+v, want today's wake onto v2", p) |
| + } |
| + b, err := rollout.OpenFailureBook(dd) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if b.HeldDown(acct, "blog", "v2", host) { |
| + t.Fatal("the console's retry was not saved") |
| + } |
| +} |
diff --git a/internal/rollout/manager.go b/internal/rollout/manager.go| index 9d6ce3ad5..5c76e01e1 100644 |
| --- a/internal/rollout/manager.go |
| +++ b/internal/rollout/manager.go |
| @@ -478,19 +478,8 @@ func (m *Manager) applyPointers(routes []registry.Route, keys []appKey) { |
| } |
| now := m.d.Now() |
| for _, k := range keys { |
| - recs := m.d.Book.ForApp(k.acct, k.app) |
| - // The marked hosts a rollback could clear, by channel. |
| - rollback := map[string][]registry.Route{} |
| - for _, rt := range routes { |
| - if !Managed(rt) || release.RouteAccount(rt) != k.acct || rt.App != k.app || rt.GoodVersion == "" || rt.Version != rt.GoodVersion { |
| - continue |
| - } |
| - if _, ok := holdRanFor(m.d.Book, k.acct, k.app, rt.Host); ok { |
| - ch := routeChannel(rt) |
| - rollback[ch] = append(rollback[ch], rt) |
| - } |
| - } |
| - if len(recs) == 0 && len(rollback) == 0 { |
| + w := pointerWorkFor(m.d.Book, routes, k) |
| + if w.empty() { |
| continue |
| } |
| m.mu.Lock() |
| @@ -502,52 +491,131 @@ func (m *Manager) applyPointers(routes []registry.Route, keys []appKey) { |
| if !due { |
| continue |
| } |
| - channels := map[string]bool{} |
| - for _, rt := range routes { |
| - if Managed(rt) && release.RouteAccount(rt) == k.acct && rt.App == k.app { |
| - channels[routeChannel(rt)] = true |
| - } |
| + cleared, unmarked := w.apply(m.ctx, m.d.Book, m.d.Pointer, routes, k, m.d.Logf) |
| + for _, build := range cleared { |
| + m.retryBuild(k, build, "") |
| } |
| - for ch := range channels { |
| - ptr, err := m.d.Pointer(m.ctx, k.acct, k.app, ch) |
| - if err != nil { |
| - m.d.Logf("rollout: %s/%s channel %s: %v (retry not checked this pass)", k.acct, k.app, ch, err) |
| + if unmarked { |
| + m.changed() |
| + } |
| + } |
| +} |
| + |
| +// pointerWork is what one app's channel pointers could clear on this box |
| +// (applyPointers): its failure records, and its marked hosts whose route |
| +// installs the good build, by channel. An app with neither costs no read. |
| +type pointerWork struct { |
| + recs []FailureRecord |
| + rollback map[string][]registry.Route |
| +} |
| + |
| +func (w pointerWork) empty() bool { return len(w.recs) == 0 && len(w.rollback) == 0 } |
| + |
| +// pointerWorkFor is k's pointerWork against book and the box's routes. |
| +func pointerWorkFor(book *FailureBook, routes []registry.Route, k appKey) pointerWork { |
| + w := pointerWork{recs: book.ForApp(k.acct, k.app), rollback: map[string][]registry.Route{}} |
| + for _, rt := range routes { |
| + if !Managed(rt) || release.RouteAccount(rt) != k.acct || rt.App != k.app || rt.GoodVersion == "" || rt.Version != rt.GoodVersion { |
| + continue |
| + } |
| + if _, ok := holdRanFor(book, k.acct, k.app, rt.Host); ok { |
| + ch := routeChannel(rt) |
| + w.rollback[ch] = append(w.rollback[ch], rt) |
| + } |
| + } |
| + return w |
| +} |
| + |
| +// apply reads the signed pointer of every channel k's managed routes follow |
| +// and applies applyPointers' two rules to the book: a record whose build a |
| +// pointer names with a newer PromotedAt is cleared (ClearOnPointer), and so |
| +// is the marker of a host whose channel's pointer names its installed good |
| +// build with a PromotedAt after the marker's At (ClearHoldRanOnPointer). A |
| +// read error is logged and that channel skipped. It reports the builds whose |
| +// records it cleared and whether it cleared a marker; what follows from |
| +// either is the caller's. |
| +func (w pointerWork) apply(ctx context.Context, book *FailureBook, read PointerRead, routes []registry.Route, k appKey, |
| + logf func(format string, args ...any)) (cleared []string, unmarked bool) { |
| + channels := map[string]bool{} |
| + for _, rt := range routes { |
| + if Managed(rt) && release.RouteAccount(rt) == k.acct && rt.App == k.app { |
| + channels[routeChannel(rt)] = true |
| + } |
| + } |
| + for ch := range channels { |
| + ptr, err := read(ctx, k.acct, k.app, ch) |
| + if err != nil { |
| + logf("rollout: %s/%s channel %s: %v (retry not checked this pass)", k.acct, k.app, ch, err) |
| + continue |
| + } |
| + if ptr == nil { |
| + continue |
| + } |
| + for _, r := range w.recs { |
| + if ptr.Version != r.Build { |
| continue |
| } |
| - if ptr == nil { |
| - continue |
| + ok, err := book.ClearOnPointer(k.acct, k.app, r.Build, ptr.PromotedAt) |
| + if err != nil { |
| + logf("rollout: %s/%s clear record for %s: %v (cleared in memory)", k.acct, k.app, r.Build, err) |
| } |
| - for _, r := range recs { |
| - if ptr.Version != r.Build { |
| - continue |
| - } |
| - ok, err := m.d.Book.ClearOnPointer(k.acct, k.app, r.Build, ptr.PromotedAt) |
| - if err != nil { |
| - m.d.Logf("rollout: %s/%s clear record for %s: %v (cleared in memory)", k.acct, k.app, r.Build, err) |
| - } |
| - if ok { |
| - m.retryBuild(k, r.Build, "") |
| - } |
| + if ok { |
| + cleared = append(cleared, r.Build) |
| } |
| - unmarked := false |
| - for _, rt := range rollback[ch] { |
| - if ptr.Version != rt.GoodVersion { |
| - continue |
| - } |
| - ok, err := m.d.Book.ClearHoldRanOnPointer(rt.Host, k.acct, k.app, ptr.PromotedAt) |
| - if err != nil { |
| - m.d.Logf("rollout: %s: clear hold-ran marker on a rollback to %s: %v (cleared in memory)", rt.Host, rt.GoodVersion, err) |
| - } |
| - if ok { |
| - m.d.Logf("rollout: %s: a signed rollback to %s (pointer of %s) cleared its hold-ran marker", rt.Host, rt.GoodVersion, ptr.PromotedAt.UTC().Format(time.RFC3339)) |
| - unmarked = true |
| - } |
| + } |
| + for _, rt := range w.rollback[ch] { |
| + if ptr.Version != rt.GoodVersion { |
| + continue |
| + } |
| + ok, err := book.ClearHoldRanOnPointer(rt.Host, k.acct, k.app, ptr.PromotedAt) |
| + if err != nil { |
| + logf("rollout: %s: clear hold-ran marker on a rollback to %s: %v (cleared in memory)", rt.Host, rt.GoodVersion, err) |
| } |
| - if unmarked { |
| - m.changed() |
| + if ok { |
| + logf("rollout: %s: a signed rollback to %s (pointer of %s) cleared its hold-ran marker", rt.Host, rt.GoodVersion, ptr.PromotedAt.UTC().Format(time.RFC3339)) |
| + unmarked = true |
| } |
| } |
| } |
| + return cleared, unmarked |
| +} |
| + |
| +// PointerRead reads a channel's signed pointer, verified (release.Channel |
| +// with the box's store and root key): Deps.Pointer's shape. |
| +type PointerRead = func(ctx context.Context, acct, app, channel string) (*release.ChannelPointer, error) |
| + |
| +// PointerReadEvery is pointerReadEvery, for the pointer pass of a box with |
| +// no Manager (ApplyPointers), which paces itself. |
| +const PointerReadEvery = pointerReadEvery |
| + |
| +// ApplyPointers is applyPointers for a box with no Manager: one that |
| +// declined CARLOS_SWAP=overlap but honours the records and hold-ran markers |
| +// an armed run left (internal/host's failureHolds; Astra R2 ruling 194 |
| +// "startup-only records", Task 10). Every app with a managed route in routes |
| +// and something its pointers could clear is read now, under the same two |
| +// rules on the same book; the caller paces the pass, at most every |
| +// PointerReadEvery. There is no scheduler to requeue a cleared build, so a |
| +// cleared record is only logged. It reports whether anything was cleared. |
| +func ApplyPointers(ctx context.Context, book *FailureBook, read PointerRead, routes []registry.Route, logf func(format string, args ...any)) bool { |
| + changed := false |
| + seen := map[appKey]bool{} |
| + for _, rt := range routes { |
| + k := appKey{release.RouteAccount(rt), rt.App} |
| + if !Managed(rt) || seen[k] { |
| + continue |
| + } |
| + seen[k] = true |
| + w := pointerWorkFor(book, routes, k) |
| + if w.empty() { |
| + continue |
| + } |
| + cleared, unmarked := w.apply(ctx, book, read, routes, k, logf) |
| + for _, build := range cleared { |
| + logf("rollout: %s/%s: a signed pointer re-promoted %s, so its failure record is cleared", k.acct, k.app, build) |
| + } |
| + changed = changed || len(cleared) > 0 || unmarked |
| + } |
| + return changed |
| } |
| // retryBuild follows a cleared record: the next attempt onto build carries |