amadan
zdd-readiness: host retries are root-signed in the console and verified on the box (Q4, spec §11)
carlos instances retry <host> now appends a retry entry carrying a release.HostRetryGrant the console signs with its root key (SignRootEnvelope; scoped signers refused), naming the account, app, host, the held build and that restriction's epoch as the box reported it (status HeldEpoch). A console with no key answers 503 and appends nothing. Every box verifies each entry against CARLOS_RELEASE_PUBKEY, the entry's account/app/host and a 24 h window before anything is cleared; unsigned, foreign or keyless entries are dropped with a log line. Manager.RetryHost(host, build, epoch) clears build's record for host and/or the marker naming build, each only while FailureBook.RestrictionEpoch still equals the grant's epoch (plan review findings 5 and 13): records carry persisted per-host and record-wide epochs, markers use their Seq, and a spent grant's clear moves the host's epoch on disk, so a stale or replayed grant clears nothing even after a reopen.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
23 files changed,
+1093
−124
instances_e2e_test.go+9 −2internal/console/apiinstances.go+5 −1internal/console/apiretry.go+45 −19internal/console/apiretry_test.go+35 −2internal/console/browserretry.go+16 −3internal/console/browserretry_test.go+21 −1internal/host/agent.go+2 −0internal/host/restart.go+15 −5internal/host/restartscope.go+49 −13internal/host/restartscope_test.go+164 −13internal/release/hostretry.go+74 −0internal/release/hostretry_test.go+168 −0internal/release/restartscope.go+17 −1internal/rollout/bookseq_test.go+5 −5internal/rollout/failures.go+163 −20internal/rollout/holdran_test.go+2 −2internal/rollout/manager.go+30 −31internal/rollout/manager_test.go+2 −2internal/rollout/retry_rearm_test.go+2 −2internal/rollout/retrygrant_test.go+252 −0internal/rollout/sim_test.go+8 −2internal/rollout/status.go+4 −0internal/status/snapshot.go+5 −0
diff --git a/instances_e2e_test.go b/instances_e2e_test.go| index 0bb510e61..51ea76704 100644 |
| --- a/instances_e2e_test.go |
| +++ b/instances_e2e_test.go |
| @@ -293,7 +293,7 @@ func TestE2EInstancesThroughConsole(t *testing.T) { |
| box := status.CollectBox(nil, status.Probes{MachineID: "e2e-box"}, time.Now()) |
| box.Metrics = &status.BoxMetrics{Rollout: &status.RolloutReport{Hosts: map[string]status.HostRollout{ |
| haHost: {Account: release.OpsAccount, App: "titogo", Installed: "v2", Serving: "v1", Good: "v1", |
| - Awake: true, HeldBy: "v2", Reason: status.ReasonHealth}, |
| + Awake: true, HeldBy: "v2", HeldEpoch: 9, Reason: status.ReasonHealth}, |
| }}} |
| if err := status.WriteBox(context.Background(), relStore, box); err != nil { |
| t.Fatal(err) |
| @@ -307,9 +307,16 @@ func TestE2EInstancesThroughConsole(t *testing.T) { |
| if err != nil || !found || len(scopes.Requests) == 0 { |
| t.Fatalf("restart scopes = %+v, %v, %v", scopes, found, err) |
| } |
| - if last := scopes.Requests[len(scopes.Requests)-1]; last.Scope.Kind != release.RestartScopeRetry || last.Scope.Value != haHost { |
| + last := scopes.Requests[len(scopes.Requests)-1] |
| + if last.Scope.Kind != release.RestartScopeRetry || last.Scope.Value != haHost { |
| t.Fatalf("last request = %+v", last) |
| } |
| + // The entry carries the console's root-signed grant for the held build |
| + // at the epoch the box reported (Task 8, spec §11): the box clears |
| + // nothing on an entry without one. |
| + if g, err := release.VerifyHostRetry(pub, last.Scope.Signed, release.OpsAccount, "titogo", haHost, time.Now()); err != nil || g.Build != "v2" || g.Epoch != 9 { |
| + t.Fatalf("signed grant = %+v, %v; want v2 at epoch 9", g, err) |
| + } |
| // The whole flow ran through the console: nothing here may have touched |
| // a local registry. |
diff --git a/internal/console/apiinstances.go b/internal/console/apiinstances.go| index b3184ca63..e4242a5dd 100644 |
| --- a/internal/console/apiinstances.go |
| +++ b/internal/console/apiinstances.go |
| @@ -364,6 +364,10 @@ type rolloutView struct { |
| // only Box, Installed and this: the box reports no per-host state. |
| Unavailable string `json:"unavailable,omitempty"` |
| account, app string |
| + // heldEpoch is the box's epoch for HeldBy's restriction |
| + // (status.HostRollout.HeldEpoch). It is not shown: the retry doors sign |
| + // it into their grant (retryHostBuild). |
| + heldEpoch uint64 |
| // refused marks a view built from an Unavailable box, which has no |
| // account or app to check (hosts are unique; the caller filtered by app). |
| refused bool |
| @@ -588,7 +592,7 @@ func (j *boxJoin) foldRollout(b status.Box, override bool) { |
| Box: b.Label, Installed: hr.Installed, Serving: hr.Serving, Good: hr.Good, Awake: hr.Awake, |
| Attempt: hr.Attempt, AttemptBuild: hr.AttemptBuild, HeldBy: hr.HeldBy, Reason: hr.Reason, |
| ConfigPending: hr.ConfigPending, SchedulerIdle: a.Queued == 0 && a.Running == 0 && a.Settling == 0, |
| - account: hr.Account, app: hr.App, |
| + account: hr.Account, app: hr.App, heldEpoch: hr.HeldEpoch, |
| }) |
| } |
| } |
diff --git a/internal/console/apiretry.go b/internal/console/apiretry.go| index 3f60f889d..e7fb218de 100644 |
| --- a/internal/console/apiretry.go |
| +++ b/internal/console/apiretry.go |
| @@ -1,12 +1,17 @@ |
| // The host-retry door (zero-downtime spec §8.4, §9.1): `carlos instances |
| // retry <host>` clears one instance's failure hold, and its next attempt |
| // runs as a canary. The app-wide retry is `carlos promote --retry`, which |
| -// re-signs the channel pointer; this door has no signing to do. |
| +// re-signs the channel pointer. |
| // |
| // The request rides the narrowed-restart log as selector kind "retry" |
| // (release.RestartScopeRetry). A box built before that kind steps over it; |
| -// one built after hands it to its rollout manager. ROLE: operate, as |
| -// `carlos restart` is. |
| +// one built after hands it to its rollout manager. Every box can write that |
| +// log (Astra impl-6 finding 5), so the entry carries a grant the root key |
| +// signs here (release.HostRetryGrant, spec §11; Paul's ruling on plan 6 Q4, |
| +// amadan #272), naming the held build and its restriction's epoch as the |
| +// box reported them, and a box clears nothing on an entry without one. A |
| +// console with no key (read-only) answers 503 and writes nothing. ROLE: |
| +// operate, as `carlos restart` is. |
| package console |
| import ( |
| @@ -26,38 +31,55 @@ const ( |
| apiRetryStore = "could not write the retry request. Try again" |
| apiRetryHostAudit = "requested a retry of %s on %s (%s)" |
| apiRetryNotHeld = "%s is not held back by a failed build, as far as its box last reported, so there is nothing to retry" |
| + // apiRetryNoKey is a read-only console's answer: a retry is a grant the |
| + // root key signs (spec §11), and there is no key here to sign it. |
| + apiRetryNoKey = "no signing key on this host. this console cannot request a retry" // TODO-COPY(zdd-readiness.retry-no-key) |
| ) |
| var ( |
| errRetryNoRecord = errors.New("retry: no such instance") |
| errRetryNotHeld = errors.New("retry: not held") |
| + errRetryNoKey = errors.New("retry: no signing key on this console") |
| ) |
| -// retryHostBuild is the build holding host back, as its box reports it. The |
| -// record must exist in this account and app (the instance family's 404), and |
| -// the report must be this app's own (boxJoin.rollout), read through the same |
| -// loader as the list and the instance page (loadRolloutJoin). A box that could not |
| -// arm its scheduler reports no hold, so its host is not held. |
| -func (s *Server) retryHostBuild(ctx context.Context, acct *Account, app, host string) (string, error) { |
| +// retryHostBuild is the build holding host back, and the epoch of that |
| +// restriction (plan review finding 13), as its box reports them. The record |
| +// must exist in this account and app (the instance family's 404), and the |
| +// report must be this app's own (boxJoin.rollout), read through the same |
| +// loader as the list and the instance page (loadRolloutJoin). A box that |
| +// could not arm its scheduler reports no hold, so its host is not held. A |
| +// box older than the epoch reports 0, which is signed as it is and which no |
| +// box accepts. |
| +func (s *Server) retryHostBuild(ctx context.Context, acct *Account, app, host string) (string, uint64, error) { |
| if _, err := provision.GetRecord(ctx, s.cfg.Releases, acct.Sqid, app, host); err != nil { |
| if errors.Is(err, os.ErrNotExist) { |
| - return "", errRetryNoRecord |
| + return "", 0, errRetryNoRecord |
| } |
| - return "", err |
| + return "", 0, err |
| } |
| join, _ := s.loadRolloutJoin(ctx, acct.Sqid, time.Now(), nil) |
| ro := join.rollout(host, acct.Sqid, app) |
| if ro == nil || ro.Unavailable != "" || ro.HeldBy == "" { |
| - return "", errRetryNotHeld |
| + return "", 0, errRetryNotHeld |
| } |
| - return ro.HeldBy, nil |
| + return ro.HeldBy, ro.heldEpoch, nil |
| } |
| -// requestHostRetry appends the retry request, by the same append the |
| -// narrowed-restart door uses, and announces it on the serial. |
| -func (s *Server) requestHostRetry(ctx context.Context, acct, app, host string) (release.RestartRequest, error) { |
| +// requestHostRetry signs the retry grant for build at epoch with the root |
| +// key, appends it as a retry request, by the same append the |
| +// narrowed-restart door uses, and announces it on the serial. With no key |
| +// it writes nothing (errRetryNoKey): an unsigned entry is one every box |
| +// drops. |
| +func (s *Server) requestHostRetry(ctx context.Context, acct, app, host, build string, epoch uint64) (release.RestartRequest, error) { |
| + if s.cfg.Key == nil { |
| + return release.RestartRequest{}, errRetryNoKey |
| + } |
| + signed, err := release.SignHostRetry(release.RootSigner(s.cfg.Key), acct, app, host, build, epoch, time.Now()) |
| + if err != nil { |
| + return release.RestartRequest{}, err |
| + } |
| req, err := release.AppendRestartScope(ctx, s.cfg.Releases, acct, app, |
| - release.RestartScope{Kind: release.RestartScopeRetry, Value: host}) |
| + release.RestartScope{Kind: release.RestartScopeRetry, Value: host, Signed: signed}) |
| if err != nil { |
| return release.RestartRequest{}, err |
| } |
| @@ -85,6 +107,10 @@ func (s *Server) handleAPIInstanceRetry(w http.ResponseWriter, r *http.Request) |
| "no deployment bucket on this host. this console cannot request a retry") |
| return |
| } |
| + if s.cfg.Key == nil { |
| + jsonError(w, http.StatusServiceUnavailable, apiRetryNoKey) |
| + return |
| + } |
| var req struct { |
| Message string `json:"message"` |
| } |
| @@ -97,7 +123,7 @@ func (s *Server) handleAPIInstanceRetry(w http.ResponseWriter, r *http.Request) |
| } |
| ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) |
| defer cancel() |
| - build, err := s.retryHostBuild(ctx, acct, app, host) |
| + build, epoch, err := s.retryHostBuild(ctx, acct, app, host) |
| switch { |
| case errors.Is(err, errRetryNoRecord): |
| http.NotFound(w, r) |
| @@ -110,7 +136,7 @@ func (s *Server) handleAPIInstanceRetry(w http.ResponseWriter, r *http.Request) |
| jsonError(w, http.StatusBadGateway, "could not read the instance") |
| return |
| } |
| - rr, err := s.requestHostRetry(ctx, acct.Sqid, app, host) |
| + rr, err := s.requestHostRetry(ctx, acct.Sqid, app, host, build, epoch) |
| if err != nil { |
| log.Printf("console: api instance retry %s/%s/%s: %v", acct.Sqid, app, host, err) |
| jsonError(w, http.StatusBadGateway, apiRetryStore) |
diff --git a/internal/console/apiretry_test.go b/internal/console/apiretry_test.go| index 24a48a30b..3e6d25f90 100644 |
| --- a/internal/console/apiretry_test.go |
| +++ b/internal/console/apiretry_test.go |
| @@ -23,11 +23,14 @@ func lastScoped(t *testing.T, fx *consoleFixture, app string) release.RestartReq |
| return g.Requests[len(g.Requests)-1] |
| } |
| +// heldEpoch is the restriction epoch writeHeldBox reports. |
| +const heldEpoch = 17 |
| + |
| func writeHeldBox(t *testing.T, fx *consoleFixture, host, app, build string) { |
| t.Helper() |
| box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) |
| box.Metrics = &status.BoxMetrics{Rollout: &status.RolloutReport{Hosts: map[string]status.HostRollout{ |
| - host: {Account: fx.sqid, App: app, Installed: build, Serving: "v1", Good: "v1", Awake: true, HeldBy: build, Reason: status.ReasonHealth}, |
| + host: {Account: fx.sqid, App: app, Installed: build, Serving: "v1", Good: "v1", Awake: true, HeldBy: build, HeldEpoch: heldEpoch, Reason: status.ReasonHealth}, |
| }}} |
| if err := status.WriteBox(context.Background(), fx.rel, box); err != nil { |
| t.Fatal(err) |
| @@ -53,9 +56,16 @@ func TestAPIInstanceRetryWritesARetryRequest(t *testing.T) { |
| if err := json.Unmarshal([]byte(body), &out); err != nil || out.Version != "v2" || out.Host != host || out.ID == "" { |
| t.Fatalf("answer = %s", body) |
| } |
| - if r := lastScoped(t, fx, "kass"); r.Scope.Kind != release.RestartScopeRetry || r.Scope.Value != host || r.ID != out.ID { |
| + r := lastScoped(t, fx, "kass") |
| + if r.Scope.Kind != release.RestartScopeRetry || r.Scope.Value != host || r.ID != out.ID { |
| t.Fatalf("stored = %+v", r) |
| } |
| + // The entry carries the root's grant for the held build at the box's |
| + // epoch (spec §11, plan review finding 13). |
| + g, err := release.VerifyHostRetry(fx.pub, r.Scope.Signed, fx.sqid, "kass", host, time.Now()) |
| + if err != nil || g.Build != "v2" || g.Epoch != heldEpoch { |
| + t.Fatalf("signed grant = %+v, %v; want v2 at epoch %d", g, err, heldEpoch) |
| + } |
| if n := serial(t, fx); n != 1 { |
| t.Fatalf("serial = %d", n) |
| } |
| @@ -125,3 +135,26 @@ func TestAPIInstanceRetryRefusesAHostThatIsNotHeld(t *testing.T) { |
| t.Fatalf("a retry was written for a host that is not held: %+v", g.Requests) |
| } |
| } |
| + |
| +// A console with no root key (read-only) cannot sign a retry: it answers |
| +// 503 and writes nothing, so no box is handed an unsigned entry it would |
| +// drop (spec §11). |
| +func TestAPIInstanceRetryWithNoKeyWritesNothing(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + const host = "web.kass.carlosapp.dev" |
| + sibling(t, fx, "kass", host, "stable") |
| + writeHeldBox(t, fx, host, "kass", "v2") |
| + fx.srv.cfg.Key = nil |
| + token := cliToken(t, fx, "paul@keymail.dev", "code-host-retry-k") |
| + u := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/kass/instances/" + host + "/retry" |
| + code, _, body := rawReq(t, http.MethodPost, u, token, `{}`) |
| + if code != http.StatusServiceUnavailable || !strings.Contains(body, apiRetryNoKey) { |
| + t.Fatalf("no key = %d %s, want 503", code, body) |
| + } |
| + if g, found, _ := release.LoadRestartScopes(context.Background(), fx.rel, fx.sqid, "kass"); found && len(g.Requests) != 0 { |
| + t.Fatalf("a console with no key wrote %+v", g.Requests) |
| + } |
| + if n := serial(t, fx); n != 0 { |
| + t.Fatalf("serial = %d, want no announcement", n) |
| + } |
| +} |
diff --git a/internal/console/browserretry.go b/internal/console/browserretry.go| index b15053519..d04ab59b1 100644 |
| --- a/internal/console/browserretry.go |
| +++ b/internal/console/browserretry.go |
| @@ -13,11 +13,18 @@ import ( |
| "github.com/carlosframework/platform/internal/provision" |
| ) |
| +// The browser door's answer on a read-only console (apiRetryNoKey's twin). |
| +const ( |
| + browserRetryNoKeyTitle = "Retry unavailable" // TODO-COPY(zdd-readiness.retry-no-key) |
| + browserRetryNoKey = "No signing key on this host, so this console cannot request a retry." // TODO-COPY(zdd-readiness.retry-no-key) |
| +) |
| + |
| // handleInstanceRetry is the instance page's Retry (spec §9.2): the browser |
| // twin of `carlos instances retry`, shaped like the restart action: session, |
| // CSRF, the byte-identical 404, a confirmation page, then the write. It |
| // reads and writes through the CLI door's helpers (retryHostBuild, |
| -// requestHostRetry), so the two cannot disagree about what is held. |
| +// requestHostRetry), so the two cannot disagree about what is held, and |
| +// both sign the same grant. |
| func (s *Server) handleInstanceRetry(w http.ResponseWriter, r *http.Request) { |
| addr, raw, acct, role, ok := s.account(w, r) |
| if !ok { |
| @@ -38,6 +45,12 @@ func (s *Server) handleInstanceRetry(w http.ResponseWriter, r *http.Request) { |
| http.NotFound(w, r) |
| return |
| } |
| + if s.cfg.Key == nil { |
| + // A retry is a grant the root key signs (spec §11): a read-only |
| + // console asks nothing, not even the confirmation. |
| + s.renderMessage(w, r, http.StatusServiceUnavailable, browserRetryNoKeyTitle, browserRetryNoKey) |
| + return |
| + } |
| back := "/a/" + url.PathEscape(acct.Sqid) + "/apps/" + url.PathEscape(app) + "/instances/" + url.PathEscape(host) |
| flash := func(kind, msg string) { |
| s.setFlash(w, kind, msg) |
| @@ -45,7 +58,7 @@ func (s *Server) handleInstanceRetry(w http.ResponseWriter, r *http.Request) { |
| } |
| ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) |
| defer cancel() |
| - build, err := s.retryHostBuild(ctx, acct, app, host) |
| + build, epoch, err := s.retryHostBuild(ctx, acct, app, host) |
| switch { |
| case errors.Is(err, errRetryNoRecord): |
| http.NotFound(w, r) |
| @@ -71,7 +84,7 @@ func (s *Server) handleInstanceRetry(w http.ResponseWriter, r *http.Request) { |
| flash("err", swapCopy.RetryLimited) |
| return |
| } |
| - if _, err := s.requestHostRetry(ctx, acct.Sqid, app, host); err != nil { |
| + if _, err := s.requestHostRetry(ctx, acct.Sqid, app, host, build, epoch); err != nil { |
| log.Printf("console: browser retry %s/%s/%s: %v", acct.Sqid, app, host, err) |
| flash("err", swapCopy.RetryFailed) |
| return |
diff --git a/internal/console/browserretry_test.go b/internal/console/browserretry_test.go| index fc6b61a00..a8a5f248c 100644 |
| --- a/internal/console/browserretry_test.go |
| +++ b/internal/console/browserretry_test.go |
| @@ -97,9 +97,29 @@ func TestBrowserRetryConfirmsThenWrites(t *testing.T) { |
| if code != http.StatusSeeOther || loc != fx.path("/apps/kass/instances/"+heldHost) { |
| t.Fatalf("retry = %d -> %q", code, loc) |
| } |
| - if r := lastScoped(t, fx, "kass"); r.Scope.Kind != release.RestartScopeRetry || r.Scope.Value != heldHost { |
| + r := lastScoped(t, fx, "kass") |
| + if r.Scope.Kind != release.RestartScopeRetry || r.Scope.Value != heldHost { |
| t.Fatalf("stored = %+v", r) |
| } |
| + g, err := release.VerifyHostRetry(fx.pub, r.Scope.Signed, fx.sqid, "kass", heldHost, time.Now()) |
| + if err != nil || g.Build != "v2" || g.Epoch != heldEpoch { |
| + t.Fatalf("signed grant = %+v, %v; want v2 at epoch %d", g, err, heldEpoch) |
| + } |
| +} |
| + |
| +// The browser twin of TestAPIInstanceRetryWithNoKeyWritesNothing. |
| +func TestBrowserRetryWithNoKeyWritesNothing(t *testing.T) { |
| + fx := heldFixture(t) |
| + fx.srv.cfg.Key = nil |
| + c := signInPaul(t, fx) |
| + csrf := sessionToken(t, fx, c) |
| + code, _, body := postGated(t, fx, c, fx.u("/apps/kass/instances/"+heldHost+"/retry"), url.Values{"csrf": {csrf}, "confirmed": {"1"}}) |
| + if code != http.StatusServiceUnavailable || !strings.Contains(body, browserRetryNoKey) { |
| + t.Fatalf("no key = %d, want 503 naming the missing key", code) |
| + } |
| + if g, found, _ := release.LoadRestartScopes(context.Background(), fx.rel, fx.sqid, "kass"); found && len(g.Requests) != 0 { |
| + t.Fatalf("a console with no key wrote %+v", g.Requests) |
| + } |
| } |
| func TestBrowserRetryRefusals(t *testing.T) { |
diff --git a/internal/host/agent.go b/internal/host/agent.go| index c8ec3e305..b4dd61dfc 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -2300,6 +2300,8 @@ func fastConvergeLoop(reg *registry.Registry, svc *activator.Service, sd *schedu |
| restartMem := newRestartMemory() |
| // `carlos instances retry` (zero-downtime spec §8.4) reaches the rollout |
| // manager through the narrowed-restart log; nil while it is not armed. |
| + // Each entry's grant is verified against the root key first (spec §11). |
| + restartMem.pub = pub |
| if m := armedRollout(); m != nil { |
| restartMem.retryHost = m.RetryHost |
| } |
diff --git a/internal/host/restart.go b/internal/host/restart.go| index 59d63153f..3950f4f92 100644 |
| --- a/internal/host/restart.go |
| +++ b/internal/host/restart.go |
| @@ -849,15 +849,25 @@ func restartRequestLeg(ctx context.Context, routes []registry.Route, store relea |
| // the first error the cause — they are the same cause in practice. |
| log.Printf("agent: restart check: %d apps unreadable: %v", len(readErrs), readErrs[0]) |
| } |
| - for _, host := range retryRequests(routes, scoped) { |
| + // A retry clears a hold only on the console's root-signed grant for this |
| + // entry (spec §11, Astra impl-6 finding 5): every box can write the log, |
| + // so an unsigned or foreign entry is dropped here, before anything is |
| + // cleared. The grant's epoch is checked against the book where the |
| + // clear is (rollout.Manager.RetryHost). |
| + for _, rq := range retryRequests(routes, scoped) { |
| if mem.retryHost == nil { |
| - log.Printf("agent: retry requested for %s, dropped: rollout manager not armed on this box, so nothing is held to clear", host) |
| + log.Printf("agent: retry requested for %s, dropped: rollout manager not armed on this box, so nothing is held to clear", rq.host) |
| continue |
| } |
| - if mem.retryHost(host) { |
| - log.Printf("agent: retry requested for %s: its failure hold is cleared and its next attempt runs as a canary", host) |
| + g, why := verifyRetry(mem.pub, rq, time.Now()) |
| + if why != "" { |
| + log.Printf("agent: retry requested for %s, dropped: %s", rq.host, why) |
| + continue |
| + } |
| + if mem.retryHost(rq.host, g.Build, g.Epoch) { |
| + log.Printf("agent: retry requested for %s: its failure hold on %s is cleared and its next attempt runs as a canary", rq.host, g.Build) |
| } else { |
| - log.Printf("agent: retry for %s: nothing held to clear", host) |
| + log.Printf("agent: retry for %s: nothing held to clear for %s at epoch %d (cleared already, or a newer failure since the grant)", rq.host, g.Build, g.Epoch) |
| } |
| } |
| scoped = withoutRetries(scoped) |
diff --git a/internal/host/restartscope.go b/internal/host/restartscope.go| index 3c8309608..63730468f 100644 |
| --- a/internal/host/restartscope.go |
| +++ b/internal/host/restartscope.go |
| @@ -28,9 +28,11 @@ package host |
| import ( |
| "context" |
| + "crypto/ed25519" |
| "fmt" |
| "log" |
| "sort" |
| + "time" |
| "github.com/carlosframework/platform/internal/registry" |
| "github.com/carlosframework/platform/internal/release" |
| @@ -52,9 +54,16 @@ type restartMemory struct { |
| scope map[[2]string]string // app -> last applied request id, or release.RestartScopeNone |
| // retryHost clears one host's failure hold (rollout.Manager.RetryHost), |
| - // for a RestartScopeRetry request. nil while the rollout manager is not |
| - // armed: there is no hold to clear, and the request is dropped. |
| - retryHost func(host string) bool |
| + // for a RestartScopeRetry request whose root-signed grant verified: the |
| + // grant's build and the restriction's epoch it was issued for. nil while |
| + // the rollout manager is not armed: there is no hold to clear, and the |
| + // request is dropped (a declined box's holds get retries in a later |
| + // task, through this same signature). |
| + retryHost func(host, build string, epoch uint64) bool |
| + // pub is the deployment root's public key (CARLOS_RELEASE_PUBKEY), which |
| + // every retry's grant is verified against (spec §11). Nil drops every |
| + // retry: a box that cannot tell a grant from a forgery clears nothing. |
| + pub ed25519.PublicKey |
| } |
| func newRestartMemory() *restartMemory { |
| @@ -173,30 +182,57 @@ func scopedHosts(routes []registry.Route, app [2]string, requests []release.Rest |
| return out |
| } |
| -// retryRequests is the host-retry half of the narrowed log: the hosts of |
| -// this box's routes, in the requesting app, that outstanding retry requests |
| -// name. A host on another box names nothing here. |
| -func retryRequests(routes []registry.Route, work []scopedRestart) []string { |
| - seen := map[string]bool{} |
| - var out []string |
| +// retryRequest is one outstanding retry entry naming a host this box |
| +// serves: the entry's account and app (the log's own), its host, and the |
| +// grant it carries, unverified. |
| +type retryRequest struct { |
| + acct, app, host string |
| + signed string |
| +} |
| + |
| +// retryRequests is the host-retry half of the narrowed log: the entries, |
| +// oldest first, whose host is one of this box's routes in the requesting |
| +// app. A host on another box names nothing here. Entries are not merged by |
| +// host: each carries its own grant, and the box verifies each (a forged |
| +// entry must not hide a real one behind it). A grant spent by an earlier |
| +// entry moves its restriction's epoch, so a second one for the same failure |
| +// clears nothing. |
| +func retryRequests(routes []registry.Route, work []scopedRestart) []retryRequest { |
| + var out []retryRequest |
| for _, w := range work { |
| for _, req := range w.requests { |
| - if req.Scope.Kind != release.RestartScopeRetry || seen[req.Scope.Value] { |
| + if req.Scope.Kind != release.RestartScopeRetry { |
| continue |
| } |
| for _, rt := range routes { |
| if rt.Host == req.Scope.Value && rt.App == w.app[1] && release.RouteAccount(rt) == w.app[0] { |
| - seen[rt.Host] = true |
| - out = append(out, rt.Host) |
| + out = append(out, retryRequest{acct: w.app[0], app: w.app[1], host: rt.Host, signed: req.Scope.Signed}) |
| break |
| } |
| } |
| } |
| } |
| - sort.Strings(out) |
| return out |
| } |
| +// verifyRetry is the box's check of one retry entry (spec §11, Paul's |
| +// ruling on plan 6 Q4): the grant must verify against the root key and |
| +// name this entry's account, app and host, within its window |
| +// (release.VerifyHostRetry). The reason it gives is an operator's log line. |
| +func verifyRetry(pub ed25519.PublicKey, rq retryRequest, now time.Time) (release.HostRetryGrant, string) { |
| + switch { |
| + case len(pub) == 0: |
| + return release.HostRetryGrant{}, "this box has no root public key to verify the grant with" |
| + case rq.signed == "": |
| + return release.HostRetryGrant{}, "unsigned (only a console's signed grant clears a hold)" |
| + } |
| + g, err := release.VerifyHostRetry(pub, rq.signed, rq.acct, rq.app, rq.host, now) |
| + if err != nil { |
| + return release.HostRetryGrant{}, fmt.Sprintf("the grant did not verify for this entry: %v", err) |
| + } |
| + return g, "" |
| +} |
| + |
| // withoutRetries drops retry requests, and apps left with none, so a serial |
| // move that carried only retries asks root for no restart pass. |
| func withoutRetries(work []scopedRestart) []scopedRestart { |
diff --git a/internal/host/restartscope_test.go b/internal/host/restartscope_test.go| index 3fd6ca894..af232d197 100644 |
| --- a/internal/host/restartscope_test.go |
| +++ b/internal/host/restartscope_test.go |
| @@ -8,17 +8,21 @@ package host |
| import ( |
| "bytes" |
| "context" |
| + "crypto/ed25519" |
| "encoding/json" |
| "errors" |
| + "fmt" |
| "log" |
| "os" |
| "path/filepath" |
| "slices" |
| "strings" |
| "testing" |
| + "time" |
| "github.com/carlosframework/platform/internal/registry" |
| "github.com/carlosframework/platform/internal/release" |
| + "github.com/carlosframework/platform/internal/rollout" |
| ) |
| // askScoped is a member's `carlos restart --host/--environment` landing in the |
| @@ -580,25 +584,172 @@ func saveMarks(t *testing.T, path string, m *restartMarks) { |
| } |
| // A retry request clears one host's hold through the rollout manager, cycles |
| -// nothing, and asks root for no restart pass. |
| +// nothing, and asks root for no restart pass. The grant's build and epoch are |
| +// what the manager is handed, once. |
| func TestRestartLegHandsARetryToTheRolloutManager(t *testing.T) { |
| ctx, s := genKit(t) |
| dd := t.TempDir() |
| routes := []registry.Route{execRoute("a.example.com", "bab", "eleven"), execRoute("b.example.com", "bab", "eleven")} |
| - mem := newRestartMemory() |
| + mem, priv := retryMemory(t) |
| var retried []string |
| - mem.retryHost = func(h string) bool { retried = append(retried, h); return true } |
| + mem.retryHost = func(h, b string, e uint64) bool { |
| + retried = append(retried, fmt.Sprintf("%s %s %d", h, b, e)) |
| + return true |
| + } |
| bounce := func(h string) bool { t.Errorf("%s was bounced for a retry", h); return true } |
| if asked := restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem); asked { |
| t.Fatal("the baselining leg asked for a root pass") |
| } |
| - askScoped(t, ctx, s, "bab", "eleven", release.RestartScopeRetry, "b.example.com") |
| - askScoped(t, ctx, s, "bab", "eleven", release.RestartScopeRetry, "elsewhere.example.com") |
| + askRetry(t, ctx, s, priv, "bab", "eleven", "b.example.com", "v2", 5) |
| + askRetry(t, ctx, s, priv, "bab", "eleven", "elsewhere.example.com", "v2", 5) |
| if asked := restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem); asked { |
| t.Fatal("a retry-only move asked for a root restart pass") |
| } |
| - if !slices.Equal(retried, []string{"b.example.com"}) { |
| - t.Fatalf("retried %v, want only this box's b.example.com", retried) |
| + if !slices.Equal(retried, []string{"b.example.com v2 5"}) { |
| + t.Fatalf("retried %v, want only this box's b.example.com, for v2 at epoch 5", retried) |
| + } |
| +} |
| + |
| +// retryMemory is a restart memory that verifies retries against a fresh |
| +// root key, returned for the test's console to sign with. |
| +func retryMemory(t *testing.T) (*restartMemory, ed25519.PrivateKey) { |
| + t.Helper() |
| + pub, priv, err := ed25519.GenerateKey(nil) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + mem := newRestartMemory() |
| + mem.pub = pub |
| + return mem, priv |
| +} |
| + |
| +// askRetry is the console's signed `carlos instances retry <host>` landing |
| +// in the bucket (internal/console requestHostRetry). |
| +func askRetry(t *testing.T, ctx context.Context, s release.Store, priv ed25519.PrivateKey, acct, app, host, build string, epoch uint64) string { |
| + t.Helper() |
| + wire, err := release.SignHostRetry(release.RootSigner(priv), acct, app, host, build, epoch, time.Now()) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + askSignedRetry(t, ctx, s, acct, app, host, wire) |
| + return wire |
| +} |
| + |
| +func askSignedRetry(t *testing.T, ctx context.Context, s release.Store, acct, app, host, wire string) { |
| + t.Helper() |
| + if _, err := release.AppendRestartScope(ctx, s, acct, app, release.RestartScope{Kind: release.RestartScopeRetry, Value: host, Signed: wire}); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +// Spec §11, Astra impl-6 finding 5: every box can write the restart-scope |
| +// log, so only a grant the root signed for this entry's account, app and |
| +// host reaches the manager. An unsigned entry (another box's, or an older |
| +// console's), one signed by another key, one whose grant names another host |
| +// or app than its entry, and any retry on a box with no root key are |
| +// dropped with a line saying why. |
| +func TestRestartLegDropsARetryThatIsNotTheRootsGrant(t *testing.T) { |
| + ctx, s := genKit(t) |
| + dd := t.TempDir() |
| + routes := []registry.Route{execRoute("a.example.com", "bab", "eleven"), execRoute("b.example.com", "bab", "eleven")} |
| + mem, priv := retryMemory(t) |
| + _, other, _ := ed25519.GenerateKey(nil) |
| + mem.retryHost = func(h, b string, e uint64) bool { t.Errorf("retryHost(%s, %s, %d) was called", h, b, e); return true } |
| + bounce := func(h string) bool { t.Errorf("%s was bounced", h); return true } |
| + restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) // baseline |
| + |
| + askScoped(t, ctx, s, "bab", "eleven", release.RestartScopeRetry, "a.example.com") |
| + askRetry(t, ctx, s, other, "bab", "eleven", "a.example.com", "v2", 5) |
| + forB, _ := release.SignHostRetry(release.RootSigner(priv), "bab", "eleven", "b.example.com", "v2", 5, time.Now()) |
| + askSignedRetry(t, ctx, s, "bab", "eleven", "a.example.com", forB) |
| + forShop, _ := release.SignHostRetry(release.RootSigner(priv), "bab", "shop", "a.example.com", "v2", 5, time.Now()) |
| + askSignedRetry(t, ctx, s, "bab", "eleven", "a.example.com", forShop) |
| + |
| + var logbuf bytes.Buffer |
| + log.SetOutput(&logbuf) |
| + defer log.SetOutput(os.Stderr) |
| + restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) |
| + if n := strings.Count(logbuf.String(), "agent: retry requested for a.example.com, dropped:"); n != 4 { |
| + t.Fatalf("log = %q, want four dropped retries", logbuf.String()) |
| + } |
| + if !strings.Contains(logbuf.String(), "unsigned") { |
| + t.Fatalf("log = %q, want the unsigned entry named as such", logbuf.String()) |
| + } |
| + |
| + // A box with no root key cannot tell a grant from a forgery. |
| + logbuf.Reset() |
| + mem.pub = nil |
| + askRetry(t, ctx, s, priv, "bab", "eleven", "a.example.com", "v2", 5) |
| + restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) |
| + if !strings.Contains(logbuf.String(), "agent: retry requested for a.example.com, dropped: this box has no root public key") { |
| + t.Fatalf("log = %q, want the keyless drop named", logbuf.String()) |
| + } |
| +} |
| + |
| +// The grants against a real failure book (plan review findings 5 and 13): a |
| +// grant whose epoch predates a second failure of the same build on the same |
| +// host, a grant for another build, and a used grant replayed after the book |
| +// is reopened each clear nothing; the grant for the current restriction |
| +// clears it. |
| +func TestRestartLegRetryGrantsAgainstTheBook(t *testing.T) { |
| + ctx, s := genKit(t) |
| + dd, bookDir := t.TempDir(), t.TempDir() |
| + const host = "a.example.com" |
| + routes := []registry.Route{execRoute(host, "bab", "eleven")} |
| + book, err := rollout.OpenFailureBook(bookDir) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + mem, priv := retryMemory(t) |
| + var cleared int |
| + mem.retryHost = func(h, b string, e uint64) bool { |
| + rec, mark, _, _ := book.RetryHostAt("bab", "eleven", b, h, e) |
| + if rec || mark { |
| + cleared++ |
| + } |
| + return rec || mark |
| + } |
| + bounce := func(h string) bool { t.Errorf("%s was bounced", h); return true } |
| + restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) // baseline |
| + held := func() bool { return book.Blocks("bab", "eleven", "v2", host, false) } |
| + |
| + book.Record(rollout.FailureRecord{Account: "bab", App: "eleven", Build: "v2", Host: host}) |
| + first := book.RestrictionEpoch("bab", "eleven", "v2", host) |
| + stale := askRetry(t, ctx, s, priv, "bab", "eleven", host, "v2", first) |
| + // The same build fails again on the same host before the box reads it. |
| + book.Record(rollout.FailureRecord{Account: "bab", App: "eleven", Build: "v2", Host: host, HeldDown: []string{host}}) |
| + restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) |
| + if cleared != 0 || !held() { |
| + t.Fatal("a grant predating the second failure cleared it") |
| + } |
| + |
| + now := book.RestrictionEpoch("bab", "eleven", "v2", host) |
| + askRetry(t, ctx, s, priv, "bab", "eleven", host, "v9", now) |
| + restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) |
| + if cleared != 0 || !held() { |
| + t.Fatal("a grant for another build cleared v2's hold") |
| + } |
| + |
| + used := askRetry(t, ctx, s, priv, "bab", "eleven", host, "v2", now) |
| + restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) |
| + if cleared != 1 || held() { |
| + t.Fatalf("the current grant: %d clears, held %v; want one clear", cleared, held()) |
| + } |
| + |
| + // The agent restarts; the retried canary fails again; the used grant |
| + // and the stale one are replayed under new entry ids. |
| + if book, err = rollout.OpenFailureBook(bookDir); err != nil { |
| + t.Fatal(err) |
| + } |
| + book.Record(rollout.FailureRecord{Account: "bab", App: "eleven", Build: "v2", Host: host, HeldDown: []string{host}, Seq: book.Snapshot()}) |
| + if !held() { |
| + t.Fatal("the failure after the retry did not hold the host") |
| + } |
| + askSignedRetry(t, ctx, s, "bab", "eleven", host, used) |
| + askSignedRetry(t, ctx, s, "bab", "eleven", host, stale) |
| + restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) |
| + if cleared != 1 || !held() { |
| + t.Fatal("a replayed grant cleared the failure that followed it") |
| } |
| } |
| @@ -666,12 +817,12 @@ func TestRestartLegSeparatesARetryFromARestartInOneWindow(t *testing.T) { |
| ctx, s := genKit(t) |
| dd := t.TempDir() |
| routes := []registry.Route{execRoute("a.example.com", "bab", "eleven"), execRoute("b.example.com", "bab", "eleven")} |
| - mem := newRestartMemory() |
| + mem, priv := retryMemory(t) |
| var retried, bounced []string |
| - mem.retryHost = func(h string) bool { retried = append(retried, h); return true } |
| + mem.retryHost = func(h, _ string, _ uint64) bool { retried = append(retried, h); return true } |
| bounce := func(h string) bool { bounced = append(bounced, h); return true } |
| restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) // baseline |
| - askScoped(t, ctx, s, "bab", "eleven", release.RestartScopeRetry, "a.example.com") |
| + askRetry(t, ctx, s, priv, "bab", "eleven", "a.example.com", "v2", 5) |
| askScoped(t, ctx, s, "bab", "eleven", release.RestartScopeHost, "b.example.com") |
| if asked := restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem); !asked { |
| t.Fatal("the restart in the window did not ask for a root pass") |
| @@ -687,11 +838,11 @@ func TestRestartLegLogsARetryWithNothingHeld(t *testing.T) { |
| ctx, s := genKit(t) |
| dd := t.TempDir() |
| routes := []registry.Route{execRoute("a.example.com", "bab", "eleven")} |
| - mem := newRestartMemory() |
| - mem.retryHost = func(string) bool { return false } |
| + mem, priv := retryMemory(t) |
| + mem.retryHost = func(string, string, uint64) bool { return false } |
| bounce := func(h string) bool { t.Errorf("%s was bounced", h); return true } |
| restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) |
| - askScoped(t, ctx, s, "bab", "eleven", release.RestartScopeRetry, "a.example.com") |
| + askRetry(t, ctx, s, priv, "bab", "eleven", "a.example.com", "v2", 5) |
| var logbuf bytes.Buffer |
| log.SetOutput(&logbuf) |
| defer log.SetOutput(os.Stderr) |
diff --git a/internal/release/hostretry.go b/internal/release/hostretry.go| new file mode 100644 |
| index 000000000..dd3abf188 |
| --- /dev/null |
| +++ b/internal/release/hostretry.go |
| @@ -0,0 +1,74 @@ |
| +package release |
| + |
| +import ( |
| + "crypto/ed25519" |
| + "errors" |
| + "time" |
| +) |
| + |
| +// HostRetryGrant is the console's signed `carlos instances retry <host>` |
| +// (zero-downtime spec §8.4, §11; Paul's ruling on plan 6 Q4, amadan #272): |
| +// a box clears a host's failure hold only on a grant the deployment root |
| +// signed, naming that account, app, host and the build that restricts it. |
| +// |
| +// The restart-scope log it rides is writable by every box (Astra impl-6 |
| +// finding 5), so an unsigned entry is a request anyone with a box could |
| +// have written, and it clears nothing. The signature closes forgery; Epoch |
| +// closes replay (plan review finding 13): it is the restriction's persisted |
| +// epoch as the box reported it (rollout.FailureBook.RestrictionEpoch), so a |
| +// grant clears the restriction the member saw and nothing that failed |
| +// after, even across an agent restart. |
| +type HostRetryGrant struct { |
| + V int `json:"v"` |
| + Account string `json:"account"` |
| + App string `json:"app"` |
| + Host string `json:"host"` |
| + // Build is the build whose restriction the grant lifts: the failure |
| + // record of that build for Host, and Host's hold-ran marker when it |
| + // names that build (plan review finding 5). It is not the build the |
| + // host should move to. |
| + Build string `json:"build"` |
| + // Epoch is the restriction's epoch the console read from the box's |
| + // report. Zero is an older box that reported none: the console still |
| + // signs it, and no box accepts it. |
| + Epoch uint64 `json:"epoch"` |
| + IssuedAt string `json:"issued_at"` // RFC 3339, UTC, whole seconds |
| +} |
| + |
| +// HostRetryGrantTTL is how long a box accepts a grant after it was issued. |
| +// It is a backstop only: the epoch is what binds a grant to one failure. |
| +const HostRetryGrantTTL = 24 * time.Hour |
| + |
| +// hostRetrySkew is how far ahead of a box's clock a grant's IssuedAt may be. |
| +const hostRetrySkew = time.Minute |
| + |
| +// ErrHostRetryGrant is a grant that verified but is not for this request. |
| +var ErrHostRetryGrant = errors.New("host retry grant: not valid for this request") |
| + |
| +// SignHostRetry signs a grant with the deployment root. A scoped release |
| +// key is refused (SignRootEnvelope): a channel delegate has no say over |
| +// another instance's failure hold. |
| +func SignHostRetry(sg Signer, acct, app, host, build string, epoch uint64, now time.Time) (string, error) { |
| + g := HostRetryGrant{V: 1, Account: acct, App: app, Host: host, Build: build, Epoch: epoch, |
| + IssuedAt: now.UTC().Truncate(time.Second).Format(time.RFC3339)} |
| + wire, err := SignRootEnvelope(sg, g) |
| + return string(wire), err |
| +} |
| + |
| +// VerifyHostRetry checks wire against the root key and the entry it rode in: |
| +// the same account, app and host, a build and a non-zero epoch, issued no |
| +// later than now plus a minute of skew and no earlier than now minus |
| +// HostRetryGrantTTL. Whether the epoch is still the restriction's is the |
| +// failure book's check, made where the clear is. |
| +func VerifyHostRetry(pub ed25519.PublicKey, wire, acct, app, host string, now time.Time) (HostRetryGrant, error) { |
| + var g HostRetryGrant |
| + if err := VerifyRootEnvelope(pub, []byte(wire), &g); err != nil { |
| + return HostRetryGrant{}, err |
| + } |
| + at, err := time.Parse(time.RFC3339, g.IssuedAt) |
| + if err != nil || g.V != 1 || g.Account != acct || g.App != app || g.Host != host || g.Build == "" || g.Epoch == 0 || |
| + at.After(now.Add(hostRetrySkew)) || at.Before(now.Add(-HostRetryGrantTTL)) { |
| + return HostRetryGrant{}, ErrHostRetryGrant |
| + } |
| + return g, nil |
| +} |
diff --git a/internal/release/hostretry_test.go b/internal/release/hostretry_test.go| new file mode 100644 |
| index 000000000..a07c5c699 |
| --- /dev/null |
| +++ b/internal/release/hostretry_test.go |
| @@ -0,0 +1,168 @@ |
| +package release |
| + |
| +import ( |
| + "crypto/ed25519" |
| + "encoding/base64" |
| + "encoding/json" |
| + "errors" |
| + "strings" |
| + "testing" |
| + "time" |
| +) |
| + |
| +func hostRetryKey(t *testing.T) (ed25519.PublicKey, ed25519.PrivateKey) { |
| + t.Helper() |
| + pub, priv, err := ed25519.GenerateKey(nil) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + return pub, priv |
| +} |
| + |
| +func TestHostRetryGrantRoundTrip(t *testing.T) { |
| + pub, priv := hostRetryKey(t) |
| + now := time.Date(2026, 10, 11, 9, 30, 15, 500, time.UTC) |
| + wire, err := SignHostRetry(RootSigner(priv), "bab", "blog", "a.example.com", "v2", 42, now) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + g, err := VerifyHostRetry(pub, wire, "bab", "blog", "a.example.com", now.Add(time.Hour)) |
| + if err != nil { |
| + t.Fatalf("verify = %v", err) |
| + } |
| + want := HostRetryGrant{V: 1, Account: "bab", App: "blog", Host: "a.example.com", Build: "v2", Epoch: 42, IssuedAt: "2026-10-11T09:30:15Z"} |
| + if g != want { |
| + t.Fatalf("grant = %+v, want %+v", g, want) |
| + } |
| + // The whole 24-hour window is the backstop: the epoch is what binds. |
| + if _, err := VerifyHostRetry(pub, wire, "bab", "blog", "a.example.com", now.Add(HostRetryGrantTTL-time.Second)); err != nil { |
| + t.Fatalf("a grant inside its window = %v", err) |
| + } |
| + if len(wire) > MaxRootEnvelopeBytes { |
| + t.Fatalf("wire is %d bytes", len(wire)) |
| + } |
| +} |
| + |
| +func TestHostRetryGrantRefusals(t *testing.T) { |
| + pub, priv := hostRetryKey(t) |
| + otherPub, _ := hostRetryKey(t) |
| + now := time.Date(2026, 10, 11, 9, 30, 0, 0, time.UTC) |
| + wire, err := SignHostRetry(RootSigner(priv), "bab", "blog", "a.example.com", "v2", 7, now) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + cases := []struct { |
| + name string |
| + pub ed25519.PublicKey |
| + wire string |
| + acct, app, host string |
| + at time.Time |
| + }{ |
| + {"wrong host", pub, wire, "bab", "blog", "b.example.com", now}, |
| + {"wrong app", pub, wire, "bab", "shop", "a.example.com", now}, |
| + {"wrong account", pub, wire, "kass", "blog", "a.example.com", now}, |
| + {"expired", pub, wire, "bab", "blog", "a.example.com", now.Add(HostRetryGrantTTL + time.Second)}, |
| + {"from the future", pub, wire, "bab", "blog", "a.example.com", now.Add(-2 * time.Minute)}, |
| + {"another root", otherPub, wire, "bab", "blog", "a.example.com", now}, |
| + {"no key", nil, wire, "bab", "blog", "a.example.com", now}, |
| + {"empty", pub, "", "bab", "blog", "a.example.com", now}, |
| + {"tampered payload", pub, tamperHostRetry(t, wire, func(g map[string]any) { g["build"] = "v3" }), "bab", "blog", "a.example.com", now}, |
| + {"tampered epoch", pub, tamperHostRetry(t, wire, func(g map[string]any) { g["epoch"] = 8 }), "bab", "blog", "a.example.com", now}, |
| + } |
| + for _, c := range cases { |
| + if g, err := VerifyHostRetry(c.pub, c.wire, c.acct, c.app, c.host, c.at); err == nil { |
| + t.Errorf("%s: verified %+v", c.name, g) |
| + } |
| + } |
| + // A minute of clock skew is allowed the console. |
| + if _, err := VerifyHostRetry(pub, wire, "bab", "blog", "a.example.com", now.Add(-30*time.Second)); err != nil { |
| + t.Fatalf("thirty seconds of skew = %v", err) |
| + } |
| +} |
| + |
| +// A grant the root signed with a zero epoch is an older box's report read by |
| +// the console: there is no restriction it can name, so no box acts on it. |
| +func TestHostRetryGrantRefusesAZeroEpoch(t *testing.T) { |
| + pub, priv := hostRetryKey(t) |
| + now := time.Now() |
| + wire, err := SignHostRetry(RootSigner(priv), "bab", "blog", "a.example.com", "v2", 0, now) |
| + if err != nil { |
| + t.Fatalf("the console signs a zero epoch: %v", err) |
| + } |
| + if _, err := VerifyHostRetry(pub, wire, "bab", "blog", "a.example.com", now); !errors.Is(err, ErrHostRetryGrant) { |
| + t.Fatalf("zero epoch = %v, want ErrHostRetryGrant", err) |
| + } |
| + wire, _ = SignHostRetry(RootSigner(priv), "bab", "blog", "a.example.com", "", 3, now) |
| + if _, err := VerifyHostRetry(pub, wire, "bab", "blog", "a.example.com", now); err == nil { |
| + t.Fatal("a grant naming no build verified") |
| + } |
| +} |
| + |
| +func TestHostRetryGrantRefusesAScopedSigner(t *testing.T) { |
| + pub, priv := hostRetryKey(t) |
| + scoped := Signer{key: priv, scope: &Grant{Name: "ci"}, grant: &signedGrant{}, root: pub} |
| + if _, err := SignHostRetry(scoped, "bab", "blog", "a.example.com", "v2", 1, time.Now()); !errors.Is(err, ErrRootEnvelope) { |
| + t.Fatalf("scoped signer = %v, want ErrRootEnvelope", err) |
| + } |
| + if _, err := SignHostRetry(Signer{}, "bab", "blog", "a.example.com", "v2", 1, time.Now()); err == nil { |
| + t.Fatal("a signer with no key signed a retry") |
| + } |
| +} |
| + |
| +// tamperHostRetry re-encodes wire's payload after edit, keeping the old |
| +// signature. |
| +func tamperHostRetry(t *testing.T, wire string, edit func(map[string]any)) string { |
| + t.Helper() |
| + var w struct { |
| + Payload string `json:"payload"` |
| + Sig string `json:"sig"` |
| + } |
| + if err := json.Unmarshal([]byte(wire), &w); err != nil { |
| + t.Fatal(err) |
| + } |
| + raw, err := base64.StdEncoding.DecodeString(w.Payload) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var g map[string]any |
| + if err := json.Unmarshal(raw, &g); err != nil { |
| + t.Fatal(err) |
| + } |
| + edit(g) |
| + // Keep the struct's field order so only the signature can refuse it. |
| + order := []string{"v", "account", "app", "host", "build", "epoch", "issued_at"} |
| + var b strings.Builder |
| + b.WriteString("{") |
| + for i, k := range order { |
| + if i > 0 { |
| + b.WriteString(",") |
| + } |
| + kv, _ := json.Marshal(k) |
| + vv, _ := json.Marshal(g[k]) |
| + b.Write(kv) |
| + b.WriteString(":") |
| + b.Write(vv) |
| + } |
| + b.WriteString("}") |
| + w.Payload = base64.StdEncoding.EncodeToString([]byte(b.String())) |
| + out, _ := json.Marshal(w) |
| + return string(out) |
| +} |
| + |
| +func TestRestartScopeSignedOnlyOnARetry(t *testing.T) { |
| + if err := ValidRestartScope(RestartScope{Kind: RestartScopeRetry, Value: "a.example.com", Signed: `{"payload":"x","sig":"y"}`}); err != nil { |
| + t.Fatalf("a signed retry = %v", err) |
| + } |
| + if err := ValidRestartScope(RestartScope{Kind: RestartScopeHost, Value: "a.example.com", Signed: "x"}); err == nil { |
| + t.Fatal("a host restart carrying a signature was valid") |
| + } |
| + if err := ValidRestartScope(RestartScope{Kind: RestartScopeRetry, Value: "a.example.com", Signed: strings.Repeat("x", MaxRootEnvelopeBytes+1)}); err == nil { |
| + t.Fatal("an oversized signature was valid") |
| + } |
| + // The field is additive: an older entry has none, and an older reader's |
| + // lax parse steps over it. |
| + raw, _ := json.Marshal(RestartScope{Kind: RestartScopeHost, Value: "a.example.com"}) |
| + if strings.Contains(string(raw), "signed") { |
| + t.Fatalf("an unsigned scope wrote %s", raw) |
| + } |
| +} |
diff --git a/internal/release/restartscope.go b/internal/release/restartscope.go| index f4de0dc4d..a503abb36 100644 |
| --- a/internal/release/restartscope.go |
| +++ b/internal/release/restartscope.go |
| @@ -76,7 +76,10 @@ const ( |
| // §8.4): Value is the host whose failure hold to clear. It is NOT a |
| // restart. Selects does not understand it, so a box built before it |
| // steps over it, and a box built after it hands it to the rollout manager |
| - // instead of cycling anything (internal/host/restartscope.go). |
| + // instead of cycling anything (internal/host/restartscope.go). Unlike a |
| + // restart, a retry lifts a restriction, so the file header's narrowing |
| + // argument does not cover it: it acts only on a root-signed grant in |
| + // Signed (HostRetryGrant, spec §11), and a box drops one without. |
| RestartScopeRetry = "retry" |
| ) |
| @@ -107,6 +110,13 @@ const RestartScopeNone = "\x00none" |
| type RestartScope struct { |
| Kind string `json:"kind"` |
| Value string `json:"value"` |
| + // Signed is a retry's root-signed grant (HostRetryGrant, spec §11): a |
| + // box clears a hold only on a grant that verifies against this entry's |
| + // account, app and host, so a box that can write this log cannot forge |
| + // a retry (Astra impl-6 finding 5). Only a retry may carry it. It is |
| + // additive: an older box's lax parse steps over it, and an older |
| + // console's unsigned retry is dropped by every box that checks. |
| + Signed string `json:"signed,omitempty"` |
| } |
| // RestartRequest is one recorded request. ID is what a box remembers; nothing |
| @@ -170,6 +180,12 @@ func ValidRestartScope(sc RestartScope) error { |
| if len(sc.Value) > 253 || strings.ContainsAny(sc.Value, "\x00\n\r") { |
| return fmt.Errorf("restart scope: %s value is not usable", sc.Kind) |
| } |
| + if sc.Signed != "" && sc.Kind != RestartScopeRetry { |
| + return fmt.Errorf("restart scope: %s carries a signature, which only a retry may", sc.Kind) |
| + } |
| + if len(sc.Signed) > MaxRootEnvelopeBytes { |
| + return fmt.Errorf("restart scope: %s signature is too large", sc.Kind) |
| + } |
| return nil |
| } |
diff --git a/internal/rollout/bookseq_test.go b/internal/rollout/bookseq_test.go| index aa44f86f1..c1f5d704f 100644 |
| --- a/internal/rollout/bookseq_test.go |
| +++ b/internal/rollout/bookseq_test.go |
| @@ -157,19 +157,19 @@ func TestASecondRetryDuringTheCanaryNeverStrandsTheHost(t *testing.T) { |
| h.pass() |
| waitFor(t, "a failure record", func() bool { _, ok := h.book.Lookup("bab", "blog", "v2"); return ok }) |
| gate := h.act.hold("b") |
| - if !h.m.RetryHost("b") { |
| + if !h.retry("b", "v2") { |
| t.Fatal("the first retry cleared nothing") |
| } |
| waitFor(t, "b's retry canary", func() bool { |
| _, started, _ := h.act.snapshot() |
| return slices.Contains(started, "b:v2") |
| }) |
| - h.m.RetryHost("b") // the second retry, while the canary runs |
| + h.retry("b", "v2") // the second retry, while the canary runs |
| close(gate) |
| waitFor(t, "b's canary to fail", func() bool { return !h.m.attemptRunning("b") && h.book.Blocks("bab", "blog", "v2", "b", false) }) |
| h.failing.Store(false) |
| - if !h.m.RetryHost("b") { |
| + if !h.retry("b", "v2") { |
| t.Fatal("a retry after the canary's failure cleared nothing: b is stranded") |
| } |
| waitFor(t, "b on v2", func() bool { return h.servingAll("v2", "b") }) |
| @@ -204,7 +204,7 @@ func TestARetryDuringThePlanLeavesTheWakesFailureStanding(t *testing.T) { |
| h.mu.Lock() |
| h.policyHook = func() { |
| once.Do(func() { |
| - if !h.m.RetryHost("w") { |
| + if !h.retry("w", "v2") { |
| t.Error("the retry during the plan cleared nothing") |
| } |
| }) |
| @@ -374,7 +374,7 @@ func preRetryOutcome(t *testing.T, retry, kind string) { |
| h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "s", PromotedAt: t0, At: t0}) |
| switch retry { |
| case "host": |
| - if !h.m.RetryHost(host) { |
| + if !h.retry(host, "v2") { |
| t.Fatalf("RetryHost(%s) cleared nothing", host) |
| } |
| case "pointer": |
diff --git a/internal/rollout/failures.go b/internal/rollout/failures.go| index 32cdf2450..f6532820f 100644 |
| --- a/internal/rollout/failures.go |
| +++ b/internal/rollout/failures.go |
| @@ -4,6 +4,7 @@ import ( |
| "encoding/json" |
| "errors" |
| "fmt" |
| + "maps" |
| "os" |
| "path/filepath" |
| "slices" |
| @@ -44,6 +45,15 @@ type FailureRecord struct { |
| // Retried are instances `carlos instances retry` cleared: the record no |
| // longer holds them, and their next attempt is a canary. |
| Retried []string `json:"retried,omitempty"` |
| + // Epochs are the book's sequence values at which this record last |
| + // changed (plan review finding 13): under "" the record as a whole |
| + // (created, or merged with a change), under a host that host's standing |
| + // in it (a hold-down added, a retry spent or granted). A signed host |
| + // retry names the epoch it was issued for (RestrictionEpoch), so it |
| + // clears the restriction the member saw and never a later failure of the |
| + // same build. Saved with the record, so a reopen keeps them; a record an |
| + // earlier agent wrote gets one when the book opens. |
| + Epochs map[string]uint64 `json:"epochs,omitempty"` |
| // MarkHoldRan asks Record to set Host's hold-ran marker onto this |
| // record's build in the same save (final review wave 2, N2): an attempt |
| // that failed in hold mode, ran its build, and was not held down. It is |
| @@ -109,7 +119,8 @@ func (r RecordResult) String() string { |
| // instance, not a failure of the build: it blocks no sibling and halts no |
| // cohort, and it lifts no failure record either (Astra R3 N7). A good-build |
| // write decided after the marker was set (a pass whose execution began after |
| -// it, Astra R3 N6), `carlos instances retry`, or a signed rollback pointer |
| +// it, Astra R3 N6), a signed `carlos instances retry` naming the marker's |
| +// build (RetryHostAt, plan review finding 5), or a signed rollback pointer |
| // newer than the marker (applyPointers, ruling 205) clears it. |
| type HoldRan struct { |
| Account string `json:"account"` |
| @@ -233,13 +244,15 @@ type FailureBook struct { |
| // time.Now; tests and the simulation set their own clock. |
| now func() time.Time |
| // seq orders the book's events that later outcomes are compared with |
| - // (Astra R3 N6, ruling 191): every marker set and every retry clear takes |
| - // the next value, and an execution takes a Snapshot (seq+1) when it |
| - // starts. An outcome is older than an event exactly when its snapshot is |
| - // not greater than the event's value. It is saved with the markers, so |
| - // snapshots after a restart are newer than every marker on disk; a value |
| - // taken by a retry clear and never saved is harmless, because clears are |
| - // not saved either. |
| + // (Astra R3 N6, ruling 191): every marker set, every retry clear and |
| + // every change to a record (its epochs, plan review finding 13) takes the |
| + // next value, and an execution takes a Snapshot (seq+1) when it starts. |
| + // An outcome is older than an event exactly when its snapshot is not |
| + // greater than the event's value. It is saved with the markers and the |
| + // records' epochs, so snapshots and epochs after a restart are newer than |
| + // every one on disk. The cleared map below is not saved; a host retry's |
| + // clear value is, as the host's epoch in its record, so a spent grant |
| + // stays spent across a restart. |
| seq uint64 |
| // cleared holds, in memory only, the sequence of the latest retry clear |
| // per (account, app, build, host): `carlos instances retry` per host |
| @@ -257,10 +270,12 @@ func (b *FailureBook) clock() time.Time { |
| return time.Now() |
| } |
| -// cloneRec copies r's host lists so the copy shares no array with r. |
| +// cloneRec copies r's host lists and epochs so the copy shares nothing |
| +// with r. |
| func cloneRec(r FailureRecord) FailureRecord { |
| r.HeldDown = slices.Clone(r.HeldDown) |
| r.Retried = slices.Clone(r.Retried) |
| + r.Epochs = maps.Clone(r.Epochs) |
| return r |
| } |
| @@ -296,12 +311,50 @@ func OpenFailureBook(dataDir string) (*FailureBook, error) { |
| // handing out a snapshot a standing marker is newer than. |
| b.seq = max(b.seq, h.Seq) |
| } |
| + for _, r := range b.recs { |
| + for _, e := range r.Epochs { // the same, for the records' epochs |
| + b.seq = max(b.seq, e) |
| + } |
| + } |
| b.mu.Lock() |
| + b.epochOlderLocked() |
| b.reconcileLocked() |
| b.mu.Unlock() |
| return b, nil |
| } |
| +// epochOlderLocked gives every record and marker an earlier agent wrote |
| +// without an epoch one now (plan review finding 13): a zero epoch is never |
| +// accepted by a signed retry, so without one its hold could not be retried |
| +// from the console. Records first, in file order, then markers by host, so |
| +// two opens of the same file hand out the same values. The book then owes a |
| +// save (dirty), which the next mutating call or Prune makes; nothing is |
| +// written here. The caller holds b.mu. |
| +func (b *FailureBook) epochOlderLocked() { |
| + for i := range b.recs { |
| + if b.recs[i].Epochs[""] == 0 { |
| + if b.recs[i].Epochs == nil { |
| + b.recs[i].Epochs = map[string]uint64{} |
| + } |
| + b.recs[i].Epochs[""] = b.bumpLocked() |
| + b.dirty = true |
| + } |
| + } |
| + hosts := make([]string, 0, len(b.held)) |
| + for host, h := range b.held { |
| + if h.Seq == 0 { |
| + hosts = append(hosts, host) |
| + } |
| + } |
| + slices.Sort(hosts) |
| + for _, host := range hosts { |
| + h := b.held[host] |
| + h.Seq = b.bumpLocked() |
| + b.held[host] = h |
| + b.dirty = true |
| + } |
| +} |
| + |
| // reconcileLocked turns each intent read from the file into the failure |
| // record its execution may have left (rule 3; spec §8.4, Astra R1 #5): its |
| // build on its host, cause release, the intent's mode, stamped with the open |
| @@ -537,26 +590,46 @@ func (b *FailureBook) recordLocked(r FailureRecord) (RecordResult, bool) { |
| i := b.find(r.Account, r.App, r.Build) |
| if i < 0 { |
| r.Retried = nil |
| + // A new record is a new restriction for every host it holds: the |
| + // record-wide epoch and the failing host's (plan review finding 13). |
| + e := b.bumpLocked() |
| + r.Epochs = map[string]uint64{"": e, r.Host: e} |
| + for _, h := range r.HeldDown { |
| + r.Epochs[h] = e |
| + } |
| b.recs = append(b.recs, cloneRec(r)) |
| return Recorded, true |
| } |
| cur := &b.recs[i] |
| - changed := false |
| + var moved []string // hosts whose standing in the record changed |
| + recChanged := false |
| if j := slices.Index(cur.Retried, r.Host); j >= 0 { |
| cur.Retried = slices.Delete(slices.Clone(cur.Retried), j, j+1) |
| - changed = true |
| + moved = append(moved, r.Host) |
| } |
| if r.PromotedAt.After(cur.PromotedAt) { |
| cur.PromotedAt = r.PromotedAt |
| - changed = true |
| + recChanged = true |
| } |
| for _, h := range r.HeldDown { |
| if !slices.Contains(cur.HeldDown, h) { |
| cur.HeldDown = append(slices.Clone(cur.HeldDown), h) |
| - changed = true |
| + moved = append(moved, h) |
| } |
| } |
| - if changed { |
| + if recChanged || len(moved) > 0 { |
| + // A merge with a change moves the record-wide epoch and the epochs |
| + // of the hosts it moved, so a grant signed before it clears nothing |
| + // (plan review finding 13). |
| + cur.Epochs = maps.Clone(cur.Epochs) |
| + if cur.Epochs == nil { |
| + cur.Epochs = map[string]uint64{} |
| + } |
| + e := b.bumpLocked() |
| + cur.Epochs[""] = e |
| + for _, h := range moved { |
| + cur.Epochs[h] = e |
| + } |
| return Merged, true |
| } |
| return Unchanged, marked |
| @@ -617,9 +690,10 @@ func (b *FailureBook) HoldRanOn(host string) (HoldRan, bool) { |
| return h, ok |
| } |
| -// ClearHoldRan drops host's hold-ran marker, whatever it is: `carlos |
| -// instances retry` named the host, the explicit recovery. It reports whether |
| -// a marker stood. |
| +// ClearHoldRan drops host's hold-ran marker, whatever it is, and reports |
| +// whether a marker stood. A host retry no longer uses it: its grant clears |
| +// only a marker naming the grant's build, at the grant's epoch |
| +// (RetryHostAt, plan review findings 5 and 13). |
| func (b *FailureBook) ClearHoldRan(host string) (bool, error) { |
| b.mu.Lock() |
| defer b.mu.Unlock() |
| @@ -702,7 +776,9 @@ func (b *FailureBook) HeldDown(acct, app, build, host string) bool { |
| } |
| // RetryHost is `carlos instances retry <host>`: it clears the record for that |
| -// host only (spec §8.4). How the request reaches the box is plan 6's. |
| +// host only (spec §8.4). It checks no grant: the box reaches it through |
| +// RetryHostAt, once the console's signed grant has verified and its epoch |
| +// is still the restriction's. |
| // |
| // A retry that clears the host also notes a host clear at the book's next |
| // sequence value (ruling 191): a failure of an execution of host that began |
| @@ -714,9 +790,21 @@ func (b *FailureBook) HeldDown(acct, app, build, host string) bool { |
| func (b *FailureBook) RetryHost(acct, app, build, host string) (bool, error) { |
| b.mu.Lock() |
| defer b.mu.Unlock() |
| + if !b.retryHostLocked(acct, app, build, host) { |
| + return false, b.retryLocked() |
| + } |
| + return true, b.saveLocked() |
| +} |
| + |
| +// retryHostLocked is RetryHost's change to memory, and reports whether it |
| +// un-retried the host. The clear's sequence value is also the host's epoch |
| +// in the record (plan review finding 13), saved with it: a grant that was |
| +// spent here never matches again, even after a reopen. The caller holds |
| +// b.mu and saves. |
| +func (b *FailureBook) retryHostLocked(acct, app, build, host string) bool { |
| i := b.find(acct, app, build) |
| if i < 0 || slices.Contains(b.recs[i].Retried, host) { |
| - return false, b.retryLocked() |
| + return false |
| } |
| b.clearedLocked(acct, app, build, host) |
| cur := &b.recs[i] |
| @@ -724,7 +812,62 @@ func (b *FailureBook) RetryHost(acct, app, build, host string) (bool, error) { |
| if j := slices.Index(cur.HeldDown, host); j >= 0 { |
| cur.HeldDown = slices.Delete(slices.Clone(cur.HeldDown), j, j+1) |
| } |
| - return true, b.saveLocked() |
| + cur.Epochs = maps.Clone(cur.Epochs) |
| + if cur.Epochs == nil { |
| + cur.Epochs = map[string]uint64{} |
| + } |
| + cur.Epochs[host] = b.cleared[clearKey(acct, app, build, host)] |
| + return true |
| +} |
| + |
| +// RestrictionEpoch is the epoch of build's restriction of host (plan review |
| +// finding 13): the larger of the build's record's record-wide and host |
| +// epochs (FailureRecord.Epochs), and host's hold-ran marker's sequence when |
| +// the marker names build. Zero when nothing of build restricts host. Status |
| +// publishes it beside HeldBy (HostRollout.HeldEpoch), the console signs it |
| +// into its retry grant, and RetryHostAt clears only while it is unchanged. |
| +func (b *FailureBook) RestrictionEpoch(acct, app, build, host string) uint64 { |
| + b.mu.Lock() |
| + defer b.mu.Unlock() |
| + return b.restrictionEpochLocked(acct, app, build, host) |
| +} |
| + |
| +func (b *FailureBook) restrictionEpochLocked(acct, app, build, host string) uint64 { |
| + var e uint64 |
| + if i := b.find(acct, app, build); i >= 0 { |
| + ep := b.recs[i].Epochs |
| + e = max(ep[""], ep[host]) |
| + } |
| + if m, ok := b.held[host]; ok && m.Account == acct && m.App == app && m.Build == build { |
| + e = max(e, m.Seq) |
| + } |
| + return e |
| +} |
| + |
| +// RetryHostAt is a signed `carlos instances retry <host>` grant (spec §8.4, |
| +// §11; HostRetryGrant) applied to the book: when epoch is build's |
| +// restriction of host now (RestrictionEpoch), it clears build's record for |
| +// host, when that record holds host (RetryHost), and host's hold-ran |
| +// marker, when the marker names build (plan review finding 5). A grant from |
| +// before the restriction last changed, a zero epoch, or one for a build that |
| +// restricts nothing clears nothing. The check and the clears are one hold of |
| +// the lock. now is the epoch at the check, for the caller's log. |
| +func (b *FailureBook) RetryHostAt(acct, app, build, host string, epoch uint64) (rec, mark bool, now uint64, err error) { |
| + b.mu.Lock() |
| + defer b.mu.Unlock() |
| + now = b.restrictionEpochLocked(acct, app, build, host) |
| + if epoch == 0 || now != epoch { |
| + return false, false, now, b.retryLocked() |
| + } |
| + if m, ok := b.held[host]; ok && m.Account == acct && m.App == app && m.Build == build { |
| + delete(b.held, host) |
| + mark = true |
| + } |
| + rec = b.retryHostLocked(acct, app, build, host) |
| + if !rec && !mark { |
| + return false, false, now, b.retryLocked() |
| + } |
| + return rec, mark, now, b.saveLocked() |
| } |
| // ClearOnPointer is `carlos promote --retry` as this box sees it: a signed |
diff --git a/internal/rollout/holdran_test.go b/internal/rollout/holdran_test.go| index 6d2fd08ac..48a1447d2 100644 |
| --- a/internal/rollout/holdran_test.go |
| +++ b/internal/rollout/holdran_test.go |
| @@ -220,7 +220,7 @@ func TestRetryHostClearsTheHoldRanMarker(t *testing.T) { |
| h.pass() |
| h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", At: t0}) |
| h.book.MarkHoldRan("a", markV2) |
| - if !h.m.RetryHost("a") { |
| + if !h.retry("a", "v2") { |
| t.Fatal("RetryHost reported nothing cleared") |
| } |
| if _, ok := h.book.HoldRanOn("a"); ok { |
| @@ -235,7 +235,7 @@ func TestRetryHostClearsTheHoldRanMarker(t *testing.T) { |
| h.pass() |
| h.book.MarkHoldRan("a", markV2) |
| h.book.MarkHoldRan("b", markV2) |
| - if !h.m.RetryHost("a") { |
| + if !h.retry("a", "v2") { |
| t.Fatal("RetryHost reported nothing cleared with a marker standing") |
| } |
| if _, ok := h.book.HoldRanOn("a"); ok { |
diff --git a/internal/rollout/manager.go b/internal/rollout/manager.go| index fa06d268c..9d6ce3ad5 100644 |
| --- a/internal/rollout/manager.go |
| +++ b/internal/rollout/manager.go |
| @@ -583,59 +583,58 @@ func (m *Manager) retryBuild(k appKey, build, host string) { |
| m.changed() |
| } |
| -// RetryHost is the box half of `carlos instances retry <host>` (spec §8.4): |
| -// the record holding host's installed build is cleared for host alone, and |
| -// its next attempt is a canary. How the request reaches the box is plan 6's. |
| +// RetryHost is the box half of `carlos instances retry <host>` (spec §8.4, |
| +// §11): a grant the console signed (release.HostRetryGrant), which the |
| +// caller has verified, naming the build whose restriction of host it lifts |
| +// and that restriction's epoch as the box last reported it (plan review |
| +// findings 5 and 13). The book clears build's record for host, when one |
| +// holds it, and host's hold-ran marker, when the marker names build, and |
| +// only while the restriction's epoch is still epoch (RetryHostAt): a grant |
| +// from before a later failure of the same build, or one replayed after it |
| +// was spent, clears nothing. True means either was cleared. |
| // |
| -// The book entry is cleared and the host's retry flag bound in one hold of |
| -// the scheduler's lock (scheduler.mu before FailureBook.mu), the book first. |
| -// Admission cannot run between the two, so the adoption onto the cleared |
| -// build is the attempt the flag lets through, as a canary, and a restart or |
| -// config attempt cannot start in between and spend the flag. The book also |
| -// notes the retry's place in its sequence there (ruling 191), in memory: an |
| -// outcome of an execution that began before it is stale, and takes the |
| +// When the record was cleared and host's slot wants build, its next attempt |
| +// is a canary. The book entry is cleared and the host's retry flag bound in |
| +// one hold of the scheduler's lock (scheduler.mu before FailureBook.mu), the |
| +// book first. Admission cannot run between the two, so the adoption onto |
| +// the cleared build is the attempt the flag lets through, as a canary, and |
| +// a restart or config attempt cannot start in between and spend the flag. |
| +// The book also notes the retry's place in its sequence there (ruling 191): |
| +// an outcome of an execution that began before it is stale, and takes the |
| // retry back no more than it halts the cohort (failCohort). No write is |
| // added under the scheduler's lock: the book saves as it did before. |
| // |
| -// It also clears host's hold-ran marker (final review wave 2, N2), whatever |
| -// build the marker names and whether or not a record held the host: the |
| -// retry is the explicit recovery spec §6.4 asks for. True means either was |
| -// cleared. |
| -func (m *Manager) RetryHost(host string) bool { |
| +// A grant naming another build than the slot's (the marker's build B while |
| +// the slot wants A) clears B's restriction and leaves A's queue alone. |
| +func (m *Manager) RetryHost(host, build string, epoch uint64) bool { |
| m.mu.Lock() |
| k, ok := m.hostApp[host] |
| m.mu.Unlock() |
| - if !ok { |
| + if !ok || build == "" { |
| return false |
| } |
| s := m.scheduler(k) |
| s.mu.Lock() |
| - build := "" |
| + want := "" |
| if h := s.q.hosts[host]; h != nil { |
| - build = h.wantBuild |
| - } |
| - var ( |
| - cleared bool |
| - err error |
| - ) |
| - if build != "" { |
| - cleared, err = m.d.Book.RetryHost(k.acct, k.app, build, host) |
| + want = h.wantBuild |
| } |
| - if cleared { |
| + cleared, unmarked, cur, err := m.d.Book.RetryHostAt(k.acct, k.app, build, host, epoch) |
| + requeue := cleared && want == build |
| + if requeue { |
| s.q.wantBuild(host, activator.CauseRetry, build, m.d.Now()) |
| s.q.retryHost(host) |
| } |
| - unmarked, merr := m.d.Book.ClearHoldRan(host) |
| s.mu.Unlock() |
| if err != nil { |
| m.d.Logf("rollout: retry %s on %s: %v (cleared in memory)", build, host, err) |
| } |
| - if merr != nil { |
| - m.d.Logf("rollout: retry on %s: clear hold-ran marker: %v (cleared in memory)", host, merr) |
| + if !cleared && !unmarked && cur != epoch { |
| + m.d.Logf("rollout: retry %s on %s: the grant is for epoch %d, the restriction's is %d, so nothing was cleared", build, host, epoch, cur) |
| } |
| - if cleared { |
| + if requeue { |
| m.retryBuild(k, build, host) |
| - } else if unmarked { |
| + } else if cleared || unmarked { |
| m.changed() |
| } |
| return cleared || unmarked |
diff --git a/internal/rollout/manager_test.go b/internal/rollout/manager_test.go| index 6ea31e715..045380285 100644 |
| --- a/internal/rollout/manager_test.go |
| +++ b/internal/rollout/manager_test.go |
| @@ -497,7 +497,7 @@ func TestRetryHostClearsOneInstance(t *testing.T) { |
| h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "c", PromotedAt: promotedAt, At: t0}) |
| h.install("v2", "a", "b") |
| h.pass() |
| - if !h.m.RetryHost("b") { |
| + if !h.retry("b", "v2") { |
| t.Fatal("RetryHost did not clear b") |
| } |
| waitFor(t, "b on v2", func() bool { return h.servingAll("v2", "b") }) |
| @@ -710,7 +710,7 @@ func TestRetryHostRunsACanaryThroughAHalt(t *testing.T) { |
| h.pass() |
| waitFor(t, "a failure record", func() bool { _, ok := h.book.Lookup("bab", "blog", "v2"); return ok }) |
| h.failing.Store(false) |
| - if !h.m.RetryHost("b") { |
| + if !h.retry("b", "v2") { |
| t.Fatal("RetryHost did not clear b") |
| } |
| waitFor(t, "b on v2", func() bool { return h.servingAll("v2", "b") }) |
diff --git a/internal/rollout/retry_rearm_test.go b/internal/rollout/retry_rearm_test.go| index b0b12db16..e0c53d29e 100644 |
| --- a/internal/rollout/retry_rearm_test.go |
| +++ b/internal/rollout/retry_rearm_test.go |
| @@ -54,7 +54,7 @@ func TestARetryAllowanceComesBackOnlyWhenNothingRan(t *testing.T) { |
| h.mismatch["v2"] = true |
| h.mu.Unlock() |
| } |
| - if !h.m.RetryHost("b") { |
| + if !h.retry("b", "v2") { |
| t.Fatal("RetryHost did not clear b") |
| } |
| if tc.staging { |
| @@ -124,7 +124,7 @@ func TestARetryAllowanceIsSpentWhenAWorkerAttemptMayHaveRun(t *testing.T) { |
| h.pass() |
| waitFor(t, "the halt", func() bool { _, ok := h.book.Lookup("bab", "blog", "v2"); return ok }) |
| waitFor(t, "a's slot to free", func() bool { return !h.m.attemptRunning("a") }) |
| - if !h.m.RetryHost("b") { |
| + if !h.retry("b", "v2") { |
| t.Fatal("RetryHost did not clear b") |
| } |
| waitFor(t, "b's first retry attempt", func() bool { return bTries.Load() >= 1 }) |
diff --git a/internal/rollout/retrygrant_test.go b/internal/rollout/retrygrant_test.go| new file mode 100644 |
| index 000000000..b2f833ef8 |
| --- /dev/null |
| +++ b/internal/rollout/retrygrant_test.go |
| @@ -0,0 +1,252 @@ |
| +package rollout |
| + |
| +import ( |
| + "encoding/json" |
| + "os" |
| + "path/filepath" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/activator" |
| +) |
| + |
| +// retry is the console's signed host retry as the box applies it: a grant |
| +// naming build, carrying the epoch the box's status publishes for it. |
| +func (h *harness) retry(host, build string) bool { |
| + return h.m.RetryHost(host, build, h.book.RestrictionEpoch("bab", "blog", build, host)) |
| +} |
| + |
| +// Plan review finding 13: a restriction's epoch is a persisted book |
| +// sequence that moves whenever the restriction changes for the host. |
| +func TestRestrictionEpochFollowsTheRecordAndTheMarker(t *testing.T) { |
| + dd := t.TempDir() |
| + b, err := OpenFailureBook(dd) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if e := b.RestrictionEpoch("bab", "blog", "v2", "a"); e != 0 { |
| + t.Fatalf("epoch with nothing recorded = %d, want 0", e) |
| + } |
| + b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "a"}) |
| + e1 := b.RestrictionEpoch("bab", "blog", "v2", "a") |
| + if e1 == 0 || b.RestrictionEpoch("bab", "blog", "v2", "b") != e1 { |
| + t.Fatalf("after the first failure: a %d, sibling b %d; want one non-zero epoch for both", e1, b.RestrictionEpoch("bab", "blog", "v2", "b")) |
| + } |
| + if res, _ := b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "a"}); res != Unchanged || |
| + b.RestrictionEpoch("bab", "blog", "v2", "a") != e1 { |
| + t.Fatalf("an unchanged failure moved the epoch to %d", b.RestrictionEpoch("bab", "blog", "v2", "a")) |
| + } |
| + b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "a", HeldDown: []string{"a"}}) |
| + e2 := b.RestrictionEpoch("bab", "blog", "v2", "a") |
| + if e2 <= e1 || b.RestrictionEpoch("bab", "blog", "v2", "b") <= e1 { |
| + t.Fatalf("a merged hold-down: a %d, b %d; both must move past %d", e2, b.RestrictionEpoch("bab", "blog", "v2", "b"), e1) |
| + } |
| + b.MarkHoldRan("c", HoldRan{Account: "bab", App: "blog", Build: "v3"}) |
| + m, _ := b.HoldRanOn("c") |
| + if e := b.RestrictionEpoch("bab", "blog", "v3", "c"); e != m.Seq || e == 0 { |
| + t.Fatalf("a marker-only restriction's epoch = %d, want the marker's sequence %d", e, m.Seq) |
| + } |
| + if e := b.RestrictionEpoch("bab", "blog", "v2", "c"); e != b.RestrictionEpoch("bab", "blog", "v2", "b") { |
| + t.Fatalf("c's epoch for v2 = %d: a marker of v3 must not count for v2", e) |
| + } |
| + if e := b.RestrictionEpoch("bab", "shop", "v3", "c"); e != 0 { |
| + t.Fatalf("another app's epoch = %d, want 0", e) |
| + } |
| + |
| + b2, err := OpenFailureBook(dd) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, c := range [][2]string{{"v2", "a"}, {"v2", "b"}, {"v3", "c"}} { |
| + if got, want := b2.RestrictionEpoch("bab", "blog", c[0], c[1]), b.RestrictionEpoch("bab", "blog", c[0], c[1]); got != want { |
| + t.Fatalf("after a reopen %s on %s has epoch %d, want %d", c[0], c[1], got, want) |
| + } |
| + } |
| +} |
| + |
| +// A book an earlier agent wrote has no epochs: the open gives each record |
| +// and marker one, newer than anything in the file, so its holds can still be |
| +// retried (a zero epoch is never accepted). |
| +func TestOpenGivesAnOlderFilesRestrictionsAnEpoch(t *testing.T) { |
| + dd := t.TempDir() |
| + raw, _ := json.Marshal(map[string]any{ |
| + "schema": 1, |
| + "records": []map[string]any{{"account": "bab", "app": "blog", "build": "v2", "host": "a"}}, |
| + "hold_ran": map[string]any{"c": map[string]any{"account": "bab", "app": "blog", "build": "v3"}}, |
| + "seq": 5, |
| + }) |
| + if err := os.WriteFile(filepath.Join(dd, FailureFile), raw, 0o644); err != nil { |
| + t.Fatal(err) |
| + } |
| + b, err := OpenFailureBook(dd) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + rec := b.RestrictionEpoch("bab", "blog", "v2", "a") |
| + mark := b.RestrictionEpoch("bab", "blog", "v3", "c") |
| + if rec <= 5 || mark <= 5 || rec == mark { |
| + t.Fatalf("epochs record %d, marker %d: want two distinct values newer than the file's 5", rec, mark) |
| + } |
| + if b.Snapshot() <= max(rec, mark) { |
| + t.Fatal("a snapshot after the open must be newer than the epochs it handed out") |
| + } |
| +} |
| + |
| +// Plan review finding 5: a grant names the build whose restriction it lifts. |
| +// One naming another build than the record's clears nothing, the marker |
| +// included. |
| +func TestARetryGrantForAnotherBuildClearsNothing(t *testing.T) { |
| + h := newHarness(t, "a", "b") |
| + h.stage("v2") |
| + h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "c", PromotedAt: promotedAt, At: t0}) |
| + h.book.MarkHoldRan("b", markV2) |
| + h.install("v2", "a", "b") |
| + h.pass() |
| + epoch := h.book.RestrictionEpoch("bab", "blog", "v2", "b") |
| + if h.m.RetryHost("b", "v9", epoch) { |
| + t.Fatal("a grant for v9 cleared something") |
| + } |
| + if !h.book.Blocks("bab", "blog", "v2", "b", false) { |
| + t.Fatal("a grant for v9 cleared v2's record for b") |
| + } |
| + if _, ok := h.book.HoldRanOn("b"); !ok { |
| + t.Fatal("a grant for v9 cleared b's v2 marker") |
| + } |
| + if h.m.RetryHost("b", "v2", 0) { |
| + t.Fatal("a zero epoch cleared something") |
| + } |
| + if !h.retry("b", "v2") { |
| + t.Fatal("the grant for v2 at its epoch cleared nothing") |
| + } |
| + if _, ok := h.book.HoldRanOn("b"); ok || h.book.Blocks("bab", "blog", "v2", "b", false) { |
| + t.Fatal("the grant for v2 left b restricted") |
| + } |
| + waitFor(t, "b on v2", func() bool { return h.servingAll("v2", "b") }) |
| +} |
| + |
| +// The marker names B while the slot wants A: a grant for B lifts the marker |
| +// and nothing of A's. |
| +func TestARetryGrantOfTheMarkersBuildClearsOnlyTheMarker(t *testing.T) { |
| + h := newHarness(t, "a") |
| + h.stage("v2") |
| + h.install("v2", "a") |
| + h.sleepHost("a") |
| + h.pass() |
| + h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", At: t0}) |
| + h.book.MarkHoldRan("a", HoldRan{Account: "bab", App: "blog", Build: "v3"}) |
| + if !h.retry("a", "v3") { |
| + t.Fatal("the grant for the marker's build cleared nothing") |
| + } |
| + if _, ok := h.book.HoldRanOn("a"); ok { |
| + t.Fatal("the marker of v3 survived its grant") |
| + } |
| + if !h.book.Blocks("bab", "blog", "v2", "a", false) { |
| + t.Fatal("a grant for v3 cleared v2's record for a") |
| + } |
| +} |
| + |
| +// Plan review finding 13: a grant is for the restriction the member saw. |
| +// One whose epoch predates a second failure of the same build on the same |
| +// host clears nothing, and a fresh grant does. |
| +func TestARetryGrantPredatingASecondFailureClearsNothing(t *testing.T) { |
| + h := newHarness(t, "a", "b") |
| + h.stage("v2") |
| + h.failing.Store(true) |
| + h.act.outcome = func(a activator.Attempt) activator.SwapOutcome { |
| + if a.Build == "v2" && h.failing.Load() { |
| + return activator.SwapFailed |
| + } |
| + return activator.SwapDone |
| + } |
| + h.install("v2", "a", "b") |
| + h.pass() |
| + waitFor(t, "a failure record", func() bool { _, ok := h.book.Lookup("bab", "blog", "v2"); return ok }) |
| + first := h.book.RestrictionEpoch("bab", "blog", "v2", "b") |
| + if !h.m.RetryHost("b", "v2", first) { |
| + t.Fatal("the first grant cleared nothing") |
| + } |
| + // b's canary fails: the same build fails again on the same host. |
| + waitFor(t, "b's canary to fail", func() bool { return !h.m.attemptRunning("b") && h.book.Blocks("bab", "blog", "v2", "b", false) }) |
| + if h.m.RetryHost("b", "v2", first) { |
| + t.Fatal("a grant from before b's second failure cleared it") |
| + } |
| + if !h.book.Blocks("bab", "blog", "v2", "b", false) { |
| + t.Fatal("b is not held after the stale grant") |
| + } |
| + h.failing.Store(false) |
| + if !h.retry("b", "v2") { |
| + t.Fatal("a fresh grant cleared nothing") |
| + } |
| + waitFor(t, "b on v2", func() bool { return h.servingAll("v2", "b") }) |
| +} |
| + |
| +// The same at the book: a merge that holds the host down after the grant |
| +// was signed moves the epoch, so the grant clears nothing. |
| +func TestARetryGrantPredatingAHoldDownClearsNothing(t *testing.T) { |
| + b := &FailureBook{} |
| + b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h"}) |
| + signed := b.RestrictionEpoch("bab", "blog", "v2", "h") |
| + b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h", HeldDown: []string{"h"}}) |
| + if rec, mark, _, _ := b.RetryHostAt("bab", "blog", "v2", "h", signed); rec || mark { |
| + t.Fatalf("a grant predating the hold-down cleared record %v, marker %v", rec, mark) |
| + } |
| + if !b.HeldDown("bab", "blog", "v2", "h") { |
| + t.Fatal("h is no longer held down") |
| + } |
| +} |
| + |
| +// A used grant replayed after the agent reopens its book clears nothing |
| +// new: the clear moved the host's epoch, and the epoch is on disk. |
| +func TestARetryGrantReplayedAfterAReopenClearsNothing(t *testing.T) { |
| + dd := t.TempDir() |
| + b, err := OpenFailureBook(dd) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h", Mode: string(activator.ModeHold), |
| + HeldDown: []string{"h"}, MarkHoldRan: true, HoldRanExec: "a1", HoldRanAt: t0}) |
| + grant := b.RestrictionEpoch("bab", "blog", "v2", "h") |
| + if rec, mark, _, err := b.RetryHostAt("bab", "blog", "v2", "h", grant); !rec || !mark || err != nil { |
| + t.Fatalf("the grant cleared record %v, marker %v (%v); want both", rec, mark, err) |
| + } |
| + b2, err := OpenFailureBook(dd) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + // The retried canary fails again, after the reopen. |
| + b2.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h", Mode: string(activator.ModeHold), |
| + HeldDown: []string{"h"}, MarkHoldRan: true, HoldRanExec: "a2", HoldRanAt: t0.Add(time.Minute), Seq: b2.Snapshot()}) |
| + if rec, mark, _, _ := b2.RetryHostAt("bab", "blog", "v2", "h", grant); rec || mark { |
| + t.Fatalf("the replayed grant cleared record %v, marker %v", rec, mark) |
| + } |
| + if !b2.HeldDown("bab", "blog", "v2", "h") { |
| + t.Fatal("the replay lifted the second hold-down") |
| + } |
| + if _, ok := b2.HoldRanOn("h"); !ok { |
| + t.Fatal("the replay lifted the second marker") |
| + } |
| +} |
| + |
| +// Status publishes the epoch the console signs, beside HeldBy, on both the |
| +// record arm and the marker arm. |
| +func TestStatusPublishesTheHeldEpoch(t *testing.T) { |
| + h := newHarness(t, "a", "b") |
| + h.stage("v2", "v3") |
| + h.install("v2", "a") |
| + h.install("v3", "b") |
| + h.sleepHost("a") |
| + h.sleepHost("b") |
| + h.pass() |
| + h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", Reason: "exited early", At: t0}) |
| + h.book.MarkHoldRan("b", HoldRan{Account: "bab", App: "blog", Build: "v4"}) |
| + r := h.m.Status() |
| + if got := r.Hosts["a"]; got.HeldBy != "v2" || got.HeldEpoch == 0 || got.HeldEpoch != h.book.RestrictionEpoch("bab", "blog", "v2", "a") { |
| + t.Fatalf("a = %+v: want held by v2 at its epoch %d", got, h.book.RestrictionEpoch("bab", "blog", "v2", "a")) |
| + } |
| + if got := r.Hosts["b"]; got.HeldBy != "v4" || got.HeldEpoch == 0 || got.HeldEpoch != h.book.RestrictionEpoch("bab", "blog", "v4", "b") { |
| + t.Fatalf("b = %+v: want held by the marker's v4 at its epoch", got) |
| + } |
| + if !h.m.RetryHost("b", r.Hosts["b"].HeldBy, r.Hosts["b"].HeldEpoch) { |
| + t.Fatal("a grant signed from the status report cleared nothing") |
| + } |
| +} |
diff --git a/internal/rollout/sim_test.go b/internal/rollout/sim_test.go| index 9fd86b94d..ec1727d29 100644 |
| --- a/internal/rollout/sim_test.go |
| +++ b/internal/rollout/sim_test.go |
| @@ -1268,7 +1268,9 @@ func (s *sim) hostRetry() { |
| // that was cleared and written again. |
| rec, recorded := s.book.Lookup("bab", "blog", b) |
| clears := recorded && !slices.Contains(rec.Retried, h.name) |
| - ok := s.m.RetryHost(h.name) |
| + // The console's grant names the build and the epoch the box reports |
| + // for it (Task 8); the sim reads the epoch just as the console would. |
| + ok := s.m.RetryHost(h.name, b, s.book.RestrictionEpoch("bab", "blog", b, h.name)) |
| if s.exempt[h.name] == nil { |
| s.exempt[h.name] = map[string]bool{} |
| } |
| @@ -1280,7 +1282,11 @@ func (s *sim) hostRetry() { |
| s.retryAt[h.name][b] = s.now |
| } |
| delete(s.heldDown[h.name], b) |
| - delete(s.holdRan, h.name) |
| + // A grant lifts the marker only when the marker names its build (plan |
| + // review finding 5). |
| + if s.holdRan[h.name] == b { |
| + delete(s.holdRan, h.name) |
| + } |
| s.logf("instances retry %s onto %s: %v", h.name, b, ok) |
| } |
diff --git a/internal/rollout/status.go b/internal/rollout/status.go| index ddc5c1eb1..bfe1f322d 100644 |
| --- a/internal/rollout/status.go |
| +++ b/internal/rollout/status.go |
| @@ -72,6 +72,9 @@ func (m *Manager) Status() status.RolloutReport { |
| if rec, ok := m.d.Book.Lookup(s.key.acct, s.key.app, h.wantBuild); ok && |
| m.d.Book.Blocks(s.key.acct, s.key.app, h.wantBuild, host, h.wantBuild == h.good) { |
| hr.Attempt, hr.AttemptBuild, hr.HeldBy, hr.Reason = status.AttemptFailed, rec.Build, rec.Build, reasonWord(rec.Reason) |
| + // The epoch the console signs into its retry grant |
| + // (plan review finding 13). |
| + hr.HeldEpoch = m.d.Book.RestrictionEpoch(s.key.acct, s.key.app, rec.Build, host) |
| if m.d.Book.HeldDown(s.key.acct, s.key.app, h.wantBuild, host) { |
| hr.Attempt = status.AttemptDown |
| } |
| @@ -91,6 +94,7 @@ func (m *Manager) Status() status.RolloutReport { |
| hr.Attempt = status.AttemptDown |
| case hr.Attempt == "": |
| hr.HeldBy = mark.Build |
| + hr.HeldEpoch = m.d.Book.RestrictionEpoch(s.key.acct, s.key.app, mark.Build, host) |
| if rec, ok := m.d.Book.Lookup(s.key.acct, s.key.app, mark.Build); ok { |
| hr.Reason = reasonWord(rec.Reason) |
| } |
diff --git a/internal/status/snapshot.go b/internal/status/snapshot.go| index 77aa2a6e0..c6bca2219 100644 |
| --- a/internal/status/snapshot.go |
| +++ b/internal/status/snapshot.go |
| @@ -292,6 +292,11 @@ type HostRollout struct { |
| ConfigPending bool `json:"config_pending,omitempty"` |
| // ConfigError is a reason word (Reason*), never raw error text. |
| ConfigError string `json:"config_error,omitempty"` |
| + // HeldEpoch is the epoch of HeldBy's restriction of this instance |
| + // (rollout.FailureBook.RestrictionEpoch): the console signs it into a |
| + // `carlos instances retry` grant, which the box honours only while the |
| + // restriction is unchanged. Absent from an older box, which is 0. |
| + HeldEpoch uint64 `json:"held_epoch,omitempty"` |
| } |
| // AppRollout counts one app's attempts. Settling is canary settles and |