amadan

carlos / platform Public

retries: signed host retries ride their own object; the console asks the box's report first

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 4d8957c1c3d38ef6bd86175b810be3840c2b03f4 parent 9f27cdf
16 files changed, +257 −52
  • internal/console/apiinstances.go +6 −2
  • internal/console/apiretry.go +51 −16
  • internal/console/apiretry_test.go +101 −14
  • internal/console/browserretry.go +10 −0
  • internal/console/browserretry_test.go +37 −3
  • internal/console/claim_invariant_fixtures_test.go +5 −5
  • internal/console/rolloutfleet_test.go +2 −1
  • internal/host/agent.go +3 −1
  • internal/host/restart.go +12 −1
  • internal/host/restartscope.go +5 −5
  • internal/host/rolloutdeclined.go +2 −1
  • internal/host/rolloutdeclined_test.go +1 −1
  • internal/provision/plan9_b8_census_test.go +1 −1
  • internal/rollout/retrygrant_test.go +7 −0
  • internal/rollout/status.go +3 −1
  • internal/status/snapshot.go +11 −0
diff --git a/internal/console/apiinstances.go b/internal/console/apiinstances.go
index 69198c09b..c385461e6 100644
--- a/internal/console/apiinstances.go
+++ b/internal/console/apiinstances.go
@@ -376,6 +376,10 @@ type rolloutView struct {
// shown: the retry doors sign both into their grant (retryHostBuild).
heldEpoch uint64
heldBook string
+ // retryV is the retry protocol the box takes (status.HostRollout.RetryV):
+ // 0 is a box built before the host-retry log, which reads no grant
+ // there, so the retry doors write it none (retryHostBuild).
+ retryV int
// refused marks a view built from an Unavailable box, which has no
// account or app to check (hosts are unique; the caller filtered by app).
refused bool
@@ -590,7 +594,7 @@ func (j *boxJoin) foldRollout(b status.Box, override bool) {
}
for host, hr := range ro.Hosts {
put(host, rolloutView{Box: b.Label, Installed: hr.Installed, Good: hr.Good, Attempt: hr.Attempt, AttemptBuild: hr.AttemptBuild,
- HeldBy: hr.HeldBy, Reason: hr.Reason, Unavailable: ro.Unavailable, account: hr.Account, app: hr.App, heldEpoch: hr.HeldEpoch, heldBook: hr.HeldBook})
+ HeldBy: hr.HeldBy, Reason: hr.Reason, Unavailable: ro.Unavailable, account: hr.Account, app: hr.App, heldEpoch: hr.HeldEpoch, heldBook: hr.HeldBook, retryV: hr.RetryV})
}
return
}
@@ -611,7 +615,7 @@ func (j *boxJoin) foldRollout(b status.Box, override bool) {
Box: b.Label, Installed: hr.Installed, Serving: hr.Serving, Good: hr.Good, Awake: hr.Awake,
Attempt: hr.Attempt, AttemptBuild: hr.AttemptBuild, HeldBy: hr.HeldBy, Reason: hr.Reason,
ConfigPending: hr.ConfigPending, Transient: hr.Transient, SchedulerIdle: a.Queued == 0 && a.Running == 0 && a.Settling == 0,
- account: hr.Account, app: hr.App, heldEpoch: hr.HeldEpoch, heldBook: hr.HeldBook,
+ account: hr.Account, app: hr.App, heldEpoch: hr.HeldEpoch, heldBook: hr.HeldBook, retryV: hr.RetryV,
})
}
}
diff --git a/internal/console/apiretry.go b/internal/console/apiretry.go
index cab729851..90ec3347e 100644
--- a/internal/console/apiretry.go
+++ b/internal/console/apiretry.go
@@ -3,15 +3,17 @@
// runs as a canary. The app-wide retry is `carlos promote --retry`, which
// re-signs the channel pointer.
//
-// The request rides the narrowed-restart log as selector kind "retry"
-// (release.RestartScopeRetry). A box built before that kind steps over it;
-// one built after hands it to its rollout manager. Every box can write that
-// log (Astra impl-6 finding 5), so the entry carries a grant the root key
-// signs here (release.HostRetryGrant, spec §11; Paul's ruling on plan 6 Q4,
-// amadan #272), naming the held build and its restriction's epoch as the
-// box reported them, and a box clears nothing on an entry without one. A
-// console with no key (read-only) answers 503 and writes nothing. ROLE:
-// operate, as `carlos restart` is.
+// The request rides the app's host-retry log (release.AppendHostRetry,
+// readiness round 2, Task 3), which no older console rewrites and no older
+// box reads. Every box can write that log (Astra impl-6 finding 5), so the
+// entry carries a grant the root key signs here (release.HostRetryGrant,
+// spec §11; Paul's ruling on plan 6 Q4, amadan #272), naming the held build
+// and its restriction's epoch and book as the box reported them, and a box
+// clears nothing on an entry without one. The door writes only for a box
+// whose report says it can take the grant (retryHostBuild): an older box,
+// or one whose failure book is not saved yet, is refused with a message
+// and nothing is written. A console with no key (read-only) answers 503
+// and writes nothing. ROLE: operate, as `carlos restart` is.
package console
import (
@@ -35,12 +37,27 @@ const (
// apiRetryNoKey is a read-only console's answer: a retry is a grant the
// root key signs (spec §11), and there is no key here to sign it.
apiRetryNoKey = "no signing key on this host. this console cannot request a retry"
+ // apiRetryUnsupported answers a box whose report has no retry_v or no
+ // held_book: it reads no grant from the host-retry log, so a retry
+ // written for it would clear nothing.
+ apiRetryUnsupported = "this instance's box is too old to take a retry. update the box, then try again" // TODO-COPY(zdd-readiness2.retry-unsupported.api)
+ // apiRetryUnsaved answers a held host reported at epoch 0: its box's
+ // failure book owes a save, and no grant clears anything until it lands.
+ apiRetryUnsaved = "this instance's box has not saved its failure yet. try again in a minute" // TODO-COPY(zdd-readiness2.retry-unsaved.api)
)
var (
errRetryNoRecord = errors.New("retry: no such instance")
errRetryNotHeld = errors.New("retry: not held")
errRetryNoKey = errors.New("retry: no signing key on this console")
+ // errRetryUnsupported is a held host whose box cannot take a retry:
+ // its report has no retry_v (a box built before the host-retry log) or
+ // no held_book (one built before the book's identity).
+ errRetryUnsupported = errors.New("retry: the box cannot take a retry")
+ // errRetryUnsaved is a held host whose box reports retry_v but epoch 0:
+ // its failure book is not saved (readiness round 2, Task 1), and a grant
+ // signed now would name no epoch the box accepts.
+ errRetryUnsaved = errors.New("retry: the box has not saved its failure book")
)
// retryHostBuild is the build holding host back, and the epoch of that
@@ -53,9 +70,16 @@ var (
// flag (a decline code, rolloutwords.UnavailableDeclines) and reports the
// host held by the failure book it still honours: that box verifies and
// applies a signed retry too (Task 10, plan review #5). A refusing box, or
-// one whose code this console does not know, is never retried here. A box
-// older than the epoch reports 0, and one older than the book reports "":
-// each is signed as it is, and no box accepts it.
+// one whose code this console does not know, is never retried here.
+//
+// Then the box's report is asked whether it can take the grant (readiness
+// round 2, Task 3), and a retry it could not take is refused with a
+// message rather than written to clear nothing: a report with no retry_v
+// is a box built before the host-retry log, which never reads it, and one
+// with no held_book is older than the book's identity
+// (errRetryUnsupported); a held host at epoch 0 is a box whose failure
+// book owes a save, and no grant clears anything until it lands
+// (errRetryUnsaved).
func (s *Server) retryHostBuild(ctx context.Context, acct *Account, app, host string) (heldRestriction, error) {
if _, err := provision.GetRecord(ctx, s.cfg.Releases, acct.Sqid, app, host); err != nil {
if errors.Is(err, os.ErrNotExist) {
@@ -68,6 +92,12 @@ func (s *Server) retryHostBuild(ctx context.Context, acct *Account, app, host st
if ro == nil || ro.HeldBy == "" || (ro.Unavailable != "" && !rolloutwords.UnavailableDeclines(ro.Unavailable)) {
return heldRestriction{}, errRetryNotHeld
}
+ switch {
+ case ro.retryV == 0 || ro.heldBook == "":
+ return heldRestriction{}, errRetryUnsupported
+ case ro.heldEpoch == 0:
+ return heldRestriction{}, errRetryUnsaved
+ }
return heldRestriction{build: ro.HeldBy, book: ro.heldBook, epoch: ro.heldEpoch}, nil
}
@@ -79,8 +109,8 @@ type heldRestriction struct {
}
// requestHostRetry signs the retry grant for held (its build, book and
-// epoch) with the root key, appends it as a retry request, by the same
-// append the narrowed-restart door uses, and announces it on the serial. With no key
+// epoch) with the root key, appends it to the app's host-retry log
+// (release.AppendHostRetry), and announces it on the serial. With no key
// it writes nothing (errRetryNoKey): an unsigned entry is one every box
// drops.
func (s *Server) requestHostRetry(ctx context.Context, acct, app, host string, held heldRestriction) (release.RestartRequest, error) {
@@ -91,8 +121,7 @@ func (s *Server) requestHostRetry(ctx context.Context, acct, app, host string, h
if err != nil {
return release.RestartRequest{}, err
}
- req, err := release.AppendRestartScope(ctx, s.cfg.Releases, acct, app,
- release.RestartScope{Kind: release.RestartScopeRetry, Value: host, Signed: signed})
+ req, err := release.AppendHostRetry(ctx, s.cfg.Releases, acct, app, host, signed)
if err != nil {
return release.RestartRequest{}, err
}
@@ -144,6 +173,12 @@ func (s *Server) handleAPIInstanceRetry(w http.ResponseWriter, r *http.Request)
case errors.Is(err, errRetryNotHeld):
jsonError(w, http.StatusConflict, fmt.Sprintf(apiRetryNotHeld, host))
return
+ case errors.Is(err, errRetryUnsupported):
+ jsonError(w, http.StatusConflict, apiRetryUnsupported)
+ return
+ case errors.Is(err, errRetryUnsaved):
+ jsonError(w, http.StatusConflict, apiRetryUnsaved)
+ return
case err != nil:
log.Printf("console: api instance retry %s/%s/%s: %v", acct.Sqid, app, host, err)
jsonError(w, http.StatusBadGateway, "could not read the instance")
diff --git a/internal/console/apiretry_test.go b/internal/console/apiretry_test.go
index 3f39b4979..66b2f39da 100644
--- a/internal/console/apiretry_test.go
+++ b/internal/console/apiretry_test.go
@@ -13,16 +13,33 @@ import (
"github.com/carlosframework/platform/internal/status"
)
-// lastScoped reads the newest request in the app's narrowed log.
+// lastScoped reads the newest request in the app's host-retry log, where
+// a retry's grant rides (readiness round 2, Task 3), and checks the
+// restart-scope log took none.
func lastScoped(t *testing.T, fx *consoleFixture, app string) release.RestartRequest {
t.Helper()
- g, found, err := release.LoadRestartScopes(context.Background(), fx.rel, fx.sqid, app)
+ g, found, err := release.LoadHostRetries(context.Background(), fx.rel, fx.sqid, app)
if err != nil || !found || len(g.Requests) == 0 {
- t.Fatalf("restart scopes = %+v, %v, %v", g, found, err)
+ t.Fatalf("host retries = %+v, %v, %v", g, found, err)
+ }
+ if old, found, _ := release.LoadRestartScopes(context.Background(), fx.rel, fx.sqid, app); found && len(old.Requests) != 0 {
+ t.Fatalf("a retry wrote the restart-scope log: %+v", old.Requests)
}
return g.Requests[len(g.Requests)-1]
}
+// noRetryWritten checks neither log holds a request for app.
+func noRetryWritten(t *testing.T, fx *consoleFixture, app string) {
+ t.Helper()
+ ctx := context.Background()
+ if g, found, _ := release.LoadHostRetries(ctx, fx.rel, fx.sqid, app); found && len(g.Requests) != 0 {
+ t.Fatalf("a retry was written: %+v", g.Requests)
+ }
+ if g, found, _ := release.LoadRestartScopes(ctx, fx.rel, fx.sqid, app); found && len(g.Requests) != 0 {
+ t.Fatalf("a request was written to the restart-scope log: %+v", g.Requests)
+ }
+}
+
// heldEpoch is the restriction epoch writeHeldBox reports, and heldBook the
// failure book it counts in (final review round 1, finding 3).
const (
@@ -34,7 +51,7 @@ func writeHeldBox(t *testing.T, fx *consoleFixture, host, app, build string) {
t.Helper()
box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now())
box.Metrics = &status.BoxMetrics{Rollout: &status.RolloutReport{Hosts: map[string]status.HostRollout{
- host: {Account: fx.sqid, App: app, Installed: build, Serving: "v1", Good: "v1", Awake: true, HeldBy: build, HeldEpoch: heldEpoch, HeldBook: heldBook, Reason: status.ReasonHealth},
+ host: {Account: fx.sqid, App: app, Installed: build, Serving: "v1", Good: "v1", Awake: true, HeldBy: build, HeldEpoch: heldEpoch, HeldBook: heldBook, RetryV: status.HostRetryV, Reason: status.ReasonHealth},
}}}
if err := status.WriteBox(context.Background(), fx.rel, box); err != nil {
t.Fatal(err)
@@ -128,7 +145,7 @@ func TestAPIInstanceRetryOfADeclinedBoxsHold(t *testing.T) {
ro := &status.RolloutReport{Unavailable: code, FailureBook: "honoured"}
if held {
ro.Hosts = map[string]status.HostRollout{host: {Account: fx.sqid, App: "kass", Installed: "v2", Good: "v1",
- Attempt: status.AttemptDown, AttemptBuild: "v2", HeldBy: "v2", HeldEpoch: heldEpoch, HeldBook: heldBook}}
+ Attempt: status.AttemptDown, AttemptBuild: "v2", HeldBy: "v2", HeldEpoch: heldEpoch, HeldBook: heldBook, RetryV: status.HostRetryV}}
}
box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now())
box.Metrics = &status.BoxMetrics{RouteVersions: map[string]string{host: "v2"}, Rollout: ro}
@@ -159,9 +176,7 @@ func TestAPIInstanceRetryOfADeclinedBoxsHold(t *testing.T) {
if code != http.StatusConflict || !strings.Contains(body, fmt.Sprintf(apiRetryNotHeld, host)) {
t.Fatalf("retry = %d %s, want 409 naming %s as not held", code, body, host)
}
- if g, found, _ := release.LoadRestartScopes(context.Background(), fx.rel, fx.sqid, "kass"); found && len(g.Requests) != 0 {
- t.Fatalf("a retry was written: %+v", g.Requests)
- }
+ noRetryWritten(t, fx, "kass")
return
}
if code != http.StatusOK {
@@ -210,9 +225,7 @@ func TestAPIInstanceRetryRefusesAHostThatIsNotHeld(t *testing.T) {
if code != http.StatusConflict || !strings.Contains(body, fmt.Sprintf(apiRetryNotHeld, host)) {
t.Fatalf("a healthy host = %d %s, want 409 naming %s as not held", code, body, host)
}
- if g, found, _ := release.LoadRestartScopes(context.Background(), fx.rel, fx.sqid, "kass"); found && len(g.Requests) != 0 {
- t.Fatalf("a retry was written for a host that is not held: %+v", g.Requests)
- }
+ noRetryWritten(t, fx, "kass")
}
// A console with no root key (read-only) cannot sign a retry: it answers
@@ -230,10 +243,84 @@ func TestAPIInstanceRetryWithNoKeyWritesNothing(t *testing.T) {
if code != http.StatusServiceUnavailable || !strings.Contains(body, apiRetryNoKey) {
t.Fatalf("no key = %d %s, want 503", code, body)
}
- if g, found, _ := release.LoadRestartScopes(context.Background(), fx.rel, fx.sqid, "kass"); found && len(g.Requests) != 0 {
- t.Fatalf("a console with no key wrote %+v", g.Requests)
- }
+ noRetryWritten(t, fx, "kass")
if n := serial(t, fx); n != 0 {
t.Fatalf("serial = %d, want no announcement", n)
}
}
+
+// writeRetryReport publishes host held by v2 with the given retry fields, on
+// an armed box or, with code set, a declined one honouring its book.
+func writeRetryReport(t *testing.T, fx *consoleFixture, host, code string, retryV int, epoch uint64, book string) {
+ t.Helper()
+ hr := status.HostRollout{Account: fx.sqid, App: "kass", Installed: "v2", Good: "v1", HeldBy: "v2", Reason: status.ReasonHealth,
+ HeldEpoch: epoch, HeldBook: book, RetryV: retryV}
+ ro := &status.RolloutReport{Hosts: map[string]status.HostRollout{host: hr}}
+ if code != "" {
+ ro.Unavailable, ro.FailureBook = code, "honoured"
+ hr.Attempt, hr.AttemptBuild, hr.Reason = status.AttemptDown, "v2", ""
+ ro.Hosts[host] = hr
+ }
+ box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now())
+ box.Metrics = &status.BoxMetrics{RouteVersions: map[string]string{host: "v2"}, Rollout: ro}
+ if err := status.WriteBox(context.Background(), fx.rel, box); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// Readiness round 2, Task 3 (plan 9's transitions): the door asks the
+// box's report before it writes a retry. A box with no retry_v (built
+// before the host-retry log) or no held_book cannot take one: 409, and
+// nothing is written or announced. A box whose failure book is not saved
+// publishes epoch 0, and no grant for it could clear anything: 409, try
+// again. A box with retry_v, an epoch and a book gets the signed grant in
+// host-retry.json, and nothing in restart-scope.json. Armed and declined
+// boxes alike.
+func TestAPIInstanceRetryAsksTheBoxsReportFirst(t *testing.T) {
+ const host = "web.kass.carlosapp.dev"
+ for _, box := range []string{"", "no-cgroup-delegation"} {
+ for _, tc := range []struct {
+ name string
+ retryV int
+ epoch uint64
+ book string
+ want string // "" is the grant written
+ }{
+ {"an older box", 0, heldEpoch, heldBook, apiRetryUnsupported},
+ {"an older box with no epoch", 0, 0, heldBook, apiRetryUnsupported},
+ {"no book", status.HostRetryV, heldEpoch, "", apiRetryUnsupported},
+ {"book not saved", status.HostRetryV, 0, heldBook, apiRetryUnsaved},
+ {"a box that takes it", status.HostRetryV, heldEpoch, heldBook, ""},
+ } {
+ t.Run(box+" "+tc.name, func(t *testing.T) {
+ fx := newConsoleFixture(t)
+ sibling(t, fx, "kass", host, "stable")
+ writeRetryReport(t, fx, host, box, tc.retryV, tc.epoch, tc.book)
+ token := cliToken(t, fx, "paul@keymail.dev", "code-host-retry-v")
+ u := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/kass/instances/" + host + "/retry"
+ code, _, body := rawReq(t, http.MethodPost, u, token, `{}`)
+ if tc.want != "" {
+ if code != http.StatusConflict || !strings.Contains(body, tc.want) {
+ t.Fatalf("retry = %d %s, want 409 %q", code, body, tc.want)
+ }
+ noRetryWritten(t, fx, "kass")
+ if n := serial(t, fx); n != 0 {
+ t.Fatalf("serial = %d, want no announcement", n)
+ }
+ return
+ }
+ if code != http.StatusOK {
+ t.Fatalf("retry = %d %s", code, body)
+ }
+ r := lastScoped(t, fx, "kass")
+ g, err := release.VerifyHostRetry(fx.pub, r.Scope.Signed, fx.sqid, "kass", host, time.Now())
+ if r.Scope.Kind != release.RestartScopeRetry || r.Scope.Value != host || err != nil || g.Build != "v2" || g.Epoch != heldEpoch || g.Book != heldBook {
+ t.Fatalf("stored %+v, grant %+v, %v", r, g, err)
+ }
+ if n := serial(t, fx); n != 1 {
+ t.Fatalf("serial = %d", n)
+ }
+ })
+ }
+ }
+}
diff --git a/internal/console/browserretry.go b/internal/console/browserretry.go
index d20bf1e42..042c50e79 100644
--- a/internal/console/browserretry.go
+++ b/internal/console/browserretry.go
@@ -17,6 +17,10 @@ import (
const (
browserRetryNoKeyTitle = "Retry unavailable"
browserRetryNoKey = "No signing key on this host, so this console cannot request a retry."
+ // The flashes for errRetryUnsupported and errRetryUnsaved
+ // (apiRetryUnsupported's and apiRetryUnsaved's twins).
+ browserRetryUnsupported = "This instance's box is too old to take a retry. Update the box, then try again." // TODO-COPY(zdd-readiness2.retry-unsupported.browser)
+ browserRetryUnsaved = "This instance's box has not saved its failure yet. Try again in a minute." // TODO-COPY(zdd-readiness2.retry-unsaved.browser)
)
// handleInstanceRetry is the instance page's Retry (spec §9.2): the browser
@@ -66,6 +70,12 @@ func (s *Server) handleInstanceRetry(w http.ResponseWriter, r *http.Request) {
case errors.Is(err, errRetryNotHeld):
flash("err", strings.ReplaceAll(swapCopy.NotHeld, "{host}", host))
return
+ case errors.Is(err, errRetryUnsupported):
+ flash("err", browserRetryUnsupported)
+ return
+ case errors.Is(err, errRetryUnsaved):
+ flash("err", browserRetryUnsaved)
+ return
case err != nil:
log.Printf("console: browser retry %s/%s/%s: %v", acct.Sqid, app, host, err)
flash("err", swapCopy.RetryFailed)
diff --git a/internal/console/browserretry_test.go b/internal/console/browserretry_test.go
index 4942ee739..8b0430a9f 100644
--- a/internal/console/browserretry_test.go
+++ b/internal/console/browserretry_test.go
@@ -3,6 +3,7 @@ package console
import (
"context"
"fmt"
+ "html"
"net/http"
"net/url"
"strings"
@@ -117,9 +118,7 @@ func TestBrowserRetryWithNoKeyWritesNothing(t *testing.T) {
if code != http.StatusServiceUnavailable || !strings.Contains(body, browserRetryNoKey) {
t.Fatalf("no key = %d, want 503 naming the missing key", code)
}
- if g, found, _ := release.LoadRestartScopes(context.Background(), fx.rel, fx.sqid, "kass"); found && len(g.Requests) != 0 {
- t.Fatalf("a console with no key wrote %+v", g.Requests)
- }
+ noRetryWritten(t, fx, "kass")
}
func TestBrowserRetryRefusals(t *testing.T) {
@@ -134,3 +133,38 @@ func TestBrowserRetryRefusals(t *testing.T) {
t.Fatalf("unknown host = %d %q", code, body)
}
}
+
+// The browser twin of TestAPIInstanceRetryAsksTheBoxsReportFirst: the
+// refusal is flashed on the instance page, before any confirmation, and
+// nothing is written.
+func TestBrowserRetryAsksTheBoxsReportFirst(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ retryV int
+ epoch uint64
+ want string
+ }{
+ {"an older box", 0, heldEpoch, browserRetryUnsupported},
+ {"book not saved", status.HostRetryV, 0, browserRetryUnsaved},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ fx := newConsoleFixture(t)
+ fx.addRoute(t, registry.Route{Host: heldHost, App: "kass", Account: fx.sqid, Kind: "instance", Status: "ready", Version: "v2", Channel: "stable"})
+ sibling(t, fx, "kass", heldHost, "stable")
+ writeRetryReport(t, fx, heldHost, "", tc.retryV, tc.epoch, heldBook)
+ c := signInPaul(t, fx)
+ csrf := sessionToken(t, fx, c)
+ endpoint := fx.u("/apps/kass/instances/" + heldHost + "/retry")
+ for _, form := range []url.Values{{"csrf": {csrf}}, {"csrf": {csrf}, "confirmed": {"1"}}} {
+ code, loc, body := postGated(t, fx, c, endpoint, form)
+ if code != http.StatusSeeOther || loc != fx.path("/apps/kass/instances/"+heldHost) {
+ t.Fatalf("retry = %d -> %q, want the flash's redirect", code, loc)
+ }
+ if page := flashAfter(t, fx, c, code, loc, body); !strings.Contains(page, html.EscapeString(tc.want)) {
+ t.Fatalf("the page after the refusal lacks %q", tc.want)
+ }
+ }
+ noRetryWritten(t, fx, "kass")
+ })
+ }
+}
diff --git a/internal/console/claim_invariant_fixtures_test.go b/internal/console/claim_invariant_fixtures_test.go
index 4d4d29378..0d2298742 100644
--- a/internal/console/claim_invariant_fixtures_test.go
+++ b/internal/console/claim_invariant_fixtures_test.go
@@ -287,7 +287,7 @@ func (cx *claimCase) heldInstance(t *testing.T, host string) {
box := status.CollectBox(nil, status.Probes{MachineID: "claim-box"}, time.Now())
box.Metrics = &status.BoxMetrics{Rollout: &status.RolloutReport{Hosts: map[string]status.HostRollout{
host: {Account: "bab", App: "titogo", Installed: "v2", Serving: "v1", Good: "v1", Awake: true,
- HeldBy: "v2", Reason: status.ReasonHealth},
+ HeldBy: "v2", Reason: status.ReasonHealth, HeldEpoch: 3, HeldBook: "claim-fixture-book", RetryV: status.HostRetryV},
}}}
if err := status.WriteBox(context.Background(), cx.rel, box); err != nil {
t.Fatal(err)
@@ -939,7 +939,7 @@ var claimFixtures = map[string]claimFixture{
setup: func(t *testing.T, cx *claimCase) { cx.heldInstance(t, "web.go.tito.io") },
do: claimForm("/a/bab/apps/titogo/instances/web.go.tito.io/retry", "confirmed", "1"),
check: func(t *testing.T, cx *claimCase, open bool) string {
- g, found, err := release.LoadRestartScopes(context.Background(), cx.rel, "bab", "titogo")
+ g, found, err := release.LoadHostRetries(context.Background(), cx.rel, "bab", "titogo")
if err != nil {
return err.Error()
}
@@ -951,7 +951,7 @@ var claimFixtures = map[string]claimFixture{
}
switch {
case open && !asked:
- return "the retry request was not written: the handler never reached the narrowed log"
+ return "the retry request was not written: the handler never reached the host-retry log"
case !open && asked:
return "a retry request was written while the app was closed under a foreign exclusive operation"
}
@@ -1262,7 +1262,7 @@ var claimFixtures = map[string]claimFixture{
setup: func(t *testing.T, cx *claimCase) { cx.heldInstance(t, "web.go.tito.io") },
do: claimAPI(http.MethodPost, "/api/cli/apps/bab/titogo/instances/web.go.tito.io/retry", `{}`),
check: func(t *testing.T, cx *claimCase, open bool) string {
- g, found, err := release.LoadRestartScopes(context.Background(), cx.rel, "bab", "titogo")
+ g, found, err := release.LoadHostRetries(context.Background(), cx.rel, "bab", "titogo")
if err != nil {
return err.Error()
}
@@ -1274,7 +1274,7 @@ var claimFixtures = map[string]claimFixture{
}
switch {
case open && !asked:
- return "the retry request was not written: the handler never reached the narrowed log"
+ return "the retry request was not written: the handler never reached the host-retry log"
case !open && asked:
return "a retry request was written while the app was closed under a foreign exclusive operation"
}
diff --git a/internal/console/rolloutfleet_test.go b/internal/console/rolloutfleet_test.go
index fedc1ddd2..29d6df9e6 100644
--- a/internal/console/rolloutfleet_test.go
+++ b/internal/console/rolloutfleet_test.go
@@ -31,7 +31,8 @@ func writeFleetHeldBox(t *testing.T, fx *consoleFixture, fleet, owner, host, app
}
box := status.CollectBox(nil, status.Probes{MachineID: fleet + "-box"}, time.Now())
box.Metrics = &status.BoxMetrics{Rollout: &status.RolloutReport{Hosts: map[string]status.HostRollout{
- host: {Account: fx.sqid, App: app, Installed: build, Serving: "v1", Good: "v1", Awake: true, HeldBy: build, Reason: status.ReasonHealth},
+ host: {Account: fx.sqid, App: app, Installed: build, Serving: "v1", Good: "v1", Awake: true, HeldBy: build, Reason: status.ReasonHealth,
+ HeldEpoch: heldEpoch, HeldBook: heldBook, RetryV: status.HostRetryV},
}}}
if err := status.WriteBox(context.Background(), release.NewPrefixStore(fx.rel, "fleets/"+fleet+"/"), box); err != nil {
t.Fatal(err)
diff --git a/internal/host/agent.go b/internal/host/agent.go
index cea8a0b21..89d942c37 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -2390,7 +2390,9 @@ func fastConvergeLoop(reg *registry.Registry, svc *activator.Service, sd *schedu
// first request to establish itself.
restartMem := newRestartMemory()
// `carlos instances retry` (zero-downtime spec §8.4) reaches the rollout
- // manager through the narrowed-restart log, or, on a box that declined
+ // manager through the host-retry log (a third read per app at arming,
+ // and on a move, only where retryHost is set; hostRetryPass) or a legacy
+ // entry in the narrowed-restart log, or, on a box that declined
// the flag but honours its failure book, that book (Task 10); nil
// otherwise. Each entry's grant is verified against the root key first
// (spec §11).
diff --git a/internal/host/restart.go b/internal/host/restart.go
index 692e1e1a4..4d6c35531 100644
--- a/internal/host/restart.go
+++ b/internal/host/restart.go
@@ -925,6 +925,12 @@ func restartRequestLeg(ctx context.Context, routes []registry.Route, store relea
// requested" and then cycling nothing unit-shaped. It is one marker touch,
// once, at arming: the pass it starts is the ordinary one, and once that pass
// has recorded the narrowed marks no later agent start asks again.
+//
+// Retry grants are the exception to "record, act on nothing" (Astra plan
+// 9): every grant in the host-retry log, and every legacy retry in the
+// narrowed log, is verified and applied here (applyRetries), so one written
+// before this agent started is not lost to the baseline. One spent or made
+// stale since clears nothing.
func restartSeed(ctx context.Context, reg *registry.Registry, store release.Store, dd string, mem *restartMemory) {
routes, err := reg.List()
if err != nil {
@@ -945,7 +951,12 @@ func restartSeed(ctx context.Context, reg *registry.Registry, store release.Stor
log.Printf("agent: retry baseline: %d apps unreadable: %v", len(grantErrs), grantErrs[0])
}
applyRetries(routes, grants, mem)
- applyRetries(routes, scoped, mem)
+ if mem.retryHost != nil {
+ // A box with nothing that could clear a hold reads every legacy
+ // retry at its start as history, silently, as it always has: its
+ // line would only say, at every start, that nothing is held.
+ applyRetries(routes, scoped, mem)
+ }
if len(readErrs) > 0 {
log.Printf("agent: restart baseline: %d apps unreadable: %v", len(readErrs), readErrs[0])
return
diff --git a/internal/host/restartscope.go b/internal/host/restartscope.go
index 4b15e5bd7..bf1945832 100644
--- a/internal/host/restartscope.go
+++ b/internal/host/restartscope.go
@@ -209,11 +209,11 @@ type retryRequest struct {
// retryRequests is the retry entries of work, from the host-retry log
// (hostRetryPass) or a legacy retry in the narrowed log (restartScopePass):
// the entries, oldest first, whose host is one of this box's routes in the
-// requesting app. A host on another box names nothing here. Entries are not merged by
-// host: each carries its own grant, and the box verifies each (a forged
-// entry must not hide a real one behind it). A grant spent by an earlier
-// entry moves its restriction's epoch, so a second one for the same failure
-// clears nothing.
+// requesting app. A host on another box names nothing here. Entries are
+// not merged by host: each carries its own grant, and the box verifies
+// each (a forged entry must not hide a real one behind it). A grant spent
+// by an earlier entry moves its restriction's epoch, so a second one for
+// the same failure clears nothing.
func retryRequests(routes []registry.Route, work []scopedRestart) []retryRequest {
var out []retryRequest
for _, w := range work {
diff --git a/internal/host/rolloutdeclined.go b/internal/host/rolloutdeclined.go
index f58b31df9..0938e0807 100644
--- a/internal/host/rolloutdeclined.go
+++ b/internal/host/rolloutdeclined.go
@@ -262,7 +262,8 @@ func (h *failureHolds) report() map[string]status.HostRollout {
}
out[rt.Host] = status.HostRollout{Account: acct, App: rt.App, Installed: rt.Version, Good: rt.GoodVersion,
Attempt: attempt, AttemptBuild: hd.by, HeldBy: hd.by, HeldEpoch: h.book.PublishedEpoch(acct, rt.App, hd.by, rt.Host),
- HeldBook: h.book.ID()} // the book the epoch counts in (final review round 1, finding 3)
+ HeldBook: h.book.ID(), // the book the epoch counts in (final review round 1, finding 3)
+ RetryV: status.HostRetryV} // grants ride the host-retry log (readiness round 2, Task 3)
}
return out
}
diff --git a/internal/host/rolloutdeclined_test.go b/internal/host/rolloutdeclined_test.go
index e7b816e5a..c198144ad 100644
--- a/internal/host/rolloutdeclined_test.go
+++ b/internal/host/rolloutdeclined_test.go
@@ -871,7 +871,7 @@ func TestDeclinedBoxProcessesAVerifiedSignedRetry(t *testing.T) {
}
hr, ok := declinedHostReport(t, host)
if !ok || hr.Account != acct || hr.App != "blog" || hr.Installed != "v2" || hr.HeldBy != "v2" || hr.HeldEpoch != epoch || hr.Attempt != status.AttemptDown ||
- hr.HeldBook == "" || hr.HeldBook != heldHolds().book.ID() {
+ hr.HeldBook == "" || hr.HeldBook != heldHolds().book.ID() || hr.RetryV != status.HostRetryV {
t.Fatalf("published %+v (%v), want v2 holding %s down at epoch %d of the box's book", hr, ok, host, epoch)
}
diff --git a/internal/provision/plan9_b8_census_test.go b/internal/provision/plan9_b8_census_test.go
index 48a98a00c..e19776e74 100644
--- a/internal/provision/plan9_b8_census_test.go
+++ b/internal/provision/plan9_b8_census_test.go
@@ -635,7 +635,7 @@ var plan9B8CensusInventory = []censusEntry{
// Registers instance and sidecar HTTP paths; record handlers use the reviewed provision writers.
{"internal/console/authcli.go", "Server.registerCLIRoutes", "key-site", "lit:instances,lit:sidecars", "d254edd637189a7e407f27c402ef1950d106ec373025c530d398ebb6397b6069"},
{"internal/console/browserrestart.go", "Server.handleBrowserRestart", "key-site", "lit:instances", "955d71813504f2cfec1e1f64817d5aa5a921aa38e1044c0e903d2cfbe4d073f4"},
- {"internal/console/browserretry.go", "Server.handleInstanceRetry", "key-site", "lit:instances", "8a950e4bf96ffd30634f3ea2ea71a37519da8a68e9ca31a4d11e68587696019a"},
+ {"internal/console/browserretry.go", "Server.handleInstanceRetry", "key-site", "lit:instances", "e51c5e8d622a727c486ec24a0433212e08a168e983e2819aa10b16fa976c38f1"},
{"internal/console/channelmove.go", "Server.handleAPIInstanceChannel", "record-writer", "key:UpdateRecord", "91a2b3157bde9af832e8d95b17d0c384910229982945818f8b52f62dafddb342"},
{"internal/console/channelmove.go", "Server.executePendingRouteChannel", "record-writer", "key:UpdateRecord", "b3b9a12213261a70b680ad5290957ebe7434eff633a73c2d80c75cc24d026d9a"},
{"internal/console/claim_alias.go", "claimAliasStore.Put", "nonrecord", "def:Put,sel:Put", "a21b7cce328ebad9955211657ca8353adc7c2495b065fe09909b93c85ea7defb"},
diff --git a/internal/rollout/retrygrant_test.go b/internal/rollout/retrygrant_test.go
index ecd7dde3c..667cb823a 100644
--- a/internal/rollout/retrygrant_test.go
+++ b/internal/rollout/retrygrant_test.go
@@ -8,6 +8,7 @@ import (
"time"
"github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/status"
)
// retry is the console's signed host retry as the box applies it: a grant
@@ -258,6 +259,12 @@ func TestStatusPublishesTheHeldEpoch(t *testing.T) {
if got := r.Hosts["b"]; got.HeldBook == "" || got.HeldBook != h.book.ID() || r.Hosts["a"].HeldBook != h.book.ID() {
t.Fatalf("held books %q and %q: want the book's identity %q on both arms", r.Hosts["a"].HeldBook, got.HeldBook, h.book.ID())
}
+ // Beside HeldBook, the retry protocol this box takes (readiness round 2,
+ // Task 3): its grants ride the host-retry log, which the console checks
+ // before it writes one.
+ if r.Hosts["a"].RetryV != status.HostRetryV || r.Hosts["b"].RetryV != status.HostRetryV || status.HostRetryV != 1 {
+ t.Fatalf("retry_v %d and %d: want %d on both arms", r.Hosts["a"].RetryV, r.Hosts["b"].RetryV, status.HostRetryV)
+ }
if !h.m.RetryHost("b", r.Hosts["b"].HeldBy, r.Hosts["b"].HeldBook, r.Hosts["b"].HeldEpoch) {
t.Fatal("a grant signed from the status report cleared nothing")
}
diff --git a/internal/rollout/status.go b/internal/rollout/status.go
index 7c6bde9b5..6d3bbe429 100644
--- a/internal/rollout/status.go
+++ b/internal/rollout/status.go
@@ -88,7 +88,8 @@ func (m *Manager) Status() status.RolloutReport {
// (plan review finding 13), 0 while the book owes a
// save (the crash-epoch hard limit, rule 2).
hr.HeldEpoch = m.d.Book.PublishedEpoch(s.key.acct, s.key.app, rec.Build, host)
- hr.HeldBook = m.d.Book.ID() // final review round 1, finding 3
+ hr.HeldBook = m.d.Book.ID() // final review round 1, finding 3
+ hr.RetryV = status.HostRetryV // grants ride the host-retry log
if m.d.Book.HeldDown(s.key.acct, s.key.app, h.wantBuild, host) {
hr.Attempt = status.AttemptDown
}
@@ -110,6 +111,7 @@ func (m *Manager) Status() status.RolloutReport {
hr.HeldBy = mark.Build
hr.HeldEpoch = m.d.Book.PublishedEpoch(s.key.acct, s.key.app, mark.Build, host)
hr.HeldBook = m.d.Book.ID()
+ hr.RetryV = status.HostRetryV
if rec, ok := m.d.Book.Lookup(s.key.acct, s.key.app, mark.Build); ok {
hr.Reason = reasonWord(rec.Reason)
}
diff --git a/internal/status/snapshot.go b/internal/status/snapshot.go
index 375ffa728..2b35c9766 100644
--- a/internal/status/snapshot.go
+++ b/internal/status/snapshot.go
@@ -338,6 +338,12 @@ type HostRollout struct {
// only for its own book (final review round 1, finding 3). Absent from
// an older box: a grant signed without it clears nothing.
HeldBook string `json:"held_book,omitempty"`
+ // RetryV is the `carlos instances retry` protocol this box takes, set
+ // beside HeldBook: HostRetryV is a root-signed, book-bound grant in the
+ // host-retry log (release.HostRetryKey). Absent (0) from a box built
+ // before that log, which reads no grant there, so the console writes
+ // it none and says the box is too old (readiness round 2, Task 3).
+ RetryV int `json:"retry_v,omitempty"`
// Transient is set when the activator owns the host's process but has no
// current generation for it: the process is starting or stopping, so the
// instance is neither asleep nor awake and no serving build is reported.
@@ -378,6 +384,11 @@ const (
AttemptDown = "down"
)
+// HostRetryV is HostRollout.RetryV for a box that reads root-signed,
+// book-bound retry grants from the host-retry log and honours a legacy
+// signed one in the restart-scope log with the same checks.
+const HostRetryV = 1
+
// The reason words: the closed vocabulary a member-facing event or status
// line may give for a failure. internal/rollout maps every diagnostic text
// onto one of them (reasonWord).