amadan
system status: say when bootstrap/carlos is not stable's build (#316)
`carlos system update` mirrors each release under bootstrap/router/ but never writes bootstrap/carlos, the binary a new, rebuilt or failover box downloads at first boot. On 2026-10-18 Tito's production copy was an unidentified September build and staging's was v0.22.2, while the fleet ran v0.28.0. The status door now streams bootstrap/carlos through sha256 and compares it with stable's signed linux-arm64 artifact (the new "bootstrap" field: current, behind or unreadable; null when there is no stable pointer, no linux-arm64 artifact or no bootstrap object). `carlos system status` prints a warning naming the mirrored key to copy over it, and is quiet otherwise. Copy approved in review. Paul's ruling: a warning first. Booting from the channel waits for install-channel, which refuses until a host floor is published; a console that overwrites bootstrap/carlos was declined (non-create-only write family, new IAM grant, never fires while a box is unreachable).
Human
Prompt
Can we take 315 and 316
From this commit's own message. The prompt ledger has no entries for it — which is what a squashed, rebased or amended commit looks like, and what every commit looks like in a repo without a ledger.
Summary
Status compares bootstrap/carlos with stable's build and warns when it lags.
Agent notes
The compare reads the whole object (~33 MB) per status call, including `carlos system update`'s preview. systemArtifactMatches now shares objectSHA256. Mutation-checked.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
4 files changed,
+184
−12
internal/console/apisystembootstrap_test.go+62 −0internal/console/apisystemupdate.go+63 −9system.go+24 −3system_test.go+35 −0
CI cancelled — superseded by a newer push — run details
CI log
=== gate === cd launchctx && go vet ./... && go test ./... -count=1 ok amadan.net/carlos/platform.git/launchctx 0.011s ? amadan.net/carlos/platform.git/launchctx/controlname [no test files] ? amadan.net/carlos/platform.git/launchctx/wireid [no test files] go vet ./... gofmt -l .: clean go test ./... -count=1 -timeout 30m amadan: job cancelled — a newer push superseded it
diff --git a/internal/console/apisystembootstrap_test.go b/internal/console/apisystembootstrap_test.go| new file mode 100644 |
| index 000000000..d051e1574 |
| --- /dev/null |
| +++ b/internal/console/apisystembootstrap_test.go |
| @@ -0,0 +1,62 @@ |
| +package console |
| + |
| +import ( |
| + "context" |
| + "crypto/sha256" |
| + "encoding/hex" |
| + "net/http" |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// Issue #316: system status compares bootstrap/carlos, the binary a new box |
| +// boots, with stable's signed linux-arm64 artifact. |
| +func TestSystemStatusReportsWhetherBootstrapIsStablesBuild(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + owner := cliToken(t, fx, "paul@keymail.dev", "code-sys-bootstrap") |
| + seedBox(t, fx, "ie", "linux-arm64", "stable", "v0.28.0") |
| + ctx := context.Background() |
| + build := []byte("stable build") |
| + sum := sha256.Sum256(build) |
| + if _, err := release.PromoteSystem(ctx, fx.rel, release.RootSigner(fx.priv), "router", "stable", release.SystemRelease{ |
| + Version: "v0.28.0", Artifacts: map[string]release.SystemArtifact{"linux-arm64": { |
| + Key: release.SystemArtifactKey("v0.28.0", "linux-arm64"), SHA256: hex.EncodeToString(sum[:])}}, |
| + }, release.SystemPromoteOptions{}); err != nil { |
| + t.Fatal(err) |
| + } |
| + bootstrap := func() any { |
| + t.Helper() |
| + st, out := getJSON(t, fx.ts.URL+"/api/cli/system/status", owner) |
| + if st != http.StatusOK { |
| + t.Fatalf("status = %d %v", st, out) |
| + } |
| + v, ok := out["bootstrap"] |
| + if !ok { |
| + t.Fatalf("no bootstrap field in %v", out) |
| + } |
| + return v |
| + } |
| + if got := bootstrap(); got != nil { |
| + t.Fatalf("absent bootstrap object: bootstrap = %v, want null", got) |
| + } |
| + put := func(data []byte) { |
| + t.Helper() |
| + _, etag, err := fx.rel.Get(ctx, bootstrapKey) |
| + if err != nil { |
| + etag = "" |
| + } |
| + if _, err := fx.rel.Put(ctx, bootstrapKey, data, etag); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + put([]byte("an older build")) |
| + row, _ := bootstrap().(map[string]any) |
| + if row["state"] != "behind" || row["version"] != "v0.28.0" || row["key"] != "bootstrap/router/v0.28.0/carlos-linux-arm64" { |
| + t.Fatalf("stale bootstrap: row = %v, want behind v0.28.0 with the mirror key", row) |
| + } |
| + put(build) |
| + if row, _ := bootstrap().(map[string]any); row["state"] != "current" { |
| + t.Fatalf("matching bootstrap: row = %v, want current", row) |
| + } |
| +} |
diff --git a/internal/console/apisystemupdate.go b/internal/console/apisystemupdate.go| index 80e0d3385..e6a58a2d4 100644 |
| --- a/internal/console/apisystemupdate.go |
| +++ b/internal/console/apisystemupdate.go |
| @@ -601,25 +601,72 @@ func (s *Server) mirrorSystemArtifact(ctx context.Context, job *systemJob, versi |
| // object beside it would make the peak two copies of a platform binary per |
| // in-flight promote, for no purpose beyond computing one sha256. |
| func (s *Server) systemArtifactMatches(ctx context.Context, key, sha string) (bool, error) { |
| - rc, _, err := release.OpenObject(ctx, s.cfg.Releases, key) |
| + got, err := s.objectSHA256(ctx, key) |
| switch { |
| case errors.Is(err, os.ErrNotExist): |
| return false, nil |
| case err != nil: |
| return false, err |
| } |
| - h := sha256.New() |
| - _, err = io.Copy(h, rc) |
| - rc.Close() |
| - if err != nil { |
| - return false, fmt.Errorf("%s: reading the mirrored object to compare it: %w", key, err) |
| - } |
| - if hex.EncodeToString(h.Sum(nil)) == sha { |
| + if got == sha { |
| return true, nil |
| } |
| return false, fmt.Errorf("%s exists with different bytes — versions are immutable; pick a new version", key) |
| } |
| +// objectSHA256 streams the object at key through a hasher; absent is |
| +// os.ErrNotExist. |
| +func (s *Server) objectSHA256(ctx context.Context, key string) (string, error) { |
| + rc, _, err := release.OpenObject(ctx, s.cfg.Releases, key) |
| + if err != nil { |
| + return "", err |
| + } |
| + defer rc.Close() |
| + h := sha256.New() |
| + if _, err := io.Copy(h, rc); err != nil { |
| + return "", fmt.Errorf("%s: reading the object to compare it: %w", key, err) |
| + } |
| + return hex.EncodeToString(h.Sum(nil)), nil |
| +} |
| + |
| +// bootstrapKey is the binary a new, rebuilt or failover box downloads at |
| +// first boot. `carlos system update` mirrors each release under |
| +// bootstrap/router/ and never writes this key, so it falls behind the fleet |
| +// unless an operator copies stable's build over it (issue #316). |
| +const bootstrapKey = "bootstrap/carlos" |
| + |
| +// bootstrapPlatform is the one platform bootstrap/carlos serves. |
| +const bootstrapPlatform = "linux-arm64" |
| + |
| +// bootstrapState is the status row for bootstrapKey against stable's signed |
| +// linux-arm64 artifact, or nil when there is nothing to compare: no verified |
| +// stable pointer, no linux-arm64 artifact in it, or no bootstrap object (a |
| +// deployment that bootstraps another way). A read failure is reported, never |
| +// fatal to the status read. |
| +func (s *Server) bootstrapState(ctx context.Context, stable *release.SystemPointer) map[string]any { |
| + if stable == nil { |
| + return nil |
| + } |
| + art, ok := stable.Current.Artifacts[bootstrapPlatform] |
| + if !ok || art.SHA256 == "" { |
| + return nil |
| + } |
| + row := map[string]any{"version": stable.Current.Version, "key": art.Key} |
| + got, err := s.objectSHA256(ctx, bootstrapKey) |
| + switch { |
| + case errors.Is(err, os.ErrNotExist): |
| + return nil |
| + case err != nil: |
| + log.Printf("console: api system status: %s: %v", bootstrapKey, err) |
| + row["state"], row["error"] = "unreadable", err.Error() |
| + case strings.EqualFold(got, art.SHA256): |
| + row["state"] = "current" |
| + default: |
| + row["state"] = "behind" |
| + } |
| + return row |
| +} |
| + |
| func (s *Server) handleAPISystemJob(w http.ResponseWriter, r *http.Request) { |
| if _, _, ok := s.apiSystemRoller(w, r); !ok { |
| return |
| @@ -806,6 +853,7 @@ func (s *Server) handleAPISystemStatus(w http.ResponseWriter, r *http.Request) { |
| ctx := r.Context() |
| pointers := map[string]any{} |
| targets := map[string]string{} |
| + var stable *release.SystemPointer |
| for _, ch := range []string{release.RungCanary, release.RungStable} { |
| p, _, err := release.GetSystemPointer(ctx, s.cfg.Releases, s.cfg.ReleasePub, release.SystemComponentRouter, ch) |
| switch { |
| @@ -824,6 +872,9 @@ func (s *Server) handleAPISystemStatus(w http.ResponseWriter, r *http.Request) { |
| pointers[ch] = map[string]any{"version": p.Current.Version, "promoted_at": p.PromotedAt, "spread": p.Spread, "source": p.Current.Source, |
| "history": len(p.History)} |
| targets[ch] = p.Current.Version |
| + if ch == release.RungStable { |
| + stable = p |
| + } |
| } |
| } |
| boxes, partial, err := s.systemBoxes(ctx, false) |
| @@ -854,5 +905,8 @@ func (s *Server) handleAPISystemStatus(w http.ResponseWriter, r *http.Request) { |
| // could not be read. Always present, so a reader never has to tell a |
| // missing key from a false one, and so "no boxes on that fleet" and |
| // "could not ask that fleet" are different answers here. |
| - writeJSON(w, http.StatusOK, map[string]any{"pointers": pointers, "boxes": rows, "partial": partial}) |
| + // |
| + // bootstrap is null when there is nothing to compare (bootstrapState). |
| + writeJSON(w, http.StatusOK, map[string]any{"pointers": pointers, "boxes": rows, "partial": partial, |
| + "bootstrap": s.bootstrapState(ctx, stable)}) |
| } |
diff --git a/system.go b/system.go| index 1828c1f5a..839a2c9c6 100644 |
| --- a/system.go |
| +++ b/system.go |
| @@ -248,9 +248,20 @@ type systemBoxRow struct { |
| // list is INCOMPLETE — a fleet's heartbeats could not be read — so a short |
| // list is never mistaken for a small fleet. |
| type systemStatusResponse struct { |
| - Pointers map[string]*systemPointerRow `json:"pointers"` |
| - Boxes []systemBoxRow `json:"boxes"` |
| - Partial bool `json:"partial"` |
| + Pointers map[string]*systemPointerRow `json:"pointers"` |
| + Boxes []systemBoxRow `json:"boxes"` |
| + Partial bool `json:"partial"` |
| + Bootstrap *systemBootstrapRow `json:"bootstrap"` |
| +} |
| + |
| +// systemBootstrapRow is bootstrap/carlos, the binary a new box boots, |
| +// against stable's signed linux-arm64 artifact (issue #316). nil when there |
| +// is nothing to compare, or the console predates the field. |
| +type systemBootstrapRow struct { |
| + Version string `json:"version"` // stable's version |
| + Key string `json:"key"` // stable's mirrored linux-arm64 artifact |
| + State string `json:"state"` // current, behind or unreadable |
| + Error string `json:"error"` |
| } |
| // ---- carlos system update ---------------------------------------------- |
| @@ -495,6 +506,16 @@ func printSystemStatus(w io.Writer, resp systemStatusResponse, channel string) { |
| fmt.Fprintln(w, strings.Join(parts, " ")) |
| } |
| } |
| + // What a new box would boot (issue #316): quiet unless it is not stable's |
| + // build, and only where stable is shown. |
| + if b := resp.Bootstrap; b != nil && (channel == "" || channel == "stable") { |
| + switch b.State { |
| + case "behind": |
| + fmt.Fprintln(w, c.warn(fmt.Sprintf("bootstrap/carlos is not stable's build (%s), so a new box would boot a different one. Copy %s over it.", b.Version, b.Key))) |
| + case "unreadable": |
| + fmt.Fprintln(w, c.warn("bootstrap/carlos could not be read, so this can't say what a new box would boot: "+b.Error)) |
| + } |
| + } |
| fmt.Fprintln(w) |
| if resp.Partial { |
| fmt.Fprintln(w, c.warn("fleet boxes could not all be read; rows may be missing")) |
diff --git a/system_test.go b/system_test.go| index 9ae34db54..68d95bc74 100644 |
| --- a/system_test.go |
| +++ b/system_test.go |
| @@ -297,3 +297,38 @@ func TestSystemFilesSeedCommandEntryRefusesBeforeEffects(t *testing.T) { |
| } |
| } |
| } |
| + |
| +// Issue #316: status says when bootstrap/carlos is not stable's build, and is |
| +// quiet when it is, when there is nothing to compare, and under --channel |
| +// canary. |
| +func TestPrintSystemStatusSaysWhenBootstrapLags(t *testing.T) { |
| + stable := map[string]*systemPointerRow{"stable": {Version: "v0.28.0"}} |
| + behind := "bootstrap/carlos is not stable's build (v0.28.0), so a new box would boot a different one. Copy bootstrap/router/v0.28.0/carlos-linux-arm64 over it." |
| + for _, tc := range []struct { |
| + name, channel, want string |
| + row *systemBootstrapRow |
| + }{ |
| + {"behind", "", behind, &systemBootstrapRow{Version: "v0.28.0", Key: "bootstrap/router/v0.28.0/carlos-linux-arm64", State: "behind"}}, |
| + {"behind, stable only", "stable", behind, &systemBootstrapRow{Version: "v0.28.0", Key: "bootstrap/router/v0.28.0/carlos-linux-arm64", State: "behind"}}, |
| + {"unreadable", "", "bootstrap/carlos could not be read, so this can't say what a new box would boot: access denied", |
| + &systemBootstrapRow{Version: "v0.28.0", State: "unreadable", Error: "access denied"}}, |
| + {"current", "", "", &systemBootstrapRow{Version: "v0.28.0", State: "current"}}, |
| + {"nothing to compare", "", "", nil}, |
| + {"canary only", "canary", "", &systemBootstrapRow{Version: "v0.28.0", State: "behind"}}, |
| + } { |
| + t.Run(tc.name, func(t *testing.T) { |
| + var b strings.Builder |
| + printSystemStatus(&b, systemStatusResponse{Pointers: stable, Bootstrap: tc.row}, tc.channel) |
| + out := b.String() |
| + if tc.want == "" { |
| + if strings.Contains(out, "bootstrap/carlos") { |
| + t.Fatalf("output mentions bootstrap/carlos:\n%s", out) |
| + } |
| + return |
| + } |
| + if !strings.Contains(out, tc.want) { |
| + t.Fatalf("output lacks %q:\n%s", tc.want, out) |
| + } |
| + }) |
| + } |
| +} |