amadan

carlos / platform Public

settings-4 task 9: boxes report build-state builds; retention protects them and pins, and fails closed

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 3a2b75f2b25826cda7335d2f2200fa4660975c7d parent f7c9259
11 files changed, +1007 −9
  • internal/host/agent.go +26 −3
  • internal/host/agent_test.go +27 −0
  • internal/host/retention.go +218 −1
  • internal/host/retention_settings_test.go +476 −0
  • internal/host/retention_test.go +3 −3
  • internal/host/versionsweep.go +46 −0
  • internal/host/versionsweep_test.go +75 −0
  • internal/status/contract_test.go +12 −1
  • internal/status/dryrun.go +33 −0
  • internal/status/snapshot.go +31 −1
  • internal/status/status_test.go +60 −0
diff --git a/internal/host/agent.go b/internal/host/agent.go
index d848e20d2..9fc30fb6c 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -645,7 +645,11 @@ func runAgent(args []string, buildClaims ClaimFactory) {
if errSink != nil {
go errshipLoop(errSink, status.Label(readMachineID()))
}
- go statusLoop(reg, svc)
+ // The build-state report (settings spec rev 11 §4.4) names the rows
+ // keepsBuildState selects, with host claims as the build-state store
+ // sees them (cfg.HostWrites.Enabled, set by configureWakeClaims).
+ buildStateClaims := cfg != nil && cfg.HostWrites.Enabled
+ go statusLoop(reg, svc, func() (map[string]status.BuildStatePins, error) { return buildStatePins(reg, buildStateClaims) })
// The memory early-warning (memory-pressure spec §2.4): per box, gated
// by CARLOS_HOST_METRICS — one log line and nothing else where unset.
go memwatchLoop()
@@ -3037,12 +3041,21 @@ func retentionLoop(reg *registry.Registry) {
log.Printf("agent: retention loop disabled: %v — retention policies will not be enforced by this box", err)
return
}
+ // The box reports (settings spec rev 11 §4.4: the build-state builds
+ // every box names) live in the status store, which is not always the
+ // deployment store above (CARLOS_STATUS_BUCKET wins). Without it no app
+ // can be pruned safely, so the loop does not arm.
+ reports, _, err := openStatusStore(ctx, os.Getenv)
+ if err != nil {
+ log.Printf("agent: retention loop disabled: the box reports cannot be read (%v). retention policies will not be enforced by this box", err)
+ return
+ }
log.Printf("agent: retention loop armed (5min + jitter) — prunes only while this box holds the housekeeper lease")
// The pass runs under the HOLDING's context, not this loop's: a sweep
// that is halfway through its deletes when renewal fails must stop, not
// finish against a lock another box now owns (housekeeper.go).
prune := housekeeperDuty{"retention pass", func(ctx context.Context) error {
- return retentionPass(ctx, store, pub, reg)
+ return retentionPass(ctx, store, reports, pub, reg)
}}
delay := time.Minute
for {
@@ -3228,7 +3241,7 @@ func upstreamStates() map[string][]status.UpstreamState {
// logging that every 5 minutes forever would bury the log in noise. Any
// OTHER acquire error (a real S3/credential problem) is logged, same as any
// other tick failure.
-func statusLoop(reg *registry.Registry, svc *activator.Service) {
+func statusLoop(reg *registry.Registry, svc *activator.Service, buildStates func() (map[string]status.BuildStatePins, error)) {
ctx := context.Background()
// bucket is read for the lease key alone (lease.Acquire below), which a
// fleet box never reaches: shouldSkipPublisherElection short-circuits the
@@ -3420,6 +3433,16 @@ func statusLoop(reg *registry.Registry, svc *activator.Service) {
p.Rollout = rolloutStatusProbe()
// Hosts held for a dev proof (D3): private Metrics only.
p.ProofWaits = proofWaitProbe()
+ // The build-state builds of the instances this box hosts (settings
+ // spec rev 11 §4.4), for the housekeeper's retention. A failed
+ // collection is reported as such: the housekeeper prunes nothing
+ // while it is fresh. Private Metrics only: hosts.
+ if pins, err := buildStates(); err != nil {
+ log.Printf("agent: status tick: build states: %v", err)
+ p.BuildStatesErr = err.Error()
+ } else {
+ p.BuildStates = pins
+ }
// A fleet box (CARLOS_FLEET set) never gets here as a contender: it
// self-reports only, never attempting the election at all — no
diff --git a/internal/host/agent_test.go b/internal/host/agent_test.go
index c5bb22dfa..c29ab77c7 100644
--- a/internal/host/agent_test.go
+++ b/internal/host/agent_test.go
@@ -1894,6 +1894,10 @@ func TestRetentionLoopIsAPlatformBoxJob(t *testing.T) {
t.Setenv("CARLOS_DEPLOYMENT_BUCKET", "")
t.Setenv("CARLOS_DEPLOYMENT_DIR", t.TempDir())
t.Setenv("CARLOS_RELEASE_PUBKEY", base64.StdEncoding.EncodeToString(pub))
+ // The box reports retention reads (settings spec rev 11 §4.4) are in
+ // the status store. Opening an S3 store makes no call, and the pass
+ // runs only under the housekeeper lease, which a test never holds.
+ t.Setenv("CARLOS_STATUS_BUCKET", "carlos-status")
// retentionLoop never returns once armed, so the arming log line is
// what proves it: run it in its own goroutine (it parks in a
@@ -1935,6 +1939,29 @@ func TestRetentionLoopIsAPlatformBoxJob(t *testing.T) {
})
}
+// Settings spec rev 11 §4.4: without the box reports no app can be pruned
+// safely, so a deployment with no status store (CARLOS_DEPLOYMENT_DIR, which
+// publishes none) does not arm retention at all.
+func TestRetentionLoopNeedsTheBoxReports(t *testing.T) {
+ pub, _, err := ed25519.GenerateKey(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ logs := captureLog(t)
+ t.Setenv("CARLOS_FLEET", "")
+ t.Setenv("CARLOS_DEPLOYMENT_BUCKET", "")
+ t.Setenv("CARLOS_STATUS_BUCKET", "")
+ t.Setenv("CARLOS_DEPLOYMENT_DIR", t.TempDir())
+ t.Setenv("CARLOS_RELEASE_PUBKEY", base64.StdEncoding.EncodeToString(pub))
+
+ retentionLoop(testRegistry(t)) // must return: it does not arm
+
+ got := logs()
+ if !strings.Contains(got, "agent: retention loop disabled: the box reports cannot be read") || strings.Contains(got, "retention loop armed") {
+ t.Fatalf("log = %q, want the loop disabled for want of box reports", got)
+ }
+}
+
// writerFunc adapts a func to io.Writer — the log sink above needs its own
// mutex because the loop under test keeps running (and may keep logging)
// after the test returns, which a bare bytes.Buffer would race on.
diff --git a/internal/host/retention.go b/internal/host/retention.go
index 6b31240ba..1daeaa424 100644
--- a/internal/host/retention.go
+++ b/internal/host/retention.go
@@ -13,21 +13,37 @@ package host
import (
"context"
"crypto/ed25519"
+ "errors"
+ "fmt"
"log"
+ "strings"
"time"
+ "github.com/carlosframework/platform/internal/provision"
"github.com/carlosframework/platform/internal/registry"
"github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/status"
)
// retentionPass runs one enforcement sweep. Per-app isolation: one app's
// error is logged and skipped, never allowed to stop another app's
// enforcement; a partial delete leaves the remainder for the next tick.
-func retentionPass(ctx context.Context, store release.Store, pub ed25519.PublicKey, reg *registry.Registry) error {
+//
+// reports is the status store the boxes publish to (openStatusStore's
+// bucket, not store's): settings spec rev 11 §4.4's build-state builds come
+// from there. A pass that cannot read the reports prunes nothing at all.
+func retentionPass(ctx context.Context, store, reports release.Store, pub ed25519.PublicKey, reg *registry.Registry) error {
routes, err := reg.List()
if err != nil {
return err
}
+ if reports == nil {
+ return errors.New("no status store to read the box reports from: pruning nothing")
+ }
+ boxes, complete, err := loadAllBoxes(ctx, reports)
+ if err != nil {
+ return fmt.Errorf("reading the box reports: pruning nothing this pass: %w", err)
+ }
type appKey struct{ acct, app string }
pins := map[appKey]map[string]bool{}
for _, rt := range routes {
@@ -44,6 +60,11 @@ func retentionPass(ctx context.Context, store release.Store, pub ed25519.PublicK
if rt.PrevVersion != "" {
pins[k][rt.PrevVersion] = true
}
+ // What a failed first wake falls back to (zero-downtime spec §8.5,
+ // zdd-5 obligation 9), however long ago it last served.
+ if rt.GoodVersion != "" {
+ pins[k][rt.GoodVersion] = true
+ }
}
for k, protected := range pins {
ret, err := release.GetRetention(ctx, store, k.acct, k.app)
@@ -54,6 +75,14 @@ func retentionPass(ctx context.Context, store release.Store, pub ed25519.PublicK
if ret == nil {
continue
}
+ extra, err := settingsProtected(ctx, store, boxes, complete, time.Now(), k.acct, k.app)
+ if err != nil {
+ log.Printf("agent: retention %s/%s: pruning nothing this pass: %v", k.acct, k.app, err)
+ continue
+ }
+ for v := range extra {
+ protected[v] = true
+ }
eligible, err := release.PruneEligible(ctx, store, pub, k.acct, k.app, ret.Keep, time.Now(), protected)
if err != nil {
log.Printf("agent: retention %s/%s: %v", k.acct, k.app, err)
@@ -75,3 +104,191 @@ func retentionPass(ctx context.Context, store release.Store, pub ed25519.PublicK
}
return nil
}
+
+// buildStatePins is this box's contribution to retention (settings spec rev
+// 11 §4.4): the build-state builds of every row keepsBuildState selects (the
+// rows refreshAll keeps a copy of), from the registry copies the agent
+// refreshes at reconcile and before every wake. claims is whether host
+// claims are on, as keepsBuildState takes it. Every such row is reported,
+// with or without builds, so a missing or unreadable copy shows as
+// incomplete rather than as nothing to protect. A failed list is an error:
+// the caller reports it (BoxMetrics.BuildStatesErr), never an empty map.
+func buildStatePins(reg routeLister, claims bool) (map[string]status.BuildStatePins, error) {
+ routes, err := reg.List()
+ if err != nil {
+ return nil, fmt.Errorf("listing routes for build states: %w", err)
+ }
+ out := map[string]status.BuildStatePins{}
+ for _, rt := range routes {
+ if !keepsBuildState(rt, claims) {
+ continue
+ }
+ p := status.BuildStatePins{Account: release.RouteAccount(rt), App: rt.App, Builds: []string{}}
+ c, err := release.ParseBuildStateCopy(rt.BuildState)
+ p.Complete = err == nil && c.Known && c.ReadErr == ""
+ if err == nil {
+ p.Builds = buildStateBuilds(c.State)
+ }
+ out[rt.Host] = p
+ }
+ if len(out) == 0 {
+ return nil, nil
+ }
+ return out, nil
+}
+
+// buildStateBuilds is the builds a build state names: last_executed, and a
+// resolved request's target when it differs. A failed or completed request's
+// T is named too: it may still be what the instance runs or recovers onto.
+func buildStateBuilds(st *release.BuildState) []string {
+ builds := []string{}
+ if st == nil {
+ return builds
+ }
+ if st.LastExecuted != "" {
+ builds = append(builds, st.LastExecuted)
+ }
+ if r := st.Resolved; r != nil && r.Target != "" && r.Target != st.LastExecuted {
+ builds = append(builds, r.Target)
+ }
+ return builds
+}
+
+// buildStateReportMaxAge is §4.4's bound: an app is not pruned while any box
+// hosting one of its instances has been silent longer than this.
+const buildStateReportMaxAge = 6 * time.Hour
+
+// loadAllBoxes reads every box report from the status store: the
+// deployment's (status/boxes/) and every fleet's (fleets/<name>/status/boxes/,
+// enumerated with ListDirs). complete is false if any report could not be
+// read; a listing that fails is an error.
+func loadAllBoxes(ctx context.Context, store release.Store) ([]status.Box, bool, error) {
+ boxes, complete, err := status.LoadBoxesComplete(ctx, store)
+ if err != nil {
+ return nil, false, err
+ }
+ fleets, err := store.ListDirs(ctx, "fleets/")
+ if err != nil {
+ return nil, false, fmt.Errorf("listing fleets: %w", err)
+ }
+ for _, f := range fleets {
+ fb, ok, err := status.LoadBoxesComplete(ctx, release.NewPrefixStore(store, f))
+ if err != nil {
+ return nil, false, fmt.Errorf("reading the reports of %s: %w", f, err)
+ }
+ complete = complete && ok
+ boxes = append(boxes, fb...)
+ }
+ return boxes, complete, nil
+}
+
+// appRecords reads every provision record of acct/app, or fails. It lists
+// the keys itself (provision.ListRecords caps on max and skips a record it
+// cannot read, ruling R-T9-STORE): a key that is a record by ListRecords'
+// own rule (instances/<host>.json, one level) and does not read, parse or
+// agree with its path is an error, so the caller prunes nothing.
+func appRecords(ctx context.Context, store release.Store, acct, app string) ([]provision.Record, error) {
+ prefix := provision.RecordsPrefix(acct, app)
+ keys, err := store.List(ctx, prefix)
+ if err != nil {
+ return nil, fmt.Errorf("listing the records of %s/%s: %w", acct, app, err)
+ }
+ var recs []provision.Record
+ for _, key := range keys {
+ rest, ok := strings.CutPrefix(key, prefix)
+ if !ok || rest == "" || strings.Contains(rest, "/") {
+ continue
+ }
+ host, ok := strings.CutSuffix(rest, ".json")
+ if !ok || host == "" {
+ continue
+ }
+ rec, err := provision.GetRecord(ctx, store, acct, app, host)
+ if err != nil {
+ return nil, fmt.Errorf("reading the record of %s: %w", host, err)
+ }
+ recs = append(recs, *rec)
+ }
+ return recs, nil
+}
+
+// settingsProtected is settings spec rev 11 §4.4's addition to an app's
+// protected set: every pin a provision record names, and every build any
+// box report names for one of the app's instances. It refuses (the caller
+// prunes nothing for the app) when a record is unreadable, when the reports
+// could not all be read, when a fresh report says its box could not collect
+// its build states, when a provisioned instance that keeps a build state is
+// named by no report, or when the newest report naming one of the app's
+// hosts is older than six hours or says that host's build state is unread.
+//
+// The newest report for a host decides freshness and completeness, so a box
+// the instance moved off does not block pruning forever. The builds of
+// every report naming the host are protected, older ones included: that can
+// only keep more.
+func settingsProtected(ctx context.Context, store release.Store, boxes []status.Box, complete bool, now time.Time, acct, app string) (map[string]bool, error) {
+ recs, err := appRecords(ctx, store, acct, app)
+ if err != nil {
+ return nil, err
+ }
+ if !complete {
+ return nil, errors.New("the box reports could not all be read")
+ }
+ out := map[string]bool{}
+ for _, r := range recs {
+ if r.Pin != "" {
+ out[r.Pin] = true
+ }
+ }
+ type newest struct {
+ at time.Time
+ pins status.BuildStatePins
+ box string
+ }
+ byHost := map[string]newest{}
+ for _, b := range boxes {
+ if b.Metrics == nil {
+ continue
+ }
+ // A box that could not collect its build states may host any of
+ // this app's instances (Astra I2 #5).
+ if b.Metrics.BuildStatesErr != "" && now.Sub(b.GeneratedAt) <= buildStateReportMaxAge {
+ return nil, fmt.Errorf("%s could not collect its build states: %s", b.Label, b.Metrics.BuildStatesErr)
+ }
+ for host, p := range b.Metrics.BuildStates {
+ if p.Account != acct || p.App != app {
+ continue
+ }
+ for _, v := range p.Builds {
+ if v != "" {
+ out[v] = true
+ }
+ }
+ if cur, ok := byHost[host]; !ok || b.GeneratedAt.After(cur.at) {
+ byHost[host] = newest{at: b.GeneratedAt, pins: p, box: b.Label}
+ }
+ }
+ }
+ // Every provisioned instance of the app that keeps a build state must
+ // be named by some report (Astra I2 #5): a missing entry is unknown,
+ // never "nothing to protect". For a valid record this is
+ // keepsBuildState's instance rule (activator.ActivatorManaged): an
+ // instance with no unit that hibernates has no addr (the record door
+ // refuses addr with hibernate), so reconcile mints it a socket.
+ for _, r := range recs {
+ if r.KindOrDefault() != "instance" || r.Unit != "" || !r.HibernateOrDefault() {
+ continue
+ }
+ if _, ok := byHost[r.Host]; !ok {
+ return nil, fmt.Errorf("no box report names the build state of %s", r.Host)
+ }
+ }
+ for host, n := range byHost {
+ if now.Sub(n.at) > buildStateReportMaxAge {
+ return nil, fmt.Errorf("%s, which hosts %s, has not reported since %s", n.box, host, n.at.UTC().Format(time.RFC3339))
+ }
+ if !n.pins.Complete {
+ return nil, fmt.Errorf("%s could not read the build state of %s", n.box, host)
+ }
+ }
+ return out, nil
+}
diff --git a/internal/host/retention_settings_test.go b/internal/host/retention_settings_test.go
new file mode 100644
index 000000000..93f75f2a2
--- /dev/null
+++ b/internal/host/retention_settings_test.go
@@ -0,0 +1,476 @@
+package host
+
+import (
+ "context"
+ "errors"
+ "path/filepath"
+ "reflect"
+ "sort"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/status"
+)
+
+func report(label string, at time.Time, pins map[string]status.BuildStatePins) status.Box {
+ return status.Box{Label: label, GeneratedAt: at, Metrics: &status.BoxMetrics{BuildStates: pins}}
+}
+
+func kmPins(builds ...string) status.BuildStatePins {
+ return status.BuildStatePins{Account: "bab", App: "km", Builds: builds, Complete: true}
+}
+
+// Astra I1 #6: a host whose build state its box could not read stops
+// pruning for the app, even in a fresh, readable report.
+func TestAnIncompleteBuildStatePrunesNothing(t *testing.T) {
+ s := recordsStore(t, provision.Record{Schema: 1, Host: "b.example.com", App: "km", Account: "bab", Channel: "stable"})
+ inc := kmPins()
+ inc.Complete = false
+ boxes := []status.Box{report("uk-1", now0.Add(-time.Minute), map[string]status.BuildStatePins{"b.example.com": inc})}
+ if _, err := settingsProtected(context.Background(), s, boxes, true, now0, "bab", "km"); err == nil {
+ t.Fatal("an unreadable build state did not stop pruning")
+ }
+}
+
+func TestAnUnreadBuildStateIsReportedIncomplete(t *testing.T) {
+ _, _, reg := testBuildStates(t) // a.example.com, never refreshed: copy unknown
+ pins, err := buildStatePins(reg, false)
+ if p, ok := pins["a.example.com"]; err != nil || !ok || p.Complete {
+ t.Fatalf("report = %+v, %v, %v; want the host listed and incomplete", p, ok, err)
+ }
+ if err := reg.SetBuildState("a.example.com", release.BuildStateCopy{Known: true, ReadErr: "timeout"}.Column()); err != nil {
+ t.Fatal(err)
+ }
+ if pins, _ := buildStatePins(reg, false); pins["a.example.com"].Complete {
+ t.Fatal("a failed refresh was reported complete")
+ }
+ if err := reg.SetBuildState("a.example.com", "{"); err != nil {
+ t.Fatal(err)
+ }
+ if pins, _ := buildStatePins(reg, false); pins["a.example.com"].Complete {
+ t.Fatal("an unparseable copy was reported complete")
+ }
+}
+
+func recordsStore(t *testing.T, recs ...provision.Record) release.Store {
+ t.Helper()
+ s := release.NewDirStore(t.TempDir())
+ for _, r := range recs {
+ if err := provision.CreateRecord(context.Background(), s, r); err != nil {
+ t.Fatal(err)
+ }
+ }
+ return s
+}
+
+var now0 = time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)
+
+// Spec rev 11 §4.4: pins from records, build-state builds from box reports.
+func TestPinsAndReportedBuildStatesAreProtected(t *testing.T) {
+ s := recordsStore(t,
+ provision.Record{Schema: 1, Host: "a.example.com", App: "km", Account: "bab", Channel: "stable", Pin: "v1"},
+ provision.Record{Schema: 1, Host: "b.example.com", App: "km", Account: "bab", Channel: "stable", Region: "uk"})
+ boxes := []status.Box{report("uk-1", now0.Add(-time.Hour), map[string]status.BuildStatePins{
+ "a.example.com": kmPins(), "b.example.com": kmPins("v2", "v3")})}
+ got, err := settingsProtected(context.Background(), s, boxes, true, now0, "bab", "km")
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, v := range []string{"v1", "v2", "v3"} {
+ if !got[v] {
+ t.Errorf("%s is not protected: %v", v, got)
+ }
+ }
+}
+
+func TestABoxSilentForSixHoursPrunesNothing(t *testing.T) {
+ s := recordsStore(t, provision.Record{Schema: 1, Host: "b.example.com", App: "km", Account: "bab", Channel: "stable"})
+ boxes := []status.Box{report("uk-1", now0.Add(-6*time.Hour-time.Minute), map[string]status.BuildStatePins{"b.example.com": kmPins("v2")})}
+ if _, err := settingsProtected(context.Background(), s, boxes, true, now0, "bab", "km"); err == nil {
+ t.Fatal("a box hosting the app that has not reported in 6h did not stop pruning")
+ }
+}
+
+func TestANewerReportForTheHostWins(t *testing.T) {
+ s := recordsStore(t, provision.Record{Schema: 1, Host: "b.example.com", App: "km", Account: "bab", Channel: "stable"})
+ boxes := []status.Box{
+ report("uk-1", now0.Add(-48*time.Hour), map[string]status.BuildStatePins{"b.example.com": kmPins("v1")}), // the box it moved off
+ report("ie-1", now0.Add(-time.Minute), map[string]status.BuildStatePins{"b.example.com": kmPins("v2")}),
+ }
+ got, err := settingsProtected(context.Background(), s, boxes, true, now0, "bab", "km")
+ if err != nil || !got["v2"] {
+ t.Fatalf("got %v, %v; want v2 from the newest report and no refusal", got, err)
+ }
+ // §4.4 protects every build named in a report: the older report's v1 is
+ // kept too, which can only keep more.
+ if !got["v1"] {
+ t.Errorf("v1, named by the older report, is not protected: %v", got)
+ }
+}
+
+// The newest report decides completeness too: an older box's incomplete
+// entry for a host that has since moved does not block, and a newer
+// incomplete one does even when an older one was complete.
+func TestTheNewestReportDecidesCompleteness(t *testing.T) {
+ s := recordsStore(t, provision.Record{Schema: 1, Host: "b.example.com", App: "km", Account: "bab", Channel: "stable"})
+ inc := kmPins()
+ inc.Complete = false
+ moved := []status.Box{
+ report("uk-1", now0.Add(-2*time.Hour), map[string]status.BuildStatePins{"b.example.com": inc}),
+ report("ie-1", now0.Add(-time.Minute), map[string]status.BuildStatePins{"b.example.com": kmPins("v2")}),
+ }
+ if _, err := settingsProtected(context.Background(), s, moved, true, now0, "bab", "km"); err != nil {
+ t.Fatalf("an older box's incomplete entry blocked: %v", err)
+ }
+ reversed := []status.Box{
+ report("uk-1", now0.Add(-time.Minute), map[string]status.BuildStatePins{"b.example.com": inc}),
+ report("ie-1", now0.Add(-2*time.Hour), map[string]status.BuildStatePins{"b.example.com": kmPins("v2")}),
+ }
+ if _, err := settingsProtected(context.Background(), s, reversed, true, now0, "bab", "km"); err == nil {
+ t.Fatal("the newest report's incomplete entry did not stop pruning")
+ }
+}
+
+func TestOtherAppsReportsDoNotBlock(t *testing.T) {
+ s := recordsStore(t, provision.Record{Schema: 1, Host: "a.example.com", App: "km", Account: "bab", Channel: "stable"})
+ boxes := []status.Box{
+ report("uk-1", now0.Add(-72*time.Hour), map[string]status.BuildStatePins{
+ "x.example.com": {Account: "bab", App: "blog", Builds: []string{"b1"}}}),
+ report("ie-1", now0.Add(-time.Minute), map[string]status.BuildStatePins{"a.example.com": kmPins("v1")}),
+ }
+ if _, err := settingsProtected(context.Background(), s, boxes, true, now0, "bab", "km"); err != nil {
+ t.Fatalf("another app's silent box blocked km: %v", err)
+ }
+}
+
+// Astra I2 #5: every provisioned instance of the app needs an entry. A
+// fresh, readable report that simply omits the host (its map left out)
+// prunes nothing, not "nothing to protect".
+func TestAnOmittedHostPrunesNothing(t *testing.T) {
+ s := recordsStore(t, provision.Record{Schema: 1, Host: "b.example.com", App: "km", Account: "bab", Channel: "stable"})
+ boxes := []status.Box{report("uk-1", now0.Add(-time.Minute), nil)}
+ if _, err := settingsProtected(context.Background(), s, boxes, true, now0, "bab", "km"); err == nil {
+ t.Fatal("a host no report names was treated as having no builds to protect")
+ }
+ // A report naming the host under another account is not this app's.
+ other := []status.Box{report("uk-1", now0.Add(-time.Minute), map[string]status.BuildStatePins{
+ "b.example.com": {Account: "zzz", App: "km", Builds: []string{"v1"}, Complete: true}})}
+ if _, err := settingsProtected(context.Background(), s, other, true, now0, "bab", "km"); err == nil {
+ t.Fatal("another account's entry for the host counted as this app's")
+ }
+}
+
+// Only a record that keeps a build state needs a report: a unit-backed or
+// always-on instance, or a static route, has none to report.
+func TestRecordsWithNoBuildStateNeedNoReport(t *testing.T) {
+ off := false
+ s := recordsStore(t,
+ provision.Record{Schema: 1, Host: "u.example.com", App: "km", Account: "bab", Channel: "stable", Unit: "km.service"},
+ provision.Record{Schema: 1, Host: "o.example.com", App: "km", Account: "bab", Channel: "stable", Hibernate: &off, Pin: "v7"},
+ provision.Record{Schema: 1, Host: "s.example.com", App: "km", Account: "bab", Channel: "stable", Kind: "static"})
+ got, err := settingsProtected(context.Background(), s, nil, true, now0, "bab", "km")
+ if err != nil {
+ t.Fatalf("records that keep no build state blocked: %v", err)
+ }
+ if !got["v7"] {
+ t.Errorf("the pin of a record that keeps no build state is not protected: %v", got)
+ }
+}
+
+// Astra I2 #5: a box that could not list its routes says so, and a fresh
+// report saying so stops pruning even when an older report from another
+// box still names the host (it may have moved onto the failing box).
+func TestACollectionFailurePrunesNothing(t *testing.T) {
+ s := recordsStore(t, provision.Record{Schema: 1, Host: "b.example.com", App: "km", Account: "bab", Channel: "stable"})
+ failed := report("uk-1", now0.Add(-time.Minute), nil)
+ failed.Metrics.BuildStatesErr = "registry: database is locked"
+ boxes := []status.Box{
+ report("ie-1", now0.Add(-2*time.Hour), map[string]status.BuildStatePins{"b.example.com": kmPins("v1")}),
+ failed,
+ }
+ if _, err := settingsProtected(context.Background(), s, boxes, true, now0, "bab", "km"); err == nil {
+ t.Fatal("a box that could not collect its build states did not stop pruning")
+ }
+}
+
+// registry.List opens its own connection, so a closed registry still lists:
+// the failing list is a fake.
+func TestTheCollectorReportsItsFailure(t *testing.T) {
+ reg := fakeLister{err: errors.New("database is locked")}
+ if pins, err := buildStatePins(reg, false); err == nil || pins != nil {
+ t.Fatalf("a failed registry list returned %v, %v; want an error", pins, err)
+ }
+}
+
+func TestAnUnreadableRecordOrReportPrunesNothing(t *testing.T) {
+ s := recordsStore(t, provision.Record{Schema: 1, Host: "a.example.com", App: "km", Account: "bab", Channel: "stable"})
+ if _, err := s.Put(context.Background(), provision.RecordKey("bab", "km", "z.example.com"), []byte("{"), ""); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := settingsProtected(context.Background(), s, nil, true, now0, "bab", "km"); err == nil {
+ t.Fatal("an unreadable record did not stop pruning")
+ }
+ clean := recordsStore(t, provision.Record{Schema: 1, Host: "a.example.com", App: "km", Account: "bab", Channel: "stable"})
+ if _, err := settingsProtected(context.Background(), clean, nil, false, now0, "bab", "km"); err == nil {
+ t.Fatal("an incomplete read of box reports did not stop pruning")
+ }
+}
+
+// Ruling R-T9-STORE: provision.ListRecords(..., 0) reports truncated whenever
+// a key exists, and it skips a record it cannot read. Every record is read,
+// and any failure (a record whose body names another host, a failed Get, a
+// failed listing) prunes nothing.
+func TestEveryRecordIsReadAndAnyFailurePrunesNothing(t *testing.T) {
+ ctx := context.Background()
+ recs := []provision.Record{
+ {Schema: 1, Host: "a.example.com", App: "km", Account: "bab", Channel: "stable", Pin: "v1"},
+ {Schema: 1, Host: "b.example.com", App: "km", Account: "bab", Channel: "stable", Pin: "v2"},
+ {Schema: 1, Host: "c.example.com", App: "km", Account: "bab", Channel: "stable", Pin: "v3"},
+ }
+ all := map[string]status.BuildStatePins{"a.example.com": kmPins(), "b.example.com": kmPins(), "c.example.com": kmPins()}
+ boxes := []status.Box{report("uk-1", now0.Add(-time.Minute), all)}
+ s := recordsStore(t, recs...)
+ got, err := settingsProtected(ctx, s, boxes, true, now0, "bab", "km")
+ if err != nil {
+ t.Fatalf("three good records refused: %v", err)
+ }
+ if !got["v1"] || !got["v2"] || !got["v3"] {
+ t.Fatalf("not every record's pin is protected: %v", got)
+ }
+
+ // A body that disagrees with its own path.
+ moved := recordsStore(t, recs...)
+ raw, _, err := moved.Get(ctx, provision.RecordKey("bab", "km", "a.example.com"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := moved.Put(ctx, provision.RecordKey("bab", "km", "d.example.com"), raw, ""); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := settingsProtected(ctx, moved, boxes, true, now0, "bab", "km"); err == nil {
+ t.Fatal("a record that disagrees with its path did not stop pruning")
+ }
+
+ // A Get that fails.
+ getFails := failingGetStore{Store: recordsStore(t, recs...), bad: provision.RecordKey("bab", "km", "b.example.com"), err: errors.New("SlowDown")}
+ if _, err := settingsProtected(ctx, getFails, boxes, true, now0, "bab", "km"); err == nil {
+ t.Fatal("a record that could not be read did not stop pruning")
+ }
+
+ // A listing that fails.
+ listFails := failingListStore{Store: recordsStore(t, recs...), prefix: provision.RecordsPrefix("bab", "km"), err: errors.New("AccessDenied")}
+ if _, err := settingsProtected(ctx, listFails, boxes, true, now0, "bab", "km"); err == nil {
+ t.Fatal("a failed record listing did not stop pruning")
+ }
+}
+
+func TestTheBoxReportsItsBuildStates(t *testing.T) {
+ _, _, reg := testBuildStates(t)
+ if err := reg.SetBuildState("a.example.com", release.BuildStateCopy{Known: true, Replica: true,
+ State: &release.BuildState{LastExecuted: "v1", Resolved: &release.ResolvedRequest{ID: 2, Target: "v2"}}}.Column()); err != nil {
+ t.Fatal(err)
+ }
+ pins, err := buildStatePins(reg, false)
+ if err != nil {
+ t.Fatal(err)
+ }
+ got := pins["a.example.com"]
+ // testBuildStates' row has no account: RouteAccount's fallback is ops,
+ // whose sqid is "bab", the account the housekeeper keys the app by.
+ if got.Account != "bab" || got.App != "km" || !got.Complete || len(got.Builds) != 2 || got.Builds[0] != "v1" || got.Builds[1] != "v2" {
+ t.Fatalf("report = %+v", got)
+ }
+}
+
+// Task 4's review: the report names exactly the rows keepsBuildState
+// selects, the rows refreshAll keeps a copy of. A unit-backed hibernating
+// instance is never refreshed, so listing it would read as incomplete
+// forever; an exec-backed worker is refreshed with host claims off, and
+// never with them on.
+func TestTheReportNamesExactlyTheRowsThatKeepABuildState(t *testing.T) {
+ reg, err := registry.Open(filepath.Join(t.TempDir(), "r.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer reg.Close()
+ rows := []registry.Route{
+ {Host: "a.example.com", Kind: "instance", App: "km", Account: "bab", Hibernate: true, Socket: "/run/carlos/a.sock"},
+ {Host: "u.example.com", Kind: "instance", App: "km", Account: "bab", Hibernate: true, Socket: "/run/carlos/u.sock", Unit: "km.service"},
+ {Host: "w.example.com", Kind: "sidecar", App: "km", Account: "bab", Socket: "/run/carlos/w.sock"},
+ {Host: "t.example.com", Kind: "instance", App: "km", Account: "bab", Hibernate: true, Socket: "/run/carlos/t.sock", Tunnel: "fleet-1"},
+ {Host: "r.example.com", Kind: "instance", App: "km", Account: "bab", Socket: "/run/carlos/r.sock"},
+ {Host: "p.example.com", Kind: "instance", App: "edge", Hibernate: true, Socket: "/run/carlos/p.sock", Platform: true},
+ }
+ for _, rt := range rows {
+ if err := reg.Upsert(rt); err != nil {
+ t.Fatal(err)
+ }
+ }
+ routes, err := reg.List()
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, claims := range []bool{false, true} {
+ pins, err := buildStatePins(reg, claims)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var got, want []string
+ for h := range pins {
+ got = append(got, h)
+ }
+ for _, rt := range routes {
+ if keepsBuildState(rt, claims) {
+ want = append(want, rt.Host)
+ }
+ }
+ sort.Strings(got)
+ sort.Strings(want)
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("claims %v: report names %v, keepsBuildState selects %v", claims, got, want)
+ }
+ wantHosts := []string{"a.example.com", "w.example.com"}
+ if claims {
+ wantHosts = []string{"a.example.com"}
+ }
+ if !reflect.DeepEqual(got, wantHosts) {
+ t.Errorf("claims %v: report names %v, want %v", claims, got, wantHosts)
+ }
+ }
+}
+
+// loadAllBoxes reads the deployment's reports and every fleet's, and says
+// when any could not be read.
+func TestLoadAllBoxesReadsEveryFleet(t *testing.T) {
+ ctx := context.Background()
+ s := release.NewDirStore(t.TempDir())
+ if err := status.WriteBox(ctx, s, report("uk-1", now0, map[string]status.BuildStatePins{"a.example.com": kmPins("v1")})); err != nil {
+ t.Fatal(err)
+ }
+ if err := status.WriteBox(ctx, release.NewPrefixStore(s, "fleets/acme/"), report("acme-1", now0, map[string]status.BuildStatePins{"b.example.com": kmPins("v2")})); err != nil {
+ t.Fatal(err)
+ }
+ boxes, complete, err := loadAllBoxes(ctx, s)
+ if err != nil || !complete {
+ t.Fatalf("loadAllBoxes = %v, %v", complete, err)
+ }
+ var labels []string
+ for _, b := range boxes {
+ labels = append(labels, b.Label)
+ }
+ sort.Strings(labels)
+ if !reflect.DeepEqual(labels, []string{"acme-1", "uk-1"}) {
+ t.Fatalf("read %v, want both the deployment's box and the fleet's", labels)
+ }
+
+ if _, err := s.Put(ctx, "fleets/beta/status/boxes/beta-1.json", []byte("{"), ""); err != nil {
+ t.Fatal(err)
+ }
+ if _, complete, err := loadAllBoxes(ctx, s); err != nil || complete {
+ t.Fatalf("a corrupt fleet report read as complete (%v, %v)", complete, err)
+ }
+
+ fleetsFail := failingListStore{Store: s, prefix: "fleets/", err: errors.New("AccessDenied")}
+ if _, _, err := loadAllBoxes(ctx, fleetsFail); err == nil {
+ t.Fatal("a failed fleets listing was not an error")
+ }
+}
+
+// retentionFixture is TestRetentionPassPrunesEligibleAndProtectsBoxPins'
+// shape with 27 versions and keep 20, so v01..v07 are the candidates, and
+// a provisioned, hibernating kass.example.com on this box at v07 (previous
+// v06, good v05) whose record pins v04.
+func retentionFixture(t *testing.T) (context.Context, *release.DirStore, *registry.Registry, func(string) bool) {
+ t.Helper()
+ ctx := context.Background()
+ dir := t.TempDir()
+ s := release.NewDirStore(dir)
+ now := time.Now()
+ base := now.Add(-30 * 24 * time.Hour)
+ vname := func(i int) string { return "v" + string(rune('0'+i/10)) + string(rune('0'+i%10)) }
+ for i := 1; i <= 27; i++ {
+ stampKey(t, s, dir, release.ManifestKey("bab", "kass", vname(i)), base.Add(time.Duration(i)*time.Hour))
+ stampKey(t, s, dir, release.ArtifactKey("bab", "kass", vname(i), "linux-arm64"), base.Add(time.Duration(i)*time.Hour))
+ }
+ if err := release.SetRetention(ctx, s, "bab", "kass", 20, "test", now); err != nil {
+ t.Fatal(err)
+ }
+ if err := provision.CreateRecord(ctx, s, provision.Record{Schema: 1, Host: "kass.example.com", App: "kass", Account: "bab", Channel: "stable", Pin: "v04"}); err != nil {
+ t.Fatal(err)
+ }
+ reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { reg.Close() })
+ if err := reg.Upsert(registry.Route{
+ Host: "kass.example.com", Kind: "instance", App: "kass", Account: "bab", Hibernate: true, Socket: "/run/carlos/kass.sock",
+ Version: "v07", PrevVersion: "v06", GoodVersion: "v05", Status: "ready",
+ }); err != nil {
+ t.Fatal(err)
+ }
+ present := func(v string) bool {
+ _, _, err := s.Get(ctx, release.ManifestKey("bab", "kass", v))
+ return err == nil
+ }
+ return ctx, s, reg, present
+}
+
+// The pass itself: this box's Version, PrevVersion and GoodVersion (zdd-5
+// obligation 9), the record's pin, and the reported last_executed and
+// resolved T are all kept; only what nothing names is pruned.
+func TestRetentionPassProtectsPinsGoodVersionAndReportedBuilds(t *testing.T) {
+ ctx, s, reg, present := retentionFixture(t)
+ _, pub := retentionKeys(t)
+ reports := release.NewDirStore(t.TempDir())
+ if err := status.WriteBox(ctx, reports, report("uk-1", time.Now().Add(-time.Minute), map[string]status.BuildStatePins{
+ "kass.example.com": {Account: "bab", App: "kass", Builds: []string{"v03", "v02"}, Complete: true}})); err != nil {
+ t.Fatal(err)
+ }
+ if err := retentionPass(ctx, s, reports, pub, reg); err != nil {
+ t.Fatal(err)
+ }
+ if present("v01") {
+ t.Error("v01 survived; nothing names it")
+ }
+ for _, v := range []string{"v02", "v03", "v04", "v05", "v06", "v07"} {
+ if !present(v) {
+ t.Errorf("%s was pruned", v)
+ }
+ }
+}
+
+// With no report naming the provisioned instance, or no readable reports at
+// all, the pass prunes nothing for the app.
+func TestRetentionPassPrunesNothingWithoutTheReports(t *testing.T) {
+ t.Run("no report names the host", func(t *testing.T) {
+ ctx, s, reg, present := retentionFixture(t)
+ _, pub := retentionKeys(t)
+ if err := retentionPass(ctx, s, release.NewDirStore(t.TempDir()), pub, reg); err != nil {
+ t.Fatal(err)
+ }
+ if !present("v01") {
+ t.Fatal("v01 was pruned with no report naming kass.example.com's build state")
+ }
+ })
+ t.Run("the reports cannot be listed", func(t *testing.T) {
+ ctx, s, reg, present := retentionFixture(t)
+ _, pub := retentionKeys(t)
+ reports := failingListStore{Store: release.NewDirStore(t.TempDir()), prefix: "status/boxes/", err: errors.New("AccessDenied")}
+ _ = retentionPass(ctx, s, reports, pub, reg)
+ if !present("v01") {
+ t.Fatal("v01 was pruned with the box reports unreadable")
+ }
+ })
+ t.Run("no status store", func(t *testing.T) {
+ ctx, s, reg, present := retentionFixture(t)
+ _, pub := retentionKeys(t)
+ _ = retentionPass(ctx, s, nil, pub, reg)
+ if !present("v01") {
+ t.Fatal("v01 was pruned with no status store to read reports from")
+ }
+ })
+}
diff --git a/internal/host/retention_test.go b/internal/host/retention_test.go
index 174049ca8..ae9547ae2 100644
--- a/internal/host/retention_test.go
+++ b/internal/host/retention_test.go
@@ -70,7 +70,7 @@ func TestRetentionPassPrunesEligibleAndProtectsBoxPins(t *testing.T) {
t.Fatal(err)
}
- if err := retentionPass(ctx, s, pub, reg); err != nil {
+ if err := retentionPass(ctx, s, release.NewDirStore(t.TempDir()), pub, reg); err != nil {
t.Fatal(err)
}
@@ -123,7 +123,7 @@ func TestRetentionPassResolvesEmptyAccountRoutesViaRouteAccount(t *testing.T) {
t.Fatal(err)
}
- if err := retentionPass(ctx, s, pub, reg); err != nil {
+ if err := retentionPass(ctx, s, release.NewDirStore(t.TempDir()), pub, reg); err != nil {
t.Fatal(err)
}
@@ -164,7 +164,7 @@ func TestRetentionPassSkipsAppsWithoutAPolicy(t *testing.T) {
}); err != nil {
t.Fatal(err)
}
- if err := retentionPass(ctx, s, pub, reg); err != nil {
+ if err := retentionPass(ctx, s, release.NewDirStore(t.TempDir()), pub, reg); err != nil {
t.Fatal(err)
}
if _, _, err := s.Get(ctx, release.ManifestKey("bab", "kass", "v01")); err != nil {
diff --git a/internal/host/versionsweep.go b/internal/host/versionsweep.go
index 6ee76ffc8..53a023125 100644
--- a/internal/host/versionsweep.go
+++ b/internal/host/versionsweep.go
@@ -1,10 +1,12 @@
package host
import (
+ "fmt"
"log"
"os"
"time"
+ "github.com/carlosframework/platform/internal/activator"
"github.com/carlosframework/platform/internal/registry"
"github.com/carlosframework/platform/internal/release"
)
@@ -74,6 +76,18 @@ func sweepStagedVersions(reg routeLister, optDir, genPath string, grace time.Dur
pinned[r.GoodVersion] = true
}
}
+ // The builds the instances' settings and build states name (ruling
+ // R-T9-STORE): a held instance's recovery onto last_executed runs its
+ // staged copy here (EvidenceStaged), so that copy is spoken for however
+ // long ago it last ran.
+ settings, err := settingsPins(routes)
+ if err != nil {
+ log.Printf("adopt: staged-version sweep skipped: %v", err)
+ return
+ }
+ for v := range settings {
+ pinned[v] = true
+ }
// Every live or pending generation the agent published (spec §8.5,
// ruling R19 item 1): a worker's captured build stays pinned after the
// live link and the row have both moved on. Neither file present is no
@@ -112,3 +126,35 @@ func sweepStagedVersions(reg routeLister, optDir, genPath string, grace time.Dur
func sweepStagedVersionsPass(reg routeLister) {
sweepStagedVersions(reg, release.OptDir(), release.GenerationsPath(dataDir()), versionSweepGrace(os.Getenv), time.Now())
}
+
+// settingsPins is every build the rows' update settings and build-state
+// copies name: a pin, last_executed and a resolved request's T. It reads
+// the registry columns root reads (ParseUpdateSettingsLenient,
+// ParseBuildStateCopy); a copy that kept its last known state after a
+// failed refresh still names it. A column that does not parse, or a held
+// instance (settings other than default) whose build state is not read
+// yet, is an incomplete pin set: its last_executed is unknown, so the
+// sweep deletes nothing.
+func settingsPins(routes []registry.Route) (map[string]bool, error) {
+ out := map[string]bool{}
+ for _, r := range routes {
+ u, err := release.ParseUpdateSettingsLenient(r.UpdateSettings)
+ if err != nil {
+ return nil, fmt.Errorf("update settings of %s: %w", r.Host, err)
+ }
+ if u.Pin != "" {
+ out[u.Pin] = true
+ }
+ c, err := release.ParseBuildStateCopy(r.BuildState)
+ if err != nil {
+ return nil, fmt.Errorf("build state of %s: %w", r.Host, err)
+ }
+ if !c.Known && !u.Default() && activator.ActivatorManaged(r) {
+ return nil, fmt.Errorf("the build state of held %s is not read yet", r.Host)
+ }
+ for _, v := range buildStateBuilds(c.State) {
+ out[v] = true
+ }
+ }
+ return out, nil
+}
diff --git a/internal/host/versionsweep_test.go b/internal/host/versionsweep_test.go
index 493173085..6960c0e83 100644
--- a/internal/host/versionsweep_test.go
+++ b/internal/host/versionsweep_test.go
@@ -270,3 +270,78 @@ func TestSweepStagedVersionsKeepsTheGoodBuild(t *testing.T) {
t.Error("gone survived; nothing names it and it is older than the grace")
}
}
+
+// settingsRow is an activator-managed instance with the given build-state
+// copy and update-settings columns.
+func settingsRow(host, buildState, settings string) registry.Route {
+ return registry.Route{Host: host, Kind: "instance", App: "blog", Account: "acct1", Hibernate: true, Socket: "/run/carlos/" + host + ".sock",
+ Version: "v9", BuildState: buildState, UpdateSettings: settings}
+}
+
+// Ruling R-T9-STORE: the box-local sweep keeps every build a build state
+// names (last_executed and a resolved request's T) and every pin, so a held
+// instance's staged recovery copy (EvidenceStaged) is never evicted.
+func TestSweepStagedVersionsKeepsBuildStateBuildsAndPins(t *testing.T) {
+ opt := t.TempDir()
+ for _, v := range []string{"v1", "v2", "v3", "v4", "v9"} {
+ stageSweepVersion(t, opt, "acct1", v, "blog", 48*time.Hour)
+ }
+ copyCol := release.BuildStateCopy{Known: true, Replica: true,
+ State: &release.BuildState{LastExecuted: "v1", Resolved: &release.ResolvedRequest{ID: 2, Target: "v2"}}}.Column()
+ reg := fakeLister{routes: []registry.Route{
+ settingsRow("a.example.com", copyCol, release.UpdateSettings{Updates: release.UpdatesManual, Channel: "stable"}.Column()),
+ settingsRow("b.example.com", release.BuildStateCopy{Known: true}.Column(), release.UpdateSettings{Updates: release.UpdatesAuto, Channel: "stable", Pin: "v3"}.Column()),
+ }}
+ sweepStagedVersions(reg, opt, noGenerations(t), 6*time.Hour, time.Now())
+ for _, v := range []string{"v1", "v2", "v3", "v9"} {
+ if !staged(t, opt, v) {
+ t.Errorf("%s was removed", v)
+ }
+ }
+ if staged(t, opt, "v4") {
+ t.Error("v4 survived; nothing names it")
+ }
+}
+
+// A copy that keeps its last known state after a failed refresh still pins
+// it: the build it names did not stop being the instance's.
+func TestSweepStagedVersionsKeepsALastKnownState(t *testing.T) {
+ opt := t.TempDir()
+ stageSweepVersion(t, opt, "acct1", "v1", "blog", 48*time.Hour)
+ col := release.BuildStateCopy{Known: true, ReadErr: "timeout", State: &release.BuildState{LastExecuted: "v1"}}.Column()
+ sweepStagedVersions(fakeLister{routes: []registry.Route{settingsRow("a.example.com", col, "")}}, opt, noGenerations(t), 6*time.Hour, time.Now())
+ if !staged(t, opt, "v1") {
+ t.Error("the last known last_executed was removed after a failed refresh")
+ }
+}
+
+// Any column the sweep cannot read is an incomplete pin set: an unparseable
+// build-state copy or update-settings column, or a held instance whose build
+// state is not read yet (its last_executed is unknown). Delete nothing.
+func TestSweepStagedVersionsDeletesNothingWhenSettingsPinsAreUnknown(t *testing.T) {
+ manual := release.UpdateSettings{Updates: release.UpdatesManual, Channel: "stable"}.Column()
+ cases := map[string]registry.Route{
+ "unparseable build state": settingsRow("a.example.com", "{", ""),
+ "unparseable update settings": settingsRow("a.example.com", "", "{"),
+ "held, build state unread": settingsRow("a.example.com", "", manual),
+ "pinned, build state unread": settingsRow("a.example.com", "", release.UpdateSettings{Updates: release.UpdatesAuto, Channel: "stable", Pin: "v1"}.Column()),
+ }
+ for name, rt := range cases {
+ t.Run(name, func(t *testing.T) {
+ opt := t.TempDir()
+ stageSweepVersion(t, opt, "acct1", "v1", "blog", 48*time.Hour)
+ sweepStagedVersions(fakeLister{routes: []registry.Route{rt}}, opt, noGenerations(t), 6*time.Hour, time.Now())
+ if !staged(t, opt, "v1") {
+ t.Error("swept with a pin set it could not complete")
+ }
+ })
+ }
+ // Default settings with an unread copy: nothing is held on its
+ // last_executed, so the sweep goes on.
+ opt := t.TempDir()
+ stageSweepVersion(t, opt, "acct1", "v1", "blog", 48*time.Hour)
+ sweepStagedVersions(fakeLister{routes: []registry.Route{settingsRow("a.example.com", "", "")}}, opt, noGenerations(t), 6*time.Hour, time.Now())
+ if staged(t, opt, "v1") {
+ t.Error("a default-settings row with an unread copy stopped the sweep")
+ }
+}
diff --git a/internal/status/contract_test.go b/internal/status/contract_test.go
index e29ee4dca..ca8855275 100644
--- a/internal/status/contract_test.go
+++ b/internal/status/contract_test.go
@@ -82,7 +82,7 @@ var allowedBoxKeys = map[string][]string{
"": {"generated_at", "label", "region", "routes", "load", "capacity", "replication", "adoption_running", "config_path_running", "restart_path_running", "expired_canaries", "metrics"},
"routes": {"static", "resident", "hibernating", "awake", "asleep"},
"replication": {"healthy", "lag_seconds"},
- "metrics": {"mem_total_bytes", "mem_avail_bytes", "load1", "disk_free_bytes", "release", "tunnel", "route_versions", "upstreams", "layout", "config_refusals", "config_refusals_total", "release_refusals", "proof_waits", "config_guard"},
+ "metrics": {"mem_total_bytes", "mem_avail_bytes", "load1", "disk_free_bytes", "release", "tunnel", "route_versions", "upstreams", "layout", "config_refusals", "config_refusals_total", "release_refusals", "proof_waits", "config_guard", "build_states", "build_states_err"},
"metrics.route_versions": {wildcardAnyKey},
"metrics.upstreams": {wildcardAnyKey},
"metrics.upstreams.*[]": {"addr", "up", "since", "fails", "last_error"},
@@ -99,6 +99,12 @@ var allowedBoxKeys = map[string][]string{
// proof_waits (D3) names the host and build the box is holding for a dev
// proof, in the private block only, pinned to the field level too.
"metrics.proof_waits[]": {"host", "version", "sha256", "prover", "reason", "since"},
+ // build_states (settings spec rev 11 §4.4) is host -> the builds that
+ // host's build state names, in the private block only, for the
+ // housekeeper's retention. Keys are hostnames; the values are pinned to
+ // the field level.
+ "metrics.build_states": {wildcardAnyKey},
+ "metrics.build_states.*": {"account", "app", "builds", "complete"},
}
// allowedFleetKeys are THE schema for Fleet's JSON output. Same friction as
@@ -344,6 +350,11 @@ func fixtureProbes() status.Probes {
{Host: "contract-proof-wait.internal", Version: "v3", SHA256: "e47fcc1e0123456789abcdef", Prover: "dev",
Reason: "no proof from dev", Since: time.Now().Add(-time.Hour)},
},
+ // One host's build-state builds, so metrics.build_states is exercised
+ // by the allowlist walk too.
+ BuildStates: map[string]status.BuildStatePins{
+ "contract-held.internal": {Account: "bab", App: "km", Builds: []string{"v1", "v2"}, Complete: true},
+ },
ConfigGuard: "bypassed (CARLOS_NO_SEALED_CONFIG)",
MemFrac: &memFrac,
DiskFrac: &diskFrac,
diff --git a/internal/status/dryrun.go b/internal/status/dryrun.go
index 83dc32d8e..5e198c134 100644
--- a/internal/status/dryrun.go
+++ b/internal/status/dryrun.go
@@ -140,6 +140,18 @@ const dryRunRolloutHost = "rollout-host.internal"
var dryRunRolloutNeedles = []string{dryRunRolloutHost}
+// dryRunBuildStateHost and dryRunBuildStateBuild are the build-state needles
+// (settings spec rev 11 §4.4): Box.Metrics.BuildStates names the hosts this
+// box runs and the builds their build states name, so the same scope split
+// applies. REQUIRED in status/boxes/*.json and BANNED in every other
+// published object.
+const (
+ dryRunBuildStateHost = "build-state-host.internal"
+ dryRunBuildStateBuild = "v-dryrun-build-state"
+)
+
+var dryRunBuildStateNeedles = []string{dryRunBuildStateHost, dryRunBuildStateBuild}
+
// ScanForLeaks reports which of banned's strings appear anywhere in j. It is
// the single implementation both DryRun's self-check and
// contract_test.go's scanForLeaks test helper run — the test package can't
@@ -246,6 +258,10 @@ func dryRunFixtures() ([]registry.Route, Probes) {
{Account: "dryrun", App: "app", Build: "v-dryrun", Host: dryRunRolloutHost, Mode: "overlap", Reason: ReasonHealth, PromotedAt: time.Now(), At: time.Now()},
},
},
+ // One host's build-state builds, for the build_states scope check.
+ BuildStates: map[string]BuildStatePins{
+ dryRunBuildStateHost: {Account: "dryrun", App: "app", Builds: []string{dryRunBuildStateBuild}, Complete: true},
+ },
// The probe table, keyed by a route host and carrying a backend
// address — see dryRunUpstreamNeedles. One up and one down upstream,
// so the down one's reason string is exercised too.
@@ -410,6 +426,9 @@ func DryRun(dir string, out io.Writer) error {
// rolloutKeySeen is the same positive half for the rollout report
// (dryRunRolloutNeedles).
rolloutKeySeen := false
+ // buildStateKeySeen is the same positive half for the build-state report
+ // (dryRunBuildStateNeedles).
+ buildStateKeySeen := false
for _, key := range keys {
body, _, err := s.Get(ctx, key)
@@ -513,6 +532,17 @@ func DryRun(dir string, out io.Writer) error {
return fmt.Errorf("dry run: %s leaked a rollout fixture %v — Rollout must never reach the public feed", key, leaks)
}
+ // dryRunBuildStateNeedles: the eighth scope split, on the same
+ // pattern. The build-state report names hosts and builds.
+ if strings.HasPrefix(key, "status/boxes/") {
+ if leaks := ScanForLeaks(body, dryRunBuildStateNeedles); len(leaks) == 0 {
+ return fmt.Errorf("dry run: %s: expected build_states fixtures %v, found none. BuildStates not attached?", key, dryRunBuildStateNeedles)
+ }
+ buildStateKeySeen = true
+ } else if leaks := ScanForLeaks(body, dryRunBuildStateNeedles); len(leaks) > 0 {
+ return fmt.Errorf("dry run: %s leaked a build_states fixture %v. BuildStates must never reach the public feed", key, leaks)
+ }
+
// Page assets (index.html, status.css) aren't JSON and simply fail
// to decode here — silently skipped, since the band-digit check
// only means anything for objects that actually carry Load/Capacity
@@ -549,6 +579,9 @@ func DryRun(dir string, out io.Writer) error {
if !rolloutKeySeen {
return fmt.Errorf("dry run: no status/boxes/*.json object was scanned — rollout scope check never ran")
}
+ if !buildStateKeySeen {
+ return fmt.Errorf("dry run: no status/boxes/*.json object was scanned. The build_states scope check never ran")
+ }
fmt.Fprintln(out, "dry-run: no leaks")
return nil
diff --git a/internal/status/snapshot.go b/internal/status/snapshot.go
index 424863e9b..487e36553 100644
--- a/internal/status/snapshot.go
+++ b/internal/status/snapshot.go
@@ -178,6 +178,11 @@ type Probes struct {
// ConfigGuard is "" in the normal mode and a words-only note when the
// config guard's explicit bypass is in effect on this box (amadan #213).
ConfigGuard string
+ // BuildStates is the build-state builds of the instances this box hosts
+ // (settings spec rev 11 §4.4), and BuildStatesErr says the box could not
+ // collect them. Private Metrics only: they name hosts.
+ BuildStates map[string]BuildStatePins
+ BuildStatesErr string
}
// RowLayout is one instance-kind route's layout, as the box's layout pass
@@ -449,6 +454,17 @@ type BoxMetrics struct {
// binary that ignores them; the console refuses those settings until
// every box reports UpdateSettingsProtocol.
UpdateSettings int `json:"update_settings,omitempty"`
+ // BuildStates is, per instance this box hosts that keeps a build state,
+ // the builds that build state names: last_executed and a resolved
+ // request's target (settings spec rev 11 §4.4). The housekeeper protects
+ // them, because it cannot read the regional data buckets the objects
+ // live in. Host-keyed and private like RouteVersions; omitted when empty.
+ BuildStates map[string]BuildStatePins `json:"build_states,omitempty"`
+ // BuildStatesErr is set when this box could not collect BuildStates at
+ // all (its registry list failed). The housekeeper prunes nothing while a
+ // fresh report says so (Astra I2 #5): an absent map is never "nothing to
+ // protect".
+ BuildStatesErr string `json:"build_states_err,omitempty"`
// RouteVersions is host -> registry.Route.Version for every non-platform
// route this box carries with a Version the adopt pass has stamped
// (issue #78's staged_version chip) — a hostname-keyed map, which is
@@ -535,6 +551,17 @@ type BoxMetrics struct {
// once it applies update settings and protects the builds they name.
const UpdateSettingsProtocol = 1
+// BuildStatePins is one instance's build-state builds in a box report.
+// Complete is false when this box could not read the instance's build state
+// (never read, a failed refresh, an unparseable copy): the housekeeper then
+// prunes nothing for the app (Astra I1 #6).
+type BuildStatePins struct {
+ Account string `json:"account"`
+ App string `json:"app"`
+ Builds []string `json:"builds"`
+ Complete bool `json:"complete"`
+}
+
// AliasReading is one alias hostname's status as CollectBox saw it this
// tick: Route is the route it points at (registry.RouteAlias.Host), DNSStatus
// and CertExpiresAt are the box-stamped health columns (Task 2/5), carried
@@ -926,7 +953,7 @@ func CollectBox(routes []registry.Route, p Probes, now time.Time) Box {
p.InstanceID != "" || p.Platform != "" || p.SystemChannel != "" || p.System != nil ||
len(routeVersions) > 0 || len(aliasReadings) > 0 || len(p.Upstreams) > 0 ||
len(routeDBBytes) > 0 || p.Layout != nil || len(p.ConfigRefusals) > 0 || len(p.ReleaseRefusals) > 0 || len(p.ProofWaits) > 0 || p.Rollout != nil || p.ConfigGuard != "" ||
- p.Alerts != nil || p.GeoIP != nil {
+ p.Alerts != nil || p.GeoIP != nil || len(p.BuildStates) > 0 || p.BuildStatesErr != "" {
box.Metrics = &BoxMetrics{
ConsoleProcesses: p.ConsoleProcesses,
ActivationID: p.ActivationID, ActivationComplete: p.ActivationComplete,
@@ -963,6 +990,9 @@ func CollectBox(routes []registry.Route, p Probes, now time.Time) Box {
ProofWaits: append([]ProofWait(nil), p.ProofWaits...),
Rollout: p.Rollout,
ConfigGuard: p.ConfigGuard,
+ // The caller builds a fresh map every tick.
+ BuildStates: p.BuildStates,
+ BuildStatesErr: p.BuildStatesErr,
}
}
diff --git a/internal/status/status_test.go b/internal/status/status_test.go
index 57c0d07eb..2801921fa 100644
--- a/internal/status/status_test.go
+++ b/internal/status/status_test.go
@@ -1615,3 +1615,63 @@ func TestCollectBoxCarriesTheConfigGuardBypass(t *testing.T) {
t.Errorf("a box in the normal mode attached metrics: %+v", quiet.Metrics)
}
}
+
+// TestCollectBoxCarriesBuildStates (settings spec rev 11 §4.4): a box names
+// the build-state builds of the instances it hosts in its private Metrics
+// block, so the housekeeper can protect them. The hostnames and builds never
+// reach the public fleet.json, and a box with none carries no key at all.
+func TestCollectBoxCarriesBuildStates(t *testing.T) {
+ pins := map[string]status.BuildStatePins{
+ "held-secret.example.com": {Account: "bab", App: "km", Builds: []string{"v1-secret-build", "v2-secret-build"}, Complete: true},
+ }
+ routes := []registry.Route{{Host: "held-secret.example.com", Kind: "instance", Hibernate: true}}
+ b := status.CollectBox(routes, status.Probes{MachineID: "m", BuildStates: pins}, time.Now())
+ if b.Metrics == nil {
+ t.Fatal("metrics block missing: build states alone should attach it")
+ }
+ if got := b.Metrics.BuildStates; !reflect.DeepEqual(got, pins) {
+ t.Fatalf("build_states = %+v, want %+v", got, pins)
+ }
+
+ s := teststore(t)
+ ctx := context.Background()
+ if err := status.WriteBox(ctx, s, b); err != nil {
+ t.Fatal(err)
+ }
+ boxes, complete, err := status.LoadBoxesComplete(ctx, s)
+ if err != nil || !complete || len(boxes) != 1 || boxes[0].Metrics == nil || !reflect.DeepEqual(boxes[0].Metrics.BuildStates, pins) {
+ t.Fatalf("read back %+v, %v, %v; want the build states", boxes, complete, err)
+ }
+ f, err := status.BuildFleet(ctx, s, routes, time.Now(), 5*time.Minute)
+ if err != nil {
+ t.Fatal(err)
+ }
+ fj, err := json.Marshal(f)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, needle := range []string{"held-secret.example.com", "v1-secret-build", "build_states"} {
+ if strings.Contains(string(fj), needle) {
+ t.Errorf("fleet.json leaked %q: %s", needle, fj)
+ }
+ }
+
+ raw, _ := json.Marshal(status.CollectBox(routes, status.Probes{MachineID: "m", MemTotalBytes: new(int64)}, time.Now()))
+ if strings.Contains(string(raw), "build_states") {
+ t.Errorf("a box with no build states publishes build_states: %s", raw)
+ }
+}
+
+// A box that could not collect its build states says so (Astra I2 #5): the
+// error alone attaches the Metrics block, so the housekeeper reads "could not
+// collect", never an absent map it might take for "nothing to protect".
+func TestCollectBoxCarriesABuildStatesFailure(t *testing.T) {
+ b := status.CollectBox(nil, status.Probes{MachineID: "m", BuildStatesErr: "listing routes for build states: database is locked"}, time.Now())
+ if b.Metrics == nil || b.Metrics.BuildStatesErr != "listing routes for build states: database is locked" {
+ t.Fatalf("metrics = %+v, want the collection failure", b.Metrics)
+ }
+ raw, _ := json.Marshal(b)
+ if !strings.Contains(string(raw), `"build_states_err"`) {
+ t.Fatalf("the failure is not published: %s", raw)
+ }
+}