| 1 | // Ambient release retention, the box half (issue #136, spec |
| 2 | // 2026-08-10-releases-retention-design.md §6): for each app this box has |
| 3 | // routes for, read the app's retention record from the bucket and delete |
| 4 | // what release.PruneEligible says is safe — protecting, on top of the |
| 5 | // bucket-derived safety list, the versions THIS box's registry still |
| 6 | // records as adopted, previous or good (the fallback). Boxes race each other harmlessly: every |
| 7 | // delete is idempotent by store contract. |
| 8 | // |
| 9 | // Hosted-apps-only is a ruling, not an accident: an app with routes on no |
| 10 | // box has nothing at stake, and whole-app trash→purge owns its end of life. |
| 11 | package host |
| 12 | |
| 13 | import ( |
| 14 | "context" |
| 15 | "crypto/ed25519" |
| 16 | "errors" |
| 17 | "fmt" |
| 18 | "log" |
| 19 | "strings" |
| 20 | "time" |
| 21 | |
| 22 | "github.com/carlosframework/platform/internal/provision" |
| 23 | "github.com/carlosframework/platform/internal/registry" |
| 24 | "github.com/carlosframework/platform/internal/release" |
| 25 | "github.com/carlosframework/platform/internal/status" |
| 26 | ) |
| 27 | |
| 28 | // retentionPass runs one enforcement sweep. Per-app isolation: one app's |
| 29 | // error is logged and skipped, never allowed to stop another app's |
| 30 | // enforcement; a partial delete leaves the remainder for the next tick. |
| 31 | // |
| 32 | // reports is where the box reports are read (settings spec rev 11 §4.4's |
| 33 | // build-state builds; retentionReports). A pass that cannot read them |
| 34 | // prunes nothing at all. |
| 35 | func retentionPass(ctx context.Context, store release.Store, reports *boxReports, pub ed25519.PublicKey, reg *registry.Registry) error { |
| 36 | routes, err := reg.List() |
| 37 | if err != nil { |
| 38 | return err |
| 39 | } |
| 40 | if reports == nil { |
| 41 | return errors.New("no box reports to read: pruning nothing") |
| 42 | } |
| 43 | boxes, complete, err := reports.load(ctx) |
| 44 | if err != nil { |
| 45 | return fmt.Errorf("reading the box reports: pruning nothing this pass: %w", err) |
| 46 | } |
| 47 | type appKey struct{ acct, app string } |
| 48 | pins := map[appKey]map[string]bool{} |
| 49 | for _, rt := range routes { |
| 50 | if rt.App == "" { |
| 51 | continue |
| 52 | } |
| 53 | k := appKey{release.RouteAccount(rt), rt.App} |
| 54 | if pins[k] == nil { |
| 55 | pins[k] = map[string]bool{} |
| 56 | } |
| 57 | if rt.Version != "" { |
| 58 | pins[k][rt.Version] = true |
| 59 | } |
| 60 | if rt.PrevVersion != "" { |
| 61 | pins[k][rt.PrevVersion] = true |
| 62 | } |
| 63 | // The good build is a planned wake's fallback (zero-downtime spec |
| 64 | // §8.5), and it can be neither Version nor PrevVersion once two |
| 65 | // releases have failed in a row. Pruned, the fallback has no artifact |
| 66 | // exactly when a failed release needs it (zdd-readiness Task 13, |
| 67 | // Astra R1 T12). Pinned whatever CARLOS_SWAP says, the plan's |
| 68 | // authorized flag-off exception: keeping one more version is the |
| 69 | // cheap side of the trade. |
| 70 | if rt.GoodVersion != "" { |
| 71 | pins[k][rt.GoodVersion] = true |
| 72 | } |
| 73 | } |
| 74 | for k, protected := range pins { |
| 75 | ret, err := release.GetRetention(ctx, store, k.acct, k.app) |
| 76 | if err != nil { |
| 77 | log.Printf("agent: retention %s/%s: %v", k.acct, k.app, err) |
| 78 | continue |
| 79 | } |
| 80 | if ret == nil { |
| 81 | continue |
| 82 | } |
| 83 | extra, err := settingsProtected(ctx, store, boxes, complete, time.Now(), k.acct, k.app) |
| 84 | if err != nil { |
| 85 | log.Printf("agent: retention %s/%s: pruning nothing this pass: %v", k.acct, k.app, err) |
| 86 | continue |
| 87 | } |
| 88 | for v := range extra { |
| 89 | protected[v] = true |
| 90 | } |
| 91 | eligible, err := release.PruneEligible(ctx, store, pub, k.acct, k.app, ret.Keep, time.Now(), protected) |
| 92 | if err != nil { |
| 93 | log.Printf("agent: retention %s/%s: %v", k.acct, k.app, err) |
| 94 | continue |
| 95 | } |
| 96 | if len(eligible) == 0 { |
| 97 | continue |
| 98 | } |
| 99 | deleted := 0 |
| 100 | for _, v := range eligible { |
| 101 | if err := release.DeleteVersion(ctx, store, k.acct, k.app, v); err != nil { |
| 102 | log.Printf("agent: retention %s/%s: delete %s: %v", k.acct, k.app, v, err) |
| 103 | break |
| 104 | } |
| 105 | deleted++ |
| 106 | } |
| 107 | log.Printf("agent: retention: pruned %d of %d eligible versions for %s/%s (keep %d)", |
| 108 | deleted, len(eligible), k.acct, k.app, ret.Keep) |
| 109 | } |
| 110 | return nil |
| 111 | } |
| 112 | |
| 113 | // buildStatePins is this box's contribution to retention (settings spec rev |
| 114 | // 11 §4.4): the build-state builds of every row keepsBuildState selects (the |
| 115 | // rows refreshAll keeps a copy of), from the registry copies the agent |
| 116 | // refreshes at reconcile and before every wake. claims is whether host |
| 117 | // claims are on, as keepsBuildState takes it. Every such row is reported, |
| 118 | // with or without builds, so a missing or unreadable copy shows as |
| 119 | // incomplete rather than as nothing to protect. A failed list is an error: |
| 120 | // the caller reports it (BoxMetrics.BuildStatesErr), never an empty map. |
| 121 | func buildStatePins(reg routeLister, claims bool) (map[string]status.BuildStatePins, error) { |
| 122 | routes, err := reg.List() |
| 123 | if err != nil { |
| 124 | return nil, fmt.Errorf("listing routes for build states: %w", err) |
| 125 | } |
| 126 | out := map[string]status.BuildStatePins{} |
| 127 | for _, rt := range routes { |
| 128 | if !keepsBuildState(rt, claims) { |
| 129 | continue |
| 130 | } |
| 131 | p := status.BuildStatePins{Account: release.RouteAccount(rt), App: rt.App, Builds: []string{}} |
| 132 | c, err := release.ParseBuildStateCopy(rt.BuildState) |
| 133 | p.Complete = err == nil && c.Known && c.ReadErr == "" |
| 134 | if err == nil { |
| 135 | p.Builds = buildStateBuilds(c.State) |
| 136 | } |
| 137 | out[rt.Host] = p |
| 138 | } |
| 139 | if len(out) == 0 { |
| 140 | return nil, nil |
| 141 | } |
| 142 | return out, nil |
| 143 | } |
| 144 | |
| 145 | // buildStateBuilds is the builds a build state names: last_executed, and a |
| 146 | // resolved request's target when it differs. A failed or completed request's |
| 147 | // T is named too: it may still be what the instance runs or recovers onto. |
| 148 | func buildStateBuilds(st *release.BuildState) []string { |
| 149 | builds := []string{} |
| 150 | if st == nil { |
| 151 | return builds |
| 152 | } |
| 153 | if st.LastExecuted != "" { |
| 154 | builds = append(builds, st.LastExecuted) |
| 155 | } |
| 156 | if r := st.Resolved; r != nil && r.Target != "" && r.Target != st.LastExecuted { |
| 157 | builds = append(builds, r.Target) |
| 158 | } |
| 159 | return builds |
| 160 | } |
| 161 | |
| 162 | // buildStateReportMaxAge is §4.4's bound: an app is not pruned while any box |
| 163 | // hosting one of its instances has been silent longer than this. |
| 164 | const buildStateReportMaxAge = 6 * time.Hour |
| 165 | |
| 166 | // boxReports is where a retention pass reads the box reports (settings spec |
| 167 | // rev 11 §4.4). At least one of status and local is set. |
| 168 | type boxReports struct { |
| 169 | // status is the store the boxes publish to (openStatusStore's bucket); |
| 170 | // nil for a sole housekeeper with none. |
| 171 | status release.Store |
| 172 | // prefix rebases the platform's own reports in status, as |
| 173 | // CARLOS_STATUS_PREFIX rebases what the status loop writes |
| 174 | // (resolveStatusPrefix). Fleet reports are not rebased: a fleet box |
| 175 | // writes at fleets/<fleet>/ from the bucket root. |
| 176 | prefix string |
| 177 | // deploy is the deployment store, read for fleets/ as well (ruling |
| 178 | // R-T9-FLEETS): a fleet box with no bucket publishes through the store |
| 179 | // door, which serves the deployment store. nil when it is the status |
| 180 | // store's own bucket. |
| 181 | deploy release.Store |
| 182 | // local is this box's own collector, read as a fresh report at every |
| 183 | // pass: set for a sole housekeeper (a directory-backed deployment, one |
| 184 | // box by construction, which publishes no status; ruling R-T9-SOLE). |
| 185 | local func() (map[string]status.BuildStatePins, error) |
| 186 | // now stamps the local report; nil is time.Now. |
| 187 | now func() time.Time |
| 188 | } |
| 189 | |
| 190 | // soleReportLabel names a sole housekeeper's own report in refusals. |
| 191 | const soleReportLabel = "this box (sole housekeeper)" |
| 192 | |
| 193 | // retentionReports is retentionLoop's report source, from the environment: |
| 194 | // the status store (rebased by CARLOS_STATUS_PREFIX) and, for fleets, the |
| 195 | // deployment store too when it is another bucket; or, on a sole |
| 196 | // housekeeper with no status store, this box's own collector and the |
| 197 | // deployment store's fleets. Anything else is an error: no app can be |
| 198 | // pruned without the reports. |
| 199 | func retentionReports(ctx context.Context, getenv func(string) string, deploy release.Store, local func() (map[string]status.BuildStatePins, error)) (*boxReports, error) { |
| 200 | st, _, err := openStatusStore(ctx, getenv) |
| 201 | switch { |
| 202 | case err == nil: |
| 203 | r := &boxReports{status: st, prefix: resolveStatusPrefix(getenv), deploy: deploy} |
| 204 | if b := getenv("CARLOS_DEPLOYMENT_BUCKET"); b != "" && b == resolveStatusBucket(getenv) { |
| 205 | r.deploy = nil // the same bucket: its fleets/ is read once |
| 206 | } |
| 207 | return r, nil |
| 208 | case errors.Is(err, errNoStatusStore) && newHousekeeper(getenv, nil).mode == housekeeperSole && local != nil: |
| 209 | return &boxReports{deploy: deploy, local: local}, nil |
| 210 | } |
| 211 | return nil, err |
| 212 | } |
| 213 | |
| 214 | // load reads every box report: the platform's (status/boxes/ under prefix, |
| 215 | // or this box's own on a sole housekeeper), and every fleet's in the status |
| 216 | // store and in the deployment store, the union (a report in both is read |
| 217 | // twice; the newest-per-host rule is indifferent to that). complete is |
| 218 | // false if any report or fleets listing could not be read. Failing to list |
| 219 | // the platform's own reports is an error. |
| 220 | func (r *boxReports) load(ctx context.Context) ([]status.Box, bool, error) { |
| 221 | if r == nil || (r.status == nil && r.local == nil) { |
| 222 | return nil, false, errors.New("no box reports to read") |
| 223 | } |
| 224 | var boxes []status.Box |
| 225 | complete := true |
| 226 | if r.status != nil { |
| 227 | root, ok, err := status.LoadBoxesComplete(ctx, release.NewPrefixStore(r.status, r.prefix)) |
| 228 | if err != nil { |
| 229 | return nil, false, err |
| 230 | } |
| 231 | boxes, complete = root, ok |
| 232 | } |
| 233 | if r.local != nil { |
| 234 | boxes = append(boxes, r.localReport()) |
| 235 | } |
| 236 | for _, s := range []release.Store{r.status, r.deploy} { |
| 237 | if s == nil { |
| 238 | continue |
| 239 | } |
| 240 | fb, ok := loadFleetReports(ctx, s) |
| 241 | complete = complete && ok |
| 242 | boxes = append(boxes, fb...) |
| 243 | } |
| 244 | return boxes, complete, nil |
| 245 | } |
| 246 | |
| 247 | // localReport is this box's own report, taken now, as the status loop |
| 248 | // would publish it: BuildStatesErr when the collector fails. |
| 249 | func (r *boxReports) localReport() status.Box { |
| 250 | now := time.Now |
| 251 | if r.now != nil { |
| 252 | now = r.now |
| 253 | } |
| 254 | m := &status.BoxMetrics{} |
| 255 | if pins, err := r.local(); err != nil { |
| 256 | m.BuildStatesErr = err.Error() |
| 257 | } else { |
| 258 | m.BuildStates = pins |
| 259 | } |
| 260 | return status.Box{Label: soleReportLabel, GeneratedAt: now(), Metrics: m} |
| 261 | } |
| 262 | |
| 263 | // loadFleetReports reads every fleet's reports in store |
| 264 | // (fleets/<name>/status/boxes/, enumerated with ListDirs). ok is false when |
| 265 | // the listing or any report could not be read. |
| 266 | func loadFleetReports(ctx context.Context, store release.Store) ([]status.Box, bool) { |
| 267 | fleets, err := store.ListDirs(ctx, "fleets/") |
| 268 | if err != nil { |
| 269 | log.Printf("agent: retention: listing fleets: %v", err) |
| 270 | return nil, false |
| 271 | } |
| 272 | var boxes []status.Box |
| 273 | ok := true |
| 274 | for _, f := range fleets { |
| 275 | fb, complete, err := status.LoadBoxesComplete(ctx, release.NewPrefixStore(store, f)) |
| 276 | if err != nil { |
| 277 | log.Printf("agent: retention: reading the reports of %s: %v", f, err) |
| 278 | ok = false |
| 279 | continue |
| 280 | } |
| 281 | ok = ok && complete |
| 282 | boxes = append(boxes, fb...) |
| 283 | } |
| 284 | return boxes, ok |
| 285 | } |
| 286 | |
| 287 | // appRecords reads every provision record of acct/app, or fails. It lists |
| 288 | // the keys itself (provision.ListRecords caps on max and skips a record it |
| 289 | // cannot read, ruling R-T9-STORE): a key that is a record by ListRecords' |
| 290 | // own rule (instances/<host>.json, one level) and does not read, parse or |
| 291 | // agree with its path is an error, so the caller prunes nothing. |
| 292 | func appRecords(ctx context.Context, store release.Store, acct, app string) ([]provision.Record, error) { |
| 293 | prefix := provision.RecordsPrefix(acct, app) |
| 294 | keys, err := store.List(ctx, prefix) |
| 295 | if err != nil { |
| 296 | return nil, fmt.Errorf("listing the records of %s/%s: %w", acct, app, err) |
| 297 | } |
| 298 | var recs []provision.Record |
| 299 | for _, key := range keys { |
| 300 | rest, ok := strings.CutPrefix(key, prefix) |
| 301 | if !ok || rest == "" || strings.Contains(rest, "/") { |
| 302 | continue |
| 303 | } |
| 304 | host, ok := strings.CutSuffix(rest, ".json") |
| 305 | if !ok || host == "" { |
| 306 | continue |
| 307 | } |
| 308 | rec, err := provision.GetRecord(ctx, store, acct, app, host) |
| 309 | if err != nil { |
| 310 | return nil, fmt.Errorf("reading the record of %s: %w", host, err) |
| 311 | } |
| 312 | recs = append(recs, *rec) |
| 313 | } |
| 314 | return recs, nil |
| 315 | } |
| 316 | |
| 317 | // settingsProtected is settings spec rev 11 §4.4's addition to an app's |
| 318 | // protected set: every pin a provision record names, and every build any |
| 319 | // box report names for one of the app's instances. It refuses (the caller |
| 320 | // prunes nothing for the app) when a record is unreadable, when the reports |
| 321 | // could not all be read, when a fresh report says its box could not collect |
| 322 | // its build states, when a provisioned instance that keeps a build state is |
| 323 | // named by no report, or when the newest report naming one of the app's |
| 324 | // hosts is older than six hours or says that host's build state is unread. |
| 325 | // |
| 326 | // The newest report for a host decides freshness and completeness, so a box |
| 327 | // the instance moved off does not block pruning forever. The builds of |
| 328 | // every report naming the host are protected, older ones included: that can |
| 329 | // only keep more. |
| 330 | func settingsProtected(ctx context.Context, store release.Store, boxes []status.Box, complete bool, now time.Time, acct, app string) (map[string]bool, error) { |
| 331 | recs, err := appRecords(ctx, store, acct, app) |
| 332 | if err != nil { |
| 333 | return nil, err |
| 334 | } |
| 335 | if !complete { |
| 336 | return nil, errors.New("the box reports could not all be read") |
| 337 | } |
| 338 | out := map[string]bool{} |
| 339 | for _, r := range recs { |
| 340 | if r.Pin != "" { |
| 341 | out[r.Pin] = true |
| 342 | } |
| 343 | } |
| 344 | type newest struct { |
| 345 | at time.Time |
| 346 | pins status.BuildStatePins |
| 347 | box string |
| 348 | } |
| 349 | byHost := map[string]newest{} |
| 350 | for _, b := range boxes { |
| 351 | if b.Metrics == nil { |
| 352 | continue |
| 353 | } |
| 354 | // A box that could not collect its build states may host any of |
| 355 | // this app's instances (Astra I2 #5). |
| 356 | if b.Metrics.BuildStatesErr != "" && now.Sub(b.GeneratedAt) <= buildStateReportMaxAge { |
| 357 | return nil, fmt.Errorf("%s could not collect its build states: %s", b.Label, b.Metrics.BuildStatesErr) |
| 358 | } |
| 359 | for host, p := range b.Metrics.BuildStates { |
| 360 | if p.Account != acct || p.App != app { |
| 361 | continue |
| 362 | } |
| 363 | for _, v := range p.Builds { |
| 364 | if v != "" { |
| 365 | out[v] = true |
| 366 | } |
| 367 | } |
| 368 | if cur, ok := byHost[host]; !ok || b.GeneratedAt.After(cur.at) { |
| 369 | byHost[host] = newest{at: b.GeneratedAt, pins: p, box: b.Label} |
| 370 | } |
| 371 | } |
| 372 | } |
| 373 | // Every provisioned instance of the app that keeps a build state must |
| 374 | // be named by some report (Astra I2 #5): a missing entry is unknown, |
| 375 | // never "nothing to protect". For a valid record this is |
| 376 | // keepsBuildState's instance rule (activator.ActivatorManaged): an |
| 377 | // instance with no unit that hibernates has no addr (the record door |
| 378 | // refuses addr with hibernate), so reconcile mints it a socket. |
| 379 | for _, r := range recs { |
| 380 | if r.KindOrDefault() != "instance" || r.Unit != "" || !r.HibernateOrDefault() { |
| 381 | continue |
| 382 | } |
| 383 | if _, ok := byHost[r.Host]; !ok { |
| 384 | return nil, fmt.Errorf("no box report names the build state of %s", r.Host) |
| 385 | } |
| 386 | } |
| 387 | for host, n := range byHost { |
| 388 | if now.Sub(n.at) > buildStateReportMaxAge { |
| 389 | return nil, fmt.Errorf("%s, which hosts %s, has not reported since %s", n.box, host, n.at.UTC().Format(time.RFC3339)) |
| 390 | } |
| 391 | if !n.pins.Complete { |
| 392 | return nil, fmt.Errorf("%s could not read the build state of %s", n.box, host) |
| 393 | } |
| 394 | } |
| 395 | return out, nil |
| 396 | } |
| 397 | |
amadan