| 1 | // Adoption — one check-and-apply sweep over the registry, shaped after |
| 2 | // Eleven updater.go's updatePass, simplified: channel pointers replace the |
| 3 | // GitHub feed, the Store replaces HTTP downloads, and per-host update prefs |
| 4 | // are the registry row itself (Channel, Hibernate). |
| 5 | package release |
| 6 | |
| 7 | import ( |
| 8 | "bytes" |
| 9 | "context" |
| 10 | "crypto/ed25519" |
| 11 | "errors" |
| 12 | "fmt" |
| 13 | "io" |
| 14 | "os" |
| 15 | "runtime" |
| 16 | "slices" |
| 17 | "strings" |
| 18 | "syscall" |
| 19 | "time" |
| 20 | |
| 21 | "github.com/carlosframework/platform/internal/caps" |
| 22 | "github.com/carlosframework/platform/internal/controlname" |
| 23 | "github.com/carlosframework/platform/internal/hostwrite" |
| 24 | "github.com/carlosframework/platform/internal/registry" |
| 25 | "github.com/carlosframework/platform/internal/safefile" |
| 26 | "github.com/carlosframework/platform/internal/updatewords" |
| 27 | ) |
| 28 | |
| 29 | // adoptPlatform picks which manifest artifact a box installs — a var so |
| 30 | // tests can exercise resolution without faking runtime.GOOS/GOARCH. |
| 31 | var adoptPlatform = runtime.GOOS + "-" + runtime.GOARCH |
| 32 | |
| 33 | // PassOption configures one call to Pass or PassFilter beyond their required |
| 34 | // arguments — NagExpiredCanaries and WithCapabilities today. The type exists |
| 35 | // so a call site opts in explicitly and a future option composes the same |
| 36 | // way, rather than every option becoming its own trailing parameter. |
| 37 | type PassOption func(*passConfig) |
| 38 | |
| 39 | type passConfig struct { |
| 40 | openRootUpdates RootUpdateOpener |
| 41 | rootUpdates *RootUpdateInputs |
| 42 | legacyRecovery func() (bool, error) |
| 43 | recovery func(context.Context, registry.Route) (bool, error) |
| 44 | rootHistory *RootHistoryPass |
| 45 | requireRootHistory bool |
| 46 | historySelected map[string]historySelection |
| 47 | runningVersion func(context.Context, registry.Route) (string, error) |
| 48 | propagateRestarts bool |
| 49 | restartAttemptSettled func(string) error |
| 50 | restartErrors *[]error |
| 51 | hostWrites hostwrite.Config |
| 52 | selectedRestart, selectedStart hostwrite.Callback |
| 53 | // selected is this pass's release work per adoptable host, captured once |
| 54 | // in PassFilter. D1's group rollback reuses a sibling's entry, so the |
| 55 | // sibling's rollback writes are checked against the same selection its |
| 56 | // forward adoption ran under, never a fresh one that could be a |
| 57 | // successor's. |
| 58 | selected map[string]hostwrite.Work |
| 59 | // ratchet is #253's per-box ratchet (WithDigestRatchet); nil admits |
| 60 | // every verified pointer, as before. |
| 61 | ratchet *DigestRatchet |
| 62 | // nag is where the canary-TTL nag (spec §5) is printed; nil (the |
| 63 | // default — every call site that does not pass NagExpiredCanaries) |
| 64 | // means the nag never fires at all, not even to a discarded writer. |
| 65 | nag io.Writer |
| 66 | |
| 67 | // capsOps arms the capability converge inside adoptHost; nil (the |
| 68 | // default — every call site that does not pass WithCapabilities, and |
| 69 | // every existing test) means converge never runs and this package's |
| 70 | // behaviour is byte-identical to before the capability design landed. |
| 71 | capsOps *caps.Ops |
| 72 | |
| 73 | // bakeDir, when non-empty, is the directory holding the bake clock |
| 74 | // (BakeDir option); empty keeps the legacy /opt-dir location. |
| 75 | bakeDir string |
| 76 | |
| 77 | // stageBlocked is called when staging fails because this process cannot |
| 78 | // write the version store (OnStageBlocked). nil — every existing caller |
| 79 | // and every existing test — means the pass reports such a failure exactly |
| 80 | // as it always has and nothing else happens. |
| 81 | stageBlocked func(host, ver string, err error) |
| 82 | |
| 83 | // restartHibernating is the callback hibernate-flagged routes get |
| 84 | // (RestartHibernating). nil — the default, and RunAdopt — keeps the |
| 85 | // original split: sleepers are repointed with restart=nil and no callback |
| 86 | // is consulted for them at all. |
| 87 | restartHibernating func(host string) error |
| 88 | |
| 89 | // unitsToRoot hands every route that names its own unit to the root pass |
| 90 | // before anything is staged or repointed (RootRestartsUnits). false, the |
| 91 | // default and RunAdopt's, restarts the named unit in this pass. |
| 92 | unitsToRoot bool |
| 93 | |
| 94 | // startUnit is consulted ONLY on adoptHost's restart-failure rollback |
| 95 | // path, after the symlink has gone back (WithStartUnit). nil — every |
| 96 | // call site that does not pass the option, and every existing test — |
| 97 | // keeps today's behaviour byte-identically: the rollback repoints and |
| 98 | // stops. |
| 99 | startUnit func(host string) error |
| 100 | |
| 101 | // rollbackRestart replaces startUnit on D1's path only — a named unit |
| 102 | // whose restart returned *UnitNotUpError. It must REPLACE whatever is |
| 103 | // running (reset-failed + restart), because a deadline failure can leave |
| 104 | // the rejected build running unready and `start` is a no-op on it. |
| 105 | rollbackRestart func(host string) error |
| 106 | |
| 107 | // proof is D3's gate (WithProofGate, proofgate.go). nil, the default for |
| 108 | // every existing caller and test, means no proof is read or required. |
| 109 | proof *ProofGate |
| 110 | |
| 111 | // pins is WithPinSource's provider: staged copies that something beyond |
| 112 | // the registry rows, the live links and the generations file still |
| 113 | // refers to. nil adds nothing. |
| 114 | pins PinSource |
| 115 | generationPins PinSource |
| 116 | |
| 117 | // stage is what ensureStagedChecked needs to judge a staged copy it is |
| 118 | // about to reuse (spec §4.2). PassFilter sets it once per pass. |
| 119 | stage stageCheck |
| 120 | |
| 121 | // settings arms the instance update settings (settings spec §4, |
| 122 | // WithUpdateSettings): adoptHost asks Target which build to install for |
| 123 | // each exec instance. resolve is root's alone: only root writes the |
| 124 | // consumed file. openFailed opens the zero-downtime failure book; |
| 125 | // PassFilter calls it once per pass into failed (or failedErr), so a |
| 126 | // long-lived option never reads a stale book. managed is |
| 127 | // rollout.Managed: an unreadable book holds those routes too. |
| 128 | settings bool |
| 129 | resolve bool |
| 130 | openFailed func() (FailedBuild, error) |
| 131 | managed func(registry.Route) bool |
| 132 | failed FailedBuild |
| 133 | failedErr error |
| 134 | // route reads one registry row now, for root's re-check under the carry |
| 135 | // fence (publishConsumed, ruling R-CARRY-FENCE); PassFilter sets it. |
| 136 | route func(host string) (*registry.Route, error) |
| 137 | // pub verifies promotion records (§4.1 evidence); PassFilter sets it. |
| 138 | pub ed25519.PublicKey |
| 139 | // tags is "<acct>/<app>" -> release version -> its newest semver tag, read |
| 140 | // once per pass for apps with a route under a major setting. |
| 141 | tags map[string]map[string]string |
| 142 | |
| 143 | // pointerGate is WithPointerGate's gate; nil (every caller but the |
| 144 | // agent's fast sweep) reads every group's pointer, as before. |
| 145 | pointerGate PointerGate |
| 146 | } |
| 147 | |
| 148 | // PointerGate lets a caller skip a channel pointer read that cannot have |
| 149 | // anything new (S3 change-index design §5). PassFilter asks it once per |
| 150 | // (account, app, channel) group, with the group's routes this pass would |
| 151 | // adopt. skip true leaves the whole group out of this pass: no pointer read |
| 152 | // and no adoption, exactly as if extra had refused its routes. Otherwise |
| 153 | // read, when non-nil, is called once with the outcome of the group's read: |
| 154 | // the verified pointer, or the read's error or the ratchet's refusal. |
| 155 | type PointerGate func(acct, app, channel string, routes []registry.Route) (skip bool, read func(ptr *ChannelPointer, err error)) |
| 156 | |
| 157 | // WithPointerGate arms g for this pass. The agent's fast sweep passes one |
| 158 | // that skips settled canary groups whose app scope has not moved. |
| 159 | func WithPointerGate(g PointerGate) PassOption { |
| 160 | return func(c *passConfig) { c.pointerGate = g } |
| 161 | } |
| 162 | |
| 163 | // gatePointers asks cfg.pointerGate about every group adopts admits, and |
| 164 | // returns adopts narrowed to the groups it did not skip, with each kept |
| 165 | // group's read callback. |
| 166 | func (cfg *passConfig) gatePointers(routes []registry.Route, adopts func(registry.Route) bool) (func(registry.Route) bool, map[[3]string]func(*ChannelPointer, error)) { |
| 167 | if cfg.pointerGate == nil { |
| 168 | return adopts, nil |
| 169 | } |
| 170 | groupOf := func(rt registry.Route) [3]string { return [3]string{RouteAccount(rt), rt.App, routeChannel(rt)} } |
| 171 | groups := map[[3]string][]registry.Route{} |
| 172 | var order [][3]string |
| 173 | for _, rt := range routes { |
| 174 | if !adopts(rt) { |
| 175 | continue |
| 176 | } |
| 177 | k := groupOf(rt) |
| 178 | if _, ok := groups[k]; !ok { |
| 179 | order = append(order, k) |
| 180 | } |
| 181 | groups[k] = append(groups[k], rt) |
| 182 | } |
| 183 | reads := map[[3]string]func(*ChannelPointer, error){} |
| 184 | skipped := map[[3]string]bool{} |
| 185 | for _, k := range order { |
| 186 | skip, read := cfg.pointerGate(k[0], k[1], k[2], groups[k]) |
| 187 | switch { |
| 188 | case skip: |
| 189 | skipped[k] = true |
| 190 | case read != nil: |
| 191 | reads[k] = read |
| 192 | } |
| 193 | } |
| 194 | if len(skipped) == 0 { |
| 195 | return adopts, reads |
| 196 | } |
| 197 | return func(rt registry.Route) bool { return adopts(rt) && !skipped[groupOf(rt)] }, reads |
| 198 | } |
| 199 | |
| 200 | // WithRunningVersion retains root's verified running build through a repoint. |
| 201 | // It must already be named by current or previous; PointLive checks that again. |
| 202 | func WithRunningVersion(running func(context.Context, registry.Route) (string, error)) PassOption { |
| 203 | return func(c *passConfig) { c.runningVersion = running } |
| 204 | } |
| 205 | |
| 206 | // FailedBuild reports whether host may not be moved onto build: the |
| 207 | // zero-downtime failure book (rollout.FailureBook), taken as a func because |
| 208 | // rollout imports this package. served is build == the route's GoodVersion. |
| 209 | type FailedBuild func(acct, app, build, host string, served bool) bool |
| 210 | |
| 211 | // ErrInstallRequestNeedsRoot is what an agent pass hands OnStageBlocked's |
| 212 | // callback when it stops for an install request only root may resolve |
| 213 | // (settings spec §4.3 rule 2; ruling R-AGENT-RELAY). Nothing failed: the |
| 214 | // callback only asks for root's pass, and the version is the request's T as |
| 215 | // this pass sees it. Never shown to members. |
| 216 | var ErrInstallRequestNeedsRoot = errors.New("release: an install request waits for root's pass to resolve it") |
| 217 | |
| 218 | // ErrUnitRestartNeedsRoot is what a RootRestartsUnits pass hands |
| 219 | // OnStageBlocked's callback for a route that names its own unit: nothing |
| 220 | // failed, and the version is the build root's pass should install. Never |
| 221 | // shown to members. |
| 222 | var ErrUnitRestartNeedsRoot = errors.New("release: a route's named unit is restarted by root's pass") |
| 223 | |
| 224 | // RootRestartsUnits is for a pass whose process cannot restart a unit an |
| 225 | // operator named (issue #315): the agent's in-process loops when they do not |
| 226 | // run as root. carlos-edge.service is User=carlos, and systemd refuses it |
| 227 | // ("Interactive authentication required") for console.service and every |
| 228 | // other route.Unit. Before this option such a pass repointed the live link, |
| 229 | // failed the restart, read the refusal as a broken build, rolled back, |
| 230 | // poisoned the version and tried to start the old build over a process that |
| 231 | // had never stopped. |
| 232 | // |
| 233 | // With it, a route that names its unit, and every route in that unit's |
| 234 | // process group (processGroup: same account, app, channel and upstream), is |
| 235 | // handed to root before anything is staged or repointed, through |
| 236 | // OnStageBlocked with ErrUnitRestartNeedsRoot, the way a build this process |
| 237 | // cannot stage is. The live links keep naming the running build, so no |
| 238 | // sibling is stamped with a build the unit does not run. Nothing is poisoned |
| 239 | // and no refusal is recorded; root's |
| 240 | // pass stages, repoints, restarts and stamps. Every hold, refusal and proof |
| 241 | // check before the hand-off still applies, so root is only asked for a build |
| 242 | // this pass would have installed. |
| 243 | // |
| 244 | // Lifecycle host writes (WithHostWrites) restart through the selected |
| 245 | // receiver instead, so the option does nothing under them. Nor for a route |
| 246 | // this pass would not restart (restart nil: a sleeper on a pass without |
| 247 | // RestartHibernating, a sidecar). |
| 248 | func RootRestartsUnits() PassOption { |
| 249 | return func(c *passConfig) { c.unitsToRoot = true } |
| 250 | } |
| 251 | |
| 252 | // NagExpiredCanaries opts a Pass/PassFilter call into spec §5's nag: one |
| 253 | // line per expired canary/* route the call sees, written to w. |
| 254 | // |
| 255 | // DO NOT pass this to a fast/frequent adoption loop. It belongs to the |
| 256 | // HOURLY ROOT adopt pass ALONE — carlos-adopt.service's RunAdopt |
| 257 | // (internal/host/agent.go), the only caller that sets it (2026-08-03 |
| 258 | // review), and every future caller must keep it that way. The same split |
| 259 | // PR #9's config sync uses, for the same reason: the in-process adoptLoop |
| 260 | // (hourly + jitter, inside the sandboxed carlos-edge.service) and |
| 261 | // fastConvergeLoop (every 2-3 SECONDS — the whole point of which is fast |
| 262 | // pickup of a freshly-promoted canary or a serial bump) must NEVER |
| 263 | // set this. An expired canary sitting still on that fast loop would |
| 264 | // otherwise nag every 2-3 seconds forever, turning a visibility feature |
| 265 | // into a journal-flooding one; gating it to the hourly root pass keeps the |
| 266 | // nag journal-friendly — once an hour, not a poll-storm of log lines. This |
| 267 | // is enforced by convention (which call sites pass the option), not by the |
| 268 | // type system — see agent.go's adoptLoop/fastPassOpts for the loud |
| 269 | // "never add this" comments at the two places it would be easiest to add |
| 270 | // by accident. |
| 271 | func NagExpiredCanaries(w io.Writer) PassOption { |
| 272 | return func(c *passConfig) { c.nag = w } |
| 273 | } |
| 274 | |
| 275 | // WithCapabilities arms the capability converge inside adoptHost (spec §4d): |
| 276 | // on a version change, a unit-backed route's drop-ins are materialized for |
| 277 | // the NEW version before its live symlink moves, and restored with the |
| 278 | // symlink when the restart fails. Unset — every existing caller and every |
| 279 | // existing test — means converge never runs and this package behaves exactly |
| 280 | // as it did before. |
| 281 | // |
| 282 | // The zero Ops is the production one: real /etc/systemd/system, real |
| 283 | // /var/lib/carlos-app, real systemctl, real useradd. A test injects temp |
| 284 | // dirs and recorders through the same value. |
| 285 | // |
| 286 | // DO NOT pass this to a fast/frequent adoption loop, and DO NOT pass it from |
| 287 | // anything running inside carlos-edge.service. It belongs to the HOURLY ROOT |
| 288 | // adopt pass ALONE — carlos-adopt.service's RunAdopt (internal/host/agent.go), |
| 289 | // the only caller that sets it — for a harder reason than the nag's: this |
| 290 | // option WRITES /etc/systemd/system, and carlos-edge.service is |
| 291 | // ProtectSystem=strict with ReadWritePaths=/data /run/carlos, so the |
| 292 | // in-process adoptLoop and fastConvergeLoop physically cannot. Arming it |
| 293 | // there would not converge anything; it would log a permission denial every |
| 294 | // 2-3 seconds forever. Enforced by convention (which call sites pass the |
| 295 | // option), not by the type system — see agent.go's adoptLoop/fastPassOpts |
| 296 | // for the loud "never add this" comments at the two places it would be |
| 297 | // easiest to add by accident. |
| 298 | // |
| 299 | // The consequence, ruled on and accepted (2026-08-03): a freshly promoted |
| 300 | // canary picked up by the fast loop runs the new binary against the unit |
| 301 | // shape the PREVIOUS version left, until the hourly root pass converges it. |
| 302 | // adoptHost prints one line per such repoint so the window is visible — |
| 303 | // see the capsOps == nil branch there. |
| 304 | func WithCapabilities(o caps.Ops) PassOption { |
| 305 | return func(c *passConfig) { c.capsOps = &o } |
| 306 | } |
| 307 | |
| 308 | // BakeDir moves this pass's bake clock into dir — in production the data |
| 309 | // dir (CARLOS_DATA_DIR, default /data), the one place both the sandboxed |
| 310 | // edge unit (ReadWritePaths=/data) and the root adopt one-shot can write. |
| 311 | // Reads migrate: a box whose clock still lives at the legacy |
| 312 | // /opt/carlos/channel-first-seen.json keeps every observation (issue #13). |
| 313 | // Unset, the clock stays at its legacy path — existing tests and callers |
| 314 | // are unaffected. |
| 315 | // |
| 316 | // The migration is durable after the FIRST pass this option is set on a box |
| 317 | // still on the legacy path: PassFilter forces a Save even when no new |
| 318 | // channel head was observed, so the new path is fully populated (and the |
| 319 | // legacy file provably safe to delete) one pass later, not whenever the |
| 320 | // fleet next happens to see a new head. |
| 321 | func BakeDir(dir string) PassOption { |
| 322 | return func(c *passConfig) { c.bakeDir = dir } |
| 323 | } |
| 324 | |
| 325 | // OnStageBlocked reports a version this pass could not stage or point live |
| 326 | // because the process cannot WRITE the install directory — never because |
| 327 | // the bucket, the signature or the manifest was bad. It is the release half |
| 328 | // of the relay carlos-adopt.path already runs for config: the agent's |
| 329 | // in-process loops are |
| 330 | // sandboxed (carlos-edge.service sets ProtectSystem=strict with no /opt write), |
| 331 | // so they can see a promotion within seconds and stage nothing, while the only |
| 332 | // process that CAN stage runs hourly off carlos-adopt.timer. Without a relay a |
| 333 | // promote is invisible for up to an hour on a box that noticed it immediately. |
| 334 | // |
| 335 | // Deliberately narrow. A read failure, a bad signature or a poisoned version |
| 336 | // must NOT fire this: none of them is fixed by re-running as root, and a |
| 337 | // callback that fired on any staging error would relay a bucket outage into a |
| 338 | // root pass every tick. Only EROFS/EPERM/EACCES — "this uid cannot write |
| 339 | // there" — describe a job a privileged pass would actually finish. |
| 340 | // |
| 341 | // Never fatal, and never a substitute for the timer: the hourly pass remains |
| 342 | // the floor, exactly as it is for config. A dropped or throttled request costs |
| 343 | // latency, not correctness. |
| 344 | func OnStageBlocked(fn func(host, ver string, err error)) PassOption { |
| 345 | return func(c *passConfig) { c.stageBlocked = fn } |
| 346 | } |
| 347 | |
| 348 | // RestartHibernating extends restart to hibernate-flagged routes, which |
| 349 | // otherwise get the pure repoint (restart=nil) and are never offered to the |
| 350 | // callback at all. |
| 351 | // |
| 352 | // It exists because the flag is not the question the restart decision |
| 353 | // actually wants answered. Hibernate=true means "this route MAY sleep", not |
| 354 | // "it is asleep right now" — an exec instance is minted hibernating |
| 355 | // (provision/reconcile.go) and then spends every serving minute awake. With |
| 356 | // the flag alone deciding, a busy exec child kept serving the binary its |
| 357 | // live symlink no longer names until it happened to go idle. Handing the |
| 358 | // callback the sleepers lets it answer the real question at the moment it |
| 359 | // matters. |
| 360 | // |
| 361 | // This does NOT weaken "an adopt pass never wakes a sleeping host" — it moves |
| 362 | // the guarantee from withholding the callback to the callback's own |
| 363 | // semantics, and only a callback with those semantics may be passed here. |
| 364 | // The agent's makeRestartRoute qualifies on both halves: activator Bounce |
| 365 | // returns false without starting anything when the host is not awake, and |
| 366 | // `systemctl try-restart` is defined as a no-op on a unit that is not |
| 367 | // running. Nothing on either path can start a process. A callback that could |
| 368 | // start one belongs on the awake loop only. (WithStartUnit narrows that |
| 369 | // invariant without breaking it: the ONE start this package can now perform |
| 370 | // fires only after this callback RETURNED AN ERROR — and no stopped unit's |
| 371 | // try-restart produces one, so a sleeper cannot arm it. See WithStartUnit.) |
| 372 | // |
| 373 | // DO NOT pass this from RunAdopt (carlos-adopt.service). That process is not |
| 374 | // the activator's parent: the exec children belong to the agent, so its only |
| 375 | // available restart is systemctl, which cannot reach them — offering it |
| 376 | // sleepers would buy nothing and cost a try-restart per sleeping host per |
| 377 | // version change. Set by the agent's in-process adoptLoop/fastConvergeLoop |
| 378 | // alone, the two loops that run inside the process owning the children. |
| 379 | // |
| 380 | // Sleepers are still adopted AFTER every awake host and still poison a bad |
| 381 | // build the same way, both unchanged: the option decides who the callback |
| 382 | // sees, never the order or the health gate. |
| 383 | func RestartHibernating(restart func(host string) error) PassOption { |
| 384 | return func(c *passConfig) { c.restartHibernating = restart } |
| 385 | } |
| 386 | |
| 387 | // WithStartUnit gives the pass a way to START a route's process after a |
| 388 | // FAILED restart has been rolled back — the one moment adoption knows the |
| 389 | // symlink is correct and the service behind it is down. Without it the |
| 390 | // rollback repoints the symlink and stops there: the unit systemd left |
| 391 | // failed stays failed, and because `systemctl try-restart` is by definition |
| 392 | // a no-op on a unit that is not running (the same property that lets a |
| 393 | // deliberately stopped service stay stopped), the NEXT pass's restart exits |
| 394 | // 0 and SetVersions stamps a build that has never run — the exact lying |
| 395 | // header the named-unit arm's returned error exists to prevent, now over a |
| 396 | // service that is not running at all. |
| 397 | // |
| 398 | // start fires ONLY on that rollback path, and only when the rollback |
| 399 | // actually HAPPENED: a first adoption whose restart fails has no previous |
| 400 | // version, and a rollback PointLive itself refused (the old version's |
| 401 | // directory gone — materialize.go's "link present, staged version gone" |
| 402 | // state) rolled nothing — in both the symlink still names the refused |
| 403 | // build, and starting there would run the exact build the failure refused. |
| 404 | // A start that also fails changes nothing about the refusal (the version |
| 405 | // stays poisoned and unstamped) but is printed, because at that point the |
| 406 | // box is most likely serving nothing and an operator must not read the |
| 407 | // rollback line as "recovered". |
| 408 | // |
| 409 | // This cannot WAKE a hibernating route, and the reasoning is the contract: |
| 410 | // the path is reached only after a restart that was ATTEMPTED and returned |
| 411 | // an error. Of the agent's dispatch arms (internal/host's makeRestartRoute) |
| 412 | // only the NAMED-unit arm returns one, and its try-restart exits 0 on a |
| 413 | // loaded-but-stopped unit — so an error means the unit was running and |
| 414 | // would not come back, or the name resolves to nothing (in which case the |
| 415 | // start reaches nothing either). The bounce and carlos-app@ arms swallow |
| 416 | // their failures and return nil, so a sleeping exec child can never route |
| 417 | // here. The converged-daemon arm (caps.Restart) is excluded at the guard |
| 418 | // itself: the restart that failed there was carlos-app@<host>'s, its |
| 419 | // rollback belongs to caps.Restore, and starting the route's NAMED unit |
| 420 | // there would start a unit that pass never touched. |
| 421 | // |
| 422 | // start must start ONLY a route that NAMES its unit (registry Unit column) |
| 423 | // and return nil for everything else — internal/host's makeStartRoute is |
| 424 | // the intended shape. It is `systemctl start`, not try-restart, because the |
| 425 | // entire premise is that the unit is down; on a unit something else already |
| 426 | // brought up (caps.Restore's socket sequence) start is an exit-0 no-op, so |
| 427 | // the two cannot fight. |
| 428 | func WithStartUnit(start func(host string) error) PassOption { |
| 429 | return func(c *passConfig) { c.startUnit = start } |
| 430 | } |
| 431 | |
| 432 | // WithRollbackRestart sets D1's rollback restart (see passConfig). |
| 433 | func WithRestartAttemptSettled(fn func(string) error) PassOption { |
| 434 | return func(c *passConfig) { c.restartAttemptSettled = fn } |
| 435 | } |
| 436 | |
| 437 | func WithRestartFailurePropagation() PassOption { |
| 438 | return func(c *passConfig) { c.propagateRestarts = true } |
| 439 | } |
| 440 | |
| 441 | func WithRollbackRestart(fn func(host string) error) PassOption { |
| 442 | return func(c *passConfig) { c.rollbackRestart = fn } |
| 443 | } |
| 444 | |
| 445 | // WithProofGate arms D3 (proofgate.go): a build of an app the policy file |
| 446 | // lists is adopted only with a verified proof for its bytes, or from this |
| 447 | // box's healthy history. Pass the SAME gate to every pass of a process, so |
| 448 | // its 30 s miss cache and its waiting list survive between ticks. |
| 449 | func WithProofGate(g *ProofGate) PassOption { |
| 450 | return func(c *passConfig) { c.proof = g } |
| 451 | } |
| 452 | |
| 453 | // WithDigestRatchet arms #253's per-box ratchet for this pass (spec §4.6): |
| 454 | // a pointer without digests is refused on a channel this box has seen with |
| 455 | // them, or everywhere under CARLOS_REQUIRE_POINTER_DIGEST. Root and the |
| 456 | // agent each pass their own (internal/host pointerRatchet). |
| 457 | func WithDigestRatchet(r *DigestRatchet) PassOption { |
| 458 | return func(c *passConfig) { c.ratchet = r } |
| 459 | } |
| 460 | |
| 461 | // WithUpdateSettings arms settings spec §4 for this pass: each exec |
| 462 | // instance installs what Target decides, with §4.1's evidence. book opens |
| 463 | // the failure book; it is called at the start of every pass, so pass the |
| 464 | // opener, never a book read once (nil: no failure records). managed is |
| 465 | // rollout.Managed (nil: none). resolve is set on root's pass only: root |
| 466 | // alone resolves install requests and writes the consumed file (§4.3 rule |
| 467 | // 2); the agent's passes apply the same decisions and write nothing. |
| 468 | // |
| 469 | // A route with default settings and no consumed file takes today's |
| 470 | // adoption unchanged and never consults the book. An unreadable book holds |
| 471 | // every route with settings and every managed route (the zero-downtime |
| 472 | // plans' refuseManaged posture). |
| 473 | func WithUpdateSettings(book func() (FailedBuild, error), managed func(registry.Route) bool, resolve bool) PassOption { |
| 474 | return func(c *passConfig) { |
| 475 | c.settings, c.openFailed, c.managed, c.resolve = true, book, managed, resolve |
| 476 | } |
| 477 | } |
| 478 | |
| 479 | // WithPinSource adds src's references to the pin set a reused staged copy is |
| 480 | // checked against (spec §4.2), on top of the live links, the registry rows |
| 481 | // and the generations file, which every pass with a data dir reads itself. |
| 482 | func WithPinSource(src PinSource) PassOption { |
| 483 | return func(c *passConfig) { c.pins = src } |
| 484 | } |
| 485 | |
| 486 | // stageBlocked reports whether err is the version store refusing a write to |
| 487 | // THIS uid, rather than any of the many other ways staging fails. errors.Is |
| 488 | // unwraps the *os.PathError that mkdir/openat return. |
| 489 | func stageBlocked(err error) bool { |
| 490 | return errors.Is(err, syscall.EROFS) || |
| 491 | errors.Is(err, syscall.EPERM) || |
| 492 | errors.Is(err, syscall.EACCES) |
| 493 | } |
| 494 | |
| 495 | // StageBlocked is stageBlocked for callers outside this package: the |
| 496 | // relay-or-fail decision a move's held stage makes (MaterializeHeld). EROFS |
| 497 | // is what ProtectSystem=strict answers a write with, and it is not |
| 498 | // fs.ErrPermission, so a caller must not substitute that test for this one. |
| 499 | func StageBlocked(err error) bool { return stageBlocked(err) } |
| 500 | |
| 501 | // Pass is one full adoption sweep: group instance routes by (App, Channel), |
| 502 | // resolve each channel pointer once, verify signatures and artifact sha256s, |
| 503 | // download/stage missing versions, and repoint live/<host> symlinks. |
| 504 | // |
| 505 | // Die-free by design: per-host and per-channel problems log and continue — |
| 506 | // an unreachable channel or one broken host must not strand the rest of the |
| 507 | // fleet unchecked. Only foundation failures (registry unreadable, bake clock |
| 508 | // unwritable) fail the pass. |
| 509 | // |
| 510 | // restart cycles a host's process onto the new symlink target (the |
| 511 | // activator's proc converge); nil means symlink-only for every host. Hosts |
| 512 | // whose route has Hibernate=true get symlink-only by default and are NEVER |
| 513 | // woken: an hourly wake would defeat hibernation, and the next wake execs |
| 514 | // the new binary anyway since the spawn resolves live/<host> at exec time. |
| 515 | // The trade (from Eleven): no health check for a sleeper — a bad build |
| 516 | // surfaces at wake, recovered by Rollback, which repoints back. Hibernating |
| 517 | // hosts are processed AFTER every awake host, so an awake host discovers a |
| 518 | // boot-broken build (and poisons it) before any sleeper is moved onto it. |
| 519 | // |
| 520 | // RestartHibernating overrides only the FIRST half of that: it gives |
| 521 | // hibernate-flagged hosts a restart callback too, because the flag means "may |
| 522 | // sleep", not "is asleep" — a warm exec child must not go on serving a binary |
| 523 | // its live symlink no longer names. It never makes the pass wake anything; |
| 524 | // the option only accepts a callback that cannot start a process — and the |
| 525 | // start WithStartUnit adds is armed only by that callback returning an error, |
| 526 | // which a sleeper's exit-0 try-restart never does (see WithStartUnit). See |
| 527 | // its doc comment. Everything else above holds unchanged: sleepers still go |
| 528 | // last, |
| 529 | // still poison a bad build, still have no health gate of their own. |
| 530 | func Pass(ctx context.Context, reg *registry.Registry, s Store, pub ed25519.PublicKey, optDir string, restart func(host string) error, opts ...PassOption) error { |
| 531 | return PassFilter(ctx, reg, s, pub, optDir, restart, nil, opts...) |
| 532 | } |
| 533 | |
| 534 | // PassFilter is Pass narrowed to the routes extra additionally accepts |
| 535 | // (nil extra makes it identical to Pass) — the seam a faster, narrower |
| 536 | // sweep composes on top of the full one without duplicating the group |
| 537 | // resolution, bake-clock, staging, or restart logic. Built for the |
| 538 | // fast-converge sweep (internal/host/agent.go's fastConvergeLoop): canary |
| 539 | // channels are individually created and inherently low-cardinality (one |
| 540 | // deliberate rollout at a time, never the whole fleet — see bake.go's |
| 541 | // "canary/* has no window"), and unstaged routes drain out of the set as |
| 542 | // they stage (with per-host backoff while they can't), so sweeping just |
| 543 | // those every few seconds instead of hourly can't turn into a fleet-wide |
| 544 | // poll storm the way sweeping every route that fast would. |
| 545 | func PassFilter(ctx context.Context, reg *registry.Registry, s Store, pub ed25519.PublicKey, optDir string, restart func(host string) error, extra func(registry.Route) bool, opts ...PassOption) error { |
| 546 | var cfg passConfig |
| 547 | for _, opt := range opts { |
| 548 | opt(&cfg) |
| 549 | } |
| 550 | if cfg.propagateRestarts || cfg.requireRootHistory { |
| 551 | cfg.restartErrors = new([]error) |
| 552 | } |
| 553 | if cfg.openRootUpdates != nil { |
| 554 | var err error |
| 555 | cfg.rootUpdates, err = cfg.openRootUpdates(ctx, s, pub) |
| 556 | if err != nil { |
| 557 | return err |
| 558 | } |
| 559 | if cfg.rootUpdates != nil && cfg.rootUpdates.Close != nil { |
| 560 | defer cfg.rootUpdates.Close() |
| 561 | } |
| 562 | } |
| 563 | cfg.pub = pub |
| 564 | cfg.route = reg.Get |
| 565 | if cfg.recovery != nil { |
| 566 | routes, err := reg.List() |
| 567 | if err != nil { |
| 568 | return err |
| 569 | } |
| 570 | if cfg.rootUpdates != nil { |
| 571 | routes, err = cfg.rootUpdates.routes(ctx, routes) |
| 572 | if err != nil { |
| 573 | return err |
| 574 | } |
| 575 | } |
| 576 | handled := map[string]bool{} |
| 577 | var failures []error |
| 578 | remaining := false |
| 579 | for _, rt := range routes { |
| 580 | if extra != nil && !extra(rt) { |
| 581 | continue |
| 582 | } |
| 583 | done, err := cfg.recovery(ctx, rt) |
| 584 | handled[rt.Host] = done |
| 585 | if err != nil { |
| 586 | failures = append(failures, err) |
| 587 | } |
| 588 | if !done && (Adoptable(rt) || SidecarAdoptable(rt)) { |
| 589 | remaining = true |
| 590 | } |
| 591 | } |
| 592 | if !remaining && len(handled) > 0 { |
| 593 | return errors.Join(failures...) |
| 594 | } |
| 595 | for _, done := range handled { |
| 596 | if done { |
| 597 | prior := extra |
| 598 | extra = func(rt registry.Route) bool { return !handled[rt.Host] && (prior == nil || prior(rt)) } |
| 599 | break |
| 600 | } |
| 601 | } |
| 602 | cfg.recovery = nil |
| 603 | opts = append(opts, func(c *passConfig) { c.recovery = nil }) |
| 604 | if cfg.legacyRecovery != nil { |
| 605 | legacy := len(routes) == 0 |
| 606 | var err error |
| 607 | if !legacy { |
| 608 | legacy, err = cfg.legacyRecovery() |
| 609 | } |
| 610 | if err != nil { |
| 611 | return errors.Join(errors.Join(failures...), err) |
| 612 | } |
| 613 | if legacy { |
| 614 | opts = append(opts, func(c *passConfig) { c.requireRootHistory = false; c.rootHistory = nil }) |
| 615 | } |
| 616 | } |
| 617 | return errors.Join(errors.Join(failures...), PassFilter(ctx, reg, s, pub, optDir, restart, extra, opts...)) |
| 618 | } |
| 619 | // Settings spec §4.3: the failure book is read fresh for every pass, so a |
| 620 | // record the agent persisted since the last one counts now. |
| 621 | if cfg.rootUpdates == nil && cfg.settings && cfg.openFailed != nil { |
| 622 | cfg.failed, cfg.failedErr = cfg.openFailed() |
| 623 | if cfg.failedErr != nil && cfg.resolve { |
| 624 | fmt.Printf(" failure book unreadable (%v): routes with update settings and managed routes install nothing this pass\n", cfg.failedErr) |
| 625 | } |
| 626 | } |
| 627 | // D3: re-read the proof policy at the start of every pass, so an edit |
| 628 | // reaches the agent and root alike without a restart (spec §4.5). |
| 629 | cfg.proof.BeginPass() |
| 630 | routes, err := reg.List() |
| 631 | if err != nil { |
| 632 | return err |
| 633 | } |
| 634 | if cfg.rootUpdates != nil { |
| 635 | routes, err = cfg.rootUpdates.routes(ctx, routes) |
| 636 | if err != nil { |
| 637 | return err |
| 638 | } |
| 639 | } |
| 640 | adopts := func(rt registry.Route) bool { |
| 641 | return (Adoptable(rt) || SidecarAdoptable(rt)) && (extra == nil || extra(rt)) |
| 642 | } |
| 643 | // WithPointerGate: a skipped group leaves the pass before anything is |
| 644 | // selected for it. |
| 645 | adopts, pointerRead := cfg.gatePointers(routes, adopts) |
| 646 | selected := make(map[string]hostwrite.Work, len(routes)) |
| 647 | cfg.historySelected = make(map[string]historySelection) |
| 648 | for _, rt := range routes { |
| 649 | if !adopts(rt) { |
| 650 | continue |
| 651 | } |
| 652 | if cfg.requireRootHistory { |
| 653 | sel, err := cfg.rootHistory.selectRoute(ctx, rt) |
| 654 | if err != nil { |
| 655 | return err |
| 656 | } |
| 657 | cfg.historySelected[rt.Host] = sel |
| 658 | } |
| 659 | work, selectErr := cfg.hostWrites.Select(ctx, rt.Host, "release") |
| 660 | if selectErr != nil { |
| 661 | return fmt.Errorf("%s: select release work: %w", rt.Host, selectErr) |
| 662 | } |
| 663 | selected[rt.Host] = work |
| 664 | } |
| 665 | cfg.selected = selected |
| 666 | |
| 667 | // The reuse check's view of what refers to a staged copy (spec §4.2). It |
| 668 | // is asked only when a staged copy turns out to differ from its canonical |
| 669 | // release, so a healthy pass never pays for it, and it reads fresh then, |
| 670 | // so a host this pass has already repointed counts. |
| 671 | cfg.stage = stageCheck{ |
| 672 | beforeWrite: func(path string) error { return cfg.generationWriteCheck(ctx, optDir, path) }, |
| 673 | verified: map[string]string{}, |
| 674 | refs: func() (StagedPaths, error) { |
| 675 | rows, err := reg.List() |
| 676 | if err != nil { |
| 677 | return nil, fmt.Errorf("list routes: %w", err) |
| 678 | } |
| 679 | // An app whose settings references are incomplete comes back |
| 680 | // with the pins (*IncompleteRefsError); stageFrom refuses that |
| 681 | // app's restage only. |
| 682 | pins, incomplete := BoxPins(optDir, rows) |
| 683 | var inc *IncompleteRefsError |
| 684 | if incomplete != nil && (pins == nil || !errors.As(incomplete, &inc)) { |
| 685 | return nil, incomplete |
| 686 | } |
| 687 | // Live and pending generations, from the file the agent's |
| 688 | // activator writes into the data dir (index amendments; its |
| 689 | // workers' captured builds since ruling R17). Every production |
| 690 | // pass passes the data dir as BakeDir: root's RunAdopt, the |
| 691 | // agent's loops, and cold materialization. A pass with no data |
| 692 | // dir (tests) has no agent beside it to ask. |
| 693 | if cfg.bakeDir != "" { |
| 694 | gen, err := GenerationPins(GenerationsPath(cfg.bakeDir), func() ([]registry.Route, error) { return rows, nil }).StagedPins(ctx, optDir) |
| 695 | if err != nil { |
| 696 | return nil, err |
| 697 | } |
| 698 | for p := range gen { |
| 699 | pins.add(p) |
| 700 | } |
| 701 | } |
| 702 | if cfg.pins != nil { |
| 703 | more, err := cfg.pins.StagedPins(ctx, optDir) |
| 704 | if err != nil { |
| 705 | return nil, err |
| 706 | } |
| 707 | for p := range more { |
| 708 | pins.add(p) |
| 709 | } |
| 710 | } |
| 711 | if inc != nil { |
| 712 | return pins, inc |
| 713 | } |
| 714 | return pins, nil |
| 715 | }, |
| 716 | } |
| 717 | |
| 718 | // Resolve every distinct (Account, App, Channel) once. Verification |
| 719 | // failures kill the group (nil entry), not the pass. |
| 720 | // |
| 721 | // The account is part of the group key, not just the app: app names are |
| 722 | // unique WITHIN an account, so two routes named the same app in different |
| 723 | // accounts are two different apps following two different channel |
| 724 | // pointers. Grouping on (app, channel) alone would resolve one of them and |
| 725 | // silently install it on both. |
| 726 | type groupKey struct{ acct, app, channel string } |
| 727 | pointers := map[groupKey]*ChannelPointer{} |
| 728 | // Whether an app has anything under its qualified prefix, cached for the |
| 729 | // pass: one listing per (account, app), not one per channel. It decides |
| 730 | // which of the two missing-pointer lines below an app gets. |
| 731 | type acctApp struct{ acct, app string } |
| 732 | // migratedState caches one pass's answer to "has this app anything under |
| 733 | // its qualified prefix", together with the listing failure when there was |
| 734 | // one — a failed listing must neither invent the migrate advice nor claim |
| 735 | // the healthy skip (2026-08-03 review). |
| 736 | type migratedState struct { |
| 737 | has bool |
| 738 | listErr error |
| 739 | } |
| 740 | migrated := map[acctApp]migratedState{} |
| 741 | for _, rt := range routes { |
| 742 | if !adopts(rt) { |
| 743 | continue |
| 744 | } |
| 745 | k := groupKey{RouteAccount(rt), rt.App, routeChannel(rt)} |
| 746 | if _, done := pointers[k]; done { |
| 747 | continue |
| 748 | } |
| 749 | ptr, err := Channel(ctx, s, pub, k.acct, k.app, k.channel) |
| 750 | read := pointerRead[[3]string{k.acct, k.app, k.channel}] |
| 751 | if err != nil { |
| 752 | if read != nil { |
| 753 | read(nil, err) |
| 754 | } |
| 755 | pointers[k] = nil // resolved-and-failed: don't retry per host |
| 756 | if errors.Is(err, os.ErrNotExist) { |
| 757 | // SKIP, never abort: one app with no pointer must not stop |
| 758 | // every other app on the box from installing. |
| 759 | // |
| 760 | // Two deployments produce a missing pointer, and only one of |
| 761 | // them should be told to migrate. A channel with nothing |
| 762 | // promoted yet is ordinary — a migrated app waiting on its |
| 763 | // first stable promotion sits here for days — and gets the |
| 764 | // plain line. An app with NOTHING under its qualified prefix |
| 765 | // upgraded without migrating, and gets the store's error, |
| 766 | // which names the verb. Advice that fires every hour on a |
| 767 | // healthy fleet is advice nobody reads. |
| 768 | ak := acctApp{k.acct, k.app} |
| 769 | st, cached := migrated[ak] |
| 770 | if !cached { |
| 771 | has, lerr := AppHasQualifiedObjects(ctx, s, k.acct, k.app) |
| 772 | // A listing we could not perform suppresses the advice |
| 773 | // rather than inventing it: a bucket this box cannot list |
| 774 | // is not evidence that an app never migrated. It is not |
| 775 | // evidence of a healthy skip either, so the failure rides |
| 776 | // along and is printed instead of a claim this box never |
| 777 | // established (realistic trigger: GetObject IAM without |
| 778 | // ListBucket). |
| 779 | st = migratedState{has: has || lerr != nil, listErr: lerr} |
| 780 | migrated[ak] = st |
| 781 | } |
| 782 | switch { |
| 783 | case st.listErr != nil: |
| 784 | fmt.Printf(" %s/%s/%s: channel state unknown (could not list the app's prefix: %v)\n", k.acct, k.app, k.channel, st.listErr) |
| 785 | case st.has: |
| 786 | fmt.Printf(" %s/%s/%s: channel has no release yet\n", k.acct, k.app, k.channel) |
| 787 | default: |
| 788 | fmt.Printf(" %s/%s/%s: channel has no release yet (%v)\n", k.acct, k.app, k.channel, err) |
| 789 | } |
| 790 | } else { |
| 791 | fmt.Printf(" %s/%s/%s: channel pointer rejected: %v\n", k.acct, k.app, k.channel, err) |
| 792 | } |
| 793 | continue |
| 794 | } |
| 795 | // Any error from Admit refuses the pointer, ErrRatchetUnpersisted |
| 796 | // included, not only ErrPointerWithoutDigests. |
| 797 | if rerr := cfg.ratchet.Admit(k.acct, ptr); rerr != nil { |
| 798 | if read != nil { |
| 799 | read(nil, rerr) |
| 800 | } |
| 801 | pointers[k] = nil // refused: the group keeps what it has |
| 802 | fmt.Printf(" %s/%s/%s: channel pointer refused: %v\n", k.acct, k.app, k.channel, rerr) |
| 803 | continue |
| 804 | } |
| 805 | if read != nil { |
| 806 | read(ptr, nil) |
| 807 | } |
| 808 | pointers[k] = ptr |
| 809 | } |
| 810 | |
| 811 | // Settings spec §3.1: a major setting holds a head whose semver tag is |
| 812 | // absent or in another major. Tags are read once per app that has a route |
| 813 | // under one. ListTags is semver-descending, so the first tag seen for a |
| 814 | // release is its newest. An unreadable list leaves the app's map empty: |
| 815 | // every head reads as untagged, so a major-held instance stays held. |
| 816 | if cfg.settings { |
| 817 | cfg.tags = map[string]map[string]string{} |
| 818 | for _, rt := range routes { |
| 819 | u, err := ParseUpdateSettingsLenient(rt.UpdateSettings) |
| 820 | if err != nil || u.Major == nil || !adopts(rt) { |
| 821 | continue |
| 822 | } |
| 823 | k := RouteAccount(rt) + "/" + rt.App |
| 824 | if _, done := cfg.tags[k]; done { |
| 825 | continue |
| 826 | } |
| 827 | m := map[string]string{} |
| 828 | tags, err := ListTags(ctx, s, RouteAccount(rt), rt.App) |
| 829 | if err != nil { |
| 830 | fmt.Printf(" %s: tags unreadable (%v): every head reads as untagged\n", k, err) |
| 831 | } |
| 832 | for _, tg := range tags { |
| 833 | if _, seen := m[tg.Release]; !seen { |
| 834 | m[tg.Release] = tg.Version |
| 835 | } |
| 836 | } |
| 837 | cfg.tags[k] = m |
| 838 | } |
| 839 | } |
| 840 | |
| 841 | // Resolve each app's pipeline once per pass — for every app with a |
| 842 | // resolved NON-CANARY head, never one read per channel or per route. |
| 843 | // canary/* lives outside the pipeline entirely (spec §2): no pipeline |
| 844 | // can name it, its window is zero either way, so a canary-only |
| 845 | // appearance has no verdict a read could change — which is also what |
| 846 | // keeps the 2-3-second canary fast sweep at zero per-tick bucket reads |
| 847 | // here, the same cost property the production-flag reads below hold. |
| 848 | // |
| 849 | // A read/verify ERROR is kept, and kept apart from verified-absent |
| 850 | // (nil, nil): the errored app's windows fall back to DefaultHolds |
| 851 | // FLOORED by the box's own ratchet memory (the appHolds loop below) — |
| 852 | // every channel the memory carries enforces max(its DefaultHolds |
| 853 | // resolution, the remembered window), so corrupting the object can |
| 854 | // never make this box looser than it already was (spec §3: corruption |
| 855 | // must never loosen; plain DefaultHolds would hand a tightened rung |
| 856 | // name like edge its ZERO legacy window back). A channel with no memory |
| 857 | // keeps the fail-closed DefaultHolds resolution — the old-binary |
| 858 | // posture (spec §3's old-binary edge), whose production: true co-write |
| 859 | // remains the mitigation for a MEMORY-LESS box only. Where memory DOES |
| 860 | // exist, the floor is enforcement in full: the bake VERDICT rides on |
| 861 | // the same memory (bakeIgnore below never ignores an app whose ratchet |
| 862 | // is still live — which is a WIDER condition than "the memory raises a |
| 863 | // window", deliberately, since an app-defined channel's ratchet |
| 864 | // normally raises nothing at all), so nothing here depends on the |
| 865 | // console having managed the co-write. |
| 866 | // The error is NEVER folded into the ratchet as a nil: nil |
| 867 | // means verified-absent only (ObservePipeline's contract), and a |
| 868 | // corrupt object must not masquerade as a deletion — a deletion RATCHETS |
| 869 | // each remembered channel down to its legacy default over one old-window |
| 870 | // cycle (ObservePipeline's p == nil branch), where an error holds every |
| 871 | // remembered window exactly where it is. |
| 872 | type pipeState struct { |
| 873 | p *Pipeline |
| 874 | err error // non-nil: the object exists but would not read or verify |
| 875 | } |
| 876 | pipes := map[acctApp]pipeState{} |
| 877 | for k, ptr := range pointers { |
| 878 | if ptr == nil { |
| 879 | continue |
| 880 | } |
| 881 | if r, ok := Rung(k.channel); ok && r == RungCanary { |
| 882 | continue |
| 883 | } |
| 884 | ak := acctApp{k.acct, k.app} |
| 885 | if _, done := pipes[ak]; done { |
| 886 | continue |
| 887 | } |
| 888 | p, perr := GetPipeline(ctx, s, pub, ak.acct, ak.app) |
| 889 | pipes[ak] = pipeState{p: p, err: perr} |
| 890 | if perr != nil { |
| 891 | // SAY SO, like the production flag below — same realistic |
| 892 | // non-tampering trigger too: on a GetObject-without-ListBucket |
| 893 | // fleet a GET of an ABSENT key answers 403, not 404, and every |
| 894 | // app would land here every pass. |
| 895 | fmt.Printf(" %s/%s: pipeline unreadable (%v) — remembered windows still apply; anything unremembered falls to legacy holds\n", ak.acct, ak.app, perr) |
| 896 | } |
| 897 | } |
| 898 | |
| 899 | // The bake clock, loaded before any of this pass's reads are folded into |
| 900 | // it. now is stamped once so every observation and verdict this pass |
| 901 | // answers the same instant. |
| 902 | now := time.Now() |
| 903 | var bake *FirstSeen |
| 904 | if cfg.requireRootHistory { |
| 905 | bindings := make([]HistoryBinding, 0, len(cfg.historySelected)) |
| 906 | for _, sel := range cfg.historySelected { |
| 907 | bindings = append(bindings, sel.binding) |
| 908 | } |
| 909 | bake, err = cfg.rootHistory.History.LoadBake(bindings, now) |
| 910 | if err != nil { |
| 911 | return err |
| 912 | } |
| 913 | } else if cfg.bakeDir != "" { |
| 914 | bake = LoadFirstSeenMigrating(FirstSeenPath(cfg.bakeDir), FirstSeenPath(optDir)) |
| 915 | } else { |
| 916 | bake = LoadFirstSeen(FirstSeenPath(optDir)) |
| 917 | } |
| 918 | // bake.migrated forces this pass's Save even if nothing new is observed |
| 919 | // below: otherwise a changed-gated Save could leave the new path absent |
| 920 | // for days on a fleet that happens to see no new channel head, and the |
| 921 | // legacy file would look safe to delete long before it actually is (see |
| 922 | // LoadFirstSeenMigrating and BakeDir's doc comments, 2026-08-03 review). |
| 923 | changed := bake.migrated |
| 924 | |
| 925 | // ORDER IS LOAD-BEARING: fold this pass's verified pipeline reads into |
| 926 | // the ratchet memory HERE, before the per-app window maps below are |
| 927 | // computed from it — a tightened window read this pass must be enforced |
| 928 | // this pass, and a pending loosening whose old window has elapsed must |
| 929 | // land before the verdicts that depend on it. Same same-pass rule as |
| 930 | // ObserveProduction's fold further down. The err == nil guard is the |
| 931 | // contract, not paranoia: nil reaches ObservePipeline only for a |
| 932 | // verified-absent object (which never loosens anything, but does give |
| 933 | // this app's pending promotions their chance to land); an errored read |
| 934 | // never ticks the memory at all. |
| 935 | for ak, ps := range pipes { |
| 936 | if ps.err == nil && bake.ObservePipeline(ak.acct, ak.app, ps.p, now) { |
| 937 | changed = true |
| 938 | } |
| 939 | } |
| 940 | |
| 941 | // One enforced-window map per app for this pass. A verified pipeline |
| 942 | // resolves through the box's ratchet memory (HoldsForApp); an ERRORED |
| 943 | // read keeps DefaultHolds floored by that same memory (never looser |
| 944 | // than what this box already enforced — see the pipes comment above); |
| 945 | // a VERIFIED-ABSENT pipeline takes the same memory floor, which is what |
| 946 | // enforces the deletion ratchet-down for as long as the memory rides |
| 947 | // above legacy; an app with no pipeline AND no memory keeps bare |
| 948 | // DefaultHolds, today's behaviour byte-for-byte. canary/* is stitched back in at zero on the verified |
| 949 | // path: it lives outside the pipeline (spec §2), so HoldsForApp's map |
| 950 | // cannot carry it, and HoldFor's fallback for a name missing from the |
| 951 | // map is the 72h fail-closed default — which would silently freeze |
| 952 | // every canary deploy of a pipelined app. Every OTHER channel outside |
| 953 | // the pipeline does take that 72h (spec §3: a channel with no pipeline |
| 954 | // entry keeps the fail-closed default), including rung names like beta |
| 955 | // whose DefaultHolds entry no longer applies. |
| 956 | // |
| 957 | // floored is the SEPARATE question the bake verdict asks: does this box |
| 958 | // hold pipeline evidence for the app at all (see bakeIgnore)? It is a |
| 959 | // membership set, never derived from the overlay above — a ratchet can |
| 960 | // be live while raising nothing, which is the ordinary state of every |
| 961 | // app-defined channel name (holdMemory.liveRatchet has the full |
| 962 | // argument). |
| 963 | appHolds := map[acctApp]map[string]time.Duration{} |
| 964 | floored := map[acctApp]bool{} |
| 965 | for ak, ps := range pipes { |
| 966 | switch { |
| 967 | case ps.err == nil && ps.p != nil: |
| 968 | h := bake.HoldsForApp(ak.acct, ak.app, ps.p) |
| 969 | h["canary/*"] = 0 |
| 970 | appHolds[ak] = h |
| 971 | floored[ak] = true |
| 972 | default: |
| 973 | // The memory floor — for an ERRORED read and for VERIFIED |
| 974 | // ABSENCE alike. Overrides overlay a COPY of DefaultHolds, |
| 975 | // so a remembered channel is only ever RAISED above its legacy |
| 976 | // resolution, an unremembered one resolves exactly as legacy |
| 977 | // (HoldFor's 72h unknown-name fallback included), and canary/* |
| 978 | // keeps its zero: no pipeline can name canary, so no ratchet |
| 979 | // memory exists to raise it. With nothing remembered above |
| 980 | // legacy, no map is built and holdsFor falls to DefaultHolds |
| 981 | // itself. The prefix match is exact by construction: neither an |
| 982 | // account, an app name nor a non-canary channel can contain |
| 983 | // clockSep or "/" (channelClockKey's own injectivity argument). |
| 984 | // |
| 985 | // For verified absence the floor is half of what makes the |
| 986 | // deletion ratchet (ObservePipeline's p == nil branch, folded |
| 987 | // above) ENFORCED: while a remembered channel's window still |
| 988 | // exceeds its legacy default, deleting the object cannot adopt |
| 989 | // a held head this pass. That covers the names whose legacy |
| 990 | // default is NARROWER than what was enforced (edge, 0); the |
| 991 | // other half — an app-defined name, whose legacy default is the |
| 992 | // 72h fallback and therefore lands instantly, raising nothing — |
| 993 | // is carried by the verdict alone (floored below, |
| 994 | // holdMemory.DeletedAt). Once every remembered channel has |
| 995 | // finished ratcheting, nothing exceeds the default, no map is |
| 996 | // built, and the app is back on the bare-DefaultHolds path. An |
| 997 | // app with no memory at all — every legacy app, every pass — |
| 998 | // takes that bare path byte-identically, and the len check |
| 999 | // keeps the fleet-wide no-pipeline case allocation-free. |
| 1000 | // |
| 1001 | // The overlay is NOT the pass's answer to "is this app |
| 1002 | // pipelined as far as this box knows" — floored below is, and |
| 1003 | // the two are deliberately separate. A raised window is only |
| 1004 | // one shape a live ratchet takes: an app-defined channel like |
| 1005 | // "production" resolves to HoldFor's 72h fail-closed default |
| 1006 | // under DefaultHolds, so a declared 24h ratchets DOWN to a |
| 1007 | // window BELOW legacy and raises nothing at all. Deriving the |
| 1008 | // verdict from this overlay would have found no evidence for |
| 1009 | // exactly that app and fallen to the production flag. |
| 1010 | if len(bake.Holds) == 0 { |
| 1011 | break |
| 1012 | } |
| 1013 | var h map[string]time.Duration |
| 1014 | prefix := holdKeyPrefix(ak.acct, ak.app) |
| 1015 | for key, mem := range bake.Holds { |
| 1016 | // mem == nil: {"holds":{"x":null}} parses cleanly into a |
| 1017 | // nil entry — skip it, never dereference; a panic here |
| 1018 | // takes down the adopt pass on every box that loads the |
| 1019 | // file. |
| 1020 | if mem == nil { |
| 1021 | continue |
| 1022 | } |
| 1023 | name := strings.TrimPrefix(key, prefix) |
| 1024 | if name == key || name == "" { |
| 1025 | continue |
| 1026 | } |
| 1027 | if mem.liveRatchet(name, bake.Jitter, now) { |
| 1028 | floored[ak] = true |
| 1029 | } |
| 1030 | if mem.Window > HoldFor(DefaultHolds, name) { |
| 1031 | if h == nil { |
| 1032 | h = make(map[string]time.Duration, len(DefaultHolds)+1) |
| 1033 | for ch, d := range DefaultHolds { |
| 1034 | h[ch] = d |
| 1035 | } |
| 1036 | } |
| 1037 | h[name] = mem.Window |
| 1038 | } |
| 1039 | } |
| 1040 | if h != nil { |
| 1041 | appHolds[ak] = h |
| 1042 | } |
| 1043 | } |
| 1044 | } |
| 1045 | holdsFor := func(ak acctApp) map[string]time.Duration { |
| 1046 | if h, ok := appHolds[ak]; ok { |
| 1047 | return h |
| 1048 | } |
| 1049 | return DefaultHolds |
| 1050 | } |
| 1051 | |
| 1052 | // Resolve each app's production flag once per pass — but ONLY for apps |
| 1053 | // with a HELD head resolved this pass (spec §6), held now meaning held |
| 1054 | // under the app's OWN window map above. That filter is a cost property, |
| 1055 | // not a nicety: any pass pays at most one read per (account, app) with a |
| 1056 | // held head, never one per channel or per route. For the fast-converge |
| 1057 | // sweep (PassFilter every 2-3 SECONDS) that usually means zero flag |
| 1058 | // reads — canary/* has a zero window on every path, and a fully-staged |
| 1059 | // box's sweep carries nothing else — but an UNSTAGED held route in the |
| 1060 | // sweep's set does resolve its held channel and pays this read too. |
| 1061 | // What keeps that bounded is the sweep's own membership, not this |
| 1062 | // filter: an unstaged host backs off 2s→5min (internal/host's |
| 1063 | // fastMembers), so a held, never-staging route costs one flag read per |
| 1064 | // backoff window at worst, not one per tick. A PIPELINED app's held |
| 1065 | // head pays the read as well even though its verdict never consults the |
| 1066 | // flag (bakeIgnore below): the fold keeps the legacy latch fresh for |
| 1067 | // the day the pipeline is deleted AND its deletion ratchet has landed, |
| 1068 | // which is when the app falls back to the flag for real. |
| 1069 | type prodState struct { |
| 1070 | flag *ProductionFlag |
| 1071 | err error // non-nil: the object exists but would not read or verify |
| 1072 | } |
| 1073 | bakeable := map[acctApp]bool{} |
| 1074 | for k, ptr := range pointers { |
| 1075 | if ptr != nil && HoldFor(holdsFor(acctApp{k.acct, k.app}), k.channel) > 0 { |
| 1076 | bakeable[acctApp{k.acct, k.app}] = true |
| 1077 | } |
| 1078 | } |
| 1079 | prod := map[acctApp]prodState{} |
| 1080 | for ak := range bakeable { |
| 1081 | // A missing flag is (nil, nil) — the default, not production. Any |
| 1082 | // OTHER failure is kept as an error and fails toward baking in |
| 1083 | // ProductionBake: an attacker who can corrupt but not sign must not |
| 1084 | // be able to shorten a window by breaking the object. |
| 1085 | f, err := GetProduction(ctx, s, pub, ak.acct, ak.app) |
| 1086 | prod[ak] = prodState{flag: f, err: err} |
| 1087 | if err != nil { |
| 1088 | // SAY SO. This failure re-imposes the full window on every held |
| 1089 | // head of the app, and the most likely cause is not tampering at |
| 1090 | // all: on a GetObject-without-ListBucket fleet (the same IAM shape |
| 1091 | // the missing-pointer branch above already handles) a GET of an |
| 1092 | // ABSENT key answers 403, not 404 — so the spec's default-OFF |
| 1093 | // silently inverts to production for every app with no flag. A |
| 1094 | // 72h hold nobody can see the reason for is the worst version of |
| 1095 | // this feature; log it like every other per-app problem here and |
| 1096 | // carry on. |
| 1097 | fmt.Printf(" %s/%s: production flag unreadable (%v) — treating as production, holds apply\n", ak.acct, ak.app, err) |
| 1098 | } |
| 1099 | } |
| 1100 | |
| 1101 | // Tick the bake clock: record every channel head this box hasn't seen at |
| 1102 | // that head before, so the supply-chain window is measured from the box's |
| 1103 | // own first sighting, never a feed timestamp (bake.go). The clock only |
| 1104 | // ever ticks for pointers that verified above. |
| 1105 | for k, ptr := range pointers { |
| 1106 | if ptr != nil && bake.Observe(k.acct, k.app, k.channel, ptr.Version, now) { |
| 1107 | changed = true |
| 1108 | } |
| 1109 | } |
| 1110 | // ORDER IS LOAD-BEARING: fold this pass's verified flag reads into the |
| 1111 | // box's production memory HERE, before any bake verdict below is taken. |
| 1112 | // That is what makes prodMemory's "a mixed-version rollout costs at most |
| 1113 | // one pass of this memory" true — a binary that wiped the latch re-latches |
| 1114 | // it from the bucket and honors it in the SAME pass, never a pass later. |
| 1115 | // |
| 1116 | // The err == nil guard is the second lock on a door ObserveProduction's |
| 1117 | // own nil-check already holds (GetProduction returns a nil flag on every |
| 1118 | // error path, and a nil flag is a no-op there). It stays because the rule |
| 1119 | // it states — the memory only ever ticks for VERIFIED documents — is |
| 1120 | // worth being true at the call site too, not only at the callee. |
| 1121 | for ak, ps := range prod { |
| 1122 | if ps.err == nil && bake.ObserveProduction(ak.acct, ak.app, ps.flag, now) { |
| 1123 | changed = true |
| 1124 | } |
| 1125 | } |
| 1126 | // bake.tainted forces a Save for a different reason than migrated: the |
| 1127 | // clock file existed but did not read or parse, so every held head on |
| 1128 | // this box is failing toward baking until that file is replaced |
| 1129 | // (ProductionBake). Left alone, the corrupt file would survive every |
| 1130 | // changed-gated pass and the box would bake forever. |
| 1131 | // |
| 1132 | // But ONLY a full pass may perform that repair. A NARROWED pass |
| 1133 | // (extra != nil — the fast-converge sweep, every 2-3 seconds) fetches |
| 1134 | // production flags only for the held heads inside its narrowed set — |
| 1135 | // often none (canary/* has no hold), sometimes a few (an unstaged |
| 1136 | // stable/beta member), never reliably ALL of the apps whose latches the |
| 1137 | // clock file carries: its in-memory Production map is PARTIAL, not |
| 1138 | // repaired. The pipeline ratchet's Holds map is partial for the same |
| 1139 | // reason (pipelines are read only for the narrowed set's own non-canary |
| 1140 | // apps), so this skip protects both memories at once. Letting it write |
| 1141 | // would overwrite the corrupt file with that partial view AND clear the |
| 1142 | // taint that is currently failing closed, |
| 1143 | // dropping the latch of every app outside its set before anything |
| 1144 | // re-observed those flags — and if a bucket flag had also been deleted, |
| 1145 | // that latch would be gone for good. So a tainted clock on a narrowed |
| 1146 | // pass skips the Save entirely, forced or ordinary: a pass that observed |
| 1147 | // only a slice of the flags cannot repair a file that carries them all. |
| 1148 | // Nothing is lost — the observations it would have recorded re-occur |
| 1149 | // within seconds, and the hourly pass does the repair with the full flag |
| 1150 | // set actually in hand. |
| 1151 | // |
| 1152 | // Residual, accepted: even the full pass only re-observes flags for apps |
| 1153 | // with a currently-resolved HELD head, so an app whose channel pointer is |
| 1154 | // simultaneously unreadable keeps its latch out of the repaired file |
| 1155 | // until that pointer heals. That is triple-tamper territory (corrupt |
| 1156 | // clock file AND unreadable pointer AND deleted flag) and sits inside the |
| 1157 | // box-file trust boundary LoadFirstSeen already documents as accepted. |
| 1158 | switch { |
| 1159 | case bake.tainted && extra != nil: |
| 1160 | // skip: see above. |
| 1161 | case changed || bake.tainted: |
| 1162 | if err := bake.Save(); err != nil { |
| 1163 | return fmt.Errorf("record bake clock: %v", err) |
| 1164 | } |
| 1165 | } |
| 1166 | |
| 1167 | // A version that failed its restart on one host isn't tried on the next |
| 1168 | // this pass — one bad build shouldn't bounce every host in turn. |
| 1169 | poisoned := map[string]bool{} |
| 1170 | |
| 1171 | // One definition of "does the window apply here", shared by the awake and |
| 1172 | // hibernating loops below — the same question must not be able to be |
| 1173 | // answered two ways for the same app depending on whether it sleeps. |
| 1174 | // |
| 1175 | // RouteAccount(rt), never the raw rt.Account column: the group and |
| 1176 | // bakeable keys were built from RouteAccount output, and an unmigrated |
| 1177 | // row can still carry an account NAME there, which resolves to ops. |
| 1178 | bakeIgnore := func(rt registry.Route) bool { |
| 1179 | ak := acctApp{RouteAccount(rt), rt.App} |
| 1180 | // A PIPELINED app never consults the flag: its bake policy is the |
| 1181 | // pipeline's alone (spec §3) — never ignore, and a zero-window |
| 1182 | // channel holds nothing anyway. |
| 1183 | // |
| 1184 | // "Pipelined" here is every app this pass found pipeline EVIDENCE |
| 1185 | // for (floored), not only the apps whose read verified this pass: |
| 1186 | // a verified pipeline, or a ratchet this box is still running for |
| 1187 | // the app (holdMemory.liveRatchet — a window that differs from the |
| 1188 | // name's legacy resolution in EITHER direction, a pending |
| 1189 | // loosening, or a deletion whose cycle has not run out). That |
| 1190 | // memory IS the evidence the app was pipelined, and it is |
| 1191 | // deliberately NOT the appHolds overlay: the overlay only ever |
| 1192 | // RAISES a window above legacy, and an app-defined channel name |
| 1193 | // resolves to HoldFor's 72h fail-closed default, so a declared 24h |
| 1194 | // on "production" raises nothing while being the very policy the |
| 1195 | // hold exists to enforce. Reading the overlay collapsed exactly |
| 1196 | // that app (PR #258 review, C1). Riding the |
| 1197 | // verdict on the console's production: true CO-WRITE instead would |
| 1198 | // make enforcement depend on a bucket write this box can never |
| 1199 | // confirm happened: an app pipelined before the co-write existed, an |
| 1200 | // app whose flag was deleted, a direct-bucket pipeline write — each |
| 1201 | // leaves a flagless app whose held heads would adopt the instant one |
| 1202 | // GET of pipeline.json errored, or the instant the object was |
| 1203 | // deleted, defeating the very floor computed above. The co-write |
| 1204 | // keeps its own job, which was always the MEMORY-LESS box (spec §3's |
| 1205 | // old-binary edge — any box that never saw the pipeline). |
| 1206 | // |
| 1207 | // The flag path is left to apps with no pipeline signal at all — |
| 1208 | // every legacy app, byte-identically — and to a once-pipelined app |
| 1209 | // whose every remembered channel has finished ratcheting back to |
| 1210 | // its legacy default: nothing is live then, the set is empty for |
| 1211 | // the app, and it is a legacy app again in full. |
| 1212 | if floored[ak] { |
| 1213 | return false |
| 1214 | } |
| 1215 | return !bake.ProductionBake(ak.acct, ak.app, prod[ak].err, bakeable[ak], DefaultHolds, now) |
| 1216 | } |
| 1217 | // routeHolds resolves the same per-app window map for a route that the |
| 1218 | // bakeable filter above used — shared for the same reason bakeIgnore is: |
| 1219 | // the same question must not be answerable two ways for one app. |
| 1220 | routeHolds := func(rt registry.Route) map[string]time.Duration { |
| 1221 | return holdsFor(acctApp{RouteAccount(rt), rt.App}) |
| 1222 | } |
| 1223 | |
| 1224 | // Awake hosts first, hibernating second: repoints have no health gate, so |
| 1225 | // let an awake host discover (and poison) a broken build before any |
| 1226 | // sleeper is moved onto it. |
| 1227 | var hibernating, sidecars []registry.Route |
| 1228 | for _, rt := range routes { |
| 1229 | if !adopts(rt) { |
| 1230 | continue |
| 1231 | } |
| 1232 | if rt.IsSidecar() { |
| 1233 | sidecars = append(sidecars, rt) |
| 1234 | continue |
| 1235 | } |
| 1236 | if rt.Hibernate { |
| 1237 | hibernating = append(hibernating, rt) |
| 1238 | continue |
| 1239 | } |
| 1240 | ptr := pointers[groupKey{RouteAccount(rt), rt.App, routeChannel(rt)}] |
| 1241 | adoptHost(ctx, selected[rt.Host], reg, s, optDir, bake, poisoned, rt, ptr, restart, now, cfg, routeHolds(rt), bakeIgnore(rt)) |
| 1242 | } |
| 1243 | for _, rt := range hibernating { |
| 1244 | ptr := pointers[groupKey{RouteAccount(rt), rt.App, routeChannel(rt)}] |
| 1245 | // cfg.restartHibernating is nil by default (and for RunAdopt): the |
| 1246 | // pure repoint, never woken. The capability converge still runs for |
| 1247 | // them (a drop-in is unit text, not a wake); see adoptHost. |
| 1248 | // |
| 1249 | // When it IS set — the agent's in-process loops — the callback is |
| 1250 | // consulted for sleepers too, and the "never woken" guarantee moves |
| 1251 | // into the callback: RestartHibernating only accepts one that cannot |
| 1252 | // start a process (Bounce is false-when-not-awake, try-restart is a |
| 1253 | // no-op on a stopped unit). The flag says a route MAY sleep, not that |
| 1254 | // it is asleep now, and a warm exec child must not keep serving a |
| 1255 | // binary its symlink no longer names. |
| 1256 | adoptHost(ctx, selected[rt.Host], reg, s, optDir, bake, poisoned, rt, ptr, cfg.restartHibernating, now, cfg, routeHolds(rt), bakeIgnore(rt)) |
| 1257 | } |
| 1258 | // Native sidecars LAST, and never through a restart callback (spec |
| 1259 | // §7.9). The pass stages, repoints and stamps. The running worker moves |
| 1260 | // when the activator's drift check sees the link change, which plan 5 |
| 1261 | // queues as a paced release attempt; a Bounce here would stop an always-on |
| 1262 | // worker with nothing to start it. Last for the reason sleepers go after |
| 1263 | // awake hosts: a repoint has no health gate, so a build an instance |
| 1264 | // poisons this pass never reaches one. |
| 1265 | for _, rt := range sidecars { |
| 1266 | ptr := pointers[groupKey{RouteAccount(rt), rt.App, routeChannel(rt)}] |
| 1267 | adoptHost(ctx, selected[rt.Host], reg, s, optDir, bake, poisoned, rt, ptr, nil, now, cfg, routeHolds(rt), bakeIgnore(rt)) |
| 1268 | } |
| 1269 | if cfg.restartErrors != nil { |
| 1270 | return errors.Join(*cfg.restartErrors...) |
| 1271 | } |
| 1272 | return nil |
| 1273 | } |
| 1274 | |
| 1275 | // Adoptable filters the routes a Pass moves: lifecycle-managed instances of |
| 1276 | // a known app that are ready (an app-less or mid-provision route has nothing |
| 1277 | // to adopt or must not be disturbed), excluding platform-minted routes. |
| 1278 | // |
| 1279 | // A platform domain (internal/domains) is an ALIAS for an app's canonical |
| 1280 | // route — a second name pointing at the same backend — not a host with a |
| 1281 | // lifecycle of its own. Adopting one would stage a redundant copy of the |
| 1282 | // artifacts, mint a live/<alias> symlink nothing ever execs, try-restart a |
| 1283 | // carlos-app@<alias> unit that does not exist, and stamp Version onto a row |
| 1284 | // the console then renders as though it were a real instance. The canonical |
| 1285 | // route it aliases is adopted normally — that IS the deploy, and the alias |
| 1286 | // serves the new version the moment it lands. |
| 1287 | // |
| 1288 | // Exported for host.fastMembers (the fast-converge sweep's membership scan), |
| 1289 | // the same reason CapabilityRoute below is: the sweep must count as |
| 1290 | // "outstanding work" exactly the routes a pass would consider, or a route |
| 1291 | // the pass refuses on shape would be swept forever. A second copy of this |
| 1292 | // predicate in another package is exactly the drift that would let the two |
| 1293 | // disagree. |
| 1294 | func Adoptable(rt registry.Route) bool { |
| 1295 | return rt.Kind == "instance" && rt.App != "" && !rt.Platform && |
| 1296 | (rt.Status == "" || rt.Status == "ready") |
| 1297 | } |
| 1298 | |
| 1299 | // SidecarAdoptable is the pass's SECOND admission predicate (zero-downtime |
| 1300 | // deploys spec §7.9): a native sidecar row (Kind "sidecar") of a known app, |
| 1301 | // not platform-minted, and ready. Its build follows the channel on its own |
| 1302 | // row, as sidecar_reconcile.go provisions it, through the same signed |
| 1303 | // selection, bake holds, staging, repoint and stamp an instance gets. |
| 1304 | // |
| 1305 | // Adoptable is deliberately NOT widened: capability drop-ins, the fast |
| 1306 | // sweep's membership, config sync, the restart passes and the console's |
| 1307 | // Restartable flag all keep their own answer for a sidecar. The pass opts in |
| 1308 | // by name, and only the pass. |
| 1309 | func SidecarAdoptable(rt registry.Route) bool { |
| 1310 | return rt.IsSidecar() && rt.App != "" && !rt.Platform && |
| 1311 | (rt.Status == "" || rt.Status == "ready") |
| 1312 | } |
| 1313 | |
| 1314 | // CapabilityRoute reports whether this route may materialize capabilities: |
| 1315 | // a lifecycle-managed instance of a known app, unit-backed, non-platform, |
| 1316 | // ready (spec §4e). It is Adoptable() plus the unit backing, deliberately — |
| 1317 | // a route this pass would not adopt is not one whose unit shape this pass |
| 1318 | // has any business rewriting. |
| 1319 | // |
| 1320 | // A platform ALIAS never materialises anything: an alias is a second name |
| 1321 | // for one process, and a second carlos-app@ unit claiming :25 would break |
| 1322 | // the real one. Adoptable() already excludes them and this depends on that. |
| 1323 | // |
| 1324 | // An EXEC-backed route (Backing == "") whose release declares capabilities |
| 1325 | // materialises nothing here and is told so once per pass by RunAdopt's |
| 1326 | // advisory, not by this function — that is the guided path through the §5 |
| 1327 | // migration. |
| 1328 | // |
| 1329 | // Exported for host.capabilityPass (the hourly grant-drift pass): the two |
| 1330 | // converge entry points must agree, to the character, on which routes may |
| 1331 | // have a root-owned unit drop-in written for them. A second copy of this |
| 1332 | // predicate in another package is exactly the drift that would let one pass |
| 1333 | // materialize a shape the other refuses. |
| 1334 | func CapabilityRoute(rt registry.Route) bool { |
| 1335 | return Adoptable(rt) && rt.Backing == registry.BackingUnit |
| 1336 | } |
| 1337 | |
| 1338 | // routeChannel is the channel a route follows; "" means stable. |
| 1339 | func routeChannel(rt registry.Route) string { |
| 1340 | if rt.Channel == "" { |
| 1341 | return "stable" |
| 1342 | } |
| 1343 | return rt.Channel |
| 1344 | } |
| 1345 | |
| 1346 | // RouteAccount is the account whose bucket keys a route's app lives under: |
| 1347 | // the sqid stamped on the row, else the built-in ops. |
| 1348 | // |
| 1349 | // The ValidAccount gate is doing migration work, not paranoia. Before the |
| 1350 | // multi-account change the account column held an account NAME ("ops", |
| 1351 | // "paul@keymail.dev"), and a box can be running this code against a registry |
| 1352 | // still carrying those. A name is not a sqid, so it reads as ops — whose keys |
| 1353 | // are tried qualified first and then fall back to the legacy unqualified |
| 1354 | // shape, which is exactly where an unmigrated app's objects still are. |
| 1355 | func RouteAccount(rt registry.Route) string { |
| 1356 | if ValidAccount(rt.Account) { |
| 1357 | return rt.Account |
| 1358 | } |
| 1359 | return OpsAccount |
| 1360 | } |
| 1361 | |
| 1362 | // adoptHost converges one host onto its channel head ptr (nil = the group's |
| 1363 | // pointer didn't resolve; skip quietly, the group phase already logged). |
| 1364 | // restart==nil is the symlink-only path (a caller with no proc manager, or a |
| 1365 | // hibernating host on a pass that did not set RestartHibernating): repoint |
| 1366 | // and record, never wake, never health-gate. A hibernating host WITH that |
| 1367 | // option set arrives here with a non-nil callback and takes the ordinary |
| 1368 | // path — safe because the option only accepts a callback that cannot start a |
| 1369 | // process, so "never woken" survives the callback being consulted. The one |
| 1370 | // start this function itself can perform (cfg.startUnit, the restart-failure |
| 1371 | // rollback below) does not weaken that: it is armed only by that callback |
| 1372 | // RETURNING AN ERROR, which a stopped unit's exit-0 try-restart never |
| 1373 | // produces — WithStartUnit's doc carries the full argument. |
| 1374 | // cfg is the call's PassOptions, already applied: cfg.nag nil (the ordinary |
| 1375 | // case) means the canary-TTL nag below never fires, never even builds its |
| 1376 | // string, and cfg.capsOps nil means the capability converge never runs at |
| 1377 | // all. See NagExpiredCanaries and WithCapabilities for who sets each and why. |
| 1378 | // |
| 1379 | // holds is the enforced per-channel window map the caller resolved for this |
| 1380 | // route's app: a verified pipeline's ratcheted HoldsForApp map (canary/* |
| 1381 | // stitched back in at zero), DefaultHolds floored by the ratchet memory for |
| 1382 | // an app whose pipeline read errored or is verified-absent mid-ratchet, else |
| 1383 | // DefaultHolds — never consulted app-blind. |
| 1384 | // |
| 1385 | // ignore is the app-level bake verdict from ProductionBake — the LEGACY |
| 1386 | // production flag's doing, never a publisher field, and constant false for |
| 1387 | // any app this box carries pipeline evidence for: a verified pipeline, or a |
| 1388 | // ratchet this box is still running for it (holdMemory.liveRatchet, which is |
| 1389 | // NOT the same question as "does the memory raise a window" — see |
| 1390 | // bakeIgnore). Such an app's windows come from holds alone (spec §3). True |
| 1391 | // means the supply-chain window does not CURRENTLY apply to this app on |
| 1392 | // this box (not declared production, or declared off and past the ratchet); |
| 1393 | // the caller computes it once per route from the flag it read this pass. |
| 1394 | func adoptHost(ctx context.Context, work hostwrite.Work, reg *registry.Registry, s Store, optDir string, bake *FirstSeen, poisoned map[string]bool, rt registry.Route, ptr *ChannelPointer, restart func(host string) error, now time.Time, cfg passConfig, holds map[string]time.Duration, ignore bool) { |
| 1395 | // settings: this route's build is the instance update settings' decision |
| 1396 | // (settings spec §4.3). Rules 1-3 need no channel head, so a route with |
| 1397 | // settings is not skipped when the pointer is unreadable (Astra I1 #9). |
| 1398 | settings := cfg.settings && ((rt.Kind == "instance" && rt.Unit == "") || cfg.rootUpdates != nil) |
| 1399 | if ptr == nil && !settings { |
| 1400 | return |
| 1401 | } |
| 1402 | nag := cfg.nag |
| 1403 | // The nag (spec §5 "Canaries expire"): visibility only, and deliberately |
| 1404 | // BEFORE the up-to-date short-circuit below — an expired canary that |
| 1405 | // isn't moving (the common case: the branch shipped once and nobody |
| 1406 | // touched it since) must still be seen every pass it's part of, not just |
| 1407 | // the passes that happen to change something. Nothing here blocks adopt, |
| 1408 | // restart or promote; it only (optionally) prints. Gated on nag != nil |
| 1409 | // rather than always printing to os.Stdout: see NagExpiredCanaries — this |
| 1410 | // must stay silent on the 2-3-second canary fast loop and speak only on |
| 1411 | // the hourly root adopt pass, or an expired canary sitting still would |
| 1412 | // nag every few seconds forever instead of once an hour. |
| 1413 | if nag != nil && ptr != nil && CanaryExpired(ptr, now) { |
| 1414 | // FormatCanaryAge(CanaryAge(...)), NOT a raw now.Sub(ptr.PromotedAt): |
| 1415 | // that would print the pointer's TOTAL age (CanaryAge + CanaryTTL), |
| 1416 | // overstating how long it's been expired by exactly one lease length |
| 1417 | // (2026-08-03 review, IMPORTANT 2) — and disagreeing with the |
| 1418 | // console's badge, which renders the same CanaryAge/FormatCanaryAge |
| 1419 | // pair (internal/console/ladder.go's canaryExpiry). One formatter, |
| 1420 | // shared, so the two operator-facing surfaces of this feature cannot |
| 1421 | // drift apart on what a duration means. |
| 1422 | fmt.Fprintf(nag, " %s: canary %s (%s) has been expired %s — still serving, nothing blocked\n", |
| 1423 | rt.Host, ptr.Channel, ptr.Version, FormatCanaryAge(CanaryAge(ptr, now))) |
| 1424 | } |
| 1425 | channel := routeChannel(rt) |
| 1426 | ver := "" |
| 1427 | if ptr != nil { |
| 1428 | ver = ptr.Version |
| 1429 | } |
| 1430 | cur, err := LiveVersion(optDir, rt.Host) |
| 1431 | if err != nil { |
| 1432 | fmt.Printf(" %s: read live link: %v — skipped\n", rt.Host, err) |
| 1433 | return |
| 1434 | } |
| 1435 | // Repair opportunity for the LIVE version's sibling, every pass, whether |
| 1436 | // or not this pass moves the symlink. C1 (2026-08-03 review): |
| 1437 | // ensureStaged's own back-fill only runs from the call below, which sits |
| 1438 | // behind the cur == ver short-circuit right after this — a version |
| 1439 | // that's already live never calls ensureStaged again, so a sibling write |
| 1440 | // that failed once (a full disk, a permissions hiccup) would otherwise |
| 1441 | // be stuck missing forever for exactly the version actually running. |
| 1442 | // This call never fails the pass: an absent or unreadable sibling is the |
| 1443 | // safe leave-alone state, so a repeated failure here is only a log line. |
| 1444 | if cur != "" && ptr != nil { |
| 1445 | backfillManifestDigest(ctx, s, optDir, RouteAccount(rt), rt.App, cur, rt.Host, ptr.DigestOf(cur), cfg.stage) |
| 1446 | } |
| 1447 | // Instance update settings (settings spec §4): one decision per route. |
| 1448 | // Everything below (bake for the decisions that take it, the proof gate, |
| 1449 | // refusals, D2, caps, restart) applies to whatever it chooses. A route |
| 1450 | // with default settings and no consumed file gets today's head. |
| 1451 | var dec TargetDecision |
| 1452 | if settings { |
| 1453 | var stop bool |
| 1454 | ver, dec, stop = settingsTarget(ctx, optDir, bake, rt, ptr, cur, now, cfg, holds, ignore) |
| 1455 | if stop { |
| 1456 | return |
| 1457 | } |
| 1458 | } |
| 1459 | // sel is the build this pass installs, as the gates below read it (Astra |
| 1460 | // I2 #8): the head itself when the decision is the head, else a pointer |
| 1461 | // naming only the selected build, the way heldProofAdmit's stand-in does |
| 1462 | // (materializeheld.go). The bake waiver, the proof gate (admit and bind), |
| 1463 | // D1, D2 and the refusal records judge the build that would run, never a |
| 1464 | // head the decision passed over. ptr stays the real head, nil or not, for |
| 1465 | // the staging digest below. |
| 1466 | sel := ptr |
| 1467 | if ptr == nil || ver != ptr.Version { |
| 1468 | sel = &ChannelPointer{Channel: channel, Version: ver} |
| 1469 | } |
| 1470 | if cfg.requireRootHistory && sel == ptr && ptr != nil { |
| 1471 | if err := cfg.recheckHistory(ctx, rt); err != nil { |
| 1472 | cfg.historyError(err) |
| 1473 | return |
| 1474 | } |
| 1475 | if err := cfg.rootHistory.History.RetireD1(cfg.historySelected[rt.Host].binding, ptr); err != nil { |
| 1476 | cfg.historyError(err) |
| 1477 | return |
| 1478 | } |
| 1479 | } |
| 1480 | if cur == ver { |
| 1481 | cfg.proof.clear(rt.Host) // live on its target: nothing left to wait for |
| 1482 | // A refusal that no longer blocks the pointer is stale even though |
| 1483 | // nothing moves: v2 was refused and rolled back to v1, then the |
| 1484 | // channel went back to v1. Without this every later pass returns |
| 1485 | // here and the watcher reports the v2 refusal forever (final review |
| 1486 | // F8). Best-effort, like the clear after a successful adoption: the |
| 1487 | // agent cannot remove root's file, so its failure is ignored and the |
| 1488 | // next root pass retries. A record that still blocks stays: the link |
| 1489 | // is on the refused build only when its rollback failed, and that is |
| 1490 | // worth showing. The record is judged against the build this route |
| 1491 | // selected (sel) and against the head (ptr): under a pin or a hold |
| 1492 | // a refusal of the head that is still current stays, so an unpin |
| 1493 | // does not retry a promotion this box already refused (Task 12a). |
| 1494 | if cfg.bakeDir != "" { |
| 1495 | if r, err := ReadRefusal(cfg.bakeDir, rt.Host); err == nil && r != nil && !r.Blocks(sel) && !r.Blocks(ptr) { |
| 1496 | _ = ClearRefusal(cfg.bakeDir, rt.Host) |
| 1497 | } |
| 1498 | } |
| 1499 | return // up to date — stay quiet on the periodic tick |
| 1500 | } |
| 1501 | // Two independent waivers, both root-signed in origin: ignore is the |
| 1502 | // app's legacy production verdict (§2/§4; constant false for a pipelined |
| 1503 | // app), ptr.FastTrack is this ONE pointer's per-promotion waiver (§5) — |
| 1504 | // deliberately unratcheted (pipelines spec §3 amendment: ratcheting it |
| 1505 | // would make it wait out the very window it exists to skip). Either one |
| 1506 | // alone releases the head. |
| 1507 | // |
| 1508 | // Each return from here to admit stops D3's gate being asked about this |
| 1509 | // head, so each one also takes the host off the proof waiting list |
| 1510 | // (cfg.proof.clear). A wait left behind for a head the channel has moved |
| 1511 | // off would stay due for ever, and the fast loop would sweep the host |
| 1512 | // every tick for the whole hold (plan 3 ruling 5). |
| 1513 | // |
| 1514 | // Under settings only rule 4 and a new held instance's first head bake |
| 1515 | // (dec.Bake): a pin, a resolved T and a recovery waive it (§4.3). Both |
| 1516 | // baking decisions name the head, so sel is ptr there. |
| 1517 | if !settings || dec.Bake { |
| 1518 | if hold := Hold(bake, RouteAccount(rt), rt.App, channel, ver, holds, ignore || sel.FastTrack, now); hold != "" { |
| 1519 | fmt.Printf(" %s: %s (%s) is %s\n", rt.Host, ver, channel, hold) |
| 1520 | cfg.proof.clear(rt.Host) |
| 1521 | return |
| 1522 | } |
| 1523 | } |
| 1524 | if poisoned[ver] { |
| 1525 | fmt.Printf(" %s: skipping %s (failed on another host this pass)\n", rt.Host, ver) |
| 1526 | cfg.proof.clear(rt.Host) |
| 1527 | return |
| 1528 | } |
| 1529 | // D1's refusal (refusal.go): this box rolled this exact promotion back |
| 1530 | // because the unit did not come up. Checked before ensureStaged so a |
| 1531 | // refused build is neither downloaded nor relayed to a root pass. |
| 1532 | // |
| 1533 | // A D3 refusal (RefusalNotProven) is the one kind that depends on the |
| 1534 | // policy: it stands only while this pass's policy still requires a proof |
| 1535 | // for the app, or failed to load (closed). Once the require entry or the |
| 1536 | // whole file is gone, the record is ignored and best-effort cleared, so |
| 1537 | // removing the requirement is a break glass that works after a binding |
| 1538 | // refusal too (final review H3). The agent cannot remove root's file; |
| 1539 | // that failure is ignored and the next root pass clears it. D1 and D2 |
| 1540 | // records never depend on the policy. |
| 1541 | if cfg.bakeDir != "" && !cfg.requireRootHistory { |
| 1542 | r, err := ReadRefusal(cfg.bakeDir, rt.Host) |
| 1543 | if err == nil && r.Blocks(sel) && r.Kind == RefusalNotProven { |
| 1544 | if required, _ := cfg.proof.Requires(rt.App); !required { |
| 1545 | if ClearRefusal(cfg.bakeDir, rt.Host) == nil { |
| 1546 | fmt.Printf(" %s: cleared the not-proven refusal of %s: this box no longer requires a proof for %s\n", rt.Host, ver, rt.App) |
| 1547 | } |
| 1548 | r = nil |
| 1549 | } |
| 1550 | } |
| 1551 | if err != nil { |
| 1552 | fmt.Printf(" %s: reading refusal record: %v\n", rt.Host, err) |
| 1553 | } else if r.Blocks(sel) { |
| 1554 | switch r.Kind { |
| 1555 | case RefusalWrongProgram: |
| 1556 | // --retry would re-stage the same bytes and be refused again, |
| 1557 | // and re-shipping this version cannot help: its manifest is |
| 1558 | // immutable, so the consent has to ride on a new version. |
| 1559 | fmt.Printf(" %s: refused: %s is the wrong program for this route (%s); still on %s; ship a new version with carlos ship --program-change and promote it, if the change is intended\n", |
| 1560 | rt.Host, ver, r.Reason, r.RestoredVersion) |
| 1561 | case RefusalNotProven: |
| 1562 | // D3's binding check (proofgate.go bind). Nothing was started |
| 1563 | // or rolled back, and --retry would re-sign the pointer over |
| 1564 | // the same staged bytes and be refused again. |
| 1565 | fmt.Printf(" %s: refused: %s's staged bytes do not match dev's proof (%s); still on %s; roll the exact bytes dev proved\n", |
| 1566 | rt.Host, ver, strings.TrimPrefix(r.Reason, "not proven: "), orNothing(r.RestoredVersion)) |
| 1567 | default: |
| 1568 | fmt.Printf(" %s: refused: %s did not come up on this box at %s (%s); retry with: carlos promote --retry %s %s\n", |
| 1569 | rt.Host, ver, r.RefusedAt.Format(time.RFC3339), r.Reason, ver, channel) |
| 1570 | } |
| 1571 | cfg.proof.clear(rt.Host) |
| 1572 | return |
| 1573 | } |
| 1574 | } |
| 1575 | if cfg.requireRootHistory { |
| 1576 | selHistory := cfg.historySelected[rt.Host] |
| 1577 | refs, err := cfg.rootHistory.History.Refusals(selHistory.binding) |
| 1578 | if err != nil { |
| 1579 | cfg.historyError(err) |
| 1580 | return |
| 1581 | } |
| 1582 | for _, ref := range refs { |
| 1583 | if ref.Kind == RefusalNotUp && ref.Blocks(sel) { |
| 1584 | return |
| 1585 | } |
| 1586 | } |
| 1587 | } |
| 1588 | // D3's cheap gate (proofgate.go, spec §4.5 check 1): before ensureStaged, |
| 1589 | // so an unproven build is neither downloaded nor relayed to root. |
| 1590 | verdict, admitted := cfg.proof.admit(ctx, s, optDir, rt, sel, cur) |
| 1591 | if !admitted { |
| 1592 | return |
| 1593 | } |
| 1594 | // A named unit this process cannot restart is root's (RootRestartsUnits), |
| 1595 | // and so is every route its process serves: ask for root's pass and touch |
| 1596 | // nothing. Silent on the tick; the requester logs each request it writes. |
| 1597 | if cfg.unitsToRoot && restart != nil && !cfg.hostWrites.Enabled && unitServed(reg, rt) { |
| 1598 | if cfg.stageBlocked != nil { |
| 1599 | cfg.stageBlocked(rt.Host, ver, ErrUnitRestartNeedsRoot) |
| 1600 | } |
| 1601 | return |
| 1602 | } |
| 1603 | // The evidence that authorizes ver (settings spec §4.1). Only today's |
| 1604 | // adoption of the head, and a request whose T is still the head, rest on |
| 1605 | // the signed pointer and its digest; everything else needs a promotion |
| 1606 | // record with a matching manifest digest (stageAuthorized). The pointer |
| 1607 | // branch is refused for any build but the head itself: a non-head build |
| 1608 | // has no pointer digest, and staging it unbound on a digested channel is |
| 1609 | // the #253 replay (invariant 3). |
| 1610 | var m *Manifest |
| 1611 | switch { |
| 1612 | case !settings || dec.Evidence == EvidencePointer: |
| 1613 | if ptr == nil || ver != ptr.Version { |
| 1614 | fmt.Printf(" %s: %s: not the signed head, and the decision named no other evidence; not staged\n", rt.Host, ver) |
| 1615 | cfg.proof.clear(rt.Host) |
| 1616 | return |
| 1617 | } |
| 1618 | m, err = ensureStagedDigest(ctx, s, optDir, RouteAccount(rt), rt.App, ver, rt.Host, ptr.DigestOf(ver), cfg.stage) |
| 1619 | case dec.Evidence == EvidenceRecord: |
| 1620 | m, err = stageAuthorized(ctx, s, cfg.pub, optDir, RouteAccount(rt), rt.App, ver, rt.Host, cfg.stage) |
| 1621 | default: |
| 1622 | fmt.Printf(" %s: %s: no evidence this pass can check; not staged\n", rt.Host, ver) |
| 1623 | cfg.proof.clear(rt.Host) |
| 1624 | return |
| 1625 | } |
| 1626 | if err != nil { |
| 1627 | fmt.Printf(" %s: %v\n", rt.Host, err) |
| 1628 | // A version this uid cannot write is the one staging failure a |
| 1629 | // privileged pass would finish; everything else (bucket, signature, |
| 1630 | // manifest) fails the same way as root. See OnStageBlocked. |
| 1631 | if cfg.stageBlocked != nil && stageBlocked(err) { |
| 1632 | // No manifest identity exists on this refusal path, so there is no |
| 1633 | // admitted host mutation to relay under lifecycle mode. |
| 1634 | if !cfg.hostWrites.Enabled { |
| 1635 | cfg.stageBlocked(rt.Host, ver, err) |
| 1636 | } |
| 1637 | } |
| 1638 | return |
| 1639 | } |
| 1640 | id, err := releaseIdentity(m) |
| 1641 | if err != nil { |
| 1642 | fmt.Printf(" %s: release identity: %v\n", rt.Host, err) |
| 1643 | return |
| 1644 | } |
| 1645 | |
| 1646 | // D2 (programhistory.go): the staged bytes, not the manifest, decide. |
| 1647 | // Skipped for non-Go binaries and when this pass has no bake dir. |
| 1648 | newProg := FileProgram(VersionPath(optDir, RouteAccount(rt), ver, rt.App)) |
| 1649 | if (cfg.bakeDir != "" || cfg.requireRootHistory) && newProg != "" { |
| 1650 | var hist []string |
| 1651 | fresh := false |
| 1652 | if cfg.requireRootHistory { |
| 1653 | hist, err = cfg.rootHistory.History.Programs(cfg.historySelected[rt.Host].binding) |
| 1654 | if err != nil { |
| 1655 | cfg.historyError(err) |
| 1656 | return |
| 1657 | } |
| 1658 | sel := cfg.historySelected[rt.Host] |
| 1659 | fresh, err = cfg.rootHistory.History.FreshInstallation(sel.binding, sel.creation) |
| 1660 | if err != nil { |
| 1661 | cfg.historyError(err) |
| 1662 | return |
| 1663 | } |
| 1664 | } else { |
| 1665 | hist = ReadPrograms(cfg.bakeDir, rt.Host) |
| 1666 | } |
| 1667 | // The live binary is what the route is running, so its program belongs |
| 1668 | // in the history whatever the file already says: a history of [A] on a |
| 1669 | // route that is live on B (a change made before this check existed, or |
| 1670 | // a record lost) must not refuse the next B build. |
| 1671 | if cur != "" && !cfg.requireRootHistory { |
| 1672 | if live := FileProgram(VersionPath(optDir, RouteAccount(rt), cur, rt.App)); live != "" && !slices.Contains(hist, live) { |
| 1673 | _ = RecordProgram(cfg.bakeDir, rt.Host, live) |
| 1674 | hist = append(slices.Clone(hist), live) |
| 1675 | } |
| 1676 | } |
| 1677 | if (len(hist) > 0 || cfg.requireRootHistory && !fresh) && !slices.Contains(hist, newProg) && m == nil { |
| 1678 | // ensureStaged has the binary but could not obtain its manifest |
| 1679 | // (no sibling, and the store did not answer), so whether the |
| 1680 | // release carries --program-change is unknown. Unknown is not a |
| 1681 | // refusal: a record here would sit at the gate above and stop the |
| 1682 | // next pass from ever reading the manifest, so an authorised |
| 1683 | // change would stay blocked after the store recovered. Not pointed |
| 1684 | // live, and the next pass asks again. |
| 1685 | fmt.Printf(" %s: %s is %s, not in this route's history, and its manifest could not be read to check for --program-change; not pointed live, will retry\n", |
| 1686 | rt.Host, ver, newProg) |
| 1687 | return |
| 1688 | } |
| 1689 | programChange := m != nil && m.ProgramChange |
| 1690 | if cfg.requireRootHistory { |
| 1691 | programChange = programChange && ptr.DigestOf(ver) != "" |
| 1692 | } |
| 1693 | if (len(hist) > 0 || cfg.requireRootHistory && !fresh) && !slices.Contains(hist, newProg) && !programChange { |
| 1694 | fmt.Printf(" %s: %s is %s, but this route has only run %s. not pointed live (ship a new version with carlos ship --program-change and promote it, if the change is intended)\n", |
| 1695 | rt.Host, ver, newProg, strings.Join(hist, ", ")) |
| 1696 | sha := m.Artifacts[adoptPlatform].SHA256 |
| 1697 | // Nothing was restarted, so the box is still on its old build. |
| 1698 | if err := cfg.recordRefusal(ctx, rt, Refusal{Host: rt.Host, App: rt.App, Version: ver, SHA256: sha, |
| 1699 | Platform: adoptPlatform, PointerPromotedAt: sel.PromotedAt, RefusedAt: now.UTC(), |
| 1700 | Reason: "wrong program: " + newProg, |
| 1701 | Kind: RefusalWrongProgram, Recovered: true, RestoredVersion: cur}); err != nil { |
| 1702 | cfg.historyError(err) |
| 1703 | fmt.Printf(" %s: writing refusal record: %v\n", rt.Host, err) |
| 1704 | } |
| 1705 | return |
| 1706 | } |
| 1707 | } |
| 1708 | |
| 1709 | // D3's binding check (spec §4.5 check 2): the staged bytes themselves |
| 1710 | // must be the proven ones, or a build this box has run well. |
| 1711 | if !cfg.bindProof(ctx, optDir, rt, sel, verdict, cur, now) { |
| 1712 | return |
| 1713 | } |
| 1714 | var historyAttempt string |
| 1715 | if cfg.requireRootHistory { |
| 1716 | if m == nil || m.Artifacts[adoptPlatform].SHA256 == "" { |
| 1717 | cfg.historyError(os.ErrPermission) |
| 1718 | return |
| 1719 | } |
| 1720 | historyIdentity := id |
| 1721 | if !cfg.hostWrites.Enabled { |
| 1722 | historyIdentity = hostwrite.Identity{} |
| 1723 | } |
| 1724 | historyAttempt, err = cfg.rootHistory.begin(ctx, rt, cfg.historySelected[rt.Host], sel, strings.ToLower(m.Artifacts[adoptPlatform].SHA256), now, historyIdentity) |
| 1725 | if err != nil { |
| 1726 | cfg.historyError(err) |
| 1727 | return |
| 1728 | } |
| 1729 | } |
| 1730 | |
| 1731 | // The capability converge for the NEW version, BEFORE PointLive (spec |
| 1732 | // §4d). m is the manifest ensureStaged has in hand — never a re-read of |
| 1733 | // the staged sibling, whose write is best-effort: a sibling that failed |
| 1734 | // to write reads back as (nil, nil), which converge would take as |
| 1735 | // "leave the drop-ins alone", and the new version would go live against |
| 1736 | // the previous version's unit shape silently. Here a manifest this pass |
| 1737 | // could not obtain REFUSES the version instead (capabilityPlan). |
| 1738 | // |
| 1739 | // A hibernating route converges here too: a drop-in is unit text, not a |
| 1740 | // wake, and nothing below starts it. (It reaches this with restart == nil |
| 1741 | // on an ordinary pass, or with the RestartHibernating callback on the |
| 1742 | // agent's loops — which cannot start anything either. In practice this |
| 1743 | // branch is root-only regardless: cfg.capsOps is set by RunAdopt alone, |
| 1744 | // and RunAdopt does not set RestartHibernating.) |
| 1745 | var ( |
| 1746 | prev caps.Snapshot |
| 1747 | plan caps.Plan |
| 1748 | converged bool |
| 1749 | ) |
| 1750 | if cfg.capsOps != nil && CapabilityRoute(rt) { |
| 1751 | // restart != nil is what makes the deferral safe: it says somebody is |
| 1752 | // going to start this daemon after the symlink moves. With no restart |
| 1753 | // callback at all (a caller with no proc manager) converge keeps |
| 1754 | // today's behaviour and brings the unit up itself. |
| 1755 | plan, err = capabilityPlan(rt, m, restart != nil) |
| 1756 | var res caps.Result |
| 1757 | if err == nil { |
| 1758 | err = withReleaseWork(ctx, work, id, func() error { |
| 1759 | if err := cfg.recheckHistory(ctx, rt); err != nil { |
| 1760 | cfg.historyError(err) |
| 1761 | return err |
| 1762 | } |
| 1763 | var convergeErr error |
| 1764 | res, convergeErr = caps.Converge(*cfg.capsOps, plan) |
| 1765 | return convergeErr |
| 1766 | }) |
| 1767 | prev = res.Prev |
| 1768 | } |
| 1769 | if err != nil { |
| 1770 | // A converge REFUSAL refuses the VERSION on this box, in the same |
| 1771 | // direction and for the same reason as an ensureStaged failure: |
| 1772 | // nothing is pointed live, so the box keeps serving what it has |
| 1773 | // rather than run a new binary against a unit shape the platform |
| 1774 | // declined to write. Converge itself has already written nothing |
| 1775 | // and logged the operator's fix. |
| 1776 | // |
| 1777 | // The version is deliberately NOT poisoned here (2026-08-03 |
| 1778 | // review). A refusal bounces nothing — no symlink moved, no unit |
| 1779 | // cycled — so there is no bad build to keep off the next host, |
| 1780 | // which is the only thing the poison exists for. And the refusal |
| 1781 | // is often HOST-shaped: a port another route's drop-in already |
| 1782 | // claims, a state-directory flip mid-migration. Poisoning on one |
| 1783 | // of those would let a single misconfigured route freeze every |
| 1784 | // other host on the box out of a version that is fine for them, |
| 1785 | // pass after pass, until an operator noticed. |
| 1786 | // |
| 1787 | // Converge's own contract: bytes are still moved only when it |
| 1788 | // reports Changed with the error — a failure AFTER its |
| 1789 | // daemon-reload, which it deliberately does not undo because the |
| 1790 | // version-change caller owns it. That is here. Without this the |
| 1791 | // NEW version's unit shape stays on disk under the OLD binary |
| 1792 | // permanently: the next pass finds no diff, snapshots those bytes |
| 1793 | // as its Prev, and every later "restore" restores the wrong |
| 1794 | // shape. Restore is NOT called for an ordinary refusal, where |
| 1795 | // Prev is empty and would be read as "this route had no drop-ins" |
| 1796 | // — that is a deletion, not a restore. |
| 1797 | if res.Changed { |
| 1798 | if err := withReleaseWork(ctx, work, id, func() error { return caps.Restore(*cfg.capsOps, prev) }); err != nil { |
| 1799 | fmt.Printf(" %s: restoring the previous version's capability drop-ins: %v\n", rt.Host, err) |
| 1800 | } |
| 1801 | } |
| 1802 | fmt.Printf(" %s: capabilities for %s were refused: %v — not pointed live\n", rt.Host, ver, err) |
| 1803 | return |
| 1804 | } |
| 1805 | converged = true |
| 1806 | } |
| 1807 | |
| 1808 | if err := withReleaseWork(ctx, work, id, func() error { |
| 1809 | if err := cfg.recheckHistory(ctx, rt); err != nil { |
| 1810 | return err |
| 1811 | } |
| 1812 | running := "" |
| 1813 | if cfg.runningVersion != nil { |
| 1814 | var err error |
| 1815 | running, err = cfg.runningVersion(ctx, rt) |
| 1816 | if err != nil { |
| 1817 | return err |
| 1818 | } |
| 1819 | } |
| 1820 | if err := PointLive(optDir, rt.Host, RouteAccount(rt), ver, rt.App, running); err != nil { |
| 1821 | return err |
| 1822 | } |
| 1823 | if running != "" { |
| 1824 | cur = running |
| 1825 | } |
| 1826 | return nil |
| 1827 | }); err != nil { |
| 1828 | cfg.historyError(err) |
| 1829 | // The symlink did not move, so the box is still serving cur — and the |
| 1830 | // unit shape on disk is the new version's. Put the old one back, or a |
| 1831 | // port the running binary never asked for stays publicly bound. |
| 1832 | // Unlike the restart-failure branch below there is no cur == "" |
| 1833 | // exception: nothing is live either way, so "restore what the symlink |
| 1834 | // names" means removing what this pass just wrote. |
| 1835 | rolled := "" |
| 1836 | if converged { |
| 1837 | if rerr := withReleaseWork(ctx, work, id, func() error { return caps.Restore(*cfg.capsOps, prev) }); rerr != nil { |
| 1838 | fmt.Printf(" %s: restoring the previous version's capability drop-ins: %v\n", rt.Host, rerr) |
| 1839 | } else { |
| 1840 | rolled = " — restored the capability drop-ins the symlink still names" |
| 1841 | } |
| 1842 | } |
| 1843 | fmt.Printf(" %s: repoint failed: %v%s\n", rt.Host, err, rolled) |
| 1844 | // A release staged by an earlier root pass can still need root to |
| 1845 | // swap its live symlink. Relay that write refusal too, or the fast |
| 1846 | // loop never requests another pass once the artifact exists. |
| 1847 | if cfg.stageBlocked != nil && stageBlocked(err) { |
| 1848 | if guardErr := withReleaseWork(ctx, work, id, func() error { return nil }); guardErr != nil { |
| 1849 | fmt.Printf(" %s: release authority changed before privileged staging relay: %v\n", rt.Host, guardErr) |
| 1850 | } else { |
| 1851 | cfg.stageBlocked(rt.Host, ver, err) |
| 1852 | } |
| 1853 | } |
| 1854 | return |
| 1855 | } |
| 1856 | if cfg.bakeDir != "" && newProg != "" && !cfg.requireRootHistory { |
| 1857 | if err := RecordProgram(cfg.bakeDir, rt.Host, newProg); err != nil { |
| 1858 | fmt.Printf(" %s: recording program history: %v\n", rt.Host, err) |
| 1859 | } |
| 1860 | } |
| 1861 | |
| 1862 | // The accepted window, made visible (2026-08-03 ruling). A pass with no |
| 1863 | // WithCapabilities — the in-process adoptLoop, and above all the 2-3 |
| 1864 | // second fastConvergeLoop, neither of which can write /etc from inside |
| 1865 | // carlos-edge.service — has just moved a unit-backed route onto a version |
| 1866 | // that declares capabilities, without converging its unit shape. |
| 1867 | // |
| 1868 | // The signal is "the new manifest declares anything at all", not "the new |
| 1869 | // set differs from what is materialized": the honest difference test needs |
| 1870 | // the route's grants and a render of both sides, and this line exists to |
| 1871 | // mark a window, not to diff it. It costs nothing — m is already in hand — |
| 1872 | // and it errs toward one extra line, once per version change per host, |
| 1873 | // never per tick. |
| 1874 | if cfg.capsOps == nil && CapabilityRoute(rt) && m != nil && m.Capabilities != nil { |
| 1875 | fmt.Printf(" %s: %s declares capabilities and this pass does not write unit configuration — it runs against the drop-ins the previous version left until the root adopt pass (carlos-adopt.service, hourly) converges them\n", rt.Host, ver) |
| 1876 | } |
| 1877 | |
| 1878 | if restart != nil { |
| 1879 | if err := withReleaseWork(ctx, work, id, func() error { return cfg.recheckHistory(ctx, rt) }); err != nil { |
| 1880 | cfg.historyError(err) |
| 1881 | fmt.Printf(" %s: release authority changed before restart: %v\n", rt.Host, err) |
| 1882 | return |
| 1883 | } |
| 1884 | // A converged DAEMON's restart is caps.Restart, not the caller's |
| 1885 | // callback: this is the only call site that knows the plan, and it is |
| 1886 | // the only restart whose error must be real. rootRestartRoute's |
| 1887 | // try-restart (and its swallow-and-continue) keeps owning every other |
| 1888 | // route, byte-identically to before. |
| 1889 | // |
| 1890 | // plan.Set is non-nil whenever converged is true (capabilityPlan |
| 1891 | // guarantees it); the nil check is here anyway, because a nil deref |
| 1892 | // on this path would panic the root adopt pass for the whole box. |
| 1893 | if cfg.restartAttemptSettled != nil { |
| 1894 | defer func() { |
| 1895 | if err := cfg.restartAttemptSettled(rt.Host); err != nil && cfg.restartErrors != nil { |
| 1896 | *cfg.restartErrors = append(*cfg.restartErrors, err) |
| 1897 | } |
| 1898 | }() |
| 1899 | } |
| 1900 | var rerr error |
| 1901 | daemonRestart := converged && plan.Set != nil && plan.Set.Daemon |
| 1902 | if cfg.hostWrites.Enabled { |
| 1903 | if cfg.selectedRestart == nil { |
| 1904 | rerr = hostwrite.ErrUnavailable |
| 1905 | } else { |
| 1906 | rerr = cfg.selectedRestart(ctx, hostwrite.Selection{Host: rt.Host, Work: work, Identity: id}) |
| 1907 | } |
| 1908 | } else if daemonRestart { |
| 1909 | rerr = caps.Restart(*cfg.capsOps, plan) |
| 1910 | } else { |
| 1911 | rerr = restart(rt.Host) |
| 1912 | } |
| 1913 | if rerr != nil { |
| 1914 | if cfg.restartErrors != nil { |
| 1915 | *cfg.restartErrors = append(*cfg.restartErrors, rerr) |
| 1916 | } |
| 1917 | // Poison the build for the rest of the pass and repoint back — |
| 1918 | // the host keeps running what it ran. The symlink goes back |
| 1919 | // FIRST: caps.Restore may start the service, and it must start |
| 1920 | // on the old binary. |
| 1921 | // |
| 1922 | // The repoint's error is CHECKED, where it was once discarded: |
| 1923 | // "link present, staged version gone" is a documented real state |
| 1924 | // (materialize.go — an operator's /opt cleanup, a partially |
| 1925 | // restored volume), and in it PointLive refuses and the symlink |
| 1926 | // STILL NAMES ver. Everything below that claims a rollback — the |
| 1927 | // restore, the summary line, above all the start — must key on |
| 1928 | // whether the rollback actually HAPPENED, not on whether one was |
| 1929 | // attempted. |
| 1930 | var temporary interface{ Temporary() bool } |
| 1931 | if !errors.As(rerr, &temporary) || !temporary.Temporary() { |
| 1932 | poisoned[ver] = true |
| 1933 | } |
| 1934 | var notUp *UnitNotUpError |
| 1935 | isNotUp := errors.As(rerr, ¬Up) |
| 1936 | rolledBack := false |
| 1937 | var rollbackErr error |
| 1938 | if cur != "" { |
| 1939 | if perr := withReleaseWork(ctx, work, id, func() error { return PointLive(optDir, rt.Host, RouteAccount(rt), cur, rt.App) }); perr != nil { |
| 1940 | fmt.Printf(" %s: could not roll the symlink back to %s: %v\n", rt.Host, cur, perr) |
| 1941 | rollbackErr = perr |
| 1942 | } else { |
| 1943 | rolledBack = true |
| 1944 | } |
| 1945 | } |
| 1946 | // The rule is one line long: restore the unit shape that matches |
| 1947 | // whatever the symlink NOW names. |
| 1948 | // |
| 1949 | // With cur == "" there was no previous version and the symlink |
| 1950 | // above could not be rolled back, so it still names ver — and the |
| 1951 | // drop-ins already match it. Restoring here would hand Restore an |
| 1952 | // empty snapshot, which it reads as "this route had no generated |
| 1953 | // drop-ins" and honours by REMOVING the ones this pass wrote, |
| 1954 | // leaving the live version running with no capability shape at |
| 1955 | // all until the hourly grant-drift pass rebuilds it. A rollback |
| 1956 | // that itself FAILED lands in the same place for the other |
| 1957 | // reason: the symlink still names ver, and putting the previous |
| 1958 | // version's drop-ins under it would be exactly the shape/binary |
| 1959 | // mismatch this whole branch exists to prevent. |
| 1960 | rolled := "" |
| 1961 | if converged && rolledBack { |
| 1962 | // Bytes are not a binding: Restore re-runs the socket |
| 1963 | // sequence itself when the socket half moves, or the MX |
| 1964 | // stays off :25 after a rollback. |
| 1965 | if err := withReleaseWork(ctx, work, id, func() error { return caps.Restore(*cfg.capsOps, prev) }); err != nil { |
| 1966 | fmt.Printf(" %s: restoring the previous version's capability drop-ins: %v\n", rt.Host, err) |
| 1967 | rolled = " but could not restore the previous version's capability drop-ins" |
| 1968 | } else { |
| 1969 | rolled = " and restored the previous version's capability drop-ins" |
| 1970 | } |
| 1971 | } else if converged { |
| 1972 | if cur == "" { |
| 1973 | rolled = "; there is no previous version to roll back to, so its capability drop-ins stand as converged" |
| 1974 | } else { |
| 1975 | rolled = "; its capability drop-ins stand as converged with the version the symlink still names" |
| 1976 | } |
| 1977 | } |
| 1978 | action := "rolled the symlink back" |
| 1979 | if cur != "" && !rolledBack { |
| 1980 | action = "could not roll the symlink back" |
| 1981 | } |
| 1982 | fmt.Printf(" %s: update to %s failed: %v — %s%s\n", rt.Host, ver, rerr, action, rolled) |
| 1983 | // Start the service again on the version the symlink NOW names — |
| 1984 | // but ONLY when that is the restored one. The rollback above put |
| 1985 | // the bytes back; without this the box is left with a correct |
| 1986 | // symlink and no process — and because try-restart is a no-op on |
| 1987 | // a unit that is not running (which is what lets a |
| 1988 | // deliberately-stopped service stay stopped), the NEXT pass's |
| 1989 | // restart would exit 0, SetVersions would stamp the build that |
| 1990 | // has never run, and the header would claim it over a service |
| 1991 | // that is down. Start, not try-restart: the whole premise here is |
| 1992 | // that the unit is not running. |
| 1993 | // |
| 1994 | // Gated on rolledBack, not cur != "": a rollback that FAILED (the |
| 1995 | // old version's directory gone — PointLive refuses) leaves the |
| 1996 | // symlink naming ver, and a start there would launch the exact |
| 1997 | // build whose restart just failed. And not for a converged daemon |
| 1998 | // (daemonRestart): the restart that failed there was |
| 1999 | // carlos-app@<host>'s, via caps.Restart; its rollback belongs to |
| 2000 | // caps.Restore above, and this callback would start rt.Unit — a |
| 2001 | // unit that pass never touched. AFTER caps.Restore, deliberately: |
| 2002 | // the drop-in shape must match the binary before anything execs |
| 2003 | // it, and on a unit Restore's socket sequence already brought up, |
| 2004 | // start is an exit-0 no-op. |
| 2005 | // |
| 2006 | // A start that also fails changes nothing about the refusal — the |
| 2007 | // version stays poisoned and unstamped — but it is printed, |
| 2008 | // because at that point the box is very likely serving nothing |
| 2009 | // and the operator must not read the rollback line as |
| 2010 | // "recovered". "May": a try-restart that failed without stopping |
| 2011 | // anything (a permission-denied systemctl) fails its start the |
| 2012 | // same way while the old process still serves — the two errors on |
| 2013 | // the line tell them apart. |
| 2014 | // |
| 2015 | // Under lifecycle host writes (hostwrite) the start goes through |
| 2016 | // the selected receiver, as the forward restart does; D1's checked |
| 2017 | // replace (rollbackRestart) is the legacy path's start for a |
| 2018 | // not-up refusal. |
| 2019 | start := cfg.startUnit |
| 2020 | if isNotUp && cfg.rollbackRestart != nil { |
| 2021 | start = cfg.rollbackRestart |
| 2022 | } |
| 2023 | // D1's group (rollback.go): siblings follow bound onto the |
| 2024 | // restored build only when bound's own rollback happened, each |
| 2025 | // under ONE release work of its own: the pass's original |
| 2026 | // selection where it has one (the one the sibling's forward |
| 2027 | // adoption ran under), else a single Select made here. Never a |
| 2028 | // Select per write: a second one could capture a successor's |
| 2029 | // installation and stamp it. |
| 2030 | var group []string |
| 2031 | if isNotUp { |
| 2032 | restored := "" |
| 2033 | if rolledBack { |
| 2034 | restored = cur |
| 2035 | } |
| 2036 | group = rollbackGroup(reg, optDir, rt, ver, restored, func(h string) (func(func() error) error, error) { |
| 2037 | w, ok := cfg.selected[h] |
| 2038 | if !ok || w == nil { |
| 2039 | var err error |
| 2040 | if w, err = cfg.hostWrites.Select(ctx, h, "release"); err != nil { |
| 2041 | return nil, err |
| 2042 | } |
| 2043 | } |
| 2044 | return func(fn func() error) error { return withReleaseWork(ctx, w, id, fn) }, nil |
| 2045 | }) |
| 2046 | if notUp.Journal != "" { |
| 2047 | fmt.Printf(" %s: %s's last lines:\n%s\n", rt.Host, notUp.Unit, indent(notUp.Journal, " ")) |
| 2048 | } |
| 2049 | } |
| 2050 | started := false |
| 2051 | var serr, authErr error |
| 2052 | if start != nil && rolledBack && !daemonRestart { |
| 2053 | // Authority lost here skips the start, and every process |
| 2054 | // mutation after it, but NOT the refusal records below: the |
| 2055 | // next pass has fresh authority and an empty poisoned map, and |
| 2056 | // without a record it would adopt the rejected build again. |
| 2057 | if guardErr := withReleaseWork(ctx, work, id, func() error { return nil }); guardErr != nil { |
| 2058 | fmt.Printf(" %s: release authority changed before rollback start: %v — not started\n", rt.Host, guardErr) |
| 2059 | authErr = guardErr |
| 2060 | } |
| 2061 | } |
| 2062 | if start != nil && rolledBack && !daemonRestart && authErr == nil { |
| 2063 | started = true |
| 2064 | if cfg.hostWrites.Enabled { |
| 2065 | if cfg.selectedStart == nil { |
| 2066 | serr = hostwrite.ErrUnavailable |
| 2067 | } else { |
| 2068 | serr = cfg.selectedStart(ctx, hostwrite.Selection{Host: rt.Host, Work: work, Identity: id}) |
| 2069 | } |
| 2070 | } else { |
| 2071 | serr = start(rt.Host) |
| 2072 | } |
| 2073 | if serr != nil { |
| 2074 | if cfg.restartErrors != nil { |
| 2075 | *cfg.restartErrors = append(*cfg.restartErrors, serr) |
| 2076 | } |
| 2077 | if rt.Unit == "" && !isNotUp { |
| 2078 | fmt.Printf(" %s: rolled back to %s but could not start it: %v\n", rt.Host, cur, serr) |
| 2079 | } else { |
| 2080 | fmt.Printf(" %s: rolled back to %s but could not start it: %v — this host may be serving nothing\n", |
| 2081 | rt.Host, cur, serr) |
| 2082 | } |
| 2083 | } |
| 2084 | } |
| 2085 | if isNotUp { |
| 2086 | // The refusals go down AFTER the restart, carrying what it |
| 2087 | // did: a record written before it would tell the watcher |
| 2088 | // "rolled back" about a box whose restored build never came |
| 2089 | // up (final review F6). |
| 2090 | rec := recovery{recovered: true} |
| 2091 | switch { |
| 2092 | case cur == "": |
| 2093 | rec = recovery{err: "there is no previous version to roll back to"} |
| 2094 | case !rolledBack: |
| 2095 | rec = recovery{err: fmt.Sprintf("could not roll the symlink back to %s: %v", cur, rollbackErr)} |
| 2096 | case authErr != nil: |
| 2097 | rec = recovery{restored: cur, err: "release authority changed before rollback start: " + authErr.Error()} |
| 2098 | case !started: |
| 2099 | rec = recovery{restored: cur, err: "nothing on this pass restarts the unit on " + cur} |
| 2100 | case serr != nil: |
| 2101 | rec = recovery{restored: cur, err: serr.Error()} |
| 2102 | default: |
| 2103 | rec.restored = cur |
| 2104 | } |
| 2105 | sha := "" |
| 2106 | if m != nil { |
| 2107 | sha = m.Artifacts[adoptPlatform].SHA256 |
| 2108 | } |
| 2109 | writeGroupRefusals(cfg, group, rt.App, sel, ver, sha, adoptPlatform, notUp, rec, now) |
| 2110 | } |
| 2111 | return |
| 2112 | } |
| 2113 | } |
| 2114 | if cfg.requireRootHistory { |
| 2115 | if err := cfg.rootHistory.complete(ctx, rt, cfg.historySelected[rt.Host], historyAttempt, ver); err != nil { |
| 2116 | cfg.historyError(err) |
| 2117 | return |
| 2118 | } |
| 2119 | } |
| 2120 | if err := withReleaseWork(ctx, work, id, func() error { return reg.SetVersions(rt.Host, ver, cur) }); err != nil { |
| 2121 | fmt.Printf(" %s: repointed but recording the version failed: %v\n", rt.Host, err) |
| 2122 | return |
| 2123 | } |
| 2124 | // A record left from an older promotion is stale once a build is live. |
| 2125 | // Best-effort: the agent cannot remove root's file, the root pass can. |
| 2126 | if cfg.bakeDir != "" { |
| 2127 | _ = ClearRefusal(cfg.bakeDir, rt.Host) |
| 2128 | } |
| 2129 | if rt.IsSidecar() { |
| 2130 | fmt.Printf(" %s: repointed %s -> %s (%s; sidecar, its worker moves onto it on the agent's drift check)\n", rt.Host, cur, ver, channel) |
| 2131 | return |
| 2132 | } |
| 2133 | if rt.Hibernate { |
| 2134 | // Two different facts, one grep. With no restart callback this pass |
| 2135 | // genuinely only moved a symlink, and the next wake is when the new |
| 2136 | // version runs. With one (RestartHibernating, the agent's loops) the |
| 2137 | // callback has ALREADY been consulted above: a warm exec child was |
| 2138 | // bounced onto ver a few lines ago, and a sleeping one was a no-op. |
| 2139 | // Which of the two happened is makeRestartRoute's to log — it prints |
| 2140 | // the bounce — so this line stops claiming the sleeping case for |
| 2141 | // both, rather than growing a third state to track. |
| 2142 | if restart != nil { |
| 2143 | fmt.Printf(" %s: repointed %s -> %s (%s; hibernating — bounced if awake, else runs it on next wake)\n", rt.Host, cur, ver, channel) |
| 2144 | } else { |
| 2145 | fmt.Printf(" %s: repointed %s -> %s (%s; hibernating — runs it on next wake)\n", rt.Host, cur, ver, channel) |
| 2146 | } |
| 2147 | } else { |
| 2148 | fmt.Printf(" %s: updated %s -> %s (%s)\n", rt.Host, cur, ver, channel) |
| 2149 | } |
| 2150 | } |
| 2151 | |
| 2152 | // capabilityPlan turns a route plus the manifest of the version being |
| 2153 | // adopted into one converge plan, or refuses. |
| 2154 | // |
| 2155 | // Two rulings live here, and both are the difference between this feature |
| 2156 | // working and this feature quietly doing the opposite of what it says: |
| 2157 | // |
| 2158 | // - A manifest this pass could not obtain (m == nil: the binary is staged, |
| 2159 | // so ensureStaged succeeded, but neither the sibling nor the bucket |
| 2160 | // could answer for its manifest) REFUSES the version. caps.Plan's nil |
| 2161 | // Set means "leave the drop-ins alone", which is the right posture for a |
| 2162 | // pass that is merely surveying the box — and exactly the wrong one on a |
| 2163 | // version change, where it would point a new binary at the previous |
| 2164 | // version's unit shape and say nothing. |
| 2165 | // |
| 2166 | // - A manifest that RESOLVED and declares NOTHING is not the same thing. |
| 2167 | // Manifest.Capabilities is a nil pointer for both a legacy manifest and |
| 2168 | // a fetch that never happened, so it is never passed through verbatim: |
| 2169 | // a resolved manifest always carries a non-nil Set, empty when it |
| 2170 | // declares nothing, which is what makes converge REMOVE stale drop-ins. |
| 2171 | // Set: m.Capabilities verbatim would mean a capability dropped from a |
| 2172 | // manifest is never removed — the exact opposite of spec §4d's removal |
| 2173 | // semantics. |
| 2174 | // |
| 2175 | // An unparseable grants column refuses too, rather than defaulting to no |
| 2176 | // grants: "no grants" is a real instruction that revokes :25 and drops a |
| 2177 | // mail server's listener, and a column this binary cannot read is not |
| 2178 | // evidence that an operator asked for that. |
| 2179 | // |
| 2180 | // canRestart arms caps.Plan.DeferRestart: this converge runs BEFORE the |
| 2181 | // symlink moves, so a daemon it started would be the OLD binary under the NEW |
| 2182 | // unit shape, and the caller would then cycle it a second time. Deferring is |
| 2183 | // only honest when somebody is actually going to do the restart — with no |
| 2184 | // restart callback at all, converge keeps bringing the unit up itself. |
| 2185 | func capabilityPlan(rt registry.Route, m *Manifest, canRestart bool) (caps.Plan, error) { |
| 2186 | if m == nil { |
| 2187 | return caps.Plan{}, fmt.Errorf("the manifest for this version could not be read or fetched, so its unit shape is unknown") |
| 2188 | } |
| 2189 | g, err := caps.ParseGrants(rt.Caps) |
| 2190 | if err != nil { |
| 2191 | return caps.Plan{}, fmt.Errorf("capability grants %q: %v", rt.Caps, err) |
| 2192 | } |
| 2193 | set := m.Capabilities |
| 2194 | if set == nil { |
| 2195 | set = &caps.Set{V: caps.Version} |
| 2196 | } |
| 2197 | return caps.Plan{ |
| 2198 | Host: rt.Host, |
| 2199 | App: rt.App, |
| 2200 | Set: set, |
| 2201 | Grants: g, |
| 2202 | Hibernate: rt.Hibernate, |
| 2203 | DeferRestart: canRestart, |
| 2204 | }, nil |
| 2205 | } |
| 2206 | |
| 2207 | // stageCheck is what ensureStagedChecked needs to judge a staged copy it is |
| 2208 | // about to reuse (spec §4.2). |
| 2209 | type stageCheck struct { |
| 2210 | beforeWrite func(string) error |
| 2211 | // refs reports the staged copies something on this box still refers to. |
| 2212 | // nil, or an error, means unknown, and unknown never permits a restage. |
| 2213 | refs func() (StagedPaths, error) |
| 2214 | // verified holds staged paths whose bytes already matched their canonical |
| 2215 | // release this pass, so N hosts moving onto one build hash it once, each |
| 2216 | // with the artifact sha it matched: a digest-bound reuse trusts the memory |
| 2217 | // only for the sha its own checked manifest names (stageFrom). nil |
| 2218 | // remembers nothing. |
| 2219 | verified map[string]string |
| 2220 | } |
| 2221 | |
| 2222 | // ensureStaged makes ver's build of app available in the version store, |
| 2223 | // downloading and sha256-verifying it if needed. Idempotent across hosts in |
| 2224 | // one pass. The manifest supplies the sha; the bytes are refused unless they |
| 2225 | // hash to it — the channel pointer's signature plus this gate is the whole |
| 2226 | // trust chain from operator to disk. host is used only to prefix this |
| 2227 | // function's own log lines, matching every other line this file prints. |
| 2228 | // |
| 2229 | // It also keeps the staged manifest sibling (versions.go's StageManifest) |
| 2230 | // current: a fresh download writes the sibling alongside the binary, and a |
| 2231 | // binary that is ALREADY staged but missing its sibling gets one written |
| 2232 | // without a re-download — the back-fill for every version staged before this |
| 2233 | // sibling existed. For copies staged by this binary or later the sibling is |
| 2234 | // the canonical manifest's exact bytes; only a copy an older binary staged |
| 2235 | // carries a lossy re-marshal (see ReadStagedManifest). The bucket object stays |
| 2236 | // canonical either way. |
| 2237 | // |
| 2238 | // The back-fill's own fetch failure is NOT fatal when the binary is already |
| 2239 | // staged (2026-08-03 review, I2): this function's whole promise is "the |
| 2240 | // binary is on disk", and a version already staged must not depend on the |
| 2241 | // bucket answering for a sibling write that can be retried. Without this, a |
| 2242 | // transient GetManifest failure would hard-fail adoption of a binary that |
| 2243 | // needs no adoption at all — and on a canary route's 2-3-second fast loop, |
| 2244 | // flood the journal every tick. adoptHost's backfillManifest (C1, same |
| 2245 | // review) is what makes leaving this non-fatal safe: every subsequent pass |
| 2246 | // gets another chance at the live version's sibling, so a transient failure |
| 2247 | // here heals on its own instead of being permanently missed. |
| 2248 | // |
| 2249 | // It RETURNS the manifest, because adoptHost's capability converge builds its |
| 2250 | // plan from it (spec §4d) and must never re-read the sibling to get one: the |
| 2251 | // sibling write is best-effort, and a failed write reads back as (nil, nil) — |
| 2252 | // "leave the drop-ins alone" — which on a version change would point a new |
| 2253 | // binary at the previous version's unit shape without a word. The one path |
| 2254 | // that has no manifest of its own to give (a staged binary whose manifest |
| 2255 | // neither disk nor bucket would answer for) returns (nil, nil) DELIBERATELY, |
| 2256 | // and adoptHost turns that into a refusal when — and only when — capabilities |
| 2257 | // are armed for the route. Every other caller keeps today's contract exactly: |
| 2258 | // an error means "the binary is not on disk", nil means it is. |
| 2259 | // |
| 2260 | // A version directory that is already staged is no longer trusted by name |
| 2261 | // (zero-downtime deploys spec §4.2). A version name can be shipped again with |
| 2262 | // different bytes once retention has pruned it, and the staged path is keyed |
| 2263 | // by name alone. So a staged binary is hashed and compared with the canonical |
| 2264 | // manifest's digest for this platform: |
| 2265 | // - a match reuses it, as before; |
| 2266 | // - a mismatch restages over it only when nothing on the box refers to it |
| 2267 | // (chk.refs: live links, Version, PrevVersion, the generations file, |
| 2268 | // and whatever WithPinSource adds); otherwise the version is refused with |
| 2269 | // ErrVersionReused, so a copy under another instance's build never |
| 2270 | // changes underneath it; |
| 2271 | // - a canonical manifest that cannot be read keeps today's answer: the copy |
| 2272 | // is used as staged, because a box cannot compare what it cannot read, |
| 2273 | // and the agent runs any attempt onto it in hold (CanonicalDemand). |
| 2274 | // |
| 2275 | // ensureStaged is ensureStagedChecked with no reference check. It has no |
| 2276 | // registry in hand, so it cannot know what refers to a staged copy and never |
| 2277 | // restages one: a copy that differs from its release is refused. It stays |
| 2278 | // refused until a pass that adopts that version restages it, or the release |
| 2279 | // is shipped under a new name. |
| 2280 | func ensureStaged(ctx context.Context, s Store, optDir, acct, app, ver, host string) (*Manifest, error) { |
| 2281 | return ensureStagedChecked(ctx, s, optDir, acct, app, ver, host, stageCheck{}) |
| 2282 | } |
| 2283 | |
| 2284 | // ensureStagedChecked is ensureStaged with the reuse check's references |
| 2285 | // supplied by chk; see ensureStaged's doc for the whole contract. |
| 2286 | func ensureStagedChecked(ctx context.Context, s Store, optDir, acct, app, ver, host string, chk stageCheck) (*Manifest, error) { |
| 2287 | path := VersionPath(optDir, acct, ver, app) |
| 2288 | _, statErr := os.Stat(path) |
| 2289 | staged := statErr == nil |
| 2290 | m, raw, err := GetManifestRaw(ctx, s, acct, app, ver) |
| 2291 | if err != nil { |
| 2292 | if !staged { |
| 2293 | return nil, fmt.Errorf("manifest for %s/%s: %w", app, ver, err) |
| 2294 | } |
| 2295 | // The binary IS staged, which is this function's whole promise, and |
| 2296 | // the canonical manifest cannot be read to compare it with. A sibling |
| 2297 | // already on disk is the manifest in hand, exactly as before this |
| 2298 | // check existed. Without one, a fetch failure is only worth a line; |
| 2299 | // backfillManifest retries on every pass. |
| 2300 | sib, serr := ReadStagedManifest(optDir, acct, ver, app) |
| 2301 | if serr == nil && sib != nil { |
| 2302 | return sib, nil |
| 2303 | } |
| 2304 | if serr != nil { |
| 2305 | fmt.Printf(" %s: staged manifest for %s: %v\n", host, ver, serr) |
| 2306 | } |
| 2307 | fmt.Printf(" %s: back-fill manifest for %s: %v\n", host, ver, err) |
| 2308 | return nil, nil |
| 2309 | } |
| 2310 | return stageFrom(ctx, s, optDir, acct, app, ver, host, chk, staged, m, raw, false) |
| 2311 | } |
| 2312 | |
| 2313 | // stageFrom is ensureStagedChecked from the point it holds the canonical |
| 2314 | // manifest: m and raw, its exact bytes, read once by the caller. staged says |
| 2315 | // whether a copy was on disk under the name. bound says m passed the |
| 2316 | // authorizing pointer's digest (ensureStagedDigest), and changes two things |
| 2317 | // only: a copy this pass already verified is reused only if it was verified |
| 2318 | // against THIS artifact sha, never another manifest's under the same name; |
| 2319 | // and the sibling is made exactly raw (syncSibling) rather than repaired only |
| 2320 | // when it does not read. Unbound, it is ensureStagedChecked as zdd-4 wrote it. |
| 2321 | func stageFrom(ctx context.Context, s Store, optDir, acct, app, ver, host string, chk stageCheck, staged bool, m *Manifest, raw []byte, bound bool) (*Manifest, error) { |
| 2322 | path := VersionPath(optDir, acct, ver, app) |
| 2323 | sibling := repairSibling |
| 2324 | if bound { |
| 2325 | sibling = syncSibling |
| 2326 | } |
| 2327 | art, hasBuild := m.Artifacts[adoptPlatform] |
| 2328 | if staged { |
| 2329 | if v := chk.verified[path]; v != "" && (!bound || hasBuild && strings.EqualFold(v, art.SHA256)) { |
| 2330 | sibling(optDir, acct, app, ver, host, raw, chk) |
| 2331 | return m, nil |
| 2332 | } |
| 2333 | sum, herr := FileSHA256(path) |
| 2334 | if herr != nil { |
| 2335 | return nil, fmt.Errorf("staged %s/%s: %w", app, ver, herr) |
| 2336 | } |
| 2337 | if hasBuild && strings.EqualFold(sum, art.SHA256) { |
| 2338 | if chk.verified != nil { |
| 2339 | chk.verified[path] = art.SHA256 |
| 2340 | } |
| 2341 | sibling(optDir, acct, app, ver, host, raw, chk) |
| 2342 | return m, nil |
| 2343 | } |
| 2344 | // Different bytes under the same name. Overwrite only a copy nothing |
| 2345 | // refers to: overwriting a referenced one would change the bytes under |
| 2346 | // another instance's build (spec §4.2). |
| 2347 | if chk.refs == nil { |
| 2348 | return nil, fmt.Errorf("%s/%s: the staged copy differs from the release and what refers to it is unknown, so it was not restaged", app, ver) |
| 2349 | } |
| 2350 | pins, perr := chk.refs() |
| 2351 | // References another app could not complete (an unread build state |
| 2352 | // of one of its held instances) do not stop this one; this app's |
| 2353 | // do (Astra round 1 finding 2). |
| 2354 | if inc := (*IncompleteRefsError)(nil); errors.As(perr, &inc) && pins != nil && !inc.Covers(acct, app) { |
| 2355 | perr = nil |
| 2356 | } |
| 2357 | if perr != nil { |
| 2358 | return nil, fmt.Errorf("%s/%s: the staged copy differs from the release and what refers to it is unknown (%v), so it was not restaged", app, ver, perr) |
| 2359 | } |
| 2360 | // By the copy's real path as well as its name, so a symlinked opt dir |
| 2361 | // cannot make a pinned copy look unpinned (Astra r3 N4). |
| 2362 | referred, rerr := pins.Refers(path) |
| 2363 | if rerr != nil { |
| 2364 | return nil, fmt.Errorf("%s/%s: the staged copy differs from the release and what refers to it is unknown (%v), so it was not restaged", app, ver, rerr) |
| 2365 | } |
| 2366 | if referred { |
| 2367 | return nil, fmt.Errorf("%s/%s: %w", app, ver, ErrVersionReused) |
| 2368 | } |
| 2369 | fmt.Printf(" %s: staged %s differs from the release in the store and nothing on this box refers to it; restaging\n", host, ver) |
| 2370 | } |
| 2371 | if !hasBuild { |
| 2372 | // A box facing a release with no build for its platform: skip, never |
| 2373 | // install the wrong arch. The next release ships it and the box |
| 2374 | // catches up. |
| 2375 | return nil, fmt.Errorf("release %s has no %s build — skipping", ver, adoptPlatform) |
| 2376 | } |
| 2377 | // Streamed to disk and hashed on the way (#269): a build read whole |
| 2378 | // outgrew the stage pass's memory cap. |
| 2379 | if chk.beforeWrite != nil { |
| 2380 | if err := chk.beforeWrite(path); err != nil { |
| 2381 | return nil, err |
| 2382 | } |
| 2383 | } |
| 2384 | err := stageArtifactFrom(optDir, acct, ver, app, art.SHA256, func() (io.ReadCloser, error) { |
| 2385 | rc, _, err := OpenQualified(ctx, s, ArtifactKey(acct, app, ver, art.File)) |
| 2386 | return rc, err |
| 2387 | }) |
| 2388 | if err != nil { |
| 2389 | return nil, err |
| 2390 | } |
| 2391 | if chk.verified != nil { |
| 2392 | chk.verified[path] = art.SHA256 |
| 2393 | } |
| 2394 | // The sibling is the CANONICAL bytes (spec §4.2), and a restage replaces |
| 2395 | // whatever sibling the stale copy had. Best-effort: a failure here is not |
| 2396 | // fatal to staging the binary, and backfillManifest gets another chance |
| 2397 | // at it on every later pass. |
| 2398 | if err := StageManifest(optDir, acct, ver, app, raw); err != nil { |
| 2399 | fmt.Printf(" %s: stage manifest sibling for %s: %v\n", host, ver, err) |
| 2400 | } |
| 2401 | return m, nil |
| 2402 | } |
| 2403 | |
| 2404 | // ensureStagedDigest is adoptHost's staging entry: ensureStagedChecked bound |
| 2405 | // to the digest the authorizing pointer entry signs for ver (#253, spec |
| 2406 | // §4.6). want "" is ensureStagedChecked exactly: a pointer from before |
| 2407 | // digests, or one written with the marker off. |
| 2408 | // |
| 2409 | // With a digest, the canonical manifest must hash to want BEFORE anything is |
| 2410 | // downloaded or restaged, and the manifest that passed is the one staging |
| 2411 | // uses (stageFrom): it is never read a second time, so a swap between the |
| 2412 | // check and the download cannot slip through. Everything after the check is |
| 2413 | // zdd-4's reuse rule (spec §4.2), judged against that checked manifest: a |
| 2414 | // staged copy whose bytes match it is reused, one that differs is restaged |
| 2415 | // only when nothing on the box refers to it and refused otherwise. Unlike the |
| 2416 | // no-digest path, a manifest that cannot be read or does not hash to want is |
| 2417 | // a refusal even when a copy is staged: that copy cannot be shown to be the |
| 2418 | // build the pointer signed. |
| 2419 | func ensureStagedDigest(ctx context.Context, s Store, optDir, acct, app, ver, host, want string, chk stageCheck) (*Manifest, error) { |
| 2420 | if want == "" { |
| 2421 | return ensureStagedChecked(ctx, s, optDir, acct, app, ver, host, chk) |
| 2422 | } |
| 2423 | m, raw, err := CheckManifestDigest(ctx, s, acct, app, ver, want) |
| 2424 | if err != nil { |
| 2425 | return nil, fmt.Errorf("manifest for %s/%s: %w", app, ver, err) |
| 2426 | } |
| 2427 | _, statErr := os.Stat(VersionPath(optDir, acct, ver, app)) |
| 2428 | return stageFrom(ctx, s, optDir, acct, app, ver, host, chk, statErr == nil, m, raw, true) |
| 2429 | } |
| 2430 | |
| 2431 | // syncSibling makes the sibling beside a digest-bound copy exactly raw, the |
| 2432 | // manifest bytes that passed the pointer's digest (spec §4.2, §4.6): left |
| 2433 | // alone when it already is, written otherwise, including over a sibling that |
| 2434 | // reads but is some other manifest's bytes (an older binary's re-marshal, or |
| 2435 | // a release since replaced under the name). Best-effort, like repairSibling. |
| 2436 | func syncSibling(optDir, acct, app, ver, host string, raw []byte, checks ...stageCheck) { |
| 2437 | if cur, err := safefile.ReadFile(StagedManifestPath(optDir, acct, ver, app)); err == nil && bytes.Equal(cur, raw) { |
| 2438 | return |
| 2439 | } |
| 2440 | if err := stageGenerationSibling(optDir, acct, app, ver, raw, checks); err != nil { |
| 2441 | fmt.Printf(" %s: stage manifest sibling for %s: %v\n", host, ver, err) |
| 2442 | } |
| 2443 | } |
| 2444 | |
| 2445 | // repairSibling writes the canonical bytes beside a reused binary whose |
| 2446 | // sibling is absent or does not read: the back-fill for versions staged |
| 2447 | // before the sibling existed, and the repair for one a different binary |
| 2448 | // wrote. A sibling that reads is left alone, as before. Best-effort. |
| 2449 | func repairSibling(optDir, acct, app, ver, host string, raw []byte, checks ...stageCheck) { |
| 2450 | if sib, err := ReadStagedManifest(optDir, acct, ver, app); err == nil && sib != nil { |
| 2451 | return |
| 2452 | } |
| 2453 | if err := stageGenerationSibling(optDir, acct, app, ver, raw, checks); err != nil { |
| 2454 | fmt.Printf(" %s: stage manifest sibling for %s: %v\n", host, ver, err) |
| 2455 | } |
| 2456 | } |
| 2457 | |
| 2458 | // backfillManifest is adoptHost's per-pass repair for a LIVE version's |
| 2459 | // staged sibling (2026-08-03 review, C1). ensureStaged's own back-fill only |
| 2460 | // fires from the call inside adoptHost that sits behind the cur == ver |
| 2461 | // short-circuit — so a version already live, once adopted, never calls |
| 2462 | // ensureStaged again on its own account, and a sibling write that failed |
| 2463 | // once would stay missing forever for exactly the version actually running. |
| 2464 | // This gives every pass another shot at it: os.Stat first (the steady-state |
| 2465 | // cost — one stat when the sibling is already there), fetch only when |
| 2466 | // absent, and never return an error — an absent or unreadable sibling is |
| 2467 | // the safe leave-alone state (ReadStagedManifest's nil-nil contract; the |
| 2468 | // converge pass treats it as "unknown", never as "no capabilities"), so a |
| 2469 | // repeated failure here is only ever worth a log line, host-prefixed like |
| 2470 | // every other line adoptHost prints. |
| 2471 | func backfillManifest(ctx context.Context, s Store, optDir, acct, app, ver, host string, checks ...stageCheck) { |
| 2472 | if _, err := os.Stat(StagedManifestPath(optDir, acct, ver, app)); err == nil { |
| 2473 | return |
| 2474 | } |
| 2475 | _, raw, err := GetManifestRaw(ctx, s, acct, app, ver) |
| 2476 | if err != nil { |
| 2477 | fmt.Printf(" %s: back-fill manifest for %s: %v\n", host, ver, err) |
| 2478 | return |
| 2479 | } |
| 2480 | if err := stageGenerationSibling(optDir, acct, app, ver, raw, checks); err != nil { |
| 2481 | fmt.Printf(" %s: stage manifest sibling for %s: %v\n", host, ver, err) |
| 2482 | } |
| 2483 | } |
| 2484 | |
| 2485 | // backfillManifestDigest is backfillManifest bound to the digest the |
| 2486 | // authorizing pointer signs for ver (#253, spec §4.6). want "" is |
| 2487 | // backfillManifest exactly. With one, the bucket manifest must hash to it |
| 2488 | // (CheckManifestDigest) and the checked canonical bytes are what is written; |
| 2489 | // a mismatch is a log line and no sibling, the same leave-alone state as any |
| 2490 | // other failure here. |
| 2491 | func backfillManifestDigest(ctx context.Context, s Store, optDir, acct, app, ver, host, want string, checks ...stageCheck) { |
| 2492 | if want == "" { |
| 2493 | backfillManifest(ctx, s, optDir, acct, app, ver, host, checks...) |
| 2494 | return |
| 2495 | } |
| 2496 | if _, err := os.Stat(StagedManifestPath(optDir, acct, ver, app)); err == nil { |
| 2497 | return |
| 2498 | } |
| 2499 | _, raw, err := CheckManifestDigest(ctx, s, acct, app, ver, want) |
| 2500 | if err != nil { |
| 2501 | fmt.Printf(" %s: back-fill manifest for %s: %v\n", host, ver, err) |
| 2502 | return |
| 2503 | } |
| 2504 | if err := stageGenerationSibling(optDir, acct, app, ver, raw, checks); err != nil { |
| 2505 | fmt.Printf(" %s: stage manifest sibling for %s: %v\n", host, ver, err) |
| 2506 | } |
| 2507 | } |
| 2508 | |
| 2509 | // settingsTarget runs Target for one exec instance (settings spec §4.3) and |
| 2510 | // does what the decision needs before staging: on root's pass, write the |
| 2511 | // consumed file (a resolution BEFORE staging); refuse what installs nothing. |
| 2512 | // It returns the version to install, the decision, and stop: nothing is |
| 2513 | // installed this pass. Every stop takes the host off the proof waiting list, |
| 2514 | // as each of adoptHost's pre-admit returns does (plan 3 ruling 5). |
| 2515 | // |
| 2516 | // A route with default settings and no consumed file is today's adoption: |
| 2517 | // the head, with bake, on the pointer's evidence, and no book, no Target and |
| 2518 | // no consumed write. Only an unreadable failure book on a managed route |
| 2519 | // changes that (the zero-downtime plans' refuseManaged posture). A consumed |
| 2520 | // file another tenant of the hostname left (Consumed.ForeignTo) is read as |
| 2521 | // none, and root's write replaces it (ruling R26). |
| 2522 | func settingsTarget(ctx context.Context, optDir string, bake *FirstSeen, rt registry.Route, ptr *ChannelPointer, cur string, now time.Time, |
| 2523 | cfg passConfig, holds map[string]time.Duration, ignore bool) (string, TargetDecision, bool) { |
| 2524 | if cfg.rootUpdates != nil { |
| 2525 | return cfg.rootUpdates.target(ctx, rt, ptr, bake, now, cfg, holds, ignore) |
| 2526 | } |
| 2527 | stop := func(dec TargetDecision) (string, TargetDecision, bool) { |
| 2528 | cfg.proof.clear(rt.Host) |
| 2529 | return "", dec, true |
| 2530 | } |
| 2531 | acct, channel := RouteAccount(rt), routeChannel(rt) |
| 2532 | u, uerr := ParseUpdateSettingsLenient(rt.UpdateSettings) |
| 2533 | plain := uerr == nil && u.Default() |
| 2534 | managed := cfg.managed != nil && cfg.managed(rt) |
| 2535 | consumed, cerr := ReadConsumed(optDir, rt.Host) |
| 2536 | // A file root wrote for another tenant of this hostname is no record of |
| 2537 | // this route's (ruling R26): every rule below decides as if there were |
| 2538 | // none, and NextConsumed starts the route's record empty. A default route |
| 2539 | // still takes the Target path over one, so root's write replaces the old |
| 2540 | // tenant's file with this route's own (the decision is today's adoption |
| 2541 | // either way). An untagged file is read as the route's own, as before. |
| 2542 | foreign := consumed.ForeignTo(rt) |
| 2543 | if foreign { |
| 2544 | consumed = nil |
| 2545 | } |
| 2546 | if plain && consumed == nil && !foreign && (cerr == nil || !controlname.ValidHostShape(rt.Host)) { |
| 2547 | if ptr == nil { |
| 2548 | return "", TargetDecision{}, true // today's quiet skip |
| 2549 | } |
| 2550 | if managed && cfg.failedErr != nil { |
| 2551 | return stop(TargetDecision{}) |
| 2552 | } |
| 2553 | return ptr.Version, TargetDecision{Version: ptr.Version, Reason: ReasonAdopt, Evidence: EvidencePointer, Bake: true, Detail: ptr.Version}, false |
| 2554 | } |
| 2555 | if cerr != nil { |
| 2556 | fmt.Printf(" %s: consumed file unreadable (%v): nothing installed\n", rt.Host, cerr) |
| 2557 | return stop(TargetDecision{}) |
| 2558 | } |
| 2559 | // A column that does not parse even leniently is held where it is, never |
| 2560 | // read as auto (ruling R-SETTINGS-UNREADABLE): it may carry a hold this |
| 2561 | // binary cannot see. Root records reason unknown, so the wake check holds |
| 2562 | // too; nothing is installed and the live link stays. |
| 2563 | if uerr != nil { |
| 2564 | dec := TargetDecision{Reason: ReasonUnknown, Detail: uerr.Error()} |
| 2565 | if cfg.resolve { |
| 2566 | fmt.Printf(" %s: update settings unreadable (%v): held, nothing installed\n", rt.Host, uerr) |
| 2567 | in := TargetInput{Route: rt, Pointer: ptr, Consumed: consumed, Installed: cur} |
| 2568 | if err := publishConsumed(ctx, optDir, cfg.route, NextConsumed(in, dec)); err != nil { |
| 2569 | fmt.Printf(" %s: recording the update decision: %v\n", rt.Host, err) |
| 2570 | } |
| 2571 | } |
| 2572 | return stop(dec) |
| 2573 | } |
| 2574 | if cfg.failedErr != nil && (!plain || managed) { |
| 2575 | return stop(TargetDecision{}) // PassFilter said why, once |
| 2576 | } |
| 2577 | bs, berr := ParseBuildStateCopy(rt.BuildState) |
| 2578 | if berr != nil { |
| 2579 | if cfg.resolve { |
| 2580 | fmt.Printf(" %s: %v: read as unknown\n", rt.Host, berr) |
| 2581 | } |
| 2582 | bs = BuildStateCopy{} |
| 2583 | } |
| 2584 | in := TargetInput{Route: rt, Pointer: ptr, Settings: u, Build: bs, Consumed: consumed, Installed: cur} |
| 2585 | if ptr != nil { |
| 2586 | in.HeadHeld = Hold(bake, acct, rt.App, channel, ptr.Version, holds, ignore || ptr.FastTrack, now) |
| 2587 | in.HeadTag = cfg.tags[acct+"/"+rt.App][ptr.Version] |
| 2588 | } |
| 2589 | // The failure records outrank every rule for a route with settings |
| 2590 | // (§4.3). A route with default settings keeps today's adoption, which |
| 2591 | // never read the book. |
| 2592 | if !plain && cfg.failed != nil { |
| 2593 | in.Failed = func(v string) bool { return cfg.failed(acct, rt.App, v, rt.Host, v == rt.GoodVersion) } |
| 2594 | } |
| 2595 | dec := Target(in) |
| 2596 | if cfg.resolve { |
| 2597 | // Under the carry fence (ruling R-CARRY-FENCE): a route a move has |
| 2598 | // held since this pass listed it gets no record, and nothing is |
| 2599 | // installed for it on this pass. |
| 2600 | if err := publishConsumed(ctx, optDir, cfg.route, NextConsumed(in, dec)); err != nil { |
| 2601 | fmt.Printf(" %s: recording the update decision: %v; nothing installed\n", rt.Host, err) |
| 2602 | return stop(dec) |
| 2603 | } |
| 2604 | } else if dec.Resolve != nil && dec.Reason == ReasonRequest { |
| 2605 | // Only root resolves (§4.3 rule 2), and only root's write makes T |
| 2606 | // the request's: the next root pass does both. A decision that |
| 2607 | // carries a Resolve for another reason (a pin, a hold, a failure: |
| 2608 | // R-TARGET-FAILEDHEAD) does not depend on it, so it goes ahead. |
| 2609 | // The stop asks root for a pass through the stage relay (ruling |
| 2610 | // R-AGENT-RELAY), the way a stage this process cannot write does, so |
| 2611 | // a cold wake waits on root's resolution rather than an hour. |
| 2612 | if cfg.stageBlocked != nil && !cfg.hostWrites.Enabled { |
| 2613 | cfg.stageBlocked(rt.Host, dec.Version, ErrInstallRequestNeedsRoot) |
| 2614 | } |
| 2615 | return stop(dec) |
| 2616 | } |
| 2617 | if dec.Version == "" { |
| 2618 | if cfg.resolve && dec.Reason != "" { |
| 2619 | fmt.Printf(" %s: %s%s: nothing installed\n", rt.Host, dec.Reason, orDetail(dec.Detail)) |
| 2620 | } |
| 2621 | return stop(dec) |
| 2622 | } |
| 2623 | return dec.Version, dec, false |
| 2624 | } |
| 2625 | |
| 2626 | // orDetail renders a decision's detail after its reason, or nothing. |
| 2627 | func orDetail(d string) string { |
| 2628 | if d == "" { |
| 2629 | return "" |
| 2630 | } |
| 2631 | return " (" + d + ")" |
| 2632 | } |
| 2633 | |
| 2634 | // stageAuthorized stages a build settings spec §4.1 authorizes by its |
| 2635 | // promotion record: the record's digest binds the manifest before anything |
| 2636 | // is downloaded or replaced (ensureStagedDigest). This box's own staged copy |
| 2637 | // is evidence on its own (§4.1's retained identity; Astra round 1 finding |
| 2638 | // 7): when its bytes still hash to the artifact digest in the copy's |
| 2639 | // sibling, the manifest it was verified against when it was staged, it is |
| 2640 | // recovered whatever the remote side says: no record (promoted before |
| 2641 | // records existed), a record or manifest that cannot be read now, or a |
| 2642 | // staging the record path refused. Without such a copy the build is |
| 2643 | // refused, and nothing is downloaded on anything weaker than the record's |
| 2644 | // digest. A failure to read the record is never read as "no record". |
| 2645 | // Nothing is backfilled. |
| 2646 | func stageAuthorized(ctx context.Context, s Store, pub ed25519.PublicKey, optDir, acct, app, ver, host string, chk stageCheck) (*Manifest, error) { |
| 2647 | rec, err := ReadPromotionRecord(ctx, s, pub, acct, app, ver) |
| 2648 | var refused error |
| 2649 | switch { |
| 2650 | case err == nil && rec.ManifestDigest == "": |
| 2651 | refused = fmt.Errorf("%s: promotion record carries no manifest digest", ver) |
| 2652 | case err == nil: |
| 2653 | m, serr := ensureStagedDigest(ctx, s, optDir, acct, app, ver, host, rec.ManifestDigest, chk) |
| 2654 | if serr == nil { |
| 2655 | return m, nil |
| 2656 | } |
| 2657 | refused = serr |
| 2658 | case errors.Is(err, ErrNoPromotionRecord): |
| 2659 | refused = fmt.Errorf("%s: %s", ver, updatewords.Text("settings.evidence.no-record")) |
| 2660 | default: |
| 2661 | refused = fmt.Errorf("%s: promotion record: %w", ver, err) |
| 2662 | } |
| 2663 | if id, m := RetainedStaged(optDir, acct, app, ver); id != nil { |
| 2664 | fmt.Printf(" %s: %s recovered from this box's verified staged copy (%v)\n", host, ver, refused) |
| 2665 | return m, nil |
| 2666 | } |
| 2667 | return nil, refused |
| 2668 | } |
| 2669 | |
amadan