| 1 | // The root pass's D3 work (spec 2026-10-06-console-release-defences-design.md |
| 2 | // §4.3, §4.5): look at every named-unit route's live build, once before |
| 3 | // release.Pass and again, only where the pass changed it, after. |
| 4 | // |
| 5 | // - A build meeting observe's conditions (bound, active in one invocation |
| 6 | // for the policy's after, hashed from its running executable, ready) |
| 7 | // enters the app's healthy history, on every box, policy file or not. |
| 8 | // That is what puts the build already live when a requirement is switched |
| 9 | // on into the history without a restart. recordLive does it BEFORE |
| 10 | // release.Pass (final review H4): on the first pass after a requirement |
| 11 | // is switched on, the pass can adopt a proven head and replace the build |
| 12 | // that has been running well for hours, and that build must already be in |
| 13 | // the history, or a later rollback to it would wait for a dev proof. |
| 14 | // - On a proving box (a prove block lists the app) the build the pass left |
| 15 | // live is minted a proof (mintAndLog, after release.Pass), unless a refusal record names it or the bucket already |
| 16 | // has one. Minting needs the root-only key, which is why it is here. |
| 17 | // - A pass that reads a require entry leaves a proof-required/<app> marker |
| 18 | // in the bake directory. On the hourly pass, an app with that marker, |
| 19 | // healthy history and no require entry gets one line saying the |
| 20 | // requirement is off: the break-glass trail. A box that never required a |
| 21 | // proof has no marker and stays quiet. |
| 22 | // |
| 23 | // A policy file that does not parse is closed (spec §4.5): nothing is |
| 24 | // minted, no marker is touched, and the pass says so every time. The live |
| 25 | // build is still recorded as healthy at the default after, because that |
| 26 | // record is evidence about the running bytes, not a reading of the policy. |
| 27 | package host |
| 28 | |
| 29 | import ( |
| 30 | "context" |
| 31 | "crypto/ed25519" |
| 32 | "errors" |
| 33 | "fmt" |
| 34 | "os" |
| 35 | "path/filepath" |
| 36 | "sort" |
| 37 | "time" |
| 38 | |
| 39 | "github.com/carlosframework/platform/internal/activator" |
| 40 | "github.com/carlosframework/platform/internal/registry" |
| 41 | "github.com/carlosframework/platform/internal/release" |
| 42 | "github.com/carlosframework/platform/internal/safefile" |
| 43 | ) |
| 44 | |
| 45 | type proofRoot struct { |
| 46 | mintCensus func() (bool, error) |
| 47 | recordProgram func(liveBuild) error |
| 48 | refusals func(release.HistoryBinding) ([]release.Refusal, error) |
| 49 | obs proofObserver |
| 50 | reg *registry.Registry |
| 51 | store release.Store |
| 52 | bakeDir string |
| 53 | healthyDir string |
| 54 | policyPath string |
| 55 | readKey func(path string) (ed25519.PrivateKey, error) |
| 56 | hourly bool // carlos-adopt.service, not -stage-only |
| 57 | logf func(format string, a ...any) |
| 58 | } |
| 59 | |
| 60 | func (r proofRoot) historyDir() string { |
| 61 | if r.healthyDir != "" { |
| 62 | return r.healthyDir |
| 63 | } |
| 64 | return r.bakeDir |
| 65 | } |
| 66 | |
| 67 | // liveObservation is one look at a route's live build: the live link's |
| 68 | // target when it was taken, and observe's answer. |
| 69 | type liveObservation struct { |
| 70 | target string |
| 71 | build liveBuild |
| 72 | err error |
| 73 | } |
| 74 | |
| 75 | // proofObservations is recordLive's result, handed to mintAndLog so a route |
| 76 | // the pass did not touch is observed once per pass, not twice. |
| 77 | type proofObservations struct { |
| 78 | pol *release.ProofPolicy |
| 79 | perr error |
| 80 | byHost map[string]liveObservation |
| 81 | } |
| 82 | |
| 83 | // run is both halves back to back, for a caller with no pass between them. |
| 84 | func (r proofRoot) run(ctx context.Context) { |
| 85 | r.mintAndLog(ctx, r.recordLive(ctx)) |
| 86 | } |
| 87 | |
| 88 | // around is RunAdopt's order: record the live builds, run the pass, then |
| 89 | // mint on what the pass left live, including routes unaffected by a pass error. |
| 90 | func (r proofRoot) around(ctx context.Context, pass func() error) error { |
| 91 | obs := r.recordLive(ctx) |
| 92 | err := pass() |
| 93 | r.mintAndLog(ctx, obs) |
| 94 | return err |
| 95 | } |
| 96 | |
| 97 | // recordLive reads the policy, observes every named-unit route's live build, |
| 98 | // and records each one that qualifies as healthy. |
| 99 | func (r proofRoot) recordLive(ctx context.Context) proofObservations { |
| 100 | pol, perr := release.LoadProofPolicy(r.policyPath) |
| 101 | if perr != nil { |
| 102 | r.logf("proof: %v; nothing is minted until it parses", perr) |
| 103 | } |
| 104 | obs := proofObservations{pol: pol, perr: perr, byHost: map[string]liveObservation{}} |
| 105 | routes, err := r.reg.List() |
| 106 | if err != nil { |
| 107 | r.logf("proof: list routes: %v", err) |
| 108 | return obs |
| 109 | } |
| 110 | for _, rt := range routes { |
| 111 | if !release.Adoptable(rt) || rt.Unit == "" || rt.Unit == activator.UnitName(rt.Host) { |
| 112 | continue |
| 113 | } |
| 114 | obs.byHost[rt.Host] = r.observeAndRecord(rt, pol.ProveAfter()) |
| 115 | } |
| 116 | return obs |
| 117 | } |
| 118 | |
| 119 | // observeAndRecord observes rt's live build and, when it qualifies, adds it |
| 120 | // to the app's healthy history. |
| 121 | func (r proofRoot) observeAndRecord(rt registry.Route, after time.Duration) liveObservation { |
| 122 | target, _ := os.Readlink(release.LiveLink(r.obs.optDir, rt.Host)) |
| 123 | b, err := r.obs.observe(rt, after) |
| 124 | o := liveObservation{target: target, build: b, err: err} |
| 125 | if err != nil { |
| 126 | return o |
| 127 | } |
| 128 | if err := release.RecordHealthy(r.historyDir(), b.binding.Route.Account, b.binding.Route.App, release.HealthyBuild{ |
| 129 | SHA256: b.SHA256, Version: b.Version, How: "live", At: r.obs.now().UTC()}); err != nil { |
| 130 | r.logf("proof: %s: recording the healthy build: %v", rt.Host, err) |
| 131 | o.err = err |
| 132 | } |
| 133 | if r.recordProgram != nil { |
| 134 | if err := r.recordProgram(b); err != nil { |
| 135 | r.logf("proof: %s: recording the installed program: %v", rt.Host, err) |
| 136 | o.err = errors.Join(o.err, err) |
| 137 | } |
| 138 | } |
| 139 | |
| 140 | return o |
| 141 | } |
| 142 | |
| 143 | // stillLive reports whether o still describes rt's live build after the |
| 144 | // pass: the live link names the same target and, for a build that passed, |
| 145 | // the unit is the same process (a rollback can put the link back where it |
| 146 | // was and restart the unit; that process has not run for after). |
| 147 | func (r proofRoot) stillLive(rt registry.Route, o liveObservation) bool { |
| 148 | if r.obs.resolve == nil { |
| 149 | return false |
| 150 | } |
| 151 | binding, err := r.obs.resolve(context.Background(), rt) |
| 152 | if err != nil || binding.Source != o.build.binding.Source { |
| 153 | return false |
| 154 | } |
| 155 | rt = binding.Route |
| 156 | target, err := os.Readlink(release.LiveLink(r.obs.optDir, rt.Host)) |
| 157 | if err != nil || target != o.target { |
| 158 | return false |
| 159 | } |
| 160 | if o.err != nil { |
| 161 | return true |
| 162 | } |
| 163 | p, err := r.obs.sys.Show(rt.Unit) |
| 164 | if err != nil || p.ActiveState != "active" || p.InvocationID != o.build.invocation || p.MainPID != o.build.pid { |
| 165 | return false |
| 166 | } |
| 167 | start, err := r.obs.sys.StartTime(p.MainPID) |
| 168 | return err == nil && start == o.build.start |
| 169 | } |
| 170 | |
| 171 | // mintAndLog is the second half: mint on a proving box, leave the |
| 172 | // requirement marker, and on the hourly pass say when a requirement is off. |
| 173 | // A route whose live build the pass changed is observed again (and recorded, |
| 174 | // if it qualifies); every other route reuses recordLive's observation. |
| 175 | func (r proofRoot) mintAndLog(ctx context.Context, obs proofObservations) { |
| 176 | pol, perr := obs.pol, obs.perr |
| 177 | mintAllowed := r.refusals == nil && r.recordProgram == nil |
| 178 | if perr == nil && pol != nil && pol.Prove != nil { |
| 179 | var err error |
| 180 | if r.mintCensus != nil { |
| 181 | mintAllowed, err = r.mintCensus() |
| 182 | } |
| 183 | if err != nil { |
| 184 | mintAllowed = false |
| 185 | r.logf("proof: reading root census: %v; not minting", err) |
| 186 | } else if !mintAllowed { |
| 187 | r.logf("proof: minting paused until this box has a root census") |
| 188 | } |
| 189 | } |
| 190 | routes, err := r.reg.List() |
| 191 | if err != nil { |
| 192 | r.logf("proof: list routes: %v", err) |
| 193 | return |
| 194 | } |
| 195 | after := pol.ProveAfter() |
| 196 | type acctApp struct{ acct, app string } |
| 197 | seen := map[acctApp]bool{} |
| 198 | for _, rt := range routes { |
| 199 | if !release.Adoptable(rt) || rt.Unit == "" || rt.Unit == activator.UnitName(rt.Host) { |
| 200 | continue |
| 201 | } |
| 202 | acct := release.RouteAccount(rt) |
| 203 | seen[acctApp{acct, rt.App}] = true |
| 204 | o, ok := obs.byHost[rt.Host] |
| 205 | if !ok || !r.stillLive(rt, o) { |
| 206 | o = r.observeAndRecord(rt, after) |
| 207 | } |
| 208 | if o.err != nil { |
| 209 | if pol.Proves(rt.App) { |
| 210 | r.logf("proof: %s: not minting yet: %v", rt.Host, o.err) |
| 211 | } |
| 212 | continue |
| 213 | } |
| 214 | if mintAllowed && perr == nil && pol.Proves(rt.App) { |
| 215 | r.mint(ctx, pol, o.build.binding, o.build) |
| 216 | } |
| 217 | } |
| 218 | if perr != nil { |
| 219 | return |
| 220 | } |
| 221 | // The marker says this box has required a proof for the app at some |
| 222 | // point. Written on every pass that reads a require entry, so a |
| 223 | // requirement switched on and off between two hourly passes still leaves |
| 224 | // its trail. |
| 225 | if pol != nil { |
| 226 | for _, e := range pol.Require { |
| 227 | r.markRequired(e.App) |
| 228 | } |
| 229 | } |
| 230 | if !r.hourly { |
| 231 | return |
| 232 | } |
| 233 | keys := make([]acctApp, 0, len(seen)) |
| 234 | for k := range seen { |
| 235 | keys = append(keys, k) |
| 236 | } |
| 237 | sort.Slice(keys, func(i, j int) bool { return keys[i].acct+"/"+keys[i].app < keys[j].acct+"/"+keys[j].app }) |
| 238 | for _, k := range keys { |
| 239 | // Only where the marker exists is "off" news: a box that never |
| 240 | // required a proof (Tito, a self-hoster) has nothing switched off. |
| 241 | if len(pol.Requires(k.app)) > 0 { |
| 242 | continue |
| 243 | } |
| 244 | if _, err := os.Stat(r.requiredMarker(k.app)); err != nil { |
| 245 | continue |
| 246 | } |
| 247 | if n := len(release.ReadHealthy(r.historyDir(), k.acct, k.app)); n > 0 { |
| 248 | r.logf("proof: requirement is off for %s on this box: %s has no require entry for it (%d healthy builds on record)", |
| 249 | k.app, r.policyPath, n) |
| 250 | } |
| 251 | } |
| 252 | } |
| 253 | |
| 254 | func (r proofRoot) requiredMarker(app string) string { |
| 255 | return filepath.Join(r.bakeDir, "proof-required", app) |
| 256 | } |
| 257 | |
| 258 | func (r proofRoot) markRequired(app string) { |
| 259 | m := r.requiredMarker(app) |
| 260 | if _, err := os.Stat(m); err == nil { |
| 261 | return |
| 262 | } |
| 263 | if err := safefile.MkdirAll(filepath.Dir(m), 0o755); err != nil { |
| 264 | r.logf("proof: writing the requirement marker %s: %v", m, err) |
| 265 | return |
| 266 | } |
| 267 | if err := safefile.WriteFile(m, nil, 0o644); err != nil { |
| 268 | r.logf("proof: writing the requirement marker %s: %v", m, err) |
| 269 | } |
| 270 | } |
| 271 | |
| 272 | func (r proofRoot) mint(ctx context.Context, pol *release.ProofPolicy, binding proofBinding, b liveBuild) { |
| 273 | rt, acct := binding.Route, binding.Route.Account |
| 274 | var refusals []release.Refusal |
| 275 | var err error |
| 276 | if r.refusals != nil { |
| 277 | refusals, err = r.refusals(release.HistoryBinding{Host: rt.Host, Account: acct, App: rt.App}) |
| 278 | } else { |
| 279 | refusals, err = release.ListRefusals(r.bakeDir) |
| 280 | } |
| 281 | if err != nil { |
| 282 | r.logf("proof: %s: reading refusal records: %v; not minting", rt.Host, err) |
| 283 | return |
| 284 | } |
| 285 | for _, ref := range refusals { |
| 286 | if ref.SHA256 == b.SHA256 { |
| 287 | r.logf("proof: %s: not minting %s: this box refused it (%s)", rt.Host, b.Version, ref.Reason) |
| 288 | return |
| 289 | } |
| 290 | } |
| 291 | // Spec §4.3's last condition, and the reason a re-mint is never tried: |
| 292 | // PutProof is create-only, and a second proof for one build would carry |
| 293 | // a different proven_at and conflict. |
| 294 | prover := pol.Prove.Prover |
| 295 | _, err = release.GetProof(ctx, r.store, acct, rt.App, b.SHA256, prover) |
| 296 | switch { |
| 297 | case err == nil: |
| 298 | return // already proven |
| 299 | case !errors.Is(err, os.ErrNotExist): |
| 300 | r.logf("proof: %s: reading the bucket for an existing proof: %v; not minting", rt.Host, err) |
| 301 | return |
| 302 | } |
| 303 | key, err := r.readKey(pol.Prove.KeyFile) |
| 304 | if err != nil { |
| 305 | r.logf("proof: %s: %v; not minting", rt.Host, err) |
| 306 | return |
| 307 | } |
| 308 | p, err := release.SignProof(key, release.Proof{App: rt.App, Platform: release.AdoptPlatform(), SHA256: b.SHA256, |
| 309 | Version: b.Version, Prover: prover, ActiveSince: b.ActiveSince.UTC().Truncate(time.Second), |
| 310 | ProvenAt: r.obs.now().UTC().Truncate(time.Second)}) |
| 311 | if err != nil { |
| 312 | r.logf("proof: %s: signing: %v", rt.Host, err) |
| 313 | return |
| 314 | } |
| 315 | if err := release.PutProof(ctx, r.store, acct, p); err != nil { |
| 316 | // A concurrent pass (the stage relay and the hourly timer can overlap |
| 317 | // when the pass lock is unavailable) minted it between the check above |
| 318 | // and this write. Its proof stands: already minted, not a failure. |
| 319 | if _, gerr := release.GetProof(ctx, r.store, acct, rt.App, b.SHA256, prover); gerr == nil { |
| 320 | r.logf("proof: %s: %s %s was already minted by another pass", rt.Host, rt.App, b.Version) |
| 321 | return |
| 322 | } |
| 323 | r.logf("proof: %s: writing the proof: %v (the agent asks again in 2 minutes)", rt.Host, err) |
| 324 | return |
| 325 | } |
| 326 | r.logf("proof: minted %s %s (sha256 %s) as %s, live since %s", rt.App, b.Version, b.SHA256, prover, |
| 327 | b.ActiveSince.UTC().Format(time.RFC3339)) |
| 328 | } |
| 329 | |
| 330 | // readProvingKey reads the root-only proving key, refusing one any other user |
| 331 | // can read (spec §4.2: the key is root 0600). |
| 332 | func readProvingKey(path string) (ed25519.PrivateKey, error) { |
| 333 | fi, err := os.Stat(path) |
| 334 | if err != nil { |
| 335 | return nil, fmt.Errorf("proving key: %w", err) |
| 336 | } |
| 337 | if perm := fi.Mode().Perm(); perm&0o077 != 0 { |
| 338 | return nil, fmt.Errorf("proving key %s is mode %04o; it must be readable by root only (0600)", path, perm) |
| 339 | } |
| 340 | b, err := safefile.ReadFile(path) |
| 341 | if err != nil { |
| 342 | return nil, fmt.Errorf("proving key: %w", err) |
| 343 | } |
| 344 | return release.ParseSigningKey(string(b), path) |
| 345 | } |
| 346 | |
| 347 | func configureRootProof(pr *proofRoot, checks rootRestartChecks, open func() (*release.RootHistoryPass, error)) (*release.RootHistoryPass, error) { |
| 348 | legacy, err := checks.allowLegacy() |
| 349 | if err == nil && legacy { |
| 350 | return nil, nil |
| 351 | } |
| 352 | pr.mintCensus = func() (bool, error) { |
| 353 | legacy, err := checks.allowLegacy() |
| 354 | return !legacy && err == nil, err |
| 355 | } |
| 356 | var history *release.RootHistoryPass |
| 357 | pr.refusals = func(b release.HistoryBinding) ([]release.Refusal, error) { |
| 358 | if history == nil { |
| 359 | return nil, os.ErrPermission |
| 360 | } |
| 361 | return checks.mintRefusals(history.History, b) |
| 362 | } |
| 363 | if err != nil { |
| 364 | return nil, err |
| 365 | } |
| 366 | history, err = open() |
| 367 | if err == nil && history != nil { |
| 368 | pr.recordProgram = func(b liveBuild) error { return recordRootProgram(history, b) } |
| 369 | } |
| 370 | return history, err |
| 371 | } |
| 372 | |
amadan