| 1 | // The dev-proof gate, the requiring half of D3 (spec |
| 2 | // 2026-10-06-console-release-defences-design.md §4.5). |
| 3 | // |
| 4 | // adoptHost asks it twice. admit runs before ensureStaged, so an unproven |
| 5 | // build is neither downloaded nor relayed to a root pass: it takes the |
| 6 | // manifest's sha256 for this platform and admits the build when that sha256 |
| 7 | // is in this box's healthy history, or a configured prover's proof for it |
| 8 | // verifies. bind runs after ensureStaged and before PointLive and hashes the |
| 9 | // staged file, because a manifest can name bytes the box is not running (a |
| 10 | // reused version name, a stale cache): the staged hash must be the proof's or |
| 11 | // in the healthy history, or the build is refused with a refusal record of |
| 12 | // kind RefusalNotProven. |
| 13 | // |
| 14 | // A missing proof is not a refusal. The box keeps looking, so the build adopts |
| 15 | // soon after its proof arrives. A miss is cached per (route, sha256) for |
| 16 | // ProofMissTTL, and the gate keeps a waiting list that the agent's fast loop |
| 17 | // (fastMembers) and `carlos status --live` read. A host leaves the list as |
| 18 | // soon as the gate stops being asked about it (adoptHost clears it on every |
| 19 | // early return before admit), so the list never holds a head the channel has |
| 20 | // moved off. |
| 21 | // |
| 22 | // One gate lives for the life of its process and is shared by every pass in |
| 23 | // it. BeginPass re-reads the policy file, so an edit reaches the agent and |
| 24 | // root alike without a restart. An absent file is D3 off: nothing is read or |
| 25 | // required. Any other failure to load it (it does not parse, or cannot be |
| 26 | // read) is D3 on and waiting for every named-unit route, because the box |
| 27 | // cannot tell which apps were listed. |
| 28 | package release |
| 29 | |
| 30 | import ( |
| 31 | "context" |
| 32 | "errors" |
| 33 | "fmt" |
| 34 | "os" |
| 35 | "sort" |
| 36 | "strings" |
| 37 | "sync" |
| 38 | "time" |
| 39 | |
| 40 | "github.com/carlosframework/platform/internal/registry" |
| 41 | ) |
| 42 | |
| 43 | const ( |
| 44 | ProofMissTTL = 30 * time.Second |
| 45 | // proofPolicyLogEvery bounds how often a broken policy file is printed |
| 46 | // by a long-lived process. A root pass is a fresh process and prints it |
| 47 | // every pass; the agent's 2-second loop prints it once a minute. |
| 48 | proofPolicyLogEvery = time.Minute |
| 49 | ) |
| 50 | |
| 51 | // ProofWait is a host held on its current build because its channel head is |
| 52 | // not proven (or the policy file cannot be loaded). |
| 53 | type ProofWait struct { |
| 54 | Host, App, Version, SHA256, Prover, Reason string |
| 55 | Since time.Time |
| 56 | } |
| 57 | |
| 58 | type promotionKey struct { |
| 59 | acct, app, version string |
| 60 | promotedAt int64 |
| 61 | } |
| 62 | |
| 63 | type missKey struct{ host, sha string } |
| 64 | |
| 65 | type ProofGate struct { |
| 66 | policyPath string |
| 67 | bakeDir string |
| 68 | healthyDir string |
| 69 | now func() time.Time |
| 70 | |
| 71 | mu sync.Mutex |
| 72 | policy *ProofPolicy |
| 73 | policyErr error |
| 74 | lastPolicyErr string |
| 75 | lastPolicyLog time.Time |
| 76 | shas map[promotionKey]string |
| 77 | misses map[missKey]time.Time |
| 78 | waits map[string]ProofWait |
| 79 | } |
| 80 | |
| 81 | func NewProofGate(policyPath, bakeDir string) *ProofGate { |
| 82 | return &ProofGate{policyPath: policyPath, bakeDir: bakeDir, healthyDir: bakeDir, now: time.Now, |
| 83 | shas: map[promotionKey]string{}, misses: map[missKey]time.Time{}, waits: map[string]ProofWait{}} |
| 84 | } |
| 85 | |
| 86 | // NewRootProofGate keeps diagnostic output in bakeDir. Health authority is |
| 87 | // always read from the root-owned history, including in the agent's gate. |
| 88 | // Do not enable a D3 requirement until both current and previous builds have |
| 89 | // root history. Service-owned history is not imported; enabling it earlier |
| 90 | // can block an otherwise authorized promote-previous recovery without proof. |
| 91 | func NewRootProofGate(policyPath, bakeDir string) *ProofGate { |
| 92 | g := NewProofGate(policyPath, bakeDir) |
| 93 | g.healthyDir = RootHealthyDir |
| 94 | return g |
| 95 | } |
| 96 | |
| 97 | // WithClock replaces the gate's clock (tests). |
| 98 | func (g *ProofGate) WithClock(now func() time.Time) *ProofGate { |
| 99 | g.now = now |
| 100 | return g |
| 101 | } |
| 102 | |
| 103 | // BeginPass re-reads the policy file. PassFilter calls it once per pass. |
| 104 | func (g *ProofGate) BeginPass() { |
| 105 | if g == nil { |
| 106 | return |
| 107 | } |
| 108 | pol, err := LoadProofPolicy(g.policyPath) |
| 109 | now := g.now() |
| 110 | say := false |
| 111 | g.mu.Lock() |
| 112 | g.policy, g.policyErr = pol, err |
| 113 | if err != nil { |
| 114 | msg := err.Error() |
| 115 | say = msg != g.lastPolicyErr || now.Sub(g.lastPolicyLog) >= proofPolicyLogEvery |
| 116 | if say { |
| 117 | g.lastPolicyErr, g.lastPolicyLog = msg, now |
| 118 | } |
| 119 | } else { |
| 120 | g.lastPolicyErr = "" |
| 121 | } |
| 122 | g.mu.Unlock() |
| 123 | if say { |
| 124 | fmt.Printf(" %v; every named-unit route waits until it is fixed\n", err) |
| 125 | } |
| 126 | } |
| 127 | |
| 128 | func (g *ProofGate) state() (*ProofPolicy, error) { |
| 129 | g.mu.Lock() |
| 130 | defer g.mu.Unlock() |
| 131 | return g.policy, g.policyErr |
| 132 | } |
| 133 | |
| 134 | // Requires reports what this pass's policy says about app: required when a |
| 135 | // require entry lists it, and policyErr when the policy failed to load, which |
| 136 | // the gate reads as required (closed) because the box cannot tell which apps |
| 137 | // were listed. A nil gate requires nothing. |
| 138 | // |
| 139 | // adoptHost's refusal gate asks it about a RefusalNotProven record: that |
| 140 | // record stands only while the box still requires a proof for the app, so |
| 141 | // removing the require entry (or the policy file) is the break glass it is |
| 142 | // meant to be (final review H3). |
| 143 | func (g *ProofGate) Requires(app string) (required bool, policyErr bool) { |
| 144 | if g == nil { |
| 145 | return false, false |
| 146 | } |
| 147 | pol, err := g.state() |
| 148 | if err != nil { |
| 149 | return true, true |
| 150 | } |
| 151 | return len(pol.Requires(app)) > 0, false |
| 152 | } |
| 153 | |
| 154 | // Waits is the waiting list, sorted by host. |
| 155 | func (g *ProofGate) Waits() []ProofWait { |
| 156 | if g == nil { |
| 157 | return nil |
| 158 | } |
| 159 | g.mu.Lock() |
| 160 | out := make([]ProofWait, 0, len(g.waits)) |
| 161 | for _, w := range g.waits { |
| 162 | out = append(out, w) |
| 163 | } |
| 164 | g.mu.Unlock() |
| 165 | sort.Slice(out, func(i, j int) bool { return out[i].Host < out[j].Host }) |
| 166 | return out |
| 167 | } |
| 168 | |
| 169 | // WaitingDue reports whether host is waiting and its cached miss has run |
| 170 | // out, which is when the fast loop should look at it again. |
| 171 | func (g *ProofGate) WaitingDue(host string) bool { |
| 172 | if g == nil { |
| 173 | return false |
| 174 | } |
| 175 | now := g.now() |
| 176 | g.mu.Lock() |
| 177 | defer g.mu.Unlock() |
| 178 | w, ok := g.waits[host] |
| 179 | if !ok { |
| 180 | return false |
| 181 | } |
| 182 | until, cached := g.misses[missKey{host, w.SHA256}] |
| 183 | return !cached || !now.Before(until) |
| 184 | } |
| 185 | |
| 186 | // clear takes host off the waiting list and forgets its cached misses. A |
| 187 | // miss that outlived its wait would hold the host silently, off the list, |
| 188 | // if its channel came back to the same build inside ProofMissTTL. |
| 189 | func (g *ProofGate) clear(host string) { |
| 190 | if g == nil { |
| 191 | return |
| 192 | } |
| 193 | g.mu.Lock() |
| 194 | delete(g.waits, host) |
| 195 | for k := range g.misses { |
| 196 | if k.host == host { |
| 197 | delete(g.misses, k) |
| 198 | } |
| 199 | } |
| 200 | g.mu.Unlock() |
| 201 | } |
| 202 | |
| 203 | // hold records host as waiting, caches the miss, and prints the line once |
| 204 | // per change of version, sha256 or reason. |
| 205 | func (g *ProofGate) hold(rt registry.Route, ver, sha, prover, reason, cur string) { |
| 206 | cur = orNothing(cur) |
| 207 | now := g.now() |
| 208 | g.mu.Lock() |
| 209 | prev, had := g.waits[rt.Host] |
| 210 | same := had && prev.Version == ver && prev.SHA256 == sha |
| 211 | w := ProofWait{Host: rt.Host, App: rt.App, Version: ver, SHA256: sha, Prover: prover, Reason: reason, Since: now} |
| 212 | if same { |
| 213 | w.Since = prev.Since |
| 214 | } |
| 215 | g.waits[rt.Host] = w |
| 216 | g.misses[missKey{rt.Host, sha}] = now.Add(ProofMissTTL) |
| 217 | g.mu.Unlock() |
| 218 | if same && prev.Reason == reason { |
| 219 | return |
| 220 | } |
| 221 | if sha == "" { |
| 222 | fmt.Printf(" %s: %s is waiting: %s; staying on %s\n", rt.Host, ver, reason, cur) |
| 223 | return |
| 224 | } |
| 225 | fmt.Printf(" %s: %s (sha256 %s) is not proven on %s; staying on %s (%s)\n", rt.Host, ver, sha, prover, cur, reason) |
| 226 | } |
| 227 | |
| 228 | // orNothing names the build a host is on for a "staying on" or "still on" |
| 229 | // line: a first adoption has none. |
| 230 | func orNothing(ver string) string { |
| 231 | if ver == "" { |
| 232 | return "nothing" |
| 233 | } |
| 234 | return ver |
| 235 | } |
| 236 | |
| 237 | // proofVerdict is admit's answer, carried to bind. |
| 238 | type proofVerdict struct { |
| 239 | required bool // the route's app is listed in a require entry |
| 240 | sha string // the manifest's sha256 for this platform |
| 241 | proof *Proof // the verified proof; nil when admitted by the healthy history |
| 242 | } |
| 243 | |
| 244 | // admit is the cheap gate (spec §4.5 check 1). false means adoptHost returns. |
| 245 | func (g *ProofGate) admit(ctx context.Context, s Store, optDir string, rt registry.Route, ptr *ChannelPointer, cur string) (proofVerdict, bool) { |
| 246 | if g == nil { |
| 247 | return proofVerdict{}, true |
| 248 | } |
| 249 | pol, perr := g.state() |
| 250 | if perr != nil { |
| 251 | // Any load error is D3 on, never "no policy" (spec §4.5): every |
| 252 | // named-unit route waits, since the box cannot tell which apps were |
| 253 | // listed. The reason is the load error itself, which names the file |
| 254 | // and says whether it did not parse or could not be read. |
| 255 | if rt.Unit == "" { |
| 256 | return proofVerdict{}, true |
| 257 | } |
| 258 | g.hold(rt, ptr.Version, "", "", perr.Error(), cur) |
| 259 | return proofVerdict{}, false |
| 260 | } |
| 261 | reqs := pol.Requires(rt.App) |
| 262 | if len(reqs) == 0 { |
| 263 | g.clear(rt.Host) |
| 264 | return proofVerdict{}, true |
| 265 | } |
| 266 | acct := RouteAccount(rt) |
| 267 | sha, err := g.manifestSHA(ctx, s, optDir, acct, rt.App, ptr) |
| 268 | if err != nil { |
| 269 | fmt.Printf(" %s: %s: reading its manifest for the proof check: %v\n", rt.Host, ptr.Version, err) |
| 270 | return proofVerdict{}, false |
| 271 | } |
| 272 | v := proofVerdict{required: true, sha: sha} |
| 273 | if sha == "" { |
| 274 | return v, true // no build for this platform: ensureStaged says so |
| 275 | } |
| 276 | if IsHealthy(g.healthyDir, acct, rt.App, sha) { |
| 277 | g.clear(rt.Host) |
| 278 | return v, true |
| 279 | } |
| 280 | g.mu.Lock() |
| 281 | until, cached := g.misses[missKey{rt.Host, sha}] |
| 282 | g.mu.Unlock() |
| 283 | if cached && g.now().Before(until) { |
| 284 | return v, false |
| 285 | } |
| 286 | var reasons, names []string |
| 287 | for _, e := range reqs { |
| 288 | names = append(names, e.Prover) |
| 289 | p, err := GetProof(ctx, s, acct, rt.App, sha, e.Prover) |
| 290 | if errors.Is(err, os.ErrNotExist) { |
| 291 | reasons = append(reasons, "no proof from "+e.Prover) |
| 292 | continue |
| 293 | } |
| 294 | if err == nil { |
| 295 | err = p.Check(e.PublicKey(), rt.App, adoptPlatform, sha, e.Prover) |
| 296 | } |
| 297 | if err != nil { |
| 298 | reasons = append(reasons, fmt.Sprintf("proof from %s: %v", e.Prover, err)) |
| 299 | continue |
| 300 | } |
| 301 | g.clear(rt.Host) |
| 302 | v.proof = p |
| 303 | return v, true |
| 304 | } |
| 305 | g.hold(rt, ptr.Version, sha, strings.Join(names, " or "), strings.Join(reasons, "; "), cur) |
| 306 | return v, false |
| 307 | } |
| 308 | |
| 309 | // manifestSHA is the manifest's sha256 for this platform, from the staged |
| 310 | // sibling when there is one and the bucket otherwise, cached per promotion. |
| 311 | func (g *ProofGate) manifestSHA(ctx context.Context, s Store, optDir, acct, app string, ptr *ChannelPointer) (string, error) { |
| 312 | k := promotionKey{acct, app, ptr.Version, ptr.PromotedAt.Unix()} |
| 313 | g.mu.Lock() |
| 314 | sha, ok := g.shas[k] |
| 315 | g.mu.Unlock() |
| 316 | if ok { |
| 317 | return sha, nil |
| 318 | } |
| 319 | m, err := ReadStagedManifest(optDir, acct, ptr.Version, app) |
| 320 | if err != nil || m == nil { |
| 321 | m, err = GetManifest(ctx, s, acct, app, ptr.Version) |
| 322 | if err != nil { |
| 323 | return "", err |
| 324 | } |
| 325 | } |
| 326 | sha = m.Artifacts[adoptPlatform].SHA256 |
| 327 | g.mu.Lock() |
| 328 | g.shas[k] = sha |
| 329 | g.mu.Unlock() |
| 330 | return sha, nil |
| 331 | } |
| 332 | |
| 333 | // bind is the staged-bytes check (spec §4.5 check 2). false means adoptHost |
| 334 | // returns. A mismatch is a RefusalNotProven record naming both hashes: |
| 335 | // nothing was restarted, so it is Recovered and still on cur, the same shape |
| 336 | // as D2's wrong-program record. |
| 337 | func (g *ProofGate) bind(optDir string, rt registry.Route, ptr *ChannelPointer, v proofVerdict, cur string, now time.Time) bool { |
| 338 | return g.bindRecording(optDir, rt, ptr, v, cur, now, func(r Refusal) error { return WriteRefusal(g.bakeDir, r) }) |
| 339 | } |
| 340 | |
| 341 | // bindRecording is bind with the refusal record's writer as a parameter: |
| 342 | // adoption's bind writes it with WriteRefusal, and a move's held stage |
| 343 | // (materializeheld.go) scopes it to the move first. |
| 344 | func (g *ProofGate) bindRecording(optDir string, rt registry.Route, ptr *ChannelPointer, v proofVerdict, cur string, now time.Time, record func(Refusal) error) bool { |
| 345 | if g == nil || !v.required { |
| 346 | return true |
| 347 | } |
| 348 | acct := RouteAccount(rt) |
| 349 | got, err := FileSHA256(VersionPath(optDir, acct, ptr.Version, rt.App)) |
| 350 | if err != nil { |
| 351 | fmt.Printf(" %s: %s: hashing the staged build for the proof check: %v; not pointed live\n", rt.Host, ptr.Version, err) |
| 352 | return false |
| 353 | } |
| 354 | if v.proof != nil && got == v.proof.SHA256 { |
| 355 | return true |
| 356 | } |
| 357 | if IsHealthy(g.healthyDir, acct, rt.App, got) { |
| 358 | return true |
| 359 | } |
| 360 | reason := fmt.Sprintf("staged sha256 %s, manifest sha256 %s", got, v.sha) |
| 361 | if v.proof != nil { |
| 362 | reason = fmt.Sprintf("staged sha256 %s, %s proved %s", got, v.proof.Prover, v.proof.SHA256) |
| 363 | } |
| 364 | fmt.Printf(" %s: %s: the staged bytes do not match the proof (%s); not pointed live\n", rt.Host, ptr.Version, reason) |
| 365 | if g.bakeDir != "" { |
| 366 | if err := record(Refusal{Host: rt.Host, App: rt.App, Version: ptr.Version, SHA256: got, |
| 367 | Platform: adoptPlatform, PointerPromotedAt: ptr.PromotedAt, RefusedAt: now.UTC(), |
| 368 | Reason: "not proven: " + reason, |
| 369 | Kind: RefusalNotProven, Recovered: true, RestoredVersion: cur}); err != nil { |
| 370 | fmt.Printf(" %s: writing refusal record: %v\n", rt.Host, err) |
| 371 | } |
| 372 | } |
| 373 | g.clear(rt.Host) |
| 374 | return false |
| 375 | } |
| 376 | |
amadan