| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/sha256" |
| 6 | "encoding/hex" |
| 7 | "fmt" |
| 8 | "os" |
| 9 | "path/filepath" |
| 10 | "testing" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/registry" |
| 14 | "github.com/carlosframework/platform/internal/release" |
| 15 | ) |
| 16 | |
| 17 | func mintAskerFixture(t *testing.T, activeFor time.Duration) (*mintAsker, *int, *time.Time, release.Store, string) { |
| 18 | t.Helper() |
| 19 | opt, etc := t.TempDir(), t.TempDir() |
| 20 | liveTo(t, opt, observedHost, "v1") |
| 21 | reg := restartTestRegistry(t) |
| 22 | if err := reg.Upsert(registry.Route{Host: observedHost, Kind: "instance", App: "console", |
| 23 | Channel: "canary/console", Unit: "console.service", Status: "ready"}); err != nil { |
| 24 | t.Fatal(err) |
| 25 | } |
| 26 | policy := filepath.Join(etc, "proof-policy.json") |
| 27 | if err := os.WriteFile(policy, []byte(provePolicy("/etc/carlos/proving.key")), 0o644); err != nil { |
| 28 | t.Fatal(err) |
| 29 | } |
| 30 | t0 := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) |
| 31 | clock := t0 |
| 32 | asked := 0 |
| 33 | store := release.NewDirStore(t.TempDir()) |
| 34 | a := &mintAsker{reg: reg, store: store, optDir: opt, policyPath: policy, |
| 35 | sys: &fakeUnits{shows: []unitProps{{ActiveState: "active", InvocationID: "i1", MainPID: 7, |
| 36 | ActiveEnterTimestamp: t0.Add(-activeFor)}}}, |
| 37 | request: func(string, string, error) { asked++ }, |
| 38 | now: func() time.Time { return clock }} |
| 39 | sha, err := release.FileSHA256(filepath.Join(opt, "versions", "v1", "console")) |
| 40 | if err != nil { |
| 41 | t.Fatal(err) |
| 42 | } |
| 43 | return a, &asked, &clock, store, sha |
| 44 | } |
| 45 | |
| 46 | // Review focus 5: the relay can drop a request and a mint write can fail, so |
| 47 | // the agent asks every 2 minutes until it sees the proof, then stops. |
| 48 | func TestMintAskerReasksUntilTheProofAppears(t *testing.T) { |
| 49 | t.Parallel() |
| 50 | a, asked, clock, store, sha := mintAskerFixture(t, 20*time.Minute) |
| 51 | ctx := context.Background() |
| 52 | t0 := *clock |
| 53 | at := func(d time.Duration) { *clock = t0.Add(d); a.tick(ctx) } |
| 54 | at(0) |
| 55 | if *asked != 1 { |
| 56 | t.Fatalf("asked %d times at first sight, want 1", *asked) |
| 57 | } |
| 58 | at(time.Minute) |
| 59 | if *asked != 1 { |
| 60 | t.Fatalf("asked %d times inside 2 minutes, want 1", *asked) |
| 61 | } |
| 62 | at(2 * time.Minute) |
| 63 | if *asked != 2 { |
| 64 | t.Fatalf("asked %d times after 2 minutes, want 2", *asked) |
| 65 | } |
| 66 | seed, _, err := release.GenerateKey() |
| 67 | if err != nil { |
| 68 | t.Fatal(err) |
| 69 | } |
| 70 | key, err := release.ParseSigningKey(seed, "test") |
| 71 | if err != nil { |
| 72 | t.Fatal(err) |
| 73 | } |
| 74 | p, err := release.SignProof(key, release.Proof{App: "console", Platform: release.AdoptPlatform(), SHA256: sha, |
| 75 | Version: "v1", Prover: "dev", ActiveSince: t0.Add(-20 * time.Minute), ProvenAt: t0}) |
| 76 | if err != nil { |
| 77 | t.Fatal(err) |
| 78 | } |
| 79 | if err := release.PutProof(ctx, store, "bab", p); err != nil { |
| 80 | t.Fatal(err) |
| 81 | } |
| 82 | at(4 * time.Minute) |
| 83 | at(6 * time.Minute) |
| 84 | if *asked != 2 { |
| 85 | t.Fatalf("asked %d times after the proof appeared, want 2", *asked) |
| 86 | } |
| 87 | pend, err := release.GetProofPending(ctx, store, "bab", "console", sha, "dev") |
| 88 | if err != nil || !pend.LiveSince.Equal(t0.Add(-20*time.Minute)) { |
| 89 | t.Fatalf("pending = %+v, %v", pend, err) |
| 90 | } |
| 91 | } |
| 92 | |
| 93 | // putDevProof puts a valid-shaped dev proof for sha in store. |
| 94 | func putDevProof(t *testing.T, store release.Store, sha string, at time.Time) { |
| 95 | t.Helper() |
| 96 | seed, _, err := release.GenerateKey() |
| 97 | if err != nil { |
| 98 | t.Fatal(err) |
| 99 | } |
| 100 | key, err := release.ParseSigningKey(seed, "test") |
| 101 | if err != nil { |
| 102 | t.Fatal(err) |
| 103 | } |
| 104 | p, err := release.SignProof(key, release.Proof{App: "console", Platform: release.AdoptPlatform(), SHA256: sha, |
| 105 | Version: "v1", Prover: "dev", ActiveSince: at.Add(-20 * time.Minute), ProvenAt: at}) |
| 106 | if err != nil { |
| 107 | t.Fatal(err) |
| 108 | } |
| 109 | if err := release.PutProof(context.Background(), store, "bab", p); err != nil { |
| 110 | t.Fatal(err) |
| 111 | } |
| 112 | } |
| 113 | |
| 114 | // Final review H5: dev proves v1, its staged copy is pruned, and the name v1 |
| 115 | // is reused for other bytes at the same path. The asker hashes the new file, |
| 116 | // finds no proof for it, and asks root again, rather than answering from the |
| 117 | // old file's hash and the proof it found for that. |
| 118 | func TestMintAskerRehashesAReusedVersionPath(t *testing.T) { |
| 119 | t.Parallel() |
| 120 | a, asked, clock, store, sha := mintAskerFixture(t, 20*time.Minute) |
| 121 | ctx := context.Background() |
| 122 | t0 := *clock |
| 123 | putDevProof(t, store, sha, t0) |
| 124 | a.tick(ctx) |
| 125 | if *asked != 0 { |
| 126 | t.Fatalf("asked %d times for a proven build, want 0", *asked) |
| 127 | } |
| 128 | staged := filepath.Join(a.optDir, "versions", "v1", "console") |
| 129 | if err := os.Remove(staged); err != nil { |
| 130 | t.Fatal(err) |
| 131 | } |
| 132 | if err := os.WriteFile(staged, []byte("other bytes under the same name"), 0o755); err != nil { |
| 133 | t.Fatal(err) |
| 134 | } |
| 135 | *clock = t0.Add(mintAskEvery) |
| 136 | a.tick(ctx) |
| 137 | if *asked != 1 { |
| 138 | t.Fatalf("asked %d times after the path's bytes changed, want 1: the new bytes have no proof", *asked) |
| 139 | } |
| 140 | newSHA := sha256Hex("other bytes under the same name") |
| 141 | if h := a.shaByPath[staged]; h.sha != newSHA { |
| 142 | t.Fatalf("cached sha256 = %q, want the new file's %q", h.sha, newSHA) |
| 143 | } |
| 144 | } |
| 145 | |
| 146 | // The same bytes rewritten in place with a new mtime are hashed again too: |
| 147 | // size and mtime are part of the file's identity, not only its inode. |
| 148 | func TestMintAskerRehashesAFileRewrittenInPlace(t *testing.T) { |
| 149 | t.Parallel() |
| 150 | a, asked, clock, store, sha := mintAskerFixture(t, 20*time.Minute) |
| 151 | ctx := context.Background() |
| 152 | t0 := *clock |
| 153 | putDevProof(t, store, sha, t0) |
| 154 | a.tick(ctx) |
| 155 | staged := filepath.Join(a.optDir, "versions", "v1", "console") |
| 156 | // liveTo's build is one byte; another byte of the same size, in place. |
| 157 | if err := os.WriteFile(staged, []byte("y"), 0o755); err != nil { |
| 158 | t.Fatal(err) |
| 159 | } |
| 160 | later := time.Now().Add(time.Hour) |
| 161 | if err := os.Chtimes(staged, later, later); err != nil { |
| 162 | t.Fatal(err) |
| 163 | } |
| 164 | *clock = t0.Add(mintAskEvery) |
| 165 | a.tick(ctx) |
| 166 | if *asked != 1 { |
| 167 | t.Fatalf("asked %d times after an in-place rewrite, want 1", *asked) |
| 168 | } |
| 169 | } |
| 170 | |
| 171 | // The proven cache is per proof object: a proof seen for one prover does not |
| 172 | // answer for another after the policy changes prover. |
| 173 | func TestMintAskerProvenCacheIsPerProver(t *testing.T) { |
| 174 | t.Parallel() |
| 175 | a, asked, clock, store, sha := mintAskerFixture(t, 20*time.Minute) |
| 176 | ctx := context.Background() |
| 177 | t0 := *clock |
| 178 | putDevProof(t, store, sha, t0) |
| 179 | a.tick(ctx) |
| 180 | if *asked != 0 { |
| 181 | t.Fatalf("asked %d times for a proven build, want 0", *asked) |
| 182 | } |
| 183 | if err := os.WriteFile(a.policyPath, []byte(`{"schema":1,"prove":{"apps":["console"],"prover":"staging","after":"15m","key_file":"/etc/carlos/proving.key"}}`), 0o644); err != nil { |
| 184 | t.Fatal(err) |
| 185 | } |
| 186 | *clock = t0.Add(mintAskEvery) |
| 187 | a.tick(ctx) |
| 188 | if *asked != 1 { |
| 189 | t.Fatalf("asked %d times once the prover is staging, want 1: dev's proof is not staging's", *asked) |
| 190 | } |
| 191 | } |
| 192 | |
| 193 | func TestMintAskerWaitsOutTheBake(t *testing.T) { |
| 194 | t.Parallel() |
| 195 | a, asked, _, store, sha := mintAskerFixture(t, 5*time.Minute) |
| 196 | a.tick(context.Background()) |
| 197 | if *asked != 0 { |
| 198 | t.Fatalf("asked %d times five minutes into a fifteen-minute bake", *asked) |
| 199 | } |
| 200 | if _, err := release.GetProofPending(context.Background(), store, "bab", "console", sha, "dev"); err != nil { |
| 201 | t.Fatalf("no pending note for a baking build: %v", err) |
| 202 | } |
| 203 | } |
| 204 | |
| 205 | // Task 2's carry: a policy file that does not parse is closed. Root mints |
| 206 | // nothing until it parses, so the asker has nothing to ask for and stays |
| 207 | // quiet rather than reading the broken file as "no prove block" and guessing. |
| 208 | func TestMintAskerAsksNothingOnABrokenPolicy(t *testing.T) { |
| 209 | t.Parallel() |
| 210 | a, asked, _, store, sha := mintAskerFixture(t, 20*time.Minute) |
| 211 | if err := os.WriteFile(a.policyPath, []byte(`{"schema":1,"prove":`), 0o644); err != nil { |
| 212 | t.Fatal(err) |
| 213 | } |
| 214 | a.tick(context.Background()) |
| 215 | if *asked != 0 { |
| 216 | t.Fatalf("asked %d times with a broken policy, want 0", *asked) |
| 217 | } |
| 218 | if _, err := release.GetProofPending(context.Background(), store, "bab", "console", sha, "dev"); err == nil { |
| 219 | t.Fatal("pending note written under a broken policy") |
| 220 | } |
| 221 | } |
| 222 | |
| 223 | // Spec §4.5, codex 14: a non-canary listed route with a live version stays in |
| 224 | // the fast set while it waits (at the gate's 30 s cadence), and leaves once |
| 225 | // it adopts. |
| 226 | func TestFastMembersKeepsAWaitingRoute(t *testing.T) { |
| 227 | t.Parallel() |
| 228 | f := newFastFixture(t) |
| 229 | ctx := context.Background() |
| 230 | host := "edge.example.com" |
| 231 | f.addRoute(t, host, "edge") |
| 232 | f.stageLive(t, host, "v1") |
| 233 | plat := release.AdoptPlatform() |
| 234 | for ver, body := range map[string]string{"v1": "staged build", "v2": "build two"} { |
| 235 | m := release.Manifest{App: "fastapp", Version: ver, Kind: "binary", Artifacts: map[string]release.Artifact{plat: {}}} |
| 236 | if err := release.Ship(ctx, f.store, "bab", m, map[string][]byte{plat: []byte(body)}); err != nil { |
| 237 | t.Fatalf("Ship %s: %v", ver, err) |
| 238 | } |
| 239 | } |
| 240 | if err := release.Promote(ctx, f.store, f.priv, "bab", "fastapp", "edge", "v2"); err != nil { |
| 241 | t.Fatal(err) |
| 242 | } |
| 243 | seed, pubB64, err := release.GenerateKey() |
| 244 | if err != nil { |
| 245 | t.Fatal(err) |
| 246 | } |
| 247 | prover, err := release.ParseSigningKey(seed, "test") |
| 248 | if err != nil { |
| 249 | t.Fatal(err) |
| 250 | } |
| 251 | policy := filepath.Join(t.TempDir(), "proof-policy.json") |
| 252 | if err := os.WriteFile(policy, []byte(fmt.Sprintf(`{"schema":1,"require":[{"app":"fastapp","prover":"dev","key":%q}]}`, pubB64)), 0o644); err != nil { |
| 253 | t.Fatal(err) |
| 254 | } |
| 255 | clock := time.Now() |
| 256 | gate := release.NewProofGate(policy, f.dataDir).WithClock(func() time.Time { return clock }) |
| 257 | d := f.deps(nil) |
| 258 | d.passOpts = append(d.passOpts, release.WithProofGate(gate)) |
| 259 | d.proof = gate |
| 260 | |
| 261 | if got := d.fastMembers(clock); len(got) != 0 { |
| 262 | t.Fatalf("members = %v before anything waits", got) |
| 263 | } |
| 264 | // The serial-triggered full pass sees v2 and finds no proof. |
| 265 | if err := release.Pass(ctx, f.reg, f.store, f.pub, f.optDir, d.restart, d.passOpts...); err != nil { |
| 266 | t.Fatal(err) |
| 267 | } |
| 268 | if v, _ := release.LiveVersion(f.optDir, host); v != "v1" { |
| 269 | t.Fatalf("live = %q, want v1 while unproven", v) |
| 270 | } |
| 271 | clock = clock.Add(31 * time.Second) |
| 272 | if got := d.fastMembers(clock); len(got) != 1 || got[0] != host { |
| 273 | t.Fatalf("members = %v, want the waiting non-canary route", got) |
| 274 | } |
| 275 | sum := sha256.Sum256([]byte("build two")) |
| 276 | p, err := release.SignProof(prover, release.Proof{App: "fastapp", Platform: plat, SHA256: hex.EncodeToString(sum[:]), |
| 277 | Version: "v2", Prover: "dev", ActiveSince: clock.Add(-time.Hour).UTC().Truncate(time.Second), |
| 278 | ProvenAt: clock.UTC().Truncate(time.Second)}) |
| 279 | if err != nil { |
| 280 | t.Fatal(err) |
| 281 | } |
| 282 | if err := release.PutProof(ctx, f.store, "bab", p); err != nil { |
| 283 | t.Fatal(err) |
| 284 | } |
| 285 | fastTick(ctx, d, clock) |
| 286 | if v, _ := release.LiveVersion(f.optDir, host); v != "v2" { |
| 287 | t.Fatalf("live = %q, want v2 once proven", v) |
| 288 | } |
| 289 | clock = clock.Add(31 * time.Second) |
| 290 | if got := d.fastMembers(clock); len(got) != 0 { |
| 291 | t.Fatalf("members = %v after adopting, want none", got) |
| 292 | } |
| 293 | } |
| 294 | |
| 295 | // Spec §4.5 for a route with no live link: a new instance of a listed app |
| 296 | // whose head is unproven waits like a staged one, and rejoins the sweep each |
| 297 | // time its 30 s miss runs out. Its unstaged backoff must not stretch that. |
| 298 | func TestFastMembersKeepsAnUnstagedWaitingRouteAtTheMissCadence(t *testing.T) { |
| 299 | t.Parallel() |
| 300 | f := newFastFixture(t) |
| 301 | ctx := context.Background() |
| 302 | host := "new.example.com" |
| 303 | f.addRoute(t, host, "edge") |
| 304 | plat := release.AdoptPlatform() |
| 305 | m := release.Manifest{App: "fastapp", Version: "v2", Kind: "binary", Artifacts: map[string]release.Artifact{plat: {}}} |
| 306 | if err := release.Ship(ctx, f.store, "bab", m, map[string][]byte{plat: []byte("build two")}); err != nil { |
| 307 | t.Fatal(err) |
| 308 | } |
| 309 | if err := release.Promote(ctx, f.store, f.priv, "bab", "fastapp", "edge", "v2"); err != nil { |
| 310 | t.Fatal(err) |
| 311 | } |
| 312 | _, pubB64, err := release.GenerateKey() |
| 313 | if err != nil { |
| 314 | t.Fatal(err) |
| 315 | } |
| 316 | policy := filepath.Join(t.TempDir(), "proof-policy.json") |
| 317 | if err := os.WriteFile(policy, []byte(fmt.Sprintf(`{"schema":1,"require":[{"app":"fastapp","prover":"dev","key":%q}]}`, pubB64)), 0o644); err != nil { |
| 318 | t.Fatal(err) |
| 319 | } |
| 320 | clock := time.Now() |
| 321 | gate := release.NewProofGate(policy, f.dataDir).WithClock(func() time.Time { return clock }) |
| 322 | d := f.deps(nil) |
| 323 | d.passOpts = append(d.passOpts, release.WithProofGate(gate)) |
| 324 | d.proof = gate |
| 325 | for i := 0; i < 8; i++ { |
| 326 | if got := d.fastMembers(clock); len(got) != 1 || got[0] != host { |
| 327 | t.Fatalf("cycle %d: members = %v, want the waiting unstaged route (step %v)", i, got, d.backoffFor(host)) |
| 328 | } |
| 329 | fastTick(ctx, d, clock) |
| 330 | if w := gate.Waits(); len(w) != 1 || w[0].Host != host { |
| 331 | t.Fatalf("cycle %d: waits = %+v, want %s waiting", i, w, host) |
| 332 | } |
| 333 | if v, _ := release.LiveVersion(f.optDir, host); v != "" { |
| 334 | t.Fatalf("cycle %d: live = %q, want nothing while unproven", i, v) |
| 335 | } |
| 336 | clock = clock.Add(release.ProofMissTTL + time.Second) |
| 337 | } |
| 338 | } |
| 339 | |
| 340 | // Ruling 5's purpose, seen from the fast loop: a wait the sweep neither |
| 341 | // renews nor clears (here the route moved to a channel with no pointer, so |
| 342 | // adoptHost returns before the gate) must not keep its host in the sweep |
| 343 | // every tick. It backs off like an unstaged route. |
| 344 | func TestFastMembersBacksOffAWaitTheSweepLeftStanding(t *testing.T) { |
| 345 | t.Parallel() |
| 346 | f := newFastFixture(t) |
| 347 | ctx := context.Background() |
| 348 | host := "edge.example.com" |
| 349 | f.addRoute(t, host, "edge") |
| 350 | f.stageLive(t, host, "v1") |
| 351 | plat := release.AdoptPlatform() |
| 352 | m := release.Manifest{App: "fastapp", Version: "v2", Kind: "binary", Artifacts: map[string]release.Artifact{plat: {}}} |
| 353 | if err := release.Ship(ctx, f.store, "bab", m, map[string][]byte{plat: []byte("build two")}); err != nil { |
| 354 | t.Fatal(err) |
| 355 | } |
| 356 | if err := release.Promote(ctx, f.store, f.priv, "bab", "fastapp", "edge", "v2"); err != nil { |
| 357 | t.Fatal(err) |
| 358 | } |
| 359 | _, pubB64, err := release.GenerateKey() |
| 360 | if err != nil { |
| 361 | t.Fatal(err) |
| 362 | } |
| 363 | policy := filepath.Join(t.TempDir(), "proof-policy.json") |
| 364 | if err := os.WriteFile(policy, []byte(fmt.Sprintf(`{"schema":1,"require":[{"app":"fastapp","prover":"dev","key":%q}]}`, pubB64)), 0o644); err != nil { |
| 365 | t.Fatal(err) |
| 366 | } |
| 367 | clock := time.Now() |
| 368 | gate := release.NewProofGate(policy, f.dataDir).WithClock(func() time.Time { return clock }) |
| 369 | d := f.deps(nil) |
| 370 | d.passOpts = append(d.passOpts, release.WithProofGate(gate)) |
| 371 | d.proof = gate |
| 372 | if err := release.Pass(ctx, f.reg, f.store, f.pub, f.optDir, d.restart, d.passOpts...); err != nil { |
| 373 | t.Fatal(err) |
| 374 | } |
| 375 | f.addRoute(t, host, "nowhere") // no pointer: the pass never reaches the gate |
| 376 | clock = clock.Add(31 * time.Second) |
| 377 | if got := d.fastMembers(clock); len(got) != 1 { |
| 378 | t.Fatalf("members = %v, want the waiting route", got) |
| 379 | } |
| 380 | fastTick(ctx, d, clock) |
| 381 | if got := d.fastMembers(clock.Add(time.Second)); len(got) != 0 { |
| 382 | t.Fatalf("members = %v a second after a sweep that left the wait standing, want none", got) |
| 383 | } |
| 384 | if d.backoffFor(host) == 0 { |
| 385 | t.Fatal("no backoff recorded for the standing wait") |
| 386 | } |
| 387 | } |
| 388 | |
amadan