| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/sha256" |
| 6 | "encoding/hex" |
| 7 | "encoding/json" |
| 8 | "errors" |
| 9 | "testing" |
| 10 | "time" |
| 11 | |
| 12 | "github.com/carlosframework/platform/internal/genselect" |
| 13 | "github.com/carlosframework/platform/internal/hostbox" |
| 14 | "github.com/carlosframework/platform/internal/hostclaim" |
| 15 | "github.com/carlosframework/platform/internal/hostfiles" |
| 16 | "github.com/carlosframework/platform/internal/hostguard" |
| 17 | "github.com/carlosframework/platform/internal/hostwrite" |
| 18 | "github.com/carlosframework/platform/internal/moves" |
| 19 | "github.com/carlosframework/platform/internal/provision" |
| 20 | "github.com/superfly/ltx" |
| 21 | ) |
| 22 | |
| 23 | // fenceWork is an admitted spawn selection whose fence can be closed: once |
| 24 | // shut, every check refuses. It counts the checks it answered. |
| 25 | type fenceWork struct { |
| 26 | shut *bool |
| 27 | checks *int |
| 28 | } |
| 29 | |
| 30 | func newFenceWork() fenceWork { return fenceWork{shut: new(bool), checks: new(int)} } |
| 31 | |
| 32 | func (w fenceWork) With(_ context.Context, _ hostwrite.Identity, fn func() error) error { |
| 33 | *w.checks++ |
| 34 | if *w.shut { |
| 35 | return hostbox.ErrChanged |
| 36 | } |
| 37 | return fn() |
| 38 | } |
| 39 | |
| 40 | func (f *poFixture) markerKey() string { return "instances/" + poHost + "/generation.json" } |
| 41 | |
| 42 | func (f *poFixture) currentMarker() (genselect.Marker, bool) { |
| 43 | f.t.Helper() |
| 44 | body, ok := f.bucket.Current("data", f.markerKey()) |
| 45 | var m genselect.Marker |
| 46 | if ok && json.Unmarshal(body, &m) != nil { |
| 47 | f.t.Fatalf("marker %s", body) |
| 48 | } |
| 49 | return m, ok |
| 50 | } |
| 51 | |
| 52 | // selectAndAdmit runs Select and the admission CAS for the claim as it is. |
| 53 | func (f *poFixture) selectAndAdmit() (hostclaim.Observed, ClaimLaunchInputs) { |
| 54 | f.t.Helper() |
| 55 | o := f.read() |
| 56 | in, err := f.owners.Select(f.t.Context(), f.route, o, poProbe) |
| 57 | if err != nil { |
| 58 | f.t.Fatal(err) |
| 59 | } |
| 60 | return f.admit(o, in), in |
| 61 | } |
| 62 | |
| 63 | // Finding 1: AdmissionEvidence, immediately before the admission's CAS, |
| 64 | // refuses a landing past its deadline, a record attempt outside what T11 |
| 65 | // admitted, an install the record does not vouch for, and a baseline whose |
| 66 | // pin is not the build's. |
| 67 | func TestMovesProbeOwnersAdmissionEvidenceChecksTheLanding(t *testing.T) { |
| 68 | past := time.Date(2026, 10, 9, 11, 59, 0, 0, time.UTC) |
| 69 | for name, tc := range map[string]struct { |
| 70 | rec func(*provision.Record) |
| 71 | plan func(*genselect.Plan) |
| 72 | }{ |
| 73 | "past the cut deadline": {rec: func(r *provision.Record) { r.Move.CutDeadline = &past }}, |
| 74 | "past the landing deadline": {rec: func(r *provision.Record) { r.Move.LandingDeadline = &past }}, |
| 75 | "no deadline": {rec: func(r *provision.Record) { r.Move.CutDeadline = nil }}, |
| 76 | "no attempt counted": {rec: func(r *provision.Record) { r.Move.Landing.Attempts = 0 }}, |
| 77 | "beyond the admitted": {rec: func(r *provision.Record) { r.Move.Landing.Attempts = 4 }}, |
| 78 | "another installed db": {rec: func(r *provision.Record) { |
| 79 | r.Move.Landing.InstalledDBSHA256 = "sha256:" + hex.EncodeToString(make([]byte, 32)) |
| 80 | }}, |
| 81 | "no installed db": {rec: func(r *provision.Record) { r.Move.Landing.InstalledDBSHA256 = "" }}, |
| 82 | "another pin version": {plan: func(p *genselect.Plan) { |
| 83 | pin := *p.Pin |
| 84 | pin.SnapshotVersion = "other" |
| 85 | p.Pin = &pin |
| 86 | }}, |
| 87 | "no pin": {plan: func(p *genselect.Plan) { p.Pin = nil }}, |
| 88 | } { |
| 89 | t.Run(name, func(t *testing.T) { |
| 90 | f := newPOFixture(t) |
| 91 | ctx := t.Context() |
| 92 | o := f.read() |
| 93 | in, err := f.owners.Select(ctx, f.route, o, poProbe) |
| 94 | if err != nil { |
| 95 | t.Fatal(err) |
| 96 | } |
| 97 | if err := f.owners.AdmissionEvidence(ctx, o, in.Admission, in.Baseline); err != nil { |
| 98 | t.Fatalf("the unchanged landing: %v", err) |
| 99 | } |
| 100 | plan := *in.Baseline |
| 101 | if tc.rec != nil { |
| 102 | tc.rec(f.rec) |
| 103 | } |
| 104 | if tc.plan != nil { |
| 105 | tc.plan(&plan) |
| 106 | } |
| 107 | if err := f.owners.AdmissionEvidence(ctx, o, in.Admission, &plan); err == nil { |
| 108 | t.Fatal("admitted") |
| 109 | } |
| 110 | }) |
| 111 | } |
| 112 | } |
| 113 | |
| 114 | // Finding 2: the prepared marker is written only under the admitted |
| 115 | // writer's live fence, checked by the store before every request attempt, |
| 116 | // and carries the install manifest's digest, verified against install.json. |
| 117 | func TestMovesProbeOwnersPreparedMarkerIsFenced(t *testing.T) { |
| 118 | f := newPOFixture(t) |
| 119 | ctx := t.Context() |
| 120 | admitted, in := f.selectAndAdmit() |
| 121 | // Group D's prepared Publish with no fence held for the session. |
| 122 | if err := f.owners.Publish(ctx, admitted, in); err == nil { |
| 123 | t.Fatal("an unfenced prepared marker was published") |
| 124 | } |
| 125 | if _, ok := f.currentMarker(); ok { |
| 126 | t.Fatal("a marker was written without the fence") |
| 127 | } |
| 128 | fence := newFenceWork() |
| 129 | if err := f.owners.Materialize(ctx, admitted, in, fence); err != nil { |
| 130 | t.Fatal(err) |
| 131 | } |
| 132 | m, ok := f.currentMarker() |
| 133 | install, _ := f.bucket.Current("data", moves.InstallManifestKey(moves.GenerationPrefix("instances/"+poHost, poOp))) |
| 134 | sum := sha256.Sum256(install) |
| 135 | if !ok || m.Op != poOp || m.ManifestSHA256 != hex.EncodeToString(sum[:]) || m.SnapshotTXID != 42 { |
| 136 | t.Fatalf("marker %+v", m) |
| 137 | } |
| 138 | // Group D's call now replays the marker this session wrote. |
| 139 | written := f.bucket.LatestID("data", f.markerKey()) |
| 140 | if err := f.owners.Publish(ctx, admitted, in); err != nil || f.bucket.LatestID("data", f.markerKey()) != written { |
| 141 | t.Fatalf("replay: %v", err) |
| 142 | } |
| 143 | if err := f.owners.PublicationEvidence(ctx, admitted); err != nil { |
| 144 | t.Fatal(err) |
| 145 | } |
| 146 | // Another writer replaced it and the session's fence has closed: nothing |
| 147 | // is written over it. |
| 148 | f.bucket.Put("data", f.markerKey(), mustJSON(t, genselect.Marker{Op: "other", Incarnation: f.inc, Host: poHost, Box: "uk-box", InstalledAt: time.Unix(3, 0).UTC()})) |
| 149 | other := f.bucket.LatestID("data", f.markerKey()) |
| 150 | *fence.shut = true |
| 151 | if err := f.owners.Publish(ctx, admitted, in); err == nil || f.bucket.LatestID("data", f.markerKey()) != other { |
| 152 | t.Fatalf("a marker written after the fence closed: %v", err) |
| 153 | } |
| 154 | } |
| 155 | |
| 156 | // Finding 2: an install.json that is not the journalled manifest is not |
| 157 | // published, and the probe does not start. |
| 158 | func TestMovesProbeOwnersPreparedMarkerNeedsTheInstallManifest(t *testing.T) { |
| 159 | f := newPOFixture(t) |
| 160 | other := f.manifest |
| 161 | other.BuiltAt = time.Unix(9, 0).UTC() |
| 162 | f.bucket.Put("data", moves.InstallManifestKey(moves.GenerationPrefix("instances/"+poHost, poOp)), mustJSON(t, other)) |
| 163 | admitted, in := f.selectAndAdmit() |
| 164 | if err := f.owners.Materialize(t.Context(), admitted, in, newFenceWork()); err == nil { |
| 165 | t.Fatal("published over an install.json the journal does not name") |
| 166 | } |
| 167 | if _, ok := f.currentMarker(); ok { |
| 168 | t.Fatal("a marker was written") |
| 169 | } |
| 170 | } |
| 171 | |
| 172 | // Finding 3: the env pin is the report of the launch Select selected: a |
| 173 | // launch environment that changed is refused though the pin and every other |
| 174 | // input are unchanged; the probe-only CARLOS_MOVE_PROBE is not part of it. |
| 175 | func TestMovesProbeOwnersEnvPinIsTheSelectedLaunch(t *testing.T) { |
| 176 | f := newPOFixture(t) |
| 177 | if _, err := f.owners.Select(t.Context(), f.route, f.read(), poProbe); err != nil { |
| 178 | t.Fatalf("the pinned launch: %v", err) |
| 179 | } |
| 180 | f.launch.Env = []string{"A=2", "CARLOS_MOVE_PROBE=0"} |
| 181 | _, err := f.owners.Select(t.Context(), f.route, f.read(), poProbe) |
| 182 | var changed *ProbeInputsChanged |
| 183 | if !errors.As(err, &changed) || changed.Reason != provision.RollbackConfigChanged { |
| 184 | t.Fatalf("a changed launch environment: %v", err) |
| 185 | } |
| 186 | f.launch.Env = []string{"A=1"} |
| 187 | if _, err := f.owners.Select(t.Context(), f.route, f.read(), poProbe); err != nil { |
| 188 | t.Fatalf("without the probe-only variable: %v", err) |
| 189 | } |
| 190 | } |
| 191 | |
| 192 | // policyWork is a held selection that asks the real MovesClaimPolicy, as |
| 193 | // hostbox.WorkSource does, before it runs anything. |
| 194 | type policyWork struct { |
| 195 | f *poFixture |
| 196 | action string |
| 197 | } |
| 198 | |
| 199 | func (w policyWork) With(ctx context.Context, id hostwrite.Identity, fn func() error) error { |
| 200 | o := w.f.read() |
| 201 | c := o.Claim |
| 202 | s := hostbox.Selection{Host: poHost, Action: w.action, Box: "uk-box", Region: "uk", Captured: hostguard.Captured{Incarnation: c.Holder.Incarnation, Stamp: c.Holder.Stamp}, |
| 203 | Kind: c.Op.Kind, Op: c.Op.ID, Phase: c.Op.Phase, Generation: c.Holder.Generation, Session: c.Holder.Session} |
| 204 | if err := MovesClaimPolicy(func(context.Context, string, string, string) (*provision.Record, error) { return w.f.rec, nil })(ctx, c, s, id); err != nil { |
| 205 | return err |
| 206 | } |
| 207 | return fn() |
| 208 | } |
| 209 | |
| 210 | // Finding 4: a render whose etags are not the pins is ProbeInputsChanged |
| 211 | // (the landing's RollbackConfigChanged), decided before any held work is |
| 212 | // selected or written, through the real policy, which would otherwise |
| 213 | // refuse the write with its own error. |
| 214 | func TestMovesProbeOwnersChangedConfigThroughThePolicy(t *testing.T) { |
| 215 | f := newPOFixture(t) |
| 216 | ctx := t.Context() |
| 217 | f.owners.Works = func(_ context.Context, _, action string, _ hostguard.Captured) (hostwrite.Work, error) { |
| 218 | f.order = append(f.order, "select:"+action) |
| 219 | return policyWork{f: f, action: action}, nil |
| 220 | } |
| 221 | admitted, in := f.selectAndAdmit() |
| 222 | if err := f.owners.Materialize(ctx, admitted, in, newFenceWork()); err != nil { |
| 223 | t.Fatalf("the pinned render through the policy: %v", err) |
| 224 | } |
| 225 | f.order = nil |
| 226 | f.renderOverlay = "e-changed" |
| 227 | err := f.owners.Materialize(ctx, admitted, in, newFenceWork()) |
| 228 | var changed *ProbeInputsChanged |
| 229 | var refusal *MovesPolicyRefusal |
| 230 | if !errors.As(err, &changed) || changed.Reason != provision.RollbackConfigChanged || errors.As(err, &refusal) { |
| 231 | t.Fatalf("a changed render: %v", err) |
| 232 | } |
| 233 | for _, step := range f.order { |
| 234 | if step == "select:held-config" || step == "select:held-release" { |
| 235 | t.Fatalf("a held work was selected for a changed render: %v", f.order) |
| 236 | } |
| 237 | } |
| 238 | } |
| 239 | |
| 240 | // Finding 8, ruled order: admit, publish the verified prepared marker under |
| 241 | // the writer's fence (superseding the earlier stay's marker by If-Match), |
| 242 | // then the start check, then the spawn. Fix round 2: the earlier stay is |
| 243 | // really in the admission history, a cold admission on this box and region |
| 244 | // that does not descend from the move's admission; it is still an earlier |
| 245 | // stay, superseded, never taken for a restart of this landing. |
| 246 | func TestMovesProbeOwnersSupersedeTheEarlierStaysMarker(t *testing.T) { |
| 247 | f := newPOFixture(t) |
| 248 | ctx := t.Context() |
| 249 | c := f.poFlipClaim() |
| 250 | c.Holder.Admissions = []hostclaim.AdmissionRecord{ |
| 251 | {Session: 5, Box: "uk-box", Generation: "earlier", Predecessor: "", Region: "uk", Mode: "cold"}, |
| 252 | {Session: 15, Box: "ie-box", Generation: "src", Predecessor: "earlier", Region: "ie", PredecessorRegion: "uk", Mode: "cold"}, |
| 253 | } |
| 254 | if err := c.Validate(); err != nil { |
| 255 | t.Fatal(err) |
| 256 | } |
| 257 | f.putClaim(c) |
| 258 | f.bucket.Put("data", f.markerKey(), mustJSON(t, genselect.Marker{Op: "earlier", Incarnation: f.inc, Host: poHost, Box: "uk-box", InstalledAt: time.Unix(1, 0).UTC(), Pin: genselect.Pin{SnapshotTXID: 3}})) |
| 259 | admitted, in := f.selectAndAdmit() |
| 260 | if err := f.owners.Materialize(ctx, admitted, in, newFenceWork()); err != nil { |
| 261 | t.Fatalf("the first probe over an earlier stay's marker: %v", err) |
| 262 | } |
| 263 | if m, _ := f.currentMarker(); m.Op != poOp { |
| 264 | t.Fatalf("the earlier stay's marker was not superseded: %+v", m) |
| 265 | } |
| 266 | if err := f.owners.Publish(ctx, admitted, in); err != nil { |
| 267 | t.Fatalf("Group D's prepared publication: %v", err) |
| 268 | } |
| 269 | if err := f.owners.PublicationEvidence(ctx, admitted); err != nil { |
| 270 | t.Fatalf("publication evidence: %v", err) |
| 271 | } |
| 272 | } |
| 273 | |
| 274 | // Fix round 2: a genuine restart after a crash is still this landing's. The |
| 275 | // first attempt published the move id; the restarted attempt fresh1 |
| 276 | // published its own marker after its root and crashed. The next attempt's |
| 277 | // baseline is the certified remote from fresh1, whose chain reaches the move |
| 278 | // id, and fresh1's marker is kept, not superseded by the prepared one. Since |
| 279 | // Task 7's fix round 1 fresh1's publication refreshed .files-gen to fresh1, |
| 280 | // so that attempt passes Task 4's files rule too. |
| 281 | func TestMovesProbeOwnersRecogniseARestartAfterACrash(t *testing.T) { |
| 282 | f := newPOFixture(t) |
| 283 | ctx := t.Context() |
| 284 | first, in := f.selectAndAdmit() |
| 285 | if err := f.owners.Materialize(ctx, first, in, newFenceWork()); err != nil { |
| 286 | t.Fatal(err) |
| 287 | } |
| 288 | again, err := f.owners.Select(ctx, f.route, first, poProbe) |
| 289 | if err != nil { |
| 290 | t.Fatal(err) |
| 291 | } |
| 292 | restarted := f.admit(first, again) |
| 293 | fresh := again.Admission.NewGeneration |
| 294 | f.restore(fresh) |
| 295 | if err := f.owners.Materialize(ctx, restarted, again, newFenceWork()); err != nil { |
| 296 | t.Fatal(err) |
| 297 | } |
| 298 | f.bucket.Put("data", "instances/"+poHost+"/g-"+fresh+"/0009/"+ltx.FormatFilename(1, 44), f.ltxHeader(1, 44)) |
| 299 | pub := ClaimPublication{Sync: func(context.Context) error { return nil }, Check: func(context.Context) error { return nil }} |
| 300 | if _, err := f.owners.PublishFenced(ctx, restarted, again, pub); err != nil { |
| 301 | t.Fatal(err) |
| 302 | } |
| 303 | // The crash: the next record attempt admits again over fresh1. |
| 304 | next, err := f.owners.Select(ctx, f.route, restarted, poProbe) |
| 305 | if err != nil { |
| 306 | t.Fatal(err) |
| 307 | } |
| 308 | if next.Baseline.Generation != fresh || next.Baseline.Marker == nil || next.Baseline.Marker.Op != fresh || next.Baseline.MaxTXID != 44 { |
| 309 | t.Fatalf("the restart's baseline is not the certified remote of %s: %+v", fresh, next.Baseline) |
| 310 | } |
| 311 | admitted := f.admit(restarted, next) |
| 312 | f.restore(admitted.Claim.Holder.Generation) |
| 313 | written := f.bucket.LatestID("data", f.markerKey()) |
| 314 | err = f.owners.Materialize(ctx, admitted, next, newFenceWork()) |
| 315 | if m, _ := f.currentMarker(); m.Op != fresh || f.bucket.LatestID("data", f.markerKey()) != written { |
| 316 | t.Fatalf("the restart's marker was superseded: %+v", m) |
| 317 | } |
| 318 | if err != nil { |
| 319 | t.Fatalf("the second restart: %v", err) |
| 320 | } |
| 321 | } |
| 322 | |
| 323 | // Fix round 2, the predicate itself: a generation is this landing's only |
| 324 | // with positive proof, the chain of cold admissions on this box and region |
| 325 | // back from it reaching the move id. |
| 326 | func TestMovesProbeOwnersLandingMarkerNeedsDescent(t *testing.T) { |
| 327 | f := newPOFixture(t) |
| 328 | v := probeLanding{op: poOp, manifest: f.manifest, holder: &hostclaim.Holder{Incarnation: f.inc, Admissions: []hostclaim.AdmissionRecord{ |
| 329 | {Session: 5, Box: "uk-box", Generation: "earlier", Region: "uk", Mode: "cold"}, |
| 330 | {Session: 15, Box: "ie-box", Generation: "src", Predecessor: "earlier", Region: "ie", Mode: "cold"}, |
| 331 | {Session: 20, Box: "uk-box", Generation: poOp, Predecessor: "src", Region: "uk", Mode: "cold", Coverage: "sealed"}, |
| 332 | {Session: 21, Box: "uk-box", Generation: "f1", Predecessor: poOp, Region: "uk", Mode: "cold"}, |
| 333 | {Session: 22, Box: "uk-box", Generation: "f2", Predecessor: "f1", Region: "uk", Mode: "cold"}, |
| 334 | {Session: 23, Box: "uk-box", Generation: "orphan", Predecessor: "gone", Region: "uk", Mode: "cold"}, |
| 335 | {Session: 24, Box: "other-box", Generation: "f3", Predecessor: "f2", Region: "uk", Mode: "cold"}, |
| 336 | }}} |
| 337 | marker := func(op string) *genselect.Marker { |
| 338 | return &genselect.Marker{Op: op, Incarnation: f.inc, Host: poHost, Box: "uk-box"} |
| 339 | } |
| 340 | for op, want := range map[string]bool{"f1": true, "f2": true, "earlier": false, "src": false, "orphan": false, "f3": false, "unknown": false} { |
| 341 | if got := f.owners.landingMarker(marker(op), poHost, v, ""); got != want { |
| 342 | t.Errorf("%s: %v, want %v", op, got, want) |
| 343 | } |
| 344 | } |
| 345 | } |
| 346 | |
| 347 | // Finding 8: a history trimmed so that the admitted generation's ancestry |
| 348 | // back to the published marker cannot be proved is refused by the start |
| 349 | // check, after the publication and before any spawn. StartCheck is not |
| 350 | // weakened. |
| 351 | func TestMovesProbeOwnersTrimmedHistoryIsRefused(t *testing.T) { |
| 352 | f := newPOFixture(t) |
| 353 | ctx := t.Context() |
| 354 | first, in := f.selectAndAdmit() |
| 355 | if err := f.owners.Materialize(ctx, first, in, newFenceWork()); err != nil { |
| 356 | t.Fatal(err) |
| 357 | } |
| 358 | // Two restarted attempts: fresh1 from the move id, fresh2 from fresh1. |
| 359 | again, err := f.owners.Select(ctx, f.route, first, poProbe) |
| 360 | if err != nil { |
| 361 | t.Fatal(err) |
| 362 | } |
| 363 | second := f.admit(first, again) |
| 364 | third, err := f.owners.Select(ctx, f.route, second, poProbe) |
| 365 | if err != nil { |
| 366 | t.Fatal(err) |
| 367 | } |
| 368 | admitted := f.admit(second, third) |
| 369 | // Trim the history to the last admission only (fresh2 from fresh1). |
| 370 | c := *admitted.Claim |
| 371 | h := *c.Holder |
| 372 | h.Admissions = h.Admissions[len(h.Admissions)-1:] |
| 373 | c.Holder = &h |
| 374 | f.putClaim(&c) |
| 375 | f.writeLocal(poOp, h.Generation, f.inc) |
| 376 | trimmed := f.read() |
| 377 | before := f.bucket.LatestID("data", f.markerKey()) |
| 378 | // C8 refuses the unprovable history first (item 1: the establishment |
| 379 | // runs before the start check, and a chain it cannot prove is |
| 380 | // ErrNoEvidence); behind it the start check stands unweakened. |
| 381 | err = f.owners.Materialize(ctx, trimmed, third, newFenceWork()) |
| 382 | if !errors.Is(err, hostfiles.ErrNoEvidence) { |
| 383 | t.Fatalf("a trimmed history through C8: %v", err) |
| 384 | } |
| 385 | f.owners.Files = noopFiles{} |
| 386 | err = f.owners.Materialize(ctx, trimmed, third, newFenceWork()) |
| 387 | if !errors.Is(err, moves.ErrStartMarker) { |
| 388 | t.Fatalf("a trimmed history: %v", err) |
| 389 | } |
| 390 | if f.bucket.LatestID("data", f.markerKey()) != before { |
| 391 | t.Fatal("the verified marker was replaced") |
| 392 | } |
| 393 | } |
| 394 | |
amadan