| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "os" |
| 8 | "path/filepath" |
| 9 | "slices" |
| 10 | "strings" |
| 11 | "testing" |
| 12 | "time" |
| 13 | |
| 14 | "github.com/carlosframework/platform/internal/hostbox" |
| 15 | "github.com/carlosframework/platform/internal/hostclaim" |
| 16 | execctl "github.com/carlosframework/platform/internal/hostexec" |
| 17 | "github.com/carlosframework/platform/internal/hostguard" |
| 18 | "github.com/carlosframework/platform/internal/release" |
| 19 | ) |
| 20 | |
| 21 | type recordingDrain struct{ calls *[]string } |
| 22 | |
| 23 | func (d recordingDrain) DrainClaim(context.Context, StopRequest, string) (DrainAcknowledgement, error) { |
| 24 | *d.calls = append(*d.calls, "producers") |
| 25 | return DrainAcknowledgement{FenceID: "fence"}, nil |
| 26 | } |
| 27 | |
| 28 | type memPositions struct { |
| 29 | calls *[]string |
| 30 | p ReleasePosition |
| 31 | ok bool |
| 32 | } |
| 33 | |
| 34 | func (m *memPositions) Record(_ context.Context, p ReleasePosition) error { |
| 35 | *m.calls = append(*m.calls, "record") |
| 36 | m.p, m.ok = p, true |
| 37 | return nil |
| 38 | } |
| 39 | func (m *memPositions) Recorded(_ context.Context, r StopRequest) (ReleasePosition, bool, error) { |
| 40 | if !m.ok || !sameStop(m.p.Request, r) { |
| 41 | return ReleasePosition{}, false, nil |
| 42 | } |
| 43 | return m.p, true, nil |
| 44 | } |
| 45 | |
| 46 | // writersStoppedFor is a complete StopWriters witness and root observation |
| 47 | // for o's writer session, by "src" (main's ClaimAssembly.WritersStopped). |
| 48 | func writersStoppedFor(o hostclaim.Observed) (ClaimWritersDrained, WritersStoppedEvidence, error) { |
| 49 | h := o.Claim.Holder |
| 50 | at := time.Date(2026, 10, 7, 9, 0, 0, 0, time.UTC) |
| 51 | target := hostguard.FenceTarget{Host: o.Host, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Box: "src", Enrollment: "enr", BootID: "boot", ControllerID: "ctl"} |
| 52 | drained := ClaimWritersDrained{Target: target, Generation: h.Generation, WriterBox: h.WriterBox, FenceClosed: true, LaunchJoined: true, WorkDrained: true, SpawnsSettled: true, |
| 53 | CgroupEmpty: true, Cgroup: "carlos/shop", DBPaths: []string{"/data/shop.db"}, CheckedAt: at} |
| 54 | seen := WritersStoppedEvidence{Host: o.Host, Incarnation: h.Incarnation, InstallationStamp: h.Stamp, WriterSession: h.Session, Box: "src", Enrollment: "enr", |
| 55 | BootID: "boot", ControllerID: "ctl", ClaimETag: o.ETag, CheckedAt: at} |
| 56 | return drained, seen, nil |
| 57 | } |
| 58 | |
| 59 | // fakeReleaseObligation records main's relocation-release CASes: Queue |
| 60 | // before root's stop, Complete with its receipt. |
| 61 | type fakeReleaseObligation struct { |
| 62 | calls *[]string |
| 63 | queueErr error |
| 64 | completeErr error |
| 65 | completed []string |
| 66 | } |
| 67 | |
| 68 | func (f *fakeReleaseObligation) Queue(_ context.Context, o hostclaim.Observed) (hostclaim.Observed, error) { |
| 69 | *f.calls = append(*f.calls, "queue") |
| 70 | return o, f.queueErr |
| 71 | } |
| 72 | |
| 73 | func (f *fakeReleaseObligation) Verify(context.Context, hostclaim.Observed, string, string) error { |
| 74 | *f.calls = append(*f.calls, "verify") |
| 75 | return nil |
| 76 | } |
| 77 | |
| 78 | func (f *fakeReleaseObligation) Complete(_ context.Context, o hostclaim.Observed, key, digest string) (hostclaim.Observed, error) { |
| 79 | *f.calls = append(*f.calls, "complete") |
| 80 | f.completed = append(f.completed, key+"@"+digest) |
| 81 | return o, f.completeErr |
| 82 | } |
| 83 | |
| 84 | func effectsFixture(t *testing.T) (*RelocationEffects, hostclaim.Observed, *[]string, *StopRequest) { |
| 85 | t.Helper() |
| 86 | calls := &[]string{} |
| 87 | seen := &StopRequest{} |
| 88 | exp := time.Date(2026, 10, 7, 10, 0, 0, 0, time.UTC) |
| 89 | e := &RelocationEffects{ |
| 90 | Claims: release.NewDirStore(t.TempDir()), |
| 91 | Participation: func(context.Context, hostclaim.Observed) (execctl.State, error) { |
| 92 | return execctl.State{Binding: execctl.Binding{Host: "shop.example", Incarnation: "inc", Op: "rel-1", Enrollment: "enr", BootID: "boot", ControllerID: "c9"}, LeaseETag: "lease-1", Expires: exp}, nil |
| 93 | }, |
| 94 | Identity: func(context.Context) (RelocationIdentity, error) { |
| 95 | return RelocationIdentity{Actor: hostbox.Actor{Box: "src", Enrollment: "enr", Revision: 4, Binding: "4"}, BootID: "boot", ControllerID: "ctl", PID: 100, ProcessStart: "5000"}, nil |
| 96 | }, |
| 97 | Producers: func(context.Context, hostclaim.Observed) (StopDrainer, error) { return recordingDrain{calls}, nil }, |
| 98 | WritersStopped: func(_ context.Context, o hostclaim.Observed) (ClaimWritersDrained, WritersStoppedEvidence, error) { |
| 99 | *calls = append(*calls, "writers-stopped") |
| 100 | return writersStoppedFor(o) |
| 101 | }, |
| 102 | Releases: &fakeReleaseObligation{calls: calls}, |
| 103 | FlushDB: func(context.Context, hostclaim.Observed) (uint64, error) { |
| 104 | *calls = append(*calls, "db") |
| 105 | return 41, nil |
| 106 | }, |
| 107 | FlushFiles: func(context.Context, hostclaim.Observed) (int64, error) { |
| 108 | *calls = append(*calls, "files") |
| 109 | return 8, nil |
| 110 | }, |
| 111 | Positions: &memPositions{calls: calls}, |
| 112 | Collect: func(ctx context.Context, r StopRequest, d StopDrainer) (StopResult, error) { |
| 113 | *seen = r |
| 114 | if _, err := d.DrainClaim(ctx, r, "fence"); err != nil { |
| 115 | return StopResult{}, err |
| 116 | } |
| 117 | *calls = append(*calls, "receipt") |
| 118 | return StopResult{Key: "control/hosts/shop.example/evidence/rel-1/src/r.json", Digest: strings.Repeat("a", 64)}, nil |
| 119 | }, |
| 120 | } |
| 121 | obs := hostclaim.Observed{Host: "shop.example", ETag: "etag-7", Claim: &hostclaim.Claim{Host: "shop.example", |
| 122 | Holder: &hostclaim.Holder{Incarnation: "inc", Stamp: 5, Session: 6, Generation: "g6", WriterBox: "src"}, |
| 123 | Op: hostclaim.Op{Kind: hostclaim.KindRelocating, ID: "rel-1", Phase: hostclaim.PhaseReleasing, Exec: "src"}}} |
| 124 | return e, obs, calls, seen |
| 125 | } |
| 126 | |
| 127 | func TestReleaseStopsWritersAndFlushesBeforeItQueues(t *testing.T) { |
| 128 | e, obs, calls, seen := effectsFixture(t) |
| 129 | r, err := e.Release(context.Background(), obs) |
| 130 | if err != nil { |
| 131 | t.Fatal(err) |
| 132 | } |
| 133 | if want := []string{"writers-stopped", "db", "files", "record", "queue", "producers", "receipt", "complete"}; !slices.Equal(*calls, want) { |
| 134 | t.Fatalf("order %v, want %v", *calls, want) |
| 135 | } |
| 136 | if r.FinalTXID != 41 || r.ManifestSeq != 8 || r.Session != 6 || r.Generation != "g6" || r.ReleasedBy != "src" || r.Receipt == "" { |
| 137 | t.Fatalf("release: %+v", r) |
| 138 | } |
| 139 | if seen.ObligationID != RelocationReleaseObligation || seen.Phase != hostclaim.PhaseReleasing || seen.ExpectedClaimETag != "etag-7" || seen.ExpectedDirectoryRevision != "4" { |
| 140 | t.Fatalf("stop request: %+v", *seen) |
| 141 | } |
| 142 | } |
| 143 | |
| 144 | func TestRelocationReleaseStopsAtTheFirstFailedStep(t *testing.T) { |
| 145 | failure := errors.New("sync failed") |
| 146 | for _, tc := range []struct { |
| 147 | name string |
| 148 | change func(*RelocationEffects) |
| 149 | want []string |
| 150 | }{ |
| 151 | {"the obligation cannot be queued", func(e *RelocationEffects) { |
| 152 | e.Releases.(*fakeReleaseObligation).queueErr = failure |
| 153 | }, []string{"writers-stopped", "db", "files", "record", "queue"}}, |
| 154 | {"the writers cannot be stopped", func(e *RelocationEffects) { |
| 155 | e.WritersStopped = func(context.Context, hostclaim.Observed) (ClaimWritersDrained, WritersStoppedEvidence, error) { |
| 156 | return ClaimWritersDrained{}, WritersStoppedEvidence{}, failure |
| 157 | } |
| 158 | }, nil}, |
| 159 | {"writers still running", func(e *RelocationEffects) { |
| 160 | e.WritersStopped = func(_ context.Context, o hostclaim.Observed) (ClaimWritersDrained, WritersStoppedEvidence, error) { |
| 161 | *e.Positions.(*memPositions).calls = append(*e.Positions.(*memPositions).calls, "writers-stopped") |
| 162 | d, ev, err := writersStoppedFor(o) |
| 163 | d.WorkDrained = false |
| 164 | return d, ev, err |
| 165 | } |
| 166 | }, []string{"writers-stopped"}}, |
| 167 | {"root observed another session", func(e *RelocationEffects) { |
| 168 | e.WritersStopped = func(_ context.Context, o hostclaim.Observed) (ClaimWritersDrained, WritersStoppedEvidence, error) { |
| 169 | *e.Positions.(*memPositions).calls = append(*e.Positions.(*memPositions).calls, "writers-stopped") |
| 170 | d, ev, err := writersStoppedFor(o) |
| 171 | ev.WriterSession++ |
| 172 | return d, ev, err |
| 173 | } |
| 174 | }, []string{"writers-stopped"}}, |
| 175 | {"the database flush", func(e *RelocationEffects) { |
| 176 | e.FlushDB = func(context.Context, hostclaim.Observed) (uint64, error) { return 0, failure } |
| 177 | }, []string{"writers-stopped"}}, |
| 178 | {"the files flush", func(e *RelocationEffects) { |
| 179 | e.FlushFiles = func(context.Context, hostclaim.Observed) (int64, error) { return 0, failure } |
| 180 | }, []string{"writers-stopped", "db"}}, |
| 181 | {"the receipt does not discharge the obligation", func(e *RelocationEffects) { |
| 182 | e.Releases.(*fakeReleaseObligation).completeErr = failure |
| 183 | }, []string{"writers-stopped", "db", "files", "record", "queue", "producers", "receipt", "complete"}}, |
| 184 | } { |
| 185 | t.Run(tc.name, func(t *testing.T) { |
| 186 | e, obs, calls, _ := effectsFixture(t) |
| 187 | tc.change(e) |
| 188 | if _, err := e.Release(context.Background(), obs); err == nil { |
| 189 | t.Fatal("released") |
| 190 | } |
| 191 | if !slices.Equal(*calls, tc.want) { |
| 192 | t.Fatalf("ran %v, want %v: nothing after a failed step", *calls, tc.want) |
| 193 | } |
| 194 | }) |
| 195 | } |
| 196 | } |
| 197 | |
| 198 | func TestRelocationReleaseReplaysOnlyRecordedPositions(t *testing.T) { |
| 199 | replayed := func(context.Context, StopRequest, StopDrainer) (StopResult, error) { |
| 200 | return StopResult{Key: "k", Digest: strings.Repeat("a", 64)}, nil // root replayed; no drain ran here |
| 201 | } |
| 202 | t.Run("no record", func(t *testing.T) { |
| 203 | // Ruling 28: without a recorded flush the release flushes first. |
| 204 | // A process with no admitted attempt cannot stop or flush the |
| 205 | // writers, so it never reaches the obligation or root. |
| 206 | e, obs, calls, _ := effectsFixture(t) |
| 207 | e.Collect = replayed |
| 208 | e.WritersStopped = func(context.Context, hostclaim.Observed) (ClaimWritersDrained, WritersStoppedEvidence, error) { |
| 209 | return ClaimWritersDrained{}, WritersStoppedEvidence{}, ErrClaimStopUnavailable |
| 210 | } |
| 211 | if _, err := e.Release(context.Background(), obs); !errors.Is(err, ErrClaimStopUnavailable) || len(*calls) != 0 { |
| 212 | t.Fatalf("got %v, ran %v", err, *calls) |
| 213 | } |
| 214 | }) |
| 215 | t.Run("the lost response", func(t *testing.T) { |
| 216 | // The first call flushed and recorded; root completed its receipt; |
| 217 | // the response was lost. The retry pairs the replayed receipt with |
| 218 | // the recorded positions. |
| 219 | e, obs, _, _ := effectsFixture(t) |
| 220 | collect := e.Collect |
| 221 | e.Collect = func(ctx context.Context, r StopRequest, d StopDrainer) (StopResult, error) { |
| 222 | _, err := collect(ctx, r, d) |
| 223 | return StopResult{}, errors.Join(err, errors.New("connection reset")) |
| 224 | } |
| 225 | if _, err := e.Release(context.Background(), obs); err == nil { |
| 226 | t.Fatal("the lost response released") |
| 227 | } |
| 228 | e.Collect = replayed |
| 229 | r, err := e.Release(context.Background(), obs) |
| 230 | if err != nil || r.FinalTXID != 41 || r.ManifestSeq != 8 { |
| 231 | t.Fatalf("replay: %+v %v", r, err) |
| 232 | } |
| 233 | }) |
| 234 | t.Run("an identity without its process", func(t *testing.T) { |
| 235 | e, obs, calls, _ := effectsFixture(t) |
| 236 | e.Identity = func(context.Context) (RelocationIdentity, error) { |
| 237 | return RelocationIdentity{Actor: hostbox.Actor{Box: "src", Enrollment: "enr", Revision: 4, Binding: "4"}, BootID: "boot", ControllerID: "ctl"}, nil |
| 238 | } |
| 239 | if _, err := e.Release(context.Background(), obs); !errors.Is(err, ErrClaimStopUnavailable) || len(*calls) != 0 { |
| 240 | t.Fatalf("released without a process identity: %v ran %v", err, *calls) |
| 241 | } |
| 242 | }) |
| 243 | t.Run("a record of another session", func(t *testing.T) { |
| 244 | e, obs, _, _ := effectsFixture(t) |
| 245 | if _, err := e.Release(context.Background(), obs); err != nil { |
| 246 | t.Fatal(err) |
| 247 | } |
| 248 | e.Collect = replayed |
| 249 | obs.Claim.Holder.Session = 7 |
| 250 | *e.Positions.(*memPositions).calls = nil |
| 251 | if _, err := e.Release(context.Background(), obs); err != nil || !slices.Contains(*e.Positions.(*memPositions).calls, "writers-stopped") { |
| 252 | t.Fatalf("another session's positions were used: %v ran %v", err, *e.Positions.(*memPositions).calls) |
| 253 | } |
| 254 | }) |
| 255 | } |
| 256 | |
| 257 | // Review Focus 3: a release never returns before root's receipt. Root |
| 258 | // fails after the producers closed (kernel verification or CompleteFence), |
| 259 | // so no receipt is durable: the release returns an error with its |
| 260 | // positions recorded, and the retry, same controller, no restart, resumes |
| 261 | // at the producers' drain with the recorded positions. No step of the |
| 262 | // release runs twice. |
| 263 | func TestReleaseNeverReturnsBeforeRootsReceipt(t *testing.T) { |
| 264 | // Round 7: the drain flushed, recorded and closed the producers; |
| 265 | // root then failed before its receipt was durable (kernel |
| 266 | // verification or CompleteFence). The retry, same controller, no |
| 267 | // restart, resumes at the producers' drain with the recorded |
| 268 | // positions: no step of the release runs again. |
| 269 | e, obs, calls, seen := effectsFixture(t) |
| 270 | collect := e.Collect |
| 271 | e.Collect = func(ctx context.Context, r StopRequest, d StopDrainer) (StopResult, error) { |
| 272 | if _, err := d.DrainClaim(ctx, r, "fence"); err != nil { |
| 273 | return StopResult{}, err |
| 274 | } |
| 275 | return StopResult{}, errors.New("verify stopped: cgroup not yet empty") // no receipt persisted |
| 276 | } |
| 277 | if _, err := e.Release(context.Background(), obs); err == nil { |
| 278 | t.Fatal("a release without a receipt succeeded") |
| 279 | } |
| 280 | if want := []string{"writers-stopped", "db", "files", "record", "queue", "producers"}; !slices.Equal(*calls, want) { |
| 281 | t.Fatalf("the first call ran %v, want %v", *calls, want) |
| 282 | } |
| 283 | *calls = nil |
| 284 | e.Collect = collect |
| 285 | // Round 8 finding 2: the roster moved on between the attempts |
| 286 | // (same boot, same controller); the retry still names the |
| 287 | // revision the stop was flushed under. |
| 288 | e.Identity = func(context.Context) (RelocationIdentity, error) { |
| 289 | return RelocationIdentity{Actor: hostbox.Actor{Box: "src", Enrollment: "enr", Revision: 5, Binding: "5"}, BootID: "boot", ControllerID: "ctl", PID: 100, ProcessStart: "5000"}, nil |
| 290 | } |
| 291 | r, err := e.Release(context.Background(), obs) |
| 292 | if err != nil || r.FinalTXID != 41 || r.ManifestSeq != 8 { |
| 293 | t.Fatalf("retry: %+v %v", r, err) |
| 294 | } |
| 295 | if want := []string{"queue", "producers", "receipt", "complete"}; !slices.Equal(*calls, want) { |
| 296 | t.Fatalf("the retry ran %v, want %v (no flush, no record, no writers stop again)", *calls, want) |
| 297 | } |
| 298 | if seen.ExpectedDirectoryRevision != "4" || seen.BootID != "boot" || seen.ControllerID != "ctl" { |
| 299 | t.Fatalf("the retry asked root under another stop: %+v", *seen) |
| 300 | } |
| 301 | if p := e.Positions.(*memPositions); !p.ok || p.p.TXID != 41 || p.p.Seq != 8 { |
| 302 | t.Fatalf("the positions were not kept for the retry: %+v", p) |
| 303 | } |
| 304 | } |
| 305 | |
| 306 | // A restarted agent asks root as the recorded owner's recovery: the |
| 307 | // request names the boot and controller the flush was made under, offers |
| 308 | // no drain, and pairs root's replayed receipt with the recorded positions. |
| 309 | // Both restarts count: new ids, and the same ids in a new process (main's |
| 310 | // claimLaunchOwner.local). |
| 311 | func TestReleaseAfterARestartRecoversTheRecordedOwner(t *testing.T) { |
| 312 | t.Run("after a restart the recorded identity asks root", func(t *testing.T) { |
| 313 | // The agent flushed and recorded under controller "ctl", root |
| 314 | // completed its receipt, the agent restarted. The new process asks |
| 315 | // root under the recorded boot and controller (so root finds that |
| 316 | // stop's durable receipt), offers no drain of its own, and pairs |
| 317 | // the replayed receipt with the recorded positions. |
| 318 | e, obs, _, _ := effectsFixture(t) |
| 319 | if _, err := e.Release(context.Background(), obs); err != nil { |
| 320 | t.Fatal(err) |
| 321 | } |
| 322 | e.Identity = func(context.Context) (RelocationIdentity, error) { |
| 323 | return RelocationIdentity{Actor: hostbox.Actor{Box: "src", Enrollment: "enr", Revision: 5, Binding: "5"}, BootID: "boot-2", ControllerID: "ctl-2", PID: 200, ProcessStart: "9000"}, nil |
| 324 | } |
| 325 | var asked StopRequest |
| 326 | recovery := false |
| 327 | e.Collect = func(_ context.Context, r StopRequest, d StopDrainer) (StopResult, error) { |
| 328 | asked = r |
| 329 | _, recovery = d.(RecoveryReleaseDrain) // Task 8a sets req.Recover from this |
| 330 | if _, err := d.DrainClaim(context.Background(), r, "fence"); !errors.Is(err, ErrClaimStopIdentity) { |
| 331 | return StopResult{}, errors.New("a restarted agent offered a drain for another process's stop") |
| 332 | } |
| 333 | return StopResult{Key: "k", Digest: strings.Repeat("a", 64)}, nil |
| 334 | } |
| 335 | r, err := e.Release(context.Background(), obs) |
| 336 | if err != nil || r.FinalTXID != 41 || r.ManifestSeq != 8 { |
| 337 | t.Fatalf("after restart: %+v %v", r, err) |
| 338 | } |
| 339 | if asked.BootID != "boot" || asked.ControllerID != "ctl" || asked.ExpectedDirectoryRevision != "4" || !recovery { |
| 340 | t.Fatalf("root was not asked as the recorded owner's recovery: %+v recovery=%v", asked, recovery) |
| 341 | } |
| 342 | }) |
| 343 | t.Run("a restarted process with the same boot and controller ids recovers", func(t *testing.T) { |
| 344 | // Fix round 1 (I-2), main's claimLaunchOwner.local: the same boot |
| 345 | // and controller ids in a restarted process are not this process. |
| 346 | // Its producers ended with the old one, so the request is a |
| 347 | // recovery under the recorded owner and no drain is offered. |
| 348 | e, obs, calls, _ := effectsFixture(t) |
| 349 | if _, err := e.Release(context.Background(), obs); err != nil { |
| 350 | t.Fatal(err) |
| 351 | } |
| 352 | *calls = nil |
| 353 | e.Identity = func(context.Context) (RelocationIdentity, error) { |
| 354 | return RelocationIdentity{Actor: hostbox.Actor{Box: "src", Enrollment: "enr", Revision: 4, Binding: "4"}, BootID: "boot", ControllerID: "ctl", PID: 101, ProcessStart: "6000"}, nil |
| 355 | } |
| 356 | var asked StopRequest |
| 357 | recovery := false |
| 358 | e.Collect = func(_ context.Context, r StopRequest, d StopDrainer) (StopResult, error) { |
| 359 | asked = r |
| 360 | _, recovery = d.(RecoveryReleaseDrain) |
| 361 | if _, err := d.DrainClaim(context.Background(), r, "fence"); !errors.Is(err, ErrClaimStopIdentity) { |
| 362 | return StopResult{}, errors.New("a restarted process offered its producers for the old process's stop") |
| 363 | } |
| 364 | return StopResult{Key: "k", Digest: strings.Repeat("a", 64)}, nil |
| 365 | } |
| 366 | r, err := e.Release(context.Background(), obs) |
| 367 | if err != nil || r.FinalTXID != 41 || r.ManifestSeq != 8 { |
| 368 | t.Fatalf("same ids, new process: %+v %v", r, err) |
| 369 | } |
| 370 | if !recovery || asked.BootID != "boot" || asked.ControllerID != "ctl" || !slices.Equal(*calls, []string{"queue", "complete"}) { |
| 371 | t.Fatalf("not a recovery: recovery=%v asked=%+v ran %v", recovery, asked, *calls) |
| 372 | } |
| 373 | }) |
| 374 | } |
| 375 | |
| 376 | func TestReleasePositionsFileBindsTheExactStop(t *testing.T) { |
| 377 | ctx := context.Background() |
| 378 | f := ReleasePositionsFile{Dir: filepath.Join(t.TempDir(), "relocate")} |
| 379 | r := StopRequest{Host: "shop.example", Incarnation: "inc", InstallationStamp: 5, WriterSession: 6, OperationID: "rel-1", Phase: "releasing", ObligationID: RelocationReleaseObligation, |
| 380 | Box: "src", Enrollment: "enr", BootID: "boot", ControllerID: "ctl", ExpectedClaimETag: "e1", ExpectedDirectoryRevision: "4"} |
| 381 | if _, ok, err := f.Recorded(ctx, r); ok || err != nil { |
| 382 | t.Fatalf("before any record: %v %v", ok, err) |
| 383 | } |
| 384 | if err := f.Record(ctx, ReleasePosition{Request: r, TXID: 41, Seq: 8}); !errors.Is(err, ErrClaimStopUnavailable) { |
| 385 | t.Fatalf("a record without its process: %v", err) |
| 386 | } |
| 387 | if err := f.Record(ctx, ReleasePosition{Request: r, PID: 100, ProcessStart: "5000", TXID: 41, Seq: 8}); err != nil { |
| 388 | t.Fatal(err) |
| 389 | } |
| 390 | moved := r |
| 391 | moved.ExpectedClaimETag, moved.ExpectedDirectoryRevision = "e2", "5" // a target replacement moved the claim |
| 392 | if rec, ok, err := f.Recorded(ctx, moved); !ok || err != nil || rec.TXID != 41 || rec.Seq != 8 || rec.Request.ControllerID != "ctl" || rec.PID != 100 || rec.ProcessStart != "5000" { |
| 393 | t.Fatalf("recorded: %+v %v %v", rec, ok, err) |
| 394 | } |
| 395 | // A restarted controller of the same installation finds the record, |
| 396 | // and gets the identity the flush was made under. |
| 397 | restarted := r |
| 398 | restarted.BootID, restarted.ControllerID = "boot-2", "ctl-2" |
| 399 | if rec, ok, err := f.Recorded(ctx, restarted); !ok || err != nil || rec.Request.BootID != "boot" || rec.Request.ControllerID != "ctl" { |
| 400 | t.Fatalf("after a restart: %+v %v %v", rec, ok, err) |
| 401 | } |
| 402 | for name, change := range map[string]func(*StopRequest){ |
| 403 | "session": func(r *StopRequest) { r.WriterSession = 7 }, |
| 404 | "stamp": func(r *StopRequest) { r.InstallationStamp = 6 }, |
| 405 | "operation": func(r *StopRequest) { r.OperationID = "rel-2" }, |
| 406 | "phase": func(r *StopRequest) { r.Phase = "probe-failed" }, |
| 407 | "box": func(r *StopRequest) { r.Box = "dst" }, |
| 408 | } { |
| 409 | other := r |
| 410 | change(&other) |
| 411 | if _, ok, _ := f.Recorded(ctx, other); ok { |
| 412 | t.Fatalf("%s: another stop's positions were returned", name) |
| 413 | } |
| 414 | } |
| 415 | if err := os.WriteFile(filepath.Join(f.Dir, "rel-1.json"), []byte("{"), 0o600); err != nil { |
| 416 | t.Fatal(err) |
| 417 | } |
| 418 | if _, _, err := f.Recorded(ctx, r); !errors.Is(err, ErrClaimStopReceipt) { |
| 419 | t.Fatalf("a damaged record: %v", err) |
| 420 | } |
| 421 | // A record that names no process would read as this process's own |
| 422 | // under claimLaunchOwner.local: it is damaged, never trusted. |
| 423 | if err := os.WriteFile(filepath.Join(f.Dir, "rel-1.json"), []byte(`{"v":1,"request":{"Host":"shop.example"},"txid":41,"seq":8}`), 0o600); err != nil { |
| 424 | t.Fatal(err) |
| 425 | } |
| 426 | if _, _, err := f.Recorded(ctx, r); !errors.Is(err, ErrClaimStopReceipt) { |
| 427 | t.Fatalf("a record without its process: %v", err) |
| 428 | } |
| 429 | } |
| 430 | |
| 431 | // Round 14 finding 2: EnsureLocal verifies the receipt's install, restores |
| 432 | // a retired one under the same stamp and verifies it again, refuses a |
| 433 | // restore that is not the receipt's, and never touches an opened P. |
| 434 | func TestEnsureLocalRepairsOnlyUnderTheActiveStampBeforePOpens(t *testing.T) { |
| 435 | ctx := context.Background() |
| 436 | e, obs, _, _ := effectsFixture(t) |
| 437 | receipt := hostclaim.LocalInstall{Box: "dst", Enrollment: "enr-d", InstallStamp: 9, FilesManifestVersion: 1, JournalSHA256: strings.Repeat("e", 64)} |
| 438 | raw, _ := json.Marshal(receipt) |
| 439 | obs.Claim.Op.Phase = hostclaim.PhaseRelocatingProbing |
| 440 | obs.Claim.Op.Journal = map[string]string{"local_install": string(raw), "prepared": "P1", "install_stamp": "9"} |
| 441 | present, installs := true, 0 |
| 442 | e.VerifyLocal = func(context.Context, hostclaim.Observed, hostclaim.LocalInstall) (bool, error) { return present, nil } |
| 443 | restored := receipt |
| 444 | e.Install = func(context.Context, hostclaim.Observed) (hostclaim.LocalInstall, error) { |
| 445 | installs++ |
| 446 | present = true |
| 447 | return restored, nil |
| 448 | } |
| 449 | if err := e.EnsureLocal(ctx, obs); err != nil || installs != 0 { |
| 450 | t.Fatalf("a complete install: %v installs=%d", err, installs) |
| 451 | } |
| 452 | present = false |
| 453 | if err := e.EnsureLocal(ctx, obs); err != nil || installs != 1 || !present { |
| 454 | t.Fatalf("a damaged install under its active stamp: %v installs=%d", err, installs) |
| 455 | } |
| 456 | present, restored.InstallStamp = false, 10 |
| 457 | if err := e.EnsureLocal(ctx, obs); !errors.Is(err, ErrClaimStopReceipt) { |
| 458 | t.Fatalf("a restore under another stamp: %v", err) |
| 459 | } |
| 460 | present, installs = false, 0 |
| 461 | obs.Claim.Op.Journal["prepared_opened"] = "P1" |
| 462 | if err := e.EnsureLocal(ctx, obs); err != nil || installs != 0 { |
| 463 | t.Fatalf("an opened P was reinstalled: %v installs=%d", err, installs) |
| 464 | } |
| 465 | } |
| 466 | |
| 467 | // Round 15 finding 1: after P opens, VerifyReceipt compares the box's |
| 468 | // durable receipt with the journaled one and nothing else: a missing or |
| 469 | // different receipt is false (the participant marks the operation stuck); |
| 470 | // changed files never trigger a restore. |
| 471 | func TestVerifyReceiptChecksTheReceiptOnly(t *testing.T) { |
| 472 | ctx := context.Background() |
| 473 | e, obs, _, _ := effectsFixture(t) |
| 474 | receipt := hostclaim.LocalInstall{Box: "dst", Enrollment: "enr-d", InstallStamp: 9, FilesManifestVersion: 1, JournalSHA256: strings.Repeat("e", 64)} |
| 475 | raw, _ := json.Marshal(receipt) |
| 476 | obs.Claim.Op.Journal = map[string]string{"local_install": string(raw), "prepared": "P1", "prepared_opened": "P1", "install_stamp": "9"} |
| 477 | stored, present := receipt, true |
| 478 | e.Receipt = func(context.Context, hostclaim.Observed) (hostclaim.LocalInstall, bool, error) { |
| 479 | return stored, present, nil |
| 480 | } |
| 481 | touched := 0 |
| 482 | e.VerifyLocal = func(context.Context, hostclaim.Observed, hostclaim.LocalInstall) (bool, error) { |
| 483 | touched++ |
| 484 | return false, nil |
| 485 | } |
| 486 | e.Install = func(context.Context, hostclaim.Observed) (hostclaim.LocalInstall, error) { |
| 487 | touched++ |
| 488 | return receipt, nil |
| 489 | } |
| 490 | if ok, err := e.VerifyReceipt(ctx, obs); !ok || err != nil || touched != 0 { |
| 491 | t.Fatalf("an intact receipt over changed files: %v %v touched=%d", ok, err, touched) |
| 492 | } |
| 493 | present = false |
| 494 | if ok, err := e.VerifyReceipt(ctx, obs); ok || err != nil { |
| 495 | t.Fatalf("a missing receipt: %v %v", ok, err) |
| 496 | } |
| 497 | present, stored.InstallStamp = true, 8 |
| 498 | if ok, err := e.VerifyReceipt(ctx, obs); ok || err != nil { |
| 499 | t.Fatalf("another installation's receipt: %v %v", ok, err) |
| 500 | } |
| 501 | } |
| 502 | |
amadan