| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/ed25519" |
| 6 | "crypto/rand" |
| 7 | "encoding/json" |
| 8 | "errors" |
| 9 | "os" |
| 10 | "path/filepath" |
| 11 | "reflect" |
| 12 | "slices" |
| 13 | "strings" |
| 14 | "testing" |
| 15 | "time" |
| 16 | |
| 17 | "github.com/carlosframework/platform/internal/activator" |
| 18 | "github.com/carlosframework/platform/internal/boxvolumes" |
| 19 | "github.com/carlosframework/platform/internal/hostactors" |
| 20 | "github.com/carlosframework/platform/internal/hostbox" |
| 21 | "github.com/carlosframework/platform/internal/hostclaim" |
| 22 | "github.com/carlosframework/platform/internal/hostguard" |
| 23 | "github.com/carlosframework/platform/internal/hostrelocate" |
| 24 | "github.com/carlosframework/platform/internal/lease" |
| 25 | "github.com/carlosframework/platform/internal/lease/leasetest" |
| 26 | "github.com/carlosframework/platform/internal/moves" |
| 27 | "github.com/carlosframework/platform/internal/provision" |
| 28 | "github.com/carlosframework/platform/internal/registry" |
| 29 | "github.com/carlosframework/platform/internal/release" |
| 30 | "github.com/carlosframework/platform/internal/termfence" |
| 31 | "github.com/superfly/ltx" |
| 32 | ) |
| 33 | |
| 34 | // passClaim writes a claim for host into store with op as given; serving |
| 35 | // claims have no op id. |
| 36 | func passClaim(t *testing.T, store release.Store, host string, op hostclaim.Op) { |
| 37 | t.Helper() |
| 38 | ctx := t.Context() |
| 39 | created := time.Date(2026, 10, 7, 1, 0, 0, 0, time.UTC) |
| 40 | nh := hostclaim.NewHolder{Family: "instance", Key: "apps/bab/jam/instances/" + host + ".json", Account: "bab", App: "jam", Host: host, Region: "ie", RecordRegion: "ie", CreatedAt: created} |
| 41 | body := []byte(`{"host":"` + host + `","account":"bab","app":"jam","region":"ie","created_at":"2026-10-07T01:00:00Z"}`) |
| 42 | obs, err := hostclaim.Read(ctx, store, host) |
| 43 | if err != nil { |
| 44 | t.Fatal(err) |
| 45 | } |
| 46 | for _, tr := range []func(*hostclaim.Claim) error{ |
| 47 | hostclaim.BeginCreateOn(nh, body, "c1", "console", created, hostclaim.ExecutorSelection{Box: "src", Enrollment: strings.Repeat("b", 32)}), |
| 48 | hostclaim.CompleteCreate("c1", "src"), |
| 49 | hostclaim.AdmitWriter(hostclaim.Admission{Box: "src", Mode: "cold", NewGeneration: "g1", Region: "ie", KeepStamp: true}), |
| 50 | func(c *hostclaim.Claim) error { |
| 51 | if op.Kind != "" { |
| 52 | c.Op = op |
| 53 | } |
| 54 | return nil |
| 55 | }, |
| 56 | } { |
| 57 | if obs, err = hostclaim.Update(ctx, store, obs, tr); err != nil { |
| 58 | t.Fatal(err) |
| 59 | } |
| 60 | } |
| 61 | } |
| 62 | |
| 63 | func relocatingOp(id string) hostclaim.Op { |
| 64 | return hostclaim.Op{Kind: hostclaim.KindRelocating, ID: id, Phase: hostclaim.PhaseInstalling, Exec: "dst", Participants: []string{"src", "dst"}, |
| 65 | Intent: map[string]string{"region": "ie", "source_box": "src", "target_box": "dst"}} |
| 66 | } |
| 67 | |
| 68 | // stepRecorder is a worker or participant that records each Step. |
| 69 | type stepRecorder struct { |
| 70 | name string |
| 71 | order *[]string |
| 72 | err error |
| 73 | } |
| 74 | |
| 75 | func (s stepRecorder) Step(_ context.Context, host string) (hostclaim.Observed, error) { |
| 76 | *s.order = append(*s.order, s.name+":"+host) |
| 77 | return hostclaim.Observed{}, s.err |
| 78 | } |
| 79 | |
| 80 | // Spec §8 step 2 and ruling 82: nothing runs with CARLOS_HOST_CLAIM off. |
| 81 | // With claim mode on, CARLOS_REGION_HA off, or the relocation owners not |
| 82 | // bound, the runtime is the completion-only one: it steps nothing, |
| 83 | // converts nothing and reads only this box's registry rows. |
| 84 | func TestRelocationRuntimeFollowsTheFlags(t *testing.T) { |
| 85 | reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 86 | if err != nil { |
| 87 | t.Fatal(err) |
| 88 | } |
| 89 | defer reg.Close() |
| 90 | local := &productionClaimLocal{legacyParticipation: true, assembly: &ClaimAssembly{Guard: hostguard.Guard{DataDir: t.TempDir()}}} |
| 91 | for _, tc := range []struct { |
| 92 | claim, ha string |
| 93 | completion bool |
| 94 | }{{"", "", false}, {"", "1", false}, {"1", "", true}, {"1", "1", true}} { |
| 95 | getenv := func(k string) string { |
| 96 | switch k { |
| 97 | case "CARLOS_HOST_CLAIM": |
| 98 | return tc.claim |
| 99 | case "CARLOS_REGION_HA": |
| 100 | return tc.ha |
| 101 | } |
| 102 | return "" |
| 103 | } |
| 104 | r, err := NewRelocationRuntime(t.Context(), getenv, release.NewDirStore(t.TempDir()), local, ClaimProviders{}, reg, &activator.Service{}, relocationRuntimeInputs{}) |
| 105 | if err != nil { |
| 106 | t.Fatalf("claim %q region HA %q: %v", tc.claim, tc.ha, err) |
| 107 | } |
| 108 | if !tc.completion { |
| 109 | if r != nil { |
| 110 | t.Fatalf("claim %q region HA %q: built %+v", tc.claim, tc.ha, r) |
| 111 | } |
| 112 | continue |
| 113 | } |
| 114 | // With region HA on, the owners are not bound here (no wake |
| 115 | // dispatcher), so this is the completion-only runtime too. |
| 116 | if r == nil || r.Worker != nil || r.Participant != nil || r.Convert != nil || r.Finish == nil || r.Hosts == nil { |
| 117 | t.Fatalf("claim %q region HA %q: not a completion-only runtime: %+v", tc.claim, tc.ha, r) |
| 118 | } |
| 119 | } |
| 120 | } |
| 121 | |
| 122 | // Ruling 82: on a box with region HA off that never completed a |
| 123 | // relocation, the completion pass writes nothing. Its hosts are its own |
| 124 | // rows; a serving claim with no completion is left alone, and so is one |
| 125 | // whose completion names another box. |
| 126 | func TestCompletionPassChangesNothingWithoutACompletion(t *testing.T) { |
| 127 | ctx := t.Context() |
| 128 | store := release.NewDirStore(t.TempDir()) |
| 129 | reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 130 | if err != nil { |
| 131 | t.Fatal(err) |
| 132 | } |
| 133 | defer reg.Close() |
| 134 | passClaim(t, store, "a.example", hostclaim.Op{}) |
| 135 | passClaim(t, store, "b.example", hostclaim.Op{}) |
| 136 | completeOn(t, store, "b.example", "rel-b", "spa", false) |
| 137 | passClaim(t, store, "c.example", hostclaim.Op{}) // no row here |
| 138 | for _, host := range []string{"a.example", "b.example"} { |
| 139 | if err := reg.Upsert(registry.Route{Host: host, Status: "ready", Socket: "/run/" + host + ".sock"}); err != nil { |
| 140 | t.Fatal(err) |
| 141 | } |
| 142 | } |
| 143 | before := map[string]string{} |
| 144 | for _, host := range []string{"a.example", "b.example", "c.example"} { |
| 145 | obs, err := hostclaim.Read(ctx, store, host) |
| 146 | if err != nil { |
| 147 | t.Fatal(err) |
| 148 | } |
| 149 | before[host] = obs.ETag |
| 150 | } |
| 151 | rowsBefore, err := reg.List() |
| 152 | if err != nil { |
| 153 | t.Fatal(err) |
| 154 | } |
| 155 | data := t.TempDir() |
| 156 | local := &productionClaimLocal{legacyParticipation: true, assembly: &ClaimAssembly{Claims: store, Guard: hostguard.Guard{DataDir: data}, stopLocal: func(context.Context) (claimStopLocal, error) { |
| 157 | return claimStopLocal{Actor: hostbox.Actor{Box: "dst"}}, nil |
| 158 | }}} |
| 159 | claimOnly := func(k string) string { |
| 160 | if k == "CARLOS_HOST_CLAIM" { |
| 161 | return "1" |
| 162 | } |
| 163 | return "" |
| 164 | } |
| 165 | svc := &activator.Service{} |
| 166 | r, err := NewRelocationRuntime(ctx, claimOnly, store, local, ClaimProviders{}, reg, svc, relocationRuntimeInputs{}) |
| 167 | if err != nil || r == nil { |
| 168 | t.Fatal(err) |
| 169 | } |
| 170 | r.once(ctx, store) |
| 171 | for host, etag := range before { |
| 172 | if obs, err := hostclaim.Read(ctx, store, host); err != nil || obs.ETag != etag { |
| 173 | t.Fatalf("%s was written: %v", host, err) |
| 174 | } |
| 175 | } |
| 176 | if rows, err := reg.List(); err != nil || !reflect.DeepEqual(rows, rowsBefore) { |
| 177 | t.Fatalf("the registry changed: %v", err) |
| 178 | } |
| 179 | if _, err := os.Stat(filepath.Join(data, "relocate")); !errors.Is(err, os.ErrNotExist) { |
| 180 | t.Fatalf("the pass made journals: %v", err) |
| 181 | } |
| 182 | if svc.Fenced("a.example") || svc.Fenced("b.example") { |
| 183 | t.Fatal("the pass fenced a host") |
| 184 | } |
| 185 | } |
| 186 | |
| 187 | // One pass steps every relocating claim (participant, then worker), hands |
| 188 | // every landing to the J3 conversion, leaves serving claims alone, and |
| 189 | // sweeps the journals of operations that are over (ruling 61). |
| 190 | func TestRelocatePassStepsEveryRelocationAndSweeps(t *testing.T) { |
| 191 | ctx := t.Context() |
| 192 | store := release.NewDirStore(t.TempDir()) |
| 193 | passClaim(t, store, "a.example", relocatingOp("rel-a")) |
| 194 | passClaim(t, store, "b.example", relocatingOp("rel-b")) |
| 195 | passClaim(t, store, "c.example", hostclaim.Op{}) |
| 196 | enrollment := map[string]string{"src": strings.Repeat("b", 32), "dst": strings.Repeat("c", 32)} |
| 197 | landingAtProbed(t, store, "d.example", "mv_abcdefghijklmnopqrstuvwxyz", time.Date(2026, 10, 7, 9, 0, 0, 0, time.UTC), enrollment) |
| 198 | data := t.TempDir() |
| 199 | stale := filepath.Join(data, "relocate", "ops", "rel-gone", "install.json") |
| 200 | if err := os.MkdirAll(filepath.Dir(stale), 0o700); err != nil { |
| 201 | t.Fatal(err) |
| 202 | } |
| 203 | if err := os.WriteFile(stale, []byte(`{"v":1,"host":"c.example","op":"rel-gone"}`), 0o600); err != nil { |
| 204 | t.Fatal(err) |
| 205 | } |
| 206 | var order, converted []string |
| 207 | r := &RelocationRuntime{Worker: stepRecorder{name: "worker", order: &order}, Participant: stepRecorder{name: "participant", order: &order}, Box: "dst", DataDir: data, |
| 208 | Convert: func(_ context.Context, _ release.Store, o hostclaim.Observed) error { |
| 209 | converted = append(converted, o.Claim.Host) |
| 210 | return nil |
| 211 | }} |
| 212 | r.once(ctx, store) |
| 213 | want := []string{"participant:a.example", "worker:a.example", "participant:b.example", "worker:b.example"} |
| 214 | if !slices.Equal(order, want) { |
| 215 | t.Fatalf("stepped %v, want %v", order, want) |
| 216 | } |
| 217 | if !slices.Equal(converted, []string{"d.example"}) { |
| 218 | t.Fatalf("offered for conversion %v, want only the landing d.example", converted) |
| 219 | } |
| 220 | if _, err := os.Stat(stale); !errors.Is(err, os.ErrNotExist) { |
| 221 | t.Fatalf("an ended op's journal survived the pass: %v", err) |
| 222 | } |
| 223 | } |
| 224 | |
| 225 | // Rulings 35 and 36: a refusal that never clears (a conversion waiting for |
| 226 | // a target) is logged once per host and cause, never per pass. |
| 227 | func TestRelocatePassAlertsOnceWhileAConversionWaitsForATarget(t *testing.T) { |
| 228 | ctx := t.Context() |
| 229 | store := release.NewDirStore(t.TempDir()) |
| 230 | passClaim(t, store, "a.example", relocatingOp("rel-a")) |
| 231 | buf := captureRacyLog(t) |
| 232 | var order []string |
| 233 | none := errors.New("no eligible box for the converted landing") |
| 234 | r := &RelocationRuntime{Worker: stepRecorder{name: "worker", order: &order, err: none}, Participant: stepRecorder{name: "participant", order: &order, err: hostrelocate.ErrNotYet}, Box: "dst"} |
| 235 | for range 3 { |
| 236 | r.once(ctx, store) |
| 237 | } |
| 238 | if n := strings.Count(buf.String(), none.Error()); n != 1 { |
| 239 | t.Fatalf("logged %d times, want once:\n%s", n, buf.String()) |
| 240 | } |
| 241 | if strings.Contains(buf.String(), hostrelocate.ErrNotYet.Error()) { |
| 242 | t.Fatalf("ErrNotYet was alerted:\n%s", buf.String()) |
| 243 | } |
| 244 | } |
| 245 | |
| 246 | // Ruling 49: one pass completes a passed probe on the target: the |
| 247 | // participant (its probe stop), then the worker (the serving CAS), then |
| 248 | // Finish for the row this box still holds relocating. |
| 249 | func TestRelocatePassStopsCompletesAndReleasesInOnePass(t *testing.T) { |
| 250 | ctx := t.Context() |
| 251 | store := release.NewDirStore(t.TempDir()) |
| 252 | passClaim(t, store, "a.example", relocatingOp("rel-a")) |
| 253 | // Ruling 80: completions are found from the claim's record. One this |
| 254 | // box finished already, and one another box finishes, are left alone. |
| 255 | passClaim(t, store, "b.example", hostclaim.Op{}) |
| 256 | completeOn(t, store, "b.example", "rel-b", "dst", true) |
| 257 | passClaim(t, store, "c.example", hostclaim.Op{}) |
| 258 | completeOn(t, store, "c.example", "rel-c", "spa", false) |
| 259 | var order []string |
| 260 | r := &RelocationRuntime{Worker: completingWorker{order: &order, store: store}, Participant: stepRecorder{name: "participant", order: &order}, Box: "dst", |
| 261 | Finish: func(_ context.Context, host, op string) error { |
| 262 | order = append(order, "finish:"+host+":"+op) |
| 263 | return nil |
| 264 | }} |
| 265 | r.once(ctx, store) |
| 266 | if want := []string{"participant:a.example", "worker:a.example", "finish:a.example:rel-a"}; !slices.Equal(order, want) { |
| 267 | t.Fatalf("order %v, want %v", order, want) |
| 268 | } |
| 269 | } |
| 270 | |
| 271 | // completeOn writes host's serving op with op's completed relocation on |
| 272 | // box, marked discharged or not. |
| 273 | func completeOn(t *testing.T, store release.Store, host, op, box string, discharged bool) hostclaim.Observed { |
| 274 | t.Helper() |
| 275 | ctx := t.Context() |
| 276 | obs, err := hostclaim.Read(ctx, store, host) |
| 277 | if err != nil { |
| 278 | t.Fatal(err) |
| 279 | } |
| 280 | done, _ := json.Marshal(hostclaim.CompletedRelocation{Op: op, Stop: hostclaim.ProbeStop{Box: box, Session: obs.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)}, |
| 281 | Phase: hostclaim.PhaseRelocatingProbing, Discharged: discharged}) |
| 282 | obs, err = hostclaim.Update(ctx, store, obs, func(c *hostclaim.Claim) error { |
| 283 | c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: box, Participants: []string{box}, Journal: map[string]string{hostclaim.JournalCompleted: string(done)}} |
| 284 | return nil |
| 285 | }) |
| 286 | if err != nil { |
| 287 | t.Fatal(err) |
| 288 | } |
| 289 | return obs |
| 290 | } |
| 291 | |
| 292 | // completingWorker is the worker as the pass sees it when its Step makes |
| 293 | // the serving CAS: the claim then serves on dst with the completion. |
| 294 | type completingWorker struct { |
| 295 | order *[]string |
| 296 | store release.Store |
| 297 | } |
| 298 | |
| 299 | func (w completingWorker) Step(ctx context.Context, host string) (hostclaim.Observed, error) { |
| 300 | *w.order = append(*w.order, "worker:"+host) |
| 301 | obs, err := hostclaim.Read(ctx, w.store, host) |
| 302 | if err != nil { |
| 303 | return obs, err |
| 304 | } |
| 305 | done, _ := json.Marshal(hostclaim.CompletedRelocation{Op: obs.Claim.Op.ID, Stop: hostclaim.ProbeStop{Box: "dst", Session: obs.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)}, |
| 306 | Phase: hostclaim.PhaseRelocatingProbing}) |
| 307 | return hostclaim.Update(ctx, w.store, obs, func(c *hostclaim.Claim) error { |
| 308 | c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "dst", Participants: []string{"dst"}, Journal: map[string]string{hostclaim.JournalCompleted: string(done)}} |
| 309 | return nil |
| 310 | }) |
| 311 | } |
| 312 | |
| 313 | // J3: the guard admits a conversion only for the move the claim's op |
| 314 | // names, committed for this incarnation with no rollback; anything else is |
| 315 | // moves' case (ErrWrongState). |
| 316 | func TestCommittedLandingRecordsAdmitOnlyACommittedMove(t *testing.T) { |
| 317 | ctx := t.Context() |
| 318 | inc, err := hostclaim.Incarnation("bab", "jam", "a.example", time.Date(2026, 10, 7, 1, 0, 0, 0, time.UTC)) |
| 319 | if err != nil { |
| 320 | t.Fatal(err) |
| 321 | } |
| 322 | committedAt := time.Date(2026, 10, 7, 5, 0, 0, 0, time.UTC) |
| 323 | flipped := committedAt.Add(-time.Hour) |
| 324 | o := hostclaim.Observed{Host: "a.example", Claim: &hostclaim.Claim{Host: "a.example", Holder: &hostclaim.Holder{Account: "bab", App: "jam", Incarnation: inc, Location: "instances/a.example"}, |
| 325 | Op: hostclaim.Op{Kind: hostclaim.KindLanding, ID: "mv-1", Phase: "probed", Journal: map[string]string{hostclaim.JournalSeal: moves.Seal{V: moves.SealVersion, N: 42, FromDB: true, SrcCovered: true, |
| 326 | SealedAt: flipped.Add(-time.Minute), LeaseUntil: flipped, Replica: []moves.SealedObject{{Key: "moves/a.example/mv-1/replica/0009/" + ltx.FormatFilename(1, 42), VersionID: "v1", Size: 100}}}.Encode()}}}} |
| 327 | record := func(change func(*provision.MoveState)) MovesRecordReader { |
| 328 | return func(context.Context, string, string, string) (*provision.Record, error) { |
| 329 | m := &provision.MoveState{ID: "mv-1", Incarnation: inc, From: "uk", To: "ie", State: provision.MoveCommitted, CommittedAt: &committedAt, FlippedAt: &flipped} |
| 330 | if change != nil { |
| 331 | change(m) |
| 332 | } |
| 333 | return &provision.Record{Host: "a.example", Account: "bab", App: "jam", Move: m}, nil |
| 334 | } |
| 335 | } |
| 336 | for name, change := range map[string]func(*provision.MoveState){ |
| 337 | "not committed": func(m *provision.MoveState) { m.State = provision.MoveLanding }, |
| 338 | "another move": func(m *provision.MoveState) { m.ID = "mv-2" }, |
| 339 | "another incarnation": func(m *provision.MoveState) { m.Incarnation = "other" }, |
| 340 | "rolled back": func(m *provision.MoveState) { m.Rollback = &provision.MoveRollback{} }, |
| 341 | "no commit time": func(m *provision.MoveState) { m.CommittedAt = nil }, |
| 342 | } { |
| 343 | if _, err := (CommittedLandingRecords{Records: record(change)}).CheckLandingCommitted(ctx, o, "mv-1"); !errors.Is(err, hostclaim.ErrWrongState) { |
| 344 | t.Fatalf("%s: %v", name, err) |
| 345 | } |
| 346 | } |
| 347 | got, err := (CommittedLandingRecords{Records: record(nil)}).CheckLandingCommitted(ctx, o, "mv-1") |
| 348 | if err != nil { |
| 349 | t.Fatalf("a committed move: %v", err) |
| 350 | } |
| 351 | source, staging, err := moves.CommitScopes(o.Claim, &flipped) |
| 352 | if err != nil || got.FromRegion != "uk" || got.ToRegion != "ie" || !reflect.DeepEqual(got.SourceScope, source) || !reflect.DeepEqual(got.StagingScope, staging) { |
| 353 | t.Fatalf("the commit's regions and scopes: %+v (%v)", got, err) |
| 354 | } |
| 355 | if _, err := (CommittedLandingRecords{Records: record(nil)}).CheckLandingCommitted(ctx, o, "mv-other"); !errors.Is(err, hostclaim.ErrWrongState) { |
| 356 | t.Fatalf("another op id: %v", err) |
| 357 | } |
| 358 | } |
| 359 | |
| 360 | // Ruling 30: the release a relocation installs is the host's channel |
| 361 | // pointer, verified with the release key; an unverifiable pointer refuses. |
| 362 | func TestRelocationReleaseIsTheChannelPointer(t *testing.T) { |
| 363 | ctx := t.Context() |
| 364 | store := release.NewDirStore(t.TempDir()) |
| 365 | pub, priv, err := ed25519.GenerateKey(rand.Reader) |
| 366 | if err != nil { |
| 367 | t.Fatal(err) |
| 368 | } |
| 369 | passClaim(t, store, "a.example", hostclaim.Op{}) |
| 370 | if err := provision.CreateRecord(ctx, store, provision.Record{Schema: 1, Host: "a.example", Slug: "a", App: "jam", Account: "bab", Region: "ie", Channel: "stable", CreatedAt: time.Date(2026, 10, 7, 1, 0, 0, 0, time.UTC)}); err != nil && !errors.Is(err, release.ErrCASConflict) { |
| 371 | t.Fatal(err) |
| 372 | } |
| 373 | plat := release.AdoptPlatform() |
| 374 | for _, v := range []string{"v1.4.0", "v1.5.0"} { |
| 375 | m := release.Manifest{App: "jam", Version: v, Kind: "binary", Artifacts: map[string]release.Artifact{plat: {}}} |
| 376 | if err := release.Ship(ctx, store, "bab", m, map[string][]byte{plat: []byte("build " + v)}); err != nil { |
| 377 | t.Fatal(err) |
| 378 | } |
| 379 | } |
| 380 | if err := release.Promote(ctx, store, priv, "bab", "jam", "stable", "v1.5.0"); err != nil { |
| 381 | t.Fatal(err) |
| 382 | } |
| 383 | got, err := channelRelease(store, pub)(ctx, "a.example") |
| 384 | if err != nil || got != "v1.5.0" { |
| 385 | t.Fatalf("release %q %v", got, err) |
| 386 | } |
| 387 | other, _, _ := ed25519.GenerateKey(rand.Reader) |
| 388 | if _, err := channelRelease(store, other)(ctx, "a.example"); err == nil { |
| 389 | t.Fatal("an unverifiable pointer answered") |
| 390 | } |
| 391 | if _, err := channelRelease(store, pub)(ctx, "absent.example"); !errors.Is(err, hostrelocate.ErrRefused) { |
| 392 | t.Fatalf("a host with no claim: %v", err) |
| 393 | } |
| 394 | } |
| 395 | |
| 396 | // Ruling 3: a terminated box's local state is discharged only with the |
| 397 | // roster's enrolled_at; without it the discharge is unqualified, with it |
| 398 | // boxvolumes.Discharge's verdict. |
| 399 | func TestDischargeIsUnqualifiedWithoutEnrolledAt(t *testing.T) { |
| 400 | ctx := t.Context() |
| 401 | store := release.NewDirStore(t.TempDir()) |
| 402 | discharge := volumeDischarge(store) |
| 403 | fence := termfence.Record{ObservedAt: time.Date(2026, 10, 7, 9, 0, 0, 0, time.UTC).Format(time.RFC3339Nano)} |
| 404 | box := hostactors.Box{BoxID: "src", Enrollment: strings.Repeat("b", 32), Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-src"}} |
| 405 | if d := discharge(ctx, box, fence); d == nil || d.Qualified || !strings.Contains(d.Reason, "enrollment time") { |
| 406 | t.Fatalf("without enrolled_at: %+v", d) |
| 407 | } |
| 408 | box.EnrolledAt = time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) |
| 409 | records, err := boxvolumes.List(ctx, store, box.Enrollment) |
| 410 | if err != nil { |
| 411 | t.Fatal(err) |
| 412 | } |
| 413 | at, _ := time.Parse(time.RFC3339Nano, fence.ObservedAt) |
| 414 | want := boxvolumes.Discharge(records, boxvolumes.Entry{Box: box.BoxID, Enrollment: box.Enrollment, Cloud: box.Cloud}, box.EnrolledAt, at) |
| 415 | if d := discharge(ctx, box, fence); d == nil || d.Qualified != want.Discharge || d.Reason != want.Reason { |
| 416 | t.Fatalf("with enrolled_at: %+v, want %+v", d, want) |
| 417 | } |
| 418 | if d := discharge(ctx, box, termfence.Record{ObservedAt: "not a time"}); d == nil || d.Qualified { |
| 419 | t.Fatalf("an unreadable observation: %+v", d) |
| 420 | } |
| 421 | } |
| 422 | |
| 423 | // Rulings 50 and 51: the data bucket is opened as a release.Store of its |
| 424 | // own (production's lease.VersionedStore is never asserted to one), the |
| 425 | // versioned client and the leases sign for the roster's cloud region, and |
| 426 | // genselect, pins and the worker use the logical region. |
| 427 | func TestRelocationRuntimeOpensADataBucketStore(t *testing.T) { |
| 428 | ctx := t.Context() |
| 429 | files := release.NewDirStore(t.TempDir()) |
| 430 | var opened, signed string |
| 431 | self := hostactors.Box{Region: "ie", Cloud: hostactors.Cloud{Region: "eu-west-1"}} |
| 432 | s, err := openRelocationStores(ctx, self, "data-ie", |
| 433 | func(_ context.Context, bucket string) (release.Store, error) { opened = bucket; return files, nil }, |
| 434 | func(_ context.Context, _ string, region string) (lease.VersionedStore, error) { |
| 435 | signed = region |
| 436 | return leasetest.NewVersionedBucket(), nil |
| 437 | }) |
| 438 | if err != nil { |
| 439 | t.Fatal(err) |
| 440 | } |
| 441 | if opened != "data-ie" || signed != "eu-west-1" || s.Files != files || s.LeaseRegion != "eu-west-1" || s.Region != "ie" || s.Bucket != "data-ie" { |
| 442 | t.Fatalf("stores %+v (opened %q, signed %q)", s, opened, signed) |
| 443 | } |
| 444 | guard := ClaimAssembly{} |
| 445 | guard.Guard.DataDir = "/data" |
| 446 | src := hostbox.WorkSource{Box: "dst", Region: "ie"} |
| 447 | cfg, err := relocationAssemblyConfig(&guard, ClaimProviders{Selector: src}, s, ®istry.Registry{}, &relocationProbeActivator{}, hostactors.Directory{}, "/opt/carlos", |
| 448 | func(context.Context, hostclaim.Observed) error { return nil }) |
| 449 | if err != nil { |
| 450 | t.Fatal(err) |
| 451 | } |
| 452 | if cfg.Data != files || cfg.Versions != s.Versions || cfg.LeaseRegion != "eu-west-1" || cfg.Region != "ie" || cfg.WorkDir != "/data/relocate/build" || cfg.ReleaseDir != "/data/relocate/release" || cfg.OptDir != "/opt/carlos" || cfg.Source.Box != "dst" { |
| 453 | t.Fatalf("config %+v", cfg) |
| 454 | } |
| 455 | if _, err := openRelocationStores(ctx, hostactors.Box{Region: "ie"}, "data-ie", func(context.Context, string) (release.Store, error) { return files, nil }, |
| 456 | func(context.Context, string, string) (lease.VersionedStore, error) { return nil, nil }); err == nil { |
| 457 | t.Fatal("opened with no cloud region to sign for") |
| 458 | } |
| 459 | } |
| 460 | |
| 461 | // Finding 11, N5, ruling 53: region HA's owners bind only with moves' C8 |
| 462 | // provider. The production gate hands the registered source main's own |
| 463 | // serving sources before it opens; with C8 bound the owners are complete |
| 464 | // and relocating binds; without it, or with region HA off, nothing binds. |
| 465 | func TestRelocationBindsWithARegisteredMovesSource(t *testing.T) { |
| 466 | ctx := t.Context() |
| 467 | actor := hostbox.Actor{Box: "dst", Bucket: "data-ie", Region: "ie"} |
| 468 | serving := func(files ClaimFilesManifest) ClaimOwnerBindings { |
| 469 | var out ClaimOwnerBindings |
| 470 | out.Serving = ClaimServingSources{Launch: func(context.Context, registry.Route) (ClaimServingLaunch, error) { return ClaimServingLaunch{}, nil }, |
| 471 | Markers: plan9D5Markers{leasetest.NewVersionedBucket()}, Files: files} |
| 472 | return out |
| 473 | } |
| 474 | source := func() *RelocationSource { |
| 475 | return &RelocationSource{DataDir: t.TempDir(), FilesStore: func(context.Context, string) (release.Store, error) { return release.NewDirStore(t.TempDir()), nil }} |
| 476 | } |
| 477 | out := serving(&roFiles{}) |
| 478 | registered := source() |
| 479 | closeSource, err := openClaimRelocationOwners(ctx, plan11Env("1", "1"), registered, release.NewDirStore(t.TempDir()), &fakeVersioned{}, actor, &out) |
| 480 | if err != nil || closeSource == nil { |
| 481 | t.Fatalf("open: %v", err) |
| 482 | } |
| 483 | // Carry 5: the source's identity check is the owners' own |
| 484 | // IdentityProvider.Verify, as main's serving owners bind it. This |
| 485 | // one's provider is unconfigured, so it refuses as Verify does. |
| 486 | registered.mu.Lock() |
| 487 | identity := registered.serving.Identity |
| 488 | registered.mu.Unlock() |
| 489 | if identity == nil { |
| 490 | t.Fatal("the source was handed no identity check") |
| 491 | } |
| 492 | if _, want := out.Identity.Verify(ctx); want == nil || !errors.Is(identity(ctx), want) { |
| 493 | t.Fatalf("the identity check is not the owners' Verify: %v", identity(ctx)) |
| 494 | } |
| 495 | if bound, err := claimRelocationC1(plan11Env("1", "1"), true, out.Relocation); !bound || err != nil || len(out.PolicyRefusals) != 2 { |
| 496 | t.Fatalf("with C8: bound %v %v refusals %d", bound, err, len(out.PolicyRefusals)) |
| 497 | } |
| 498 | without := serving(nil) |
| 499 | if _, err := openClaimRelocationOwners(ctx, plan11Env("1", "1"), source(), release.NewDirStore(t.TempDir()), &fakeVersioned{}, actor, &without); err != nil || !without.Relocation.zero() { |
| 500 | t.Fatalf("without C8: %+v %v", without.Relocation, err) |
| 501 | } |
| 502 | off := serving(&roFiles{}) |
| 503 | if closeSource, err := openClaimRelocationOwners(ctx, plan11Env("1", ""), source(), release.NewDirStore(t.TempDir()), &fakeVersioned{}, actor, &off); err != nil || closeSource != nil || !off.Relocation.zero() { |
| 504 | t.Fatalf("region HA off: %+v %v", off.Relocation, err) |
| 505 | } |
| 506 | // Without the bound owners there is no relocation pass, only the |
| 507 | // completion-only runtime (ruling 82). |
| 508 | reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 509 | if err != nil { |
| 510 | t.Fatal(err) |
| 511 | } |
| 512 | defer reg.Close() |
| 513 | local := &productionClaimLocal{legacyParticipation: true, assembly: &ClaimAssembly{Wake: &ClaimWakeAssembly{Inputs: ClaimOwnerDispatch{}}, Guard: hostguard.Guard{DataDir: t.TempDir()}}} |
| 514 | r, err := NewRelocationRuntime(ctx, plan11Env("1", "1"), release.NewDirStore(t.TempDir()), local, ClaimProviders{}, reg, &activator.Service{}, relocationRuntimeInputs{}) |
| 515 | if err != nil || r == nil || r.Worker != nil || r.Participant != nil || r.Finish == nil { |
| 516 | t.Fatalf("a runtime without the relocation owners: %+v %v", r, err) |
| 517 | } |
| 518 | } |
| 519 | |