| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/ed25519" |
| 6 | "crypto/rand" |
| 7 | "encoding/json" |
| 8 | "errors" |
| 9 | "strings" |
| 10 | "testing" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/hostactors" |
| 14 | "github.com/carlosframework/platform/internal/hostclaim" |
| 15 | "github.com/carlosframework/platform/internal/hostfence" |
| 16 | "github.com/carlosframework/platform/internal/hostrelocate" |
| 17 | "github.com/carlosframework/platform/internal/moves" |
| 18 | "github.com/carlosframework/platform/internal/park" |
| 19 | "github.com/carlosframework/platform/internal/provision" |
| 20 | "github.com/carlosframework/platform/internal/release" |
| 21 | "github.com/carlosframework/platform/internal/termfence" |
| 22 | "github.com/superfly/ltx" |
| 23 | ) |
| 24 | |
| 25 | // landingWorld is a landing claim at probed whose target dst is |
| 26 | // terminated (its fence in the roster's bucket), the authoritative record |
| 27 | // of its move (committed or still landing) and a signed roster in which |
| 28 | // wrk, the converting worker, is as change leaves it. |
| 29 | type landingWorld struct { |
| 30 | store release.Store |
| 31 | directory hostactors.Directory |
| 32 | obs hostclaim.Observed |
| 33 | now time.Time |
| 34 | enrollment map[string]string |
| 35 | } |
| 36 | |
| 37 | func newLandingWorld(t *testing.T, committed bool, change func(*hostactors.Roster)) landingWorld { |
| 38 | t.Helper() |
| 39 | return newLandingWorldHome(t, committed, change, "") |
| 40 | } |
| 41 | |
| 42 | // newLandingWorldHome is newLandingWorld with the landing into the home |
| 43 | // when home is the fleet's home region (ie): the signed roster designates |
| 44 | // it, the move is admitted under it (MoveAdmission.HomeRegion, its "to" |
| 45 | // park.HomeRegion, as BeginClaimMove writes it) and the record's To is the |
| 46 | // home's placement code "". |
| 47 | func newLandingWorldHome(t *testing.T, committed bool, change func(*hostactors.Roster), home string) landingWorld { |
| 48 | t.Helper() |
| 49 | ctx := t.Context() |
| 50 | const host, id = "shop.example", "mv_abcdefghijklmnopqrstuvwxyz" |
| 51 | w := landingWorld{store: release.NewDirStore(t.TempDir()), now: time.Date(2026, 10, 7, 9, 0, 0, 0, time.UTC), |
| 52 | enrollment: map[string]string{"src": strings.Repeat("b", 32), "dst": strings.Repeat("c", 32), "spa": strings.Repeat("d", 32), "wrk": strings.Repeat("e", 32)}} |
| 53 | pub, key, err := ed25519.GenerateKey(rand.Reader) |
| 54 | if err != nil { |
| 55 | t.Fatal(err) |
| 56 | } |
| 57 | roster := hostactors.Roster{V: 1, DeploymentBucket: "bucket", Revision: 1, |
| 58 | Regions: map[string]hostactors.Region{"ie": {Account: "123", AWSRegion: "eu-west-1"}, "uk": {Account: "123", AWSRegion: "eu-west-2"}}, |
| 59 | Boxes: map[string]hostactors.Box{}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}} |
| 60 | for box, enr := range w.enrollment { |
| 61 | roster.Boxes[box] = hostactors.Box{Enrollment: enr, Namespace: "platform", BoxID: box, Label: box, Region: "ie", Pool: "default", |
| 62 | Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-" + box}, EvidenceSHA256: strings.Repeat("c", 64), State: "active"} |
| 63 | } |
| 64 | roster.HomeRegion = home |
| 65 | if change != nil { |
| 66 | change(&roster) |
| 67 | } |
| 68 | raw, err := release.SignRootEnvelope(release.RootSigner(key), roster) |
| 69 | if err != nil { |
| 70 | t.Fatal(err) |
| 71 | } |
| 72 | if _, err := w.store.Put(ctx, hostactors.Key, raw, ""); err != nil { |
| 73 | t.Fatal(err) |
| 74 | } |
| 75 | w.directory = hostactors.Directory{Store: w.store, PublicKey: pub, Bucket: "bucket"} |
| 76 | w.obs = landingAtProbedHome(t, w.store, host, id, w.now, w.enrollment, home) |
| 77 | dst := roster.Boxes["dst"] |
| 78 | fence, _ := json.Marshal(termfence.Record{V: 1, Box: "dst", Enrollment: dst.Enrollment, Cloud: dst.Cloud, State: "terminated", ObservedAt: w.now.Add(-time.Minute).Format(time.RFC3339Nano), |
| 79 | Via: termfence.ViaDescribe, RequestID: "req", WrittenBy: termfence.WrittenBy{Box: "spa", Enrollment: w.enrollment["spa"]}}) |
| 80 | if _, err := w.store.Put(ctx, hostfence.FenceKey(dst.Enrollment), fence, ""); err != nil { |
| 81 | t.Fatal(err) |
| 82 | } |
| 83 | created := time.Date(2026, 10, 4, 1, 0, 0, 0, time.UTC) |
| 84 | state, committedAt, flipped := provision.MoveLanding, (*time.Time)(nil), w.now.Add(-90*time.Minute) |
| 85 | if committed { |
| 86 | at := w.now.Add(-30 * time.Minute) |
| 87 | state, committedAt = provision.MoveCommitted, &at |
| 88 | } |
| 89 | to := "ie" |
| 90 | if home != "" { |
| 91 | to = "" // the home's placement code |
| 92 | } |
| 93 | record := provision.Record{Schema: 1, Host: host, Slug: "shop", App: "jam", Account: "bab", Region: to, Channel: "stable", CreatedAt: created, |
| 94 | Move: &provision.MoveState{ID: id, Incarnation: w.obs.Claim.Holder.Incarnation, From: "uk", To: to, State: state, CommittedAt: committedAt, FlippedAt: &flipped}} |
| 95 | if err := provision.CreateRecord(ctx, w.store, record); err != nil { |
| 96 | t.Fatal(err) |
| 97 | } |
| 98 | return w |
| 99 | } |
| 100 | |
| 101 | // homeWorker is worker with the box's verified home binding. |
| 102 | func (w landingWorld) homeWorker(enrollment, region, home string) hostrelocate.Worker { |
| 103 | x := w.worker(enrollment, region) |
| 104 | x.HomeRegion = home |
| 105 | return x |
| 106 | } |
| 107 | |
| 108 | // worker is the converting worker: wrk, running as enrollment, in region. |
| 109 | func (w landingWorld) worker(enrollment, region string) hostrelocate.Worker { |
| 110 | now := func() time.Time { return w.now } |
| 111 | return hostrelocate.Worker{Claims: w.store, Directory: w.directory, Fences: hostfence.ClaimFenceAuthority{Directory: w.directory, Now: now}, |
| 112 | Discharge: func(context.Context, hostactors.Box, termfence.Record) *hostclaim.LocalDischarge { return nil }, |
| 113 | Self: termfence.WrittenBy{Box: "wrk", Enrollment: enrollment}, Region: region, NewID: newClaimID, Now: now} |
| 114 | } |
| 115 | |
| 116 | // J3: the pass converts a landing whose target box is terminated into a |
| 117 | // failover relocation when the move committed, through |
| 118 | // CommittedLandingRecords over the authoritative record; a move that has |
| 119 | // not committed is moves' case, and nothing is written. |
| 120 | func TestRelocatePassConvertsAFencedCommittedLanding(t *testing.T) { |
| 121 | for _, committed := range []bool{true, false} { |
| 122 | name := "uncommitted" |
| 123 | if committed { |
| 124 | name = "committed" |
| 125 | } |
| 126 | t.Run(name, func(t *testing.T) { |
| 127 | ctx := t.Context() |
| 128 | w := newLandingWorld(t, committed, nil) |
| 129 | if err := convertFencedLanding(w.worker(w.enrollment["wrk"], "ie"), w.store)(ctx, w.store, w.obs); err != nil { |
| 130 | t.Fatalf("convert: %v", err) |
| 131 | } |
| 132 | after, err := hostclaim.Read(ctx, w.store, w.obs.Host) |
| 133 | if err != nil { |
| 134 | t.Fatal(err) |
| 135 | } |
| 136 | if !committed { |
| 137 | if after.ETag != w.obs.ETag || after.Claim.Op.Kind != hostclaim.KindLanding { |
| 138 | t.Fatalf("an uncommitted landing was written: %+v", after.Claim.Op) |
| 139 | } |
| 140 | return |
| 141 | } |
| 142 | c := after.Claim |
| 143 | if c.Op.Kind != hostclaim.KindRelocating || c.Op.Phase != hostclaim.PhaseStarting || c.Op.Intent["source_box"] != "dst" || c.Op.Journal["conversion"] != "committed-landing" { |
| 144 | t.Fatalf("not converted: %+v", c.Op) |
| 145 | } |
| 146 | }) |
| 147 | } |
| 148 | } |
| 149 | |
| 150 | // Astra round 6, finding 4: the conversion is the worker's, under Step's |
| 151 | // own checks of the caller. A worker whose roster entry is another |
| 152 | // enrollment (revoked and re-enrolled), one that is not active, one that |
| 153 | // is fenced, or one enrolled outside the region it claims, is refused |
| 154 | // with ErrNotWorker; a worker of another region leaves the landing alone. |
| 155 | // None of them writes the claim, though the move committed and its target |
| 156 | // is fenced. |
| 157 | func TestRelocatePassConversionRefusesAnUnauthorizedWorker(t *testing.T) { |
| 158 | t.Parallel() |
| 159 | type tc struct { |
| 160 | name string |
| 161 | change func(*hostactors.Roster) |
| 162 | enrollment, region string |
| 163 | want error |
| 164 | } |
| 165 | moved := func(state, region string) func(*hostactors.Roster) { |
| 166 | return func(r *hostactors.Roster) { |
| 167 | b := r.Boxes["wrk"] |
| 168 | if state != "" { |
| 169 | b.State = state |
| 170 | } |
| 171 | if region != "" { |
| 172 | b.Region = region |
| 173 | } |
| 174 | r.Boxes["wrk"] = b |
| 175 | } |
| 176 | } |
| 177 | for _, c := range []tc{ |
| 178 | {name: "a revoked enrollment", enrollment: strings.Repeat("f", 32), region: "ie", want: hostrelocate.ErrNotWorker}, |
| 179 | {name: "a retired worker", change: moved("retired", ""), region: "ie", want: hostrelocate.ErrNotWorker}, |
| 180 | {name: "enrolled in another region", change: moved("", "uk"), region: "ie", want: hostrelocate.ErrNotWorker}, |
| 181 | {name: "a worker of another region", change: moved("", "uk"), region: "uk"}, |
| 182 | } { |
| 183 | t.Run(c.name, func(t *testing.T) { |
| 184 | ctx := t.Context() |
| 185 | w := newLandingWorld(t, true, c.change) |
| 186 | if c.enrollment == "" { |
| 187 | c.enrollment = w.enrollment["wrk"] |
| 188 | } |
| 189 | err := convertFencedLanding(w.worker(c.enrollment, c.region), w.store)(ctx, w.store, w.obs) |
| 190 | if c.want == nil && err != nil || c.want != nil && !errors.Is(err, c.want) { |
| 191 | t.Fatalf("got %v, want %v", err, c.want) |
| 192 | } |
| 193 | after, err := hostclaim.Read(ctx, w.store, w.obs.Host) |
| 194 | if err != nil { |
| 195 | t.Fatal(err) |
| 196 | } |
| 197 | if after.ETag != w.obs.ETag || after.Claim.Op.Kind != hostclaim.KindLanding { |
| 198 | t.Fatalf("an unauthorized worker wrote the claim: %+v", after.Claim.Op) |
| 199 | } |
| 200 | }) |
| 201 | } |
| 202 | t.Run("a fenced worker", func(t *testing.T) { |
| 203 | ctx := t.Context() |
| 204 | w := newLandingWorld(t, true, nil) |
| 205 | o, err := w.directory.Read(ctx) |
| 206 | if err != nil { |
| 207 | t.Fatal(err) |
| 208 | } |
| 209 | wrk := o.Roster.Boxes["wrk"] |
| 210 | fence, _ := json.Marshal(termfence.Record{V: 1, Box: "wrk", Enrollment: wrk.Enrollment, Cloud: wrk.Cloud, State: "terminated", ObservedAt: w.now.Add(-time.Minute).Format(time.RFC3339Nano), |
| 211 | Via: termfence.ViaDescribe, RequestID: "req", WrittenBy: termfence.WrittenBy{Box: "spa", Enrollment: w.enrollment["spa"]}}) |
| 212 | if _, err := w.store.Put(ctx, hostfence.FenceKey(wrk.Enrollment), fence, ""); err != nil { |
| 213 | t.Fatal(err) |
| 214 | } |
| 215 | if err := convertFencedLanding(w.worker(w.enrollment["wrk"], "ie"), w.store)(ctx, w.store, w.obs); !errors.Is(err, hostrelocate.ErrNotWorker) { |
| 216 | t.Fatalf("got %v", err) |
| 217 | } |
| 218 | if after, err := hostclaim.Read(ctx, w.store, w.obs.Host); err != nil || after.ETag != w.obs.ETag { |
| 219 | t.Fatalf("a fenced worker wrote the claim: %v", err) |
| 220 | } |
| 221 | }) |
| 222 | } |
| 223 | |
| 224 | // landingAtProbed is a landing claim as moves leaves it at probed: the |
| 225 | // move from src to dst flipped, the target's probe admitted. Its seal is a |
| 226 | // real one, so moves.CommitScopes reads it. |
| 227 | func landingAtProbed(t *testing.T, store release.Store, host, id string, now time.Time, enrollment map[string]string) hostclaim.Observed { |
| 228 | t.Helper() |
| 229 | return landingAtProbedHome(t, store, host, id, now, enrollment, "") |
| 230 | } |
| 231 | |
| 232 | // landingAtProbedHome is landingAtProbed into the home when home is set: |
| 233 | // the move is admitted under the home binding with "to" park.HomeRegion. |
| 234 | func landingAtProbedHome(t *testing.T, store release.Store, host, id string, now time.Time, enrollment map[string]string, home string) hostclaim.Observed { |
| 235 | t.Helper() |
| 236 | ctx := t.Context() |
| 237 | created := time.Date(2026, 10, 4, 1, 0, 0, 0, time.UTC) |
| 238 | nh := hostclaim.NewHolder{Family: "instance", Key: provision.RecordKey("bab", "jam", host), Account: "bab", App: "jam", Host: host, Region: "ie", RecordRegion: "ie", CreatedAt: created} |
| 239 | body := []byte(`{"host":"` + host + `","account":"bab","app":"jam","region":"ie","created_at":"2026-10-04T01:00:00Z"}`) |
| 240 | o, err := hostclaim.Read(ctx, store, host) |
| 241 | if err != nil { |
| 242 | t.Fatal(err) |
| 243 | } |
| 244 | step := func(tr func(*hostclaim.Claim) error) { |
| 245 | t.Helper() |
| 246 | if o, err = hostclaim.Update(ctx, store, o, tr); err != nil { |
| 247 | t.Fatal(err) |
| 248 | } |
| 249 | } |
| 250 | seal := moves.Seal{V: moves.SealVersion, N: 7, FromDB: true, SrcCovered: true, SealedAt: now.Add(-2 * time.Hour), LeaseUntil: now.Add(-time.Hour), |
| 251 | Replica: []moves.SealedObject{{Key: "moves/" + host + "/" + id + "/replica/0009/" + ltx.FormatFilename(1, 7), VersionID: "v1", Size: 100}}} |
| 252 | step(hostclaim.BeginCreateOn(nh, body, "c1", "console", created, hostclaim.ExecutorSelection{Box: "src", Enrollment: enrollment["src"]})) |
| 253 | step(hostclaim.CompleteCreate("c1", "src")) |
| 254 | step(hostclaim.AdmitWriter(hostclaim.Admission{Box: "src", Mode: "cold", NewGeneration: "g1", Region: "ie", KeepStamp: true})) |
| 255 | to := "ie" |
| 256 | if home != "" { |
| 257 | to = park.HomeRegion |
| 258 | } |
| 259 | step(hostclaim.BeginAdmittedMove(id, "ops", "src", "dst", to, now.Add(-3*time.Hour), hostclaim.MoveAdmission{HomeRegion: home, AppID: "app-canonical", AdmissionID: id, SourceEnrollment: enrollment["src"], TargetEnrollment: enrollment["dst"]})) |
| 260 | step(hostclaim.ExecToNone(id, "src", "moves/"+host+"/"+id+"/stopped-source.json", hostclaim.JournalEvidenceSource)) |
| 261 | step(hostclaim.SetJournal(id, map[string]string{hostclaim.JournalSeal: seal.Encode(), hostclaim.JournalCutSession: "1"})) |
| 262 | step(hostclaim.SetMovePhase(id, hostclaim.KindMoving, hostclaim.PhaseMoving, hostclaim.PhaseSealed)) |
| 263 | step(hostclaim.Flip(id, "dst", o.Claim.Holder.Stamp, o.Claim.Holder.Generation)) |
| 264 | for _, p := range [][2]string{{"checking", "parking"}, {"parking", "building"}, {"building", "published"}, {"published", "local"}, {"local", "configured"}, {"configured", "restored"}, {"restored", "probing"}, {"probing", "probed"}} { |
| 265 | step(hostclaim.SetMovePhase(id, hostclaim.KindLanding, p[0], p[1])) |
| 266 | } |
| 267 | step(func(c *hostclaim.Claim) error { |
| 268 | h := c.Holder |
| 269 | h.Session++ |
| 270 | h.Admissions = append(h.Admissions, hostclaim.AdmissionRecord{Region: "ie", PredecessorRegion: "ie", Session: h.Session, Box: "dst", Generation: "gen-landed", Predecessor: h.Generation, Mode: "cold", Coverage: "sealed"}) |
| 271 | h.Generation, h.WriterBox = "gen-landed", "dst" |
| 272 | return nil |
| 273 | }) |
| 274 | return o |
| 275 | } |
| 276 | |
| 277 | // Main's home-region parks name a move into the home by park.HomeRegion |
| 278 | // ("to") while the roster, the fence binding and the region worker name it |
| 279 | // by its roster name (ie). Under the binding the box and the move agree on |
| 280 | // (the signed roster's home), the home's worker converts the committed |
| 281 | // landing: the relocation's region is the roster identity (as |
| 282 | // ConvertMovingToRelocating's is), the move's home binding is kept, and the |
| 283 | // commit's parks are queued in engine regions exactly as moves' commit |
| 284 | // queues them (the source's location in uk, the staging lane in |
| 285 | // park.HomeRegion). A box with no binding, or a binding the roster does not |
| 286 | // designate, writes nothing. |
| 287 | func TestRelocatePassConvertsACommittedLandingIntoTheHome(t *testing.T) { |
| 288 | ctx := t.Context() |
| 289 | w := newLandingWorldHome(t, true, nil, "ie") |
| 290 | if err := convertFencedLanding(w.homeWorker(w.enrollment["wrk"], "ie", "ie"), w.store)(ctx, w.store, w.obs); err != nil { |
| 291 | t.Fatalf("convert: %v", err) |
| 292 | } |
| 293 | after, err := hostclaim.Read(ctx, w.store, w.obs.Host) |
| 294 | if err != nil { |
| 295 | t.Fatal(err) |
| 296 | } |
| 297 | c := after.Claim |
| 298 | if c.Op.Kind != hostclaim.KindRelocating || c.Op.Phase != hostclaim.PhaseStarting || c.Op.Intent["source_box"] != "dst" || c.Op.Journal["conversion"] != "committed-landing" { |
| 299 | t.Fatalf("not converted: %+v", c.Op) |
| 300 | } |
| 301 | if c.Op.Intent["region"] != "ie" || c.Op.Intent["home_region"] != "ie" || c.Op.Intent["move_to_region"] != park.HomeRegion || c.Op.Journal["move_from_region"] != "uk" { |
| 302 | t.Fatalf("intent %v journal from %q", c.Op.Intent, c.Op.Journal["move_from_region"]) |
| 303 | } |
| 304 | parks := map[string]int{} |
| 305 | for _, o := range c.Obligations { |
| 306 | if o.Kind == "park-replica" { |
| 307 | parks[o.Region+" "+o.Owner]++ |
| 308 | } |
| 309 | } |
| 310 | if parks["uk region:uk"] != 1 || parks[park.HomeRegion+" region:"+park.HomeRegion] != 1 || len(parks) != 2 { |
| 311 | t.Fatalf("the commit's parks: %v", parks) |
| 312 | } |
| 313 | |
| 314 | for _, tc := range []struct { |
| 315 | name string |
| 316 | change func(*hostactors.Roster) |
| 317 | home string |
| 318 | want error |
| 319 | }{ |
| 320 | {name: "the box holds no home binding", want: park.ErrUnsafe}, |
| 321 | {name: "the roster designates no home", change: func(r *hostactors.Roster) { r.HomeRegion = "" }, home: "ie", want: hostrelocate.ErrNotWorker}, |
| 322 | } { |
| 323 | t.Run(tc.name, func(t *testing.T) { |
| 324 | w := newLandingWorldHome(t, true, tc.change, "ie") |
| 325 | err := convertFencedLanding(w.homeWorker(w.enrollment["wrk"], "ie", tc.home), w.store)(ctx, w.store, w.obs) |
| 326 | if !errors.Is(err, tc.want) { |
| 327 | t.Fatalf("got %v, want %v", err, tc.want) |
| 328 | } |
| 329 | if after, err := hostclaim.Read(ctx, w.store, w.obs.Host); err != nil || after.ETag != w.obs.ETag { |
| 330 | t.Fatalf("the claim was written: %v", err) |
| 331 | } |
| 332 | }) |
| 333 | } |
| 334 | } |
| 335 | |
| 336 | // CommittedLandingRecords answers the record's placement codes as engine |
| 337 | // regions (park.PlacementRegion), the regions moves' commit queues its |
| 338 | // parks in: the home's "" is park.HomeRegion, and a code that is no |
| 339 | // placement refuses. |
| 340 | func TestCommittedLandingRecordsAnswersEngineRegions(t *testing.T) { |
| 341 | ctx := t.Context() |
| 342 | for _, tc := range []struct { |
| 343 | name, from, to string |
| 344 | wantFrom, wantTo string |
| 345 | refused bool |
| 346 | }{ |
| 347 | {name: "named", from: "uk", to: "ie", wantFrom: "uk", wantTo: "ie"}, |
| 348 | {name: "into the home", from: "uk", to: "", wantFrom: "uk", wantTo: park.HomeRegion}, |
| 349 | {name: "out of the home", from: "", to: "uk", wantFrom: park.HomeRegion, wantTo: "uk"}, |
| 350 | {name: "not a placement code", from: "UK", to: "ie", refused: true}, |
| 351 | } { |
| 352 | t.Run(tc.name, func(t *testing.T) { |
| 353 | w := newLandingWorld(t, true, nil) |
| 354 | c := w.obs.Claim |
| 355 | read := func(ctx context.Context, account, app, host string) (*provision.Record, error) { |
| 356 | rec, err := provision.GetRecord(ctx, w.store, account, app, host) |
| 357 | if err != nil { |
| 358 | return nil, err |
| 359 | } |
| 360 | rec.Move.From, rec.Move.To = tc.from, tc.to |
| 361 | return rec, nil |
| 362 | } |
| 363 | got, err := CommittedLandingRecords{Records: read}.CheckLandingCommitted(ctx, w.obs, c.Op.ID) |
| 364 | if tc.refused { |
| 365 | if !errors.Is(err, hostclaim.ErrWrongState) { |
| 366 | t.Fatalf("got %+v, %v", got, err) |
| 367 | } |
| 368 | return |
| 369 | } |
| 370 | if err != nil || got.FromRegion != tc.wantFrom || got.ToRegion != tc.wantTo { |
| 371 | t.Fatalf("got %+v, %v", got, err) |
| 372 | } |
| 373 | }) |
| 374 | } |
| 375 | } |
| 376 | |
amadan