| 1 | package hostops |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/ed25519" |
| 6 | "crypto/rand" |
| 7 | "encoding/json" |
| 8 | "errors" |
| 9 | "strings" |
| 10 | "testing" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/hostactors" |
| 14 | "github.com/carlosframework/platform/internal/hostclaim" |
| 15 | "github.com/carlosframework/platform/internal/provision" |
| 16 | "github.com/carlosframework/platform/internal/release" |
| 17 | ) |
| 18 | |
| 19 | const moveStartID = "mv_abcdefghijklmnopqrstuvwxyz" |
| 20 | |
| 21 | // moveStartStore counts claim CASes and can run a hook on the first record |
| 22 | // read, which BeginClaimMove makes after its admission-free prechecks. |
| 23 | type moveStartStore struct { |
| 24 | release.Store |
| 25 | claimKey, recordKey string |
| 26 | claimPuts int |
| 27 | onRecordRead func() |
| 28 | } |
| 29 | |
| 30 | func (s *moveStartStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| 31 | if key == s.recordKey && s.onRecordRead != nil { |
| 32 | f := s.onRecordRead |
| 33 | s.onRecordRead = nil |
| 34 | f() |
| 35 | } |
| 36 | return s.Store.Get(ctx, key) |
| 37 | } |
| 38 | |
| 39 | func (s *moveStartStore) Put(ctx context.Context, key string, body []byte, match string) (string, error) { |
| 40 | if key == s.claimKey { |
| 41 | s.claimPuts++ |
| 42 | } |
| 43 | return s.Store.Put(ctx, key, body, match) |
| 44 | } |
| 45 | |
| 46 | type moveStartFixture struct { |
| 47 | raw *moveStartStore |
| 48 | w *ControlWriter |
| 49 | authority ProductionAuthority |
| 50 | host, inc string |
| 51 | now time.Time |
| 52 | enIE string |
| 53 | enUK string |
| 54 | pub ed25519.PublicKey |
| 55 | } |
| 56 | |
| 57 | // productionData is ConfigureClaimRuntime's data handle over f.raw: the |
| 58 | // mutation boundary over the conditional record store, checked by the |
| 59 | // production checker with the real ControlWriter over the same raw store. |
| 60 | func (f *moveStartFixture) productionData(t *testing.T) release.Store { |
| 61 | t.Helper() |
| 62 | records, err := NewConditionalRecordStore(f.raw, f.pub) |
| 63 | if err != nil { |
| 64 | t.Fatal(err) |
| 65 | } |
| 66 | check, err := NewProductionMutationCheck(f.raw, f.w) |
| 67 | if err != nil { |
| 68 | t.Fatal(err) |
| 69 | } |
| 70 | data, err := NewMutationBoundary(true, records, check) |
| 71 | if err != nil { |
| 72 | t.Fatal(err) |
| 73 | } |
| 74 | return data |
| 75 | } |
| 76 | |
| 77 | func moveRosterBox(id, region, enrollment, instance string) hostactors.Box { |
| 78 | return hostactors.Box{Enrollment: enrollment, Namespace: "platform", BoxID: id, Label: id, Region: region, Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: region, InstanceID: instance}, EvidenceSHA256: strings.Repeat("c", 64), State: "active"} |
| 79 | } |
| 80 | |
| 81 | func newMoveStartFixture(t *testing.T, mutateRoster func(*hostactors.Roster)) *moveStartFixture { |
| 82 | t.Helper() |
| 83 | ctx := context.Background() |
| 84 | now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) |
| 85 | host := "shop.example.com" |
| 86 | f := &moveStartFixture{host: host, now: now, enIE: strings.Repeat("a", 32), enUK: strings.Repeat("b", 32)} |
| 87 | claimKey, _ := hostclaim.Key(host) |
| 88 | f.raw = &moveStartStore{Store: release.NewDirStore(t.TempDir()), claimKey: claimKey, recordKey: provision.RecordKey("bab", "app", host)} |
| 89 | pub, key, err := ed25519.GenerateKey(rand.Reader) |
| 90 | if err != nil { |
| 91 | t.Fatal(err) |
| 92 | } |
| 93 | f.pub = pub |
| 94 | roster := hostactors.Roster{V: 1, DeploymentBucket: "bucket", Revision: 3, Boxes: map[string]hostactors.Box{ |
| 95 | "ie-box": moveRosterBox("ie-box", "ie", f.enIE, "i-ie"), |
| 96 | "uk-box": moveRosterBox("uk-box", "uk", f.enUK, "i-uk"), |
| 97 | }, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}} |
| 98 | if mutateRoster != nil { |
| 99 | mutateRoster(&roster) |
| 100 | } |
| 101 | signed, err := release.SignRootEnvelope(release.RootSigner(key), roster) |
| 102 | if err != nil { |
| 103 | t.Fatal(err) |
| 104 | } |
| 105 | if _, err = f.raw.Store.Put(ctx, hostactors.Key, signed, ""); err != nil { |
| 106 | t.Fatal(err) |
| 107 | } |
| 108 | if f.inc, err = hostclaim.Incarnation("bab", "app", host, now); err != nil { |
| 109 | t.Fatal(err) |
| 110 | } |
| 111 | rec, _ := json.Marshal(provision.Record{Schema: 1, Host: host, Slug: "app", Account: "bab", App: "app", Channel: "edge", Region: "ie", Acquisition: "acq", CreatedAt: now}) |
| 112 | if _, err = f.raw.Store.Put(ctx, f.raw.recordKey, rec, ""); err != nil { |
| 113 | t.Fatal(err) |
| 114 | } |
| 115 | c := hostclaim.Claim{V: 2, Host: host, Gen: 4, Holder: &hostclaim.Holder{Family: "instance", Account: "bab", App: "app", Key: f.raw.recordKey, Acquisition: "acq", Incarnation: f.inc, CreatedAt: now, Location: "instances/" + host}, |
| 116 | Op: hostclaim.Op{Kind: hostclaim.KindServing, Exec: "ie-box"}, Touched: hostclaim.Touched{Complete: true, Regions: []string{"ie"}, Boxes: []string{"ie-box"}}} |
| 117 | cb, _ := json.Marshal(c) |
| 118 | if _, err = f.raw.Store.Put(ctx, claimKey, cb, ""); err != nil { |
| 119 | t.Fatal(err) |
| 120 | } |
| 121 | admission, _ := json.Marshal(AppAdmission{V: 1, AppID: "canonical", Revision: 1, Epoch: 1, State: "open", Registrations: map[string]Registration{}, Effects: map[string]Effect{}}) |
| 122 | if _, err = f.raw.Store.Put(ctx, "apps/bab/app/admission.json", admission, ""); err != nil { |
| 123 | t.Fatal(err) |
| 124 | } |
| 125 | clock := func() time.Time { return f.now } |
| 126 | if f.w, err = NewControlWriter(f.raw, pub, clock); err != nil { |
| 127 | t.Fatal(err) |
| 128 | } |
| 129 | f.authority = ProductionAuthority{Store: f.raw, Control: f.w, Directory: hostactors.Directory{Store: f.raw, PublicKey: pub, Bucket: "bucket"}, ConsoleID: "console", Enrollment: strings.Repeat("e", 32), Now: clock} |
| 130 | return f |
| 131 | } |
| 132 | |
| 133 | func (f *moveStartFixture) read(t *testing.T) hostclaim.Observed { |
| 134 | t.Helper() |
| 135 | o, err := hostclaim.Read(context.Background(), f.raw, f.host) |
| 136 | if err != nil { |
| 137 | t.Fatal(err) |
| 138 | } |
| 139 | return o |
| 140 | } |
| 141 | |
| 142 | func (f *moveStartFixture) begin(h *StartHandle, o hostclaim.Observed, target, to string) (hostclaim.Observed, error) { |
| 143 | f.raw.claimPuts = 0 |
| 144 | return BeginClaimMove(context.Background(), f.raw, f.authority.Directory, h, o, moveStartID, "operator", target, to, f.now) |
| 145 | } |
| 146 | |
| 147 | // BeginClaimMove persists admission_id from the writer-minted exact "move" |
| 148 | // registration alongside app_id, and both enrollments from the signed |
| 149 | // roster, in one claim CAS. |
| 150 | func TestBeginClaimMovePersistsAdmissionAndRosterEnrollment(t *testing.T) { |
| 151 | ctx := context.Background() |
| 152 | f := newMoveStartFixture(t, nil) |
| 153 | o := f.read(t) |
| 154 | h, err := f.authority.RegisterMove(ctx, o, "canonical", moveStartID) |
| 155 | if err != nil { |
| 156 | t.Fatal(err) |
| 157 | } |
| 158 | moving, err := f.begin(h, o, "uk-box", "uk") |
| 159 | if err != nil { |
| 160 | t.Fatal(err) |
| 161 | } |
| 162 | in := moving.Claim.Op.Intent |
| 163 | want := map[string]string{"to": "uk", "source": "ie-box", "target": "uk-box", "app_id": "canonical", "admission_id": moveStartID, "source_enrollment": f.enIE, "target_enrollment": f.enUK} |
| 164 | for k, v := range want { |
| 165 | if in[k] != v { |
| 166 | t.Fatalf("intent %s = %q, want %q (%+v)", k, in[k], v, in) |
| 167 | } |
| 168 | } |
| 169 | if len(in) != len(want) || in["source_box"] != "" || in["source_region"] != "" { |
| 170 | t.Fatalf("extra intent keys: %+v", in) |
| 171 | } |
| 172 | if f.raw.claimPuts != 1 || moving.Claim.Op.Kind != hostclaim.KindMoving || moving.Claim.Gen != o.Claim.Gen+1 { |
| 173 | t.Fatalf("claim CASes=%d op=%+v", f.raw.claimPuts, moving.Claim.Op) |
| 174 | } |
| 175 | a, err := ReadAdmission(ctx, f.raw, "bab", "app", "canonical") |
| 176 | if err != nil { |
| 177 | t.Fatal(err) |
| 178 | } |
| 179 | reg := a.Value.Registrations[h.ID()] |
| 180 | if reg.Kind != RegistrationMove || reg.Host != f.host || reg.Incarnation != f.inc || reg.Claim != SavedObservation(o) { |
| 181 | t.Fatalf("registration not the exact move registration: %+v", reg) |
| 182 | } |
| 183 | // The record start is one issued effect of the same registration and |
| 184 | // passes the lifecycle record writer's admission reread. |
| 185 | rec, err := StartClaimMoveRecord(ctx, conditionalRecordStore{raw: f.raw}, h, moving, provision.MoveStart{ID: moveStartID, Incarnation: f.inc, From: "ie", To: "uk", Cut: "auto", By: "operator", Now: f.now}) |
| 186 | if err != nil { |
| 187 | t.Fatal(err) |
| 188 | } |
| 189 | if rec.Move == nil || rec.Move.ID != moveStartID || rec.Move.State != provision.MovePreparing { |
| 190 | t.Fatalf("record not started: %+v", rec.Move) |
| 191 | } |
| 192 | // The start CAS output converts under C7 only with the admission it |
| 193 | // carries: the conversion's required keys are present. |
| 194 | if in["source"] == "" || in["source_enrollment"] == "" || in["app_id"] == "" || in["admission_id"] == "" { |
| 195 | t.Fatal("start output is not convertible") |
| 196 | } |
| 197 | } |
| 198 | |
| 199 | func TestBeginClaimMoveRefusesWithoutExactLiveAdmission(t *testing.T) { |
| 200 | for _, tc := range []string{"closed-before-cas", "drained-before-cas", "expired", "other-observation", "wrong-kind", "foreign-handle", "nil-handle"} { |
| 201 | t.Run(tc, func(t *testing.T) { |
| 202 | ctx := context.Background() |
| 203 | f := newMoveStartFixture(t, nil) |
| 204 | o := f.read(t) |
| 205 | var h *StartHandle |
| 206 | var err error |
| 207 | if tc == "wrong-kind" { |
| 208 | // A writer-minted registration under the move id, of another kind. |
| 209 | reg := Registration{Kind: "wake", Host: f.host, Owner: RegistrationOwner{Console: "console", Enrollment: strings.Repeat("e", 32), Controller: "controller"}, AppID: "canonical", Epoch: 1, Claim: SavedObservation(o), Incarnation: f.inc, CreatedAt: f.now} |
| 210 | h, err = f.w.RegisterStart(ctx, "bab", "app", "canonical", moveStartID, reg, f.now) |
| 211 | } else { |
| 212 | h, err = f.authority.RegisterMove(ctx, o, "canonical", moveStartID) |
| 213 | } |
| 214 | if err != nil { |
| 215 | t.Fatal(err) |
| 216 | } |
| 217 | switch tc { |
| 218 | case "closed-before-cas": |
| 219 | // Closed between the prechecks and the claim CAS: the final |
| 220 | // admission reread is inside the transition. |
| 221 | f.raw.onRecordRead = func() { |
| 222 | if _, err := f.w.BeginExclusive(ctx, "bab", "app", "canonical", AppOperation{ID: "delete", Kind: OpDeleting, AppID: "canonical"}, RegistrationOwner{Console: "console", Enrollment: strings.Repeat("e", 32), Controller: "controller"}); err != nil { |
| 223 | t.Error(err) |
| 224 | } |
| 225 | } |
| 226 | case "drained-before-cas": |
| 227 | f.raw.onRecordRead = func() { f.now = f.now.Add(RegistrationTTL) } |
| 228 | case "expired": |
| 229 | f.now = f.now.Add(RegistrationTTL) |
| 230 | case "other-observation": |
| 231 | // A registration minted for an older claim observation. |
| 232 | var err error |
| 233 | if o, err = hostclaim.Update(ctx, f.raw.Store, o, func(c *hostclaim.Claim) error { return nil }); err != nil { |
| 234 | t.Fatal(err) |
| 235 | } |
| 236 | case "foreign-handle": |
| 237 | h = &StartHandle{w: f.w, account: "bab", app: "app", appID: "canonical", id: h.ID(), owner: RegistrationOwner{Console: "other", Enrollment: strings.Repeat("e", 32), Controller: "forged"}, epoch: 1} |
| 238 | case "nil-handle": |
| 239 | h = nil |
| 240 | } |
| 241 | before := f.read(t) |
| 242 | if _, err := f.begin(h, o, "uk-box", "uk"); err == nil { |
| 243 | t.Fatal("began a move without its exact live admission") |
| 244 | } |
| 245 | after := f.read(t) |
| 246 | if f.raw.claimPuts != 0 || after.ETag != before.ETag { |
| 247 | t.Fatalf("refusal wrote the claim: puts=%d", f.raw.claimPuts) |
| 248 | } |
| 249 | }) |
| 250 | } |
| 251 | } |
| 252 | |
| 253 | func TestBeginClaimMoveTakesEnrollmentsOnlyFromTheSignedRoster(t *testing.T) { |
| 254 | for _, tc := range []string{"target-wrong-region", "source-wrong-region", "target-quarantined", "unknown-target", "same-box", "fenced-source"} { |
| 255 | t.Run(tc, func(t *testing.T) { |
| 256 | ctx := context.Background() |
| 257 | var mutate func(*hostactors.Roster) |
| 258 | target, to := "uk-box", "uk" |
| 259 | switch tc { |
| 260 | case "target-wrong-region": |
| 261 | to = "de" |
| 262 | case "source-wrong-region": |
| 263 | mutate = func(r *hostactors.Roster) { |
| 264 | b := r.Boxes["ie-box"] |
| 265 | b.Region, b.Cloud.Region = "fr", "fr" |
| 266 | r.Boxes["ie-box"] = b |
| 267 | } |
| 268 | case "target-quarantined": |
| 269 | mutate = func(r *hostactors.Roster) { |
| 270 | b := r.Boxes["uk-box"] |
| 271 | b.State, b.QuarantineRevision, b.LossIDs = "quarantined", 1, []string{"loss"} |
| 272 | r.Boxes["uk-box"] = b |
| 273 | } |
| 274 | case "unknown-target": |
| 275 | target = "us-box" |
| 276 | case "same-box": |
| 277 | target, to = "ie-box", "ie" |
| 278 | } |
| 279 | f := newMoveStartFixture(t, mutate) |
| 280 | o := f.read(t) |
| 281 | if tc == "fenced-source" { |
| 282 | // Any fence object for the source's enrollment fails Lookup closed. |
| 283 | if _, err := f.raw.Store.Put(ctx, "control/box-fences/"+f.enIE+".json", []byte(`{}`), ""); err != nil { |
| 284 | t.Fatal(err) |
| 285 | } |
| 286 | } |
| 287 | h, err := f.authority.RegisterMove(ctx, o, "canonical", moveStartID) |
| 288 | if err != nil { |
| 289 | t.Fatal(err) |
| 290 | } |
| 291 | if _, err := f.begin(h, o, target, to); err == nil { |
| 292 | t.Fatal("accepted") |
| 293 | } |
| 294 | if f.raw.claimPuts != 0 { |
| 295 | t.Fatal("refusal wrote the claim") |
| 296 | } |
| 297 | }) |
| 298 | } |
| 299 | } |
| 300 | |
| 301 | // The registration id is the move id, so the settle sweep keeps it past its |
| 302 | // TTL while the move holds the claim; recovery's admission reread depends on |
| 303 | // it (moveAdmission). |
| 304 | func TestClaimMoveRegistrationOutlivesItsTTL(t *testing.T) { |
| 305 | ctx := context.Background() |
| 306 | f := newMoveStartFixture(t, nil) |
| 307 | o := f.read(t) |
| 308 | h, err := f.authority.RegisterMove(ctx, o, "canonical", moveStartID) |
| 309 | if err != nil { |
| 310 | t.Fatal(err) |
| 311 | } |
| 312 | if _, err = f.begin(h, o, "uk-box", "uk"); err != nil { |
| 313 | t.Fatal(err) |
| 314 | } |
| 315 | f.now = f.now.Add(RegistrationTTL + 2*RegistrationSkew) |
| 316 | if err := f.w.SettleRegistration(ctx, "bab", "app", "canonical", moveStartID, f.now); !errors.Is(err, ErrPending) { |
| 317 | t.Fatalf("settled a move that still holds the claim: %v", err) |
| 318 | } |
| 319 | a, err := ReadAdmission(ctx, f.raw, "bab", "app", "canonical") |
| 320 | if err != nil { |
| 321 | t.Fatal(err) |
| 322 | } |
| 323 | if _, ok := a.Value.Registrations[moveStartID]; !ok { |
| 324 | t.Fatal("move registration drained under a moving claim") |
| 325 | } |
| 326 | if _, err := f.authority.RegisterMove(ctx, o, "canonical", "not-a-move-id"); !errors.Is(err, ErrIdentity) { |
| 327 | t.Fatalf("registered under a non-move id: %v", err) |
| 328 | } |
| 329 | } |
| 330 | |
| 331 | // The record start runs only under the registration the claim names. |
| 332 | func TestStartClaimMoveRecordIsBoundToItsRegistration(t *testing.T) { |
| 333 | ctx := context.Background() |
| 334 | f := newMoveStartFixture(t, nil) |
| 335 | o := f.read(t) |
| 336 | h, err := f.authority.RegisterMove(ctx, o, "canonical", moveStartID) |
| 337 | if err != nil { |
| 338 | t.Fatal(err) |
| 339 | } |
| 340 | moving, err := f.begin(h, o, "uk-box", "uk") |
| 341 | if err != nil { |
| 342 | t.Fatal(err) |
| 343 | } |
| 344 | moving.Claim.Op.Intent["admission_id"] = "other" |
| 345 | if _, err := StartClaimMoveRecord(ctx, conditionalRecordStore{raw: f.raw}, h, moving, provision.MoveStart{ID: moveStartID, Incarnation: f.inc, From: "ie", To: "uk", Cut: "auto", Now: f.now}); !errors.Is(err, ErrIdentity) { |
| 346 | t.Fatalf("record start under another registration: %v", err) |
| 347 | } |
| 348 | } |
| 349 | |
| 350 | // The start CAS and the record start through the production data handle: |
| 351 | // the claim CAS needs the registration's lifecycle grant and the record Put |
| 352 | // its issued effect. A caller holding only the data handle (no grant, or a |
| 353 | // grant but no issued effect) writes neither. |
| 354 | func TestClaimMoveStartThroughProductionBoundary(t *testing.T) { |
| 355 | ctx := context.Background() |
| 356 | f := newMoveStartFixture(t, nil) |
| 357 | data := f.productionData(t) |
| 358 | dir := hostactors.Directory{Store: data, PublicKey: f.pub, Bucket: "bucket"} |
| 359 | authority := f.authority |
| 360 | authority.Store, authority.Directory = data, dir |
| 361 | o := f.read(t) |
| 362 | h, err := authority.RegisterMove(ctx, o, "canonical", moveStartID) |
| 363 | if err != nil { |
| 364 | t.Fatal(err) |
| 365 | } |
| 366 | recordTag := func() string { |
| 367 | t.Helper() |
| 368 | _, tag, err := provision.GetRecordETag(ctx, f.raw, "bab", "app", f.host) |
| 369 | if err != nil { |
| 370 | t.Fatal(err) |
| 371 | } |
| 372 | return tag |
| 373 | } |
| 374 | f.raw.claimPuts = 0 |
| 375 | admitted := hostclaim.MoveAdmission{AppID: "canonical", AdmissionID: moveStartID, SourceEnrollment: f.enIE, TargetEnrollment: f.enUK} |
| 376 | if _, err := hostclaim.Update(ctx, data, o, hostclaim.BeginAdmittedMove(moveStartID, "operator", "ie-box", "uk-box", "uk", f.now, admitted)); !errors.Is(err, ErrPending) { |
| 377 | t.Fatalf("unbound start CAS: %v", err) |
| 378 | } |
| 379 | if f.raw.claimPuts != 0 || f.read(t).ETag != o.ETag { |
| 380 | t.Fatal("unbound start CAS reached the claim") |
| 381 | } |
| 382 | moving, err := BeginClaimMove(ctx, data, dir, h, o, moveStartID, "operator", "uk-box", "uk", f.now) |
| 383 | if err != nil { |
| 384 | t.Fatal(err) |
| 385 | } |
| 386 | if f.raw.claimPuts != 1 || moving.Claim.Op.Kind != hostclaim.KindMoving || f.read(t).ETag != moving.ETag { |
| 387 | t.Fatalf("granted start CAS: puts=%d op=%+v", f.raw.claimPuts, moving.Claim.Op) |
| 388 | } |
| 389 | start := provision.MoveStart{ID: moveStartID, Incarnation: f.inc, From: "ie", To: "uk", Cut: "auto", By: "operator", Now: f.now} |
| 390 | before := recordTag() |
| 391 | for name, c := range map[string]context.Context{"no-grant": ctx, "grant-without-effect": h.Lifecycle(ctx)} { |
| 392 | if _, err := setLifecycleRecordMove(c, data, moving, provision.MoveStartTransition(start)); !errors.Is(err, ErrPending) { |
| 393 | t.Fatalf("%s: unbound record start: %v", name, err) |
| 394 | } |
| 395 | } |
| 396 | if recordTag() != before { |
| 397 | t.Fatal("unbound record start wrote the record") |
| 398 | } |
| 399 | rec, err := StartClaimMoveRecord(ctx, data, h, moving, start) |
| 400 | if err != nil { |
| 401 | t.Fatal(err) |
| 402 | } |
| 403 | if rec.Move == nil || rec.Move.ID != moveStartID || rec.Move.State != provision.MovePreparing { |
| 404 | t.Fatalf("record not started: %+v", rec.Move) |
| 405 | } |
| 406 | a, err := ReadAdmission(ctx, f.raw, "bab", "app", "canonical") |
| 407 | if err != nil { |
| 408 | t.Fatal(err) |
| 409 | } |
| 410 | if e := a.Value.Effects[moveStartID]; e.State != "complete" || e.OperationID != moveStartID || e.Resource != f.raw.recordKey || e.Owner != h.Owner() { |
| 411 | t.Fatalf("record start effect: %+v", e) |
| 412 | } |
| 413 | // A replay of the completed effect writes nothing and reads the result. |
| 414 | started := recordTag() |
| 415 | if rec, err = StartClaimMoveRecord(ctx, data, h, moving, start); err != nil || rec.Move.ID != moveStartID || recordTag() != started { |
| 416 | t.Fatalf("replay: %v", err) |
| 417 | } |
| 418 | } |
| 419 | |
amadan