| 1 | package provision |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "encoding/json" |
| 7 | "errors" |
| 8 | "reflect" |
| 9 | "strings" |
| 10 | "testing" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/release" |
| 14 | "github.com/carlosframework/platform/internal/seeds" |
| 15 | ) |
| 16 | |
| 17 | const ( |
| 18 | offID = "mv_abcdefghijklmnopqrstuvwxyz" |
| 19 | otherOff = "mv_zyxwvutsrqponmlkjihgfedcba" |
| 20 | offSealer = "box-a" |
| 21 | ) |
| 22 | |
| 23 | var offBoxes = []string{"box-a", "box-b"} |
| 24 | |
| 25 | // offlineRecord creates a record that carries everything an offline move must |
| 26 | // leave alone: an acquisition marker, a pool, an environment, a pinned |
| 27 | // channel, a hibernate flag and a steering rule. Its region is "" (home). |
| 28 | func offlineRecord(t *testing.T, s release.Store) Record { |
| 29 | t.Helper() |
| 30 | hib := true |
| 31 | rec := Record{ |
| 32 | Schema: 1, Host: "a.go.example.com", Slug: "s", App: "app", Account: "bab", |
| 33 | Pool: "main", Region: "", Channel: "stable", Environment: "shop", |
| 34 | Acquisition: "acq_0123456789abcdef", Hibernate: &hib, Steering: "nearest", |
| 35 | CreatedAt: time.Date(2026, 9, 1, 8, 0, 0, 0, time.UTC), CreatedBy: "ops@example.com", |
| 36 | } |
| 37 | if err := CreateRecord(context.Background(), s, rec); err != nil { |
| 38 | t.Fatal(err) |
| 39 | } |
| 40 | return rec |
| 41 | } |
| 42 | |
| 43 | // offlineStep is one transition by name, so a test can walk the path and try |
| 44 | // every step from every state. |
| 45 | type offlineStep struct { |
| 46 | name string |
| 47 | mut func() func(*Record) error |
| 48 | from string // the one state it leaves |
| 49 | } |
| 50 | |
| 51 | func offlineSteps(now time.Time) []offlineStep { |
| 52 | src := MoveSource{Replication: "cold", NextDue: 99, FinalTXID: 42, Release: "v0.28.4", EnvDigest: "d1", Files: &MoveFilesDigest{Plain: "p", Sealed: "s", Bytes: 7, Entries: 2}} |
| 53 | return []offlineStep{ |
| 54 | {"start", func() func(*Record) error { return OfflineStart(offID, "", "uk", "ops@example.com", now) }, ""}, |
| 55 | {"seal", func() func(*Record) error { return OfflineSeal(offID, offSealer, offBoxes) }, OfflineStopping}, |
| 56 | {"sealed", func() func(*Record) error { return OfflineSealedBy(offID, offSealer, src) }, OfflineSealing}, |
| 57 | {"flip", func() func(*Record) error { return OfflineFlip(offID) }, OfflineSealed}, |
| 58 | {"landed", func() func(*Record) error { return OfflineLandedBy(offID, "uk") }, OfflineLanding}, |
| 59 | {"open", func() func(*Record) error { return OfflineOpenTransition(offID, now.Add(30*time.Minute)) }, OfflineLanded}, |
| 60 | {"finishing", func() func(*Record) error { return OfflineFinishingTransition(offID) }, OfflineOpen}, |
| 61 | {"finish", func() func(*Record) error { return OfflineFinish(offID, now.Add(time.Hour)) }, OfflineFinishing}, |
| 62 | } |
| 63 | } |
| 64 | |
| 65 | func offlineWalk(t *testing.T, s release.Store, rec Record, steps []offlineStep) *Record { |
| 66 | t.Helper() |
| 67 | var got *Record |
| 68 | for _, st := range steps { |
| 69 | var err error |
| 70 | got, err = SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, st.mut()) |
| 71 | if err != nil { |
| 72 | t.Fatalf("%s: %v", st.name, err) |
| 73 | } |
| 74 | } |
| 75 | return got |
| 76 | } |
| 77 | |
| 78 | func offlineGet(t *testing.T, s release.Store, rec Record) Record { |
| 79 | t.Helper() |
| 80 | got, err := GetRecord(context.Background(), s, rec.Account, rec.App, rec.Host) |
| 81 | if err != nil { |
| 82 | t.Fatal(err) |
| 83 | } |
| 84 | return *got |
| 85 | } |
| 86 | |
| 87 | // asMap is a record as the generic JSON object every other reader sees. |
| 88 | func asMap(t *testing.T, rec Record) map[string]any { |
| 89 | t.Helper() |
| 90 | raw, err := json.Marshal(rec) |
| 91 | if err != nil { |
| 92 | t.Fatal(err) |
| 93 | } |
| 94 | var m map[string]any |
| 95 | if err := json.Unmarshal(raw, &m); err != nil { |
| 96 | t.Fatal(err) |
| 97 | } |
| 98 | return m |
| 99 | } |
| 100 | |
| 101 | func TestOfflineFullLegalPathThroughTheWriter(t *testing.T) { |
| 102 | s := release.NewDirStore(t.TempDir()) |
| 103 | rec := offlineRecord(t, s) |
| 104 | now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| 105 | steps := offlineSteps(now) |
| 106 | wantState := []string{OfflineStopping, OfflineSealing, OfflineSealed, OfflineLanding, OfflineLanded, OfflineOpen, OfflineFinishing} |
| 107 | wantRegion := []string{"", "", "", "uk", "uk", "uk", "uk"} |
| 108 | |
| 109 | for i, st := range steps[:len(steps)-1] { |
| 110 | got, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, st.mut()) |
| 111 | if err != nil { |
| 112 | t.Fatalf("%s: %v", st.name, err) |
| 113 | } |
| 114 | if got.Move == nil || !got.Move.Offline() || got.Move.State != wantState[i] || got.Region != wantRegion[i] { |
| 115 | t.Fatalf("%s: move %+v region %q", st.name, got.Move, got.Region) |
| 116 | } |
| 117 | if got.Move.Legacy() { |
| 118 | t.Fatalf("%s: an offline move reads as legacy", st.name) |
| 119 | } |
| 120 | if got.Move.UpdatedAt.IsZero() { |
| 121 | t.Fatalf("%s: updated_at not stamped", st.name) |
| 122 | } |
| 123 | // What the box and the console read back is what the writer wrote. |
| 124 | if again := offlineGet(t, s, rec); !reflect.DeepEqual(again.Move, got.Move) || again.Region != got.Region { |
| 125 | t.Fatalf("%s: stored %+v, returned %+v", st.name, again.Move, got.Move) |
| 126 | } |
| 127 | } |
| 128 | mid := offlineGet(t, s, rec) |
| 129 | m := mid.Move |
| 130 | if m.ID != offID || m.From != "" || m.To != "uk" || m.RequestedBy != "ops@example.com" || !m.RequestedAt.Equal(now) || |
| 131 | m.Sealer != offSealer || !reflect.DeepEqual(m.Boxes, offBoxes) || m.Incarnation != "" || m.Cut != "" || |
| 132 | m.Source == nil || m.Source.Release != "v0.28.4" || m.Source.EnvDigest != "d1" || m.Source.FinalTXID != 42 || |
| 133 | m.Source.Files == nil || m.Source.Files.Entries != 2 || m.OpenedAt == nil { |
| 134 | t.Fatalf("the journal at finishing: %+v source %+v", m, m.Source) |
| 135 | } |
| 136 | |
| 137 | got, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, steps[len(steps)-1].mut()) |
| 138 | if err != nil { |
| 139 | t.Fatal(err) |
| 140 | } |
| 141 | if got.Move != nil || got.LastMove == nil || got.LastMove.ID != offID || got.LastMove.Outcome != "done" || !got.LastMove.At.Equal(now.Add(time.Hour)) || got.LastMove.OfflineS < 0 { |
| 142 | t.Fatalf("finish: move %+v last %+v", got.Move, got.LastMove) |
| 143 | } |
| 144 | if got.Region != "uk" { |
| 145 | t.Fatalf("finish: region %q", got.Region) |
| 146 | } |
| 147 | } |
| 148 | |
| 149 | // Review Focus 8 (§9.2): across a whole move the record differs only in |
| 150 | // region, move, last_move, record_seq (advanced) and move.updated_at. The |
| 151 | // record here carries an acquisition marker, a pool, an environment and the |
| 152 | // rest of what ordinary writers must leave alone; the comparison is of the |
| 153 | // whole JSON object, so a field a transition touched by accident shows. |
| 154 | func TestOfflineRecordDiffersOnlyWhereTheMoveSaysSo(t *testing.T) { |
| 155 | for _, via := range []string{"done", "aborted after the flip", "aborted before the flip"} { |
| 156 | t.Run(via, func(t *testing.T) { |
| 157 | // The doors call the transitions with a plain context. A counter that |
| 158 | // is already present advances without any context asking for it, so |
| 159 | // the record starts with one: a first write under the sequence rule |
| 160 | // puts record_seq into presence, as on any record a writer has touched. |
| 161 | ctx := context.Background() |
| 162 | s := release.NewDirStore(t.TempDir()) |
| 163 | rec := offlineRecord(t, s) |
| 164 | if _, err := updateRecord(WithRecordSequence(ctx), s, rec.Account, rec.App, rec.Host, func(*Record) error { return nil }); err != nil { |
| 165 | t.Fatal(err) |
| 166 | } |
| 167 | before := offlineGet(t, s, rec) |
| 168 | if before.RecordSeq == 0 { |
| 169 | t.Fatal("the record starts without record_seq") |
| 170 | } |
| 171 | beforeMap := asMap(t, before) |
| 172 | |
| 173 | now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| 174 | steps := offlineSteps(now) |
| 175 | upto := len(steps) |
| 176 | switch via { |
| 177 | case "aborted after the flip": |
| 178 | upto = 5 // through landed |
| 179 | case "aborted before the flip": |
| 180 | upto = 3 // through sealed |
| 181 | } |
| 182 | seqs := []uint64{} |
| 183 | for _, st := range steps[:upto] { |
| 184 | got, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, st.mut()) |
| 185 | if err != nil { |
| 186 | t.Fatalf("%s: %v", st.name, err) |
| 187 | } |
| 188 | seqs = append(seqs, got.RecordSeq) |
| 189 | } |
| 190 | if via != "done" { |
| 191 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineAbort(offID)); err != nil { |
| 192 | t.Fatal(err) |
| 193 | } |
| 194 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineFinish(offID, now.Add(time.Hour))); err != nil { |
| 195 | t.Fatal(err) |
| 196 | } |
| 197 | } |
| 198 | after := offlineGet(t, s, rec) |
| 199 | if after.Move != nil || after.LastMove == nil { |
| 200 | t.Fatalf("the move did not end: %+v %+v", after.Move, after.LastMove) |
| 201 | } |
| 202 | for i := 1; i < len(seqs); i++ { |
| 203 | if seqs[i] <= seqs[i-1] { |
| 204 | t.Fatalf("record_seq did not advance across transitions: %v", seqs) |
| 205 | } |
| 206 | } |
| 207 | if after.RecordSeq <= before.RecordSeq { |
| 208 | t.Fatalf("record_seq %d -> %d", before.RecordSeq, after.RecordSeq) |
| 209 | } |
| 210 | |
| 211 | afterMap := asMap(t, after) |
| 212 | allowed := map[string]bool{"region": true, "move": true, "last_move": true, "record_seq": true} |
| 213 | keys := map[string]bool{} |
| 214 | for k := range beforeMap { |
| 215 | keys[k] = true |
| 216 | } |
| 217 | for k := range afterMap { |
| 218 | keys[k] = true |
| 219 | } |
| 220 | for k := range keys { |
| 221 | if reflect.DeepEqual(beforeMap[k], afterMap[k]) { |
| 222 | continue |
| 223 | } |
| 224 | if !allowed[k] { |
| 225 | t.Errorf("field %q changed: %v -> %v", k, beforeMap[k], afterMap[k]) |
| 226 | } |
| 227 | } |
| 228 | if _, ok := afterMap["move"]; ok { |
| 229 | t.Errorf("move left on the record at rest: %v", afterMap["move"]) |
| 230 | } |
| 231 | wantRegion := "" |
| 232 | if via == "done" { |
| 233 | wantRegion = "uk" |
| 234 | } |
| 235 | if after.Region != wantRegion { |
| 236 | t.Errorf("region %q, want %q", after.Region, wantRegion) |
| 237 | } |
| 238 | for _, k := range []string{"acq", "pool", "environment", "channel", "hibernate", "steering", "created_at", "created_by", "slug"} { |
| 239 | if _, ok := afterMap[k]; !ok { |
| 240 | t.Errorf("field %q is gone", k) |
| 241 | } |
| 242 | } |
| 243 | }) |
| 244 | } |
| 245 | } |
| 246 | |
| 247 | // Every transition, tried from every state but its own, is refused with |
| 248 | // ErrMoveConflict and leaves the record as it was. |
| 249 | func TestOfflineEveryIllegalPairIsRefused(t *testing.T) { |
| 250 | now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| 251 | steps := offlineSteps(now) |
| 252 | // Each state is reached by walking the first n steps; reverting by an |
| 253 | // abort from stopping. |
| 254 | type at struct { |
| 255 | state string |
| 256 | walk []offlineStep |
| 257 | abort bool |
| 258 | } |
| 259 | states := []at{ |
| 260 | {OfflineStopping, steps[:1], false}, {OfflineSealing, steps[:2], false}, {OfflineSealed, steps[:3], false}, |
| 261 | {OfflineLanding, steps[:4], false}, {OfflineLanded, steps[:5], false}, {OfflineOpen, steps[:6], false}, |
| 262 | {OfflineFinishing, steps[:7], false}, {OfflineReverting, steps[:1], true}, |
| 263 | } |
| 264 | |
| 265 | for _, at := range states { |
| 266 | for _, st := range steps[1:] { // start has its own test |
| 267 | legal := st.from == at.state |
| 268 | s := release.NewDirStore(t.TempDir()) |
| 269 | rec := offlineRecord(t, s) |
| 270 | offlineWalk(t, s, rec, at.walk) |
| 271 | if at.abort { |
| 272 | if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, OfflineAbort(offID)); err != nil { |
| 273 | t.Fatal(err) |
| 274 | } |
| 275 | } |
| 276 | before := offlineGet(t, s, rec) |
| 277 | _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, st.mut()) |
| 278 | switch { |
| 279 | case legal && err != nil: |
| 280 | t.Errorf("%s from %s: %v", st.name, at.state, err) |
| 281 | case !legal && st.name == "finish" && at.state == OfflineReverting: |
| 282 | if err != nil { |
| 283 | t.Errorf("finish from reverting: %v", err) |
| 284 | } |
| 285 | case !legal && !errors.Is(err, ErrMoveConflict): |
| 286 | t.Errorf("%s from %s: err %v, want ErrMoveConflict", st.name, at.state, err) |
| 287 | } |
| 288 | if err != nil { |
| 289 | if after := offlineGet(t, s, rec); !reflect.DeepEqual(after, before) { |
| 290 | t.Errorf("%s from %s: a refusal changed the record", st.name, at.state) |
| 291 | } |
| 292 | } |
| 293 | } |
| 294 | } |
| 295 | } |
| 296 | |
| 297 | func TestOfflineWrongIDIsRefusedEverywhere(t *testing.T) { |
| 298 | now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| 299 | steps := offlineSteps(now) |
| 300 | wrong := map[string]func() func(*Record) error{ |
| 301 | "seal": func() func(*Record) error { return OfflineSeal(otherOff, offSealer, offBoxes) }, |
| 302 | "sealed": func() func(*Record) error { return OfflineSealedBy(otherOff, offSealer, MoveSource{}) }, |
| 303 | "flip": func() func(*Record) error { return OfflineFlip(otherOff) }, |
| 304 | "landed": func() func(*Record) error { return OfflineLandedBy(otherOff, "uk") }, |
| 305 | "open": func() func(*Record) error { return OfflineOpenTransition(otherOff, now) }, |
| 306 | "finishing": func() func(*Record) error { return OfflineFinishingTransition(otherOff) }, |
| 307 | "finish": func() func(*Record) error { return OfflineFinish(otherOff, now) }, |
| 308 | "abort": func() func(*Record) error { return OfflineAbort(otherOff) }, |
| 309 | } |
| 310 | // Each wrong-id step is tried in the state where its right-id twin is legal. |
| 311 | for i, st := range steps[1:] { |
| 312 | s := release.NewDirStore(t.TempDir()) |
| 313 | rec := offlineRecord(t, s) |
| 314 | offlineWalk(t, s, rec, steps[:i+1]) |
| 315 | before := offlineGet(t, s, rec) |
| 316 | if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, wrong[st.name]()); !errors.Is(err, ErrMoveConflict) { |
| 317 | t.Errorf("%s with another move's id: %v", st.name, err) |
| 318 | } |
| 319 | if after := offlineGet(t, s, rec); !reflect.DeepEqual(after, before) { |
| 320 | t.Errorf("%s with another move's id changed the record", st.name) |
| 321 | } |
| 322 | } |
| 323 | // Abort from a legal state with the wrong id, too. |
| 324 | s := release.NewDirStore(t.TempDir()) |
| 325 | rec := offlineRecord(t, s) |
| 326 | offlineWalk(t, s, rec, steps[:1]) |
| 327 | if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, wrong["abort"]()); !errors.Is(err, ErrMoveConflict) { |
| 328 | t.Errorf("abort with another move's id: %v", err) |
| 329 | } |
| 330 | // No move at all. |
| 331 | s = release.NewDirStore(t.TempDir()) |
| 332 | rec = offlineRecord(t, s) |
| 333 | for name, mk := range wrong { |
| 334 | if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, mk()); !errors.Is(err, ErrMoveConflict) { |
| 335 | t.Errorf("%s on a record with no move: %v", name, err) |
| 336 | } |
| 337 | } |
| 338 | } |
| 339 | |
| 340 | func TestOfflineStartRefusals(t *testing.T) { |
| 341 | ctx := context.Background() |
| 342 | now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| 343 | |
| 344 | start := func(t *testing.T, s release.Store, rec Record, id, from, to string) (*Record, error) { |
| 345 | t.Helper() |
| 346 | return SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStart(id, from, to, "ops@example.com", now)) |
| 347 | } |
| 348 | t.Run("stamps the head and leaves the region", func(t *testing.T) { |
| 349 | s := release.NewDirStore(t.TempDir()) |
| 350 | rec := offlineRecord(t, s) |
| 351 | got, err := start(t, s, rec, offID, "", "uk") |
| 352 | if err != nil { |
| 353 | t.Fatal(err) |
| 354 | } |
| 355 | m := got.Move |
| 356 | if m.Kind != MoveKindOffline || m.ID != offID || m.From != "" || m.To != "uk" || m.State != OfflineStopping || m.RequestedBy != "ops@example.com" || !m.RequestedAt.Equal(now) || got.Region != "" { |
| 357 | t.Fatalf("head %+v region %q", m, got.Region) |
| 358 | } |
| 359 | if m.Cut != "" || m.Incarnation != "" || m.Inputs != nil || m.Source != nil || m.CutPhase != "" || m.Sealer != "" || m.Boxes != nil { |
| 360 | t.Fatalf("start stamped more than the head: %+v", m) |
| 361 | } |
| 362 | }) |
| 363 | t.Run("in flight", func(t *testing.T) { |
| 364 | s := release.NewDirStore(t.TempDir()) |
| 365 | rec := offlineRecord(t, s) |
| 366 | if _, err := start(t, s, rec, offID, "", "uk"); err != nil { |
| 367 | t.Fatal(err) |
| 368 | } |
| 369 | if _, err := start(t, s, rec, otherOff, "", "us"); !errors.Is(err, ErrMoveConflict) { |
| 370 | t.Fatalf("second start: %v", err) |
| 371 | } |
| 372 | if got := offlineGet(t, s, rec); got.Move.ID != offID || got.Move.To != "uk" { |
| 373 | t.Fatalf("a refused start replaced the move: %+v", got.Move) |
| 374 | } |
| 375 | }) |
| 376 | t.Run("a legacy or Phase 1 move in flight", func(t *testing.T) { |
| 377 | for name, mut := range map[string]func(*Record) error{ |
| 378 | "legacy": MoveRequest("uk", now), |
| 379 | "phase1": MoveStartTransition(MoveStart{ID: offID, Incarnation: strings.Repeat("ab", 32), From: "", To: "uk", Cut: "auto", Now: now}), |
| 380 | } { |
| 381 | s := release.NewDirStore(t.TempDir()) |
| 382 | rec := offlineRecord(t, s) |
| 383 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, mut); err != nil { |
| 384 | t.Fatalf("%s: %v", name, err) |
| 385 | } |
| 386 | if _, err := start(t, s, rec, otherOff, "", "us"); !errors.Is(err, ErrMoveConflict) { |
| 387 | t.Fatalf("%s: offline start over it: %v", name, err) |
| 388 | } |
| 389 | } |
| 390 | }) |
| 391 | t.Run("same region, wrong from, bad id", func(t *testing.T) { |
| 392 | s := release.NewDirStore(t.TempDir()) |
| 393 | rec := offlineRecord(t, s) |
| 394 | for name, c := range map[string][3]string{ |
| 395 | "same region": {offID, "", ""}, |
| 396 | "wrong from": {offID, "us", "uk"}, |
| 397 | "short id": {"mv_short", "", "uk"}, |
| 398 | "empty id": {"", "", "uk"}, |
| 399 | "rollback id": {"rb_abcdefghijklmnopqrstuvwxyz", "", "uk"}, |
| 400 | } { |
| 401 | if _, err := start(t, s, rec, c[0], c[1], c[2]); !errors.Is(err, ErrMoveConflict) { |
| 402 | t.Errorf("%s: %v", name, err) |
| 403 | } |
| 404 | } |
| 405 | if got := offlineGet(t, s, rec); got.Move != nil { |
| 406 | t.Fatalf("a refused start left a journal: %+v", got.Move) |
| 407 | } |
| 408 | }) |
| 409 | t.Run("replica", func(t *testing.T) { |
| 410 | s := release.NewDirStore(t.TempDir()) |
| 411 | r := Record{Schema: 1, Host: "r.go.example.com", Slug: "s", App: "app", Account: "bab", Region: "", Channel: "stable", CreatedAt: now, Role: "replica"} |
| 412 | if err := CreateRecord(ctx, s, r); err != nil { |
| 413 | t.Fatal(err) |
| 414 | } |
| 415 | if _, err := start(t, s, r, offID, "", "uk"); !errors.Is(err, ErrMoveConflict) { |
| 416 | t.Fatalf("replica: %v", err) |
| 417 | } |
| 418 | }) |
| 419 | t.Run("seed", func(t *testing.T) { |
| 420 | s := release.NewDirStore(t.TempDir()) |
| 421 | rec := offlineRecord(t, s) |
| 422 | // A seeded record that is not placed: the writer under test refuses |
| 423 | // before any journal exists, as MoveRequest does. |
| 424 | unplaced := rec |
| 425 | unplaced.Seed = &SeedSource{ID: "sd_1", Kind: seeds.KindReplica, State: seeds.StateReady, By: "ops@example.com", At: now} |
| 426 | err := OfflineStart(offID, "", "uk", "ops@example.com", now)(&unplaced) |
| 427 | var refusal *SeedMoveError |
| 428 | if !errors.Is(err, ErrMoveConflict) || !errors.As(err, &refusal) { |
| 429 | t.Fatalf("seed: %v", err) |
| 430 | } |
| 431 | if unplaced.Move != nil { |
| 432 | t.Fatal("a refused start left a journal") |
| 433 | } |
| 434 | }) |
| 435 | t.Run("open run", func(t *testing.T) { |
| 436 | s := release.NewDirStore(t.TempDir()) |
| 437 | rec := offlineRecord(t, s) |
| 438 | running := rec |
| 439 | running.ActiveRun = &RunState{ID: "run_1", State: RunPicked, Deadline: now.Add(time.Hour)} |
| 440 | err := OfflineStart(offID, "", "uk", "ops@example.com", now)(&running) |
| 441 | var open *MoveRunOpenError |
| 442 | if !errors.As(err, &open) { |
| 443 | t.Fatalf("open run: %v", err) |
| 444 | } |
| 445 | if running.Move != nil { |
| 446 | t.Fatal("a refused start left a journal") |
| 447 | } |
| 448 | }) |
| 449 | } |
| 450 | |
| 451 | func TestOfflineSealAndSealedByChecks(t *testing.T) { |
| 452 | now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| 453 | steps := offlineSteps(now) |
| 454 | ctx := context.Background() |
| 455 | |
| 456 | s := release.NewDirStore(t.TempDir()) |
| 457 | rec := offlineRecord(t, s) |
| 458 | offlineWalk(t, s, rec, steps[:1]) |
| 459 | for name, mut := range map[string]func(*Record) error{ |
| 460 | "no sealer": OfflineSeal(offID, "", offBoxes), |
| 461 | "no boxes": OfflineSeal(offID, offSealer, nil), |
| 462 | } { |
| 463 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, mut); !errors.Is(err, ErrMoveConflict) { |
| 464 | t.Errorf("%s: %v", name, err) |
| 465 | } |
| 466 | } |
| 467 | // The sealer is one of the boxes. |
| 468 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineSeal(offID, "box-z", offBoxes)); !errors.Is(err, ErrMoveConflict) || !strings.Contains(err.Error(), refusedOfflineSeal) { |
| 469 | t.Errorf("a sealer outside the boxes: %v", err) |
| 470 | } |
| 471 | // The seal copies the caller's slice. |
| 472 | boxes := []string{"box-a", "box-b"} |
| 473 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineSeal(offID, offSealer, boxes)); err != nil { |
| 474 | t.Fatal(err) |
| 475 | } |
| 476 | boxes[0] = "changed" |
| 477 | if got := offlineGet(t, s, rec); got.Move.Boxes[0] != "box-a" { |
| 478 | t.Fatalf("the journal aliases the caller's slice: %v", got.Move.Boxes) |
| 479 | } |
| 480 | |
| 481 | // Only the named sealer records what it sealed, and the source is copied. |
| 482 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineSealedBy(offID, "box-b", MoveSource{})); !errors.Is(err, ErrMoveConflict) { |
| 483 | t.Fatalf("another box sealing: %v", err) |
| 484 | } |
| 485 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineSealedBy(offID, "", MoveSource{})); !errors.Is(err, ErrMoveConflict) { |
| 486 | t.Fatalf("no sealer: %v", err) |
| 487 | } |
| 488 | files := &MoveFilesDigest{Plain: "p", Bytes: 1} |
| 489 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineSealedBy(offID, offSealer, MoveSource{Files: files, Release: "v1"})); err != nil { |
| 490 | t.Fatal(err) |
| 491 | } |
| 492 | files.Plain = "changed" |
| 493 | if got := offlineGet(t, s, rec); got.Move.Source.Files.Plain != "p" || got.Move.Source.Release != "v1" { |
| 494 | t.Fatalf("the journal aliases the caller's source: %+v", got.Move.Source) |
| 495 | } |
| 496 | } |
| 497 | |
| 498 | func TestOfflineFlipAndLandedGuardTheRegion(t *testing.T) { |
| 499 | now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| 500 | steps := offlineSteps(now) |
| 501 | ctx := context.Background() |
| 502 | |
| 503 | // A record whose region moved under the move is not flipped. |
| 504 | s := release.NewDirStore(t.TempDir()) |
| 505 | rec := offlineRecord(t, s) |
| 506 | offlineWalk(t, s, rec, steps[:3]) |
| 507 | if _, err := updateRecord(ctx, s, rec.Account, rec.App, rec.Host, func(r *Record) error { r.Region = "us"; return nil }); err != nil { |
| 508 | t.Fatal(err) |
| 509 | } |
| 510 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineFlip(offID)); !errors.Is(err, ErrMoveConflict) { |
| 511 | t.Fatalf("flip over a changed region: %v", err) |
| 512 | } |
| 513 | |
| 514 | // landed is guarded by the record's region == to. |
| 515 | s = release.NewDirStore(t.TempDir()) |
| 516 | rec = offlineRecord(t, s) |
| 517 | offlineWalk(t, s, rec, steps[:4]) // through flip: region uk, landing |
| 518 | for _, to := range []string{"", "us"} { |
| 519 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineLandedBy(offID, to)); !errors.Is(err, ErrMoveConflict) { |
| 520 | t.Errorf("landed by %q: %v", to, err) |
| 521 | } |
| 522 | } |
| 523 | // An abort puts the region back; a landing that raced it is refused. |
| 524 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineAbort(offID)); err != nil { |
| 525 | t.Fatal(err) |
| 526 | } |
| 527 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineLandedBy(offID, "uk")); !errors.Is(err, ErrMoveConflict) { |
| 528 | t.Fatalf("landed after abort: %v", err) |
| 529 | } |
| 530 | } |
| 531 | |
| 532 | func TestOfflineAbort(t *testing.T) { |
| 533 | ctx := context.Background() |
| 534 | now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| 535 | steps := offlineSteps(now) |
| 536 | |
| 537 | // Legal from stopping (1 step walked) through landed (5), region back |
| 538 | // from landing and landed only. |
| 539 | for walked, c := range map[int]struct { |
| 540 | state string |
| 541 | region string |
| 542 | }{ |
| 543 | 1: {OfflineStopping, ""}, 2: {OfflineSealing, ""}, 3: {OfflineSealed, ""}, |
| 544 | 4: {OfflineLanding, "uk"}, 5: {OfflineLanded, "uk"}, |
| 545 | } { |
| 546 | s := release.NewDirStore(t.TempDir()) |
| 547 | rec := offlineRecord(t, s) |
| 548 | got := offlineWalk(t, s, rec, steps[:walked]) |
| 549 | if got.Move.State != c.state || got.Region != c.region { |
| 550 | t.Fatalf("setup %s: %s %q", c.state, got.Move.State, got.Region) |
| 551 | } |
| 552 | got, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineAbort(offID)) |
| 553 | if err != nil { |
| 554 | t.Fatalf("abort from %s: %v", c.state, err) |
| 555 | } |
| 556 | if got.Move.State != OfflineReverting || got.Region != "" || !got.Move.Offline() || got.Move.ID != offID { |
| 557 | t.Fatalf("abort from %s: %+v region %q", c.state, got.Move, got.Region) |
| 558 | } |
| 559 | // A second abort is a success that writes nothing: the stored record |
| 560 | // is the same bytes, record_seq and updated_at included. |
| 561 | key := RecordKey(rec.Account, rec.App, rec.Host) |
| 562 | rawBefore, etagBefore, err := s.Get(ctx, key) |
| 563 | if err != nil { |
| 564 | t.Fatal(err) |
| 565 | } |
| 566 | again, err := SetRecordMove(WithRecordSequence(ctx), s, rec.Account, rec.App, rec.Host, OfflineAbort(offID)) |
| 567 | rawAfter, etagAfter, gerr := s.Get(ctx, key) |
| 568 | if gerr != nil || !bytes.Equal(rawBefore, rawAfter) || etagBefore != etagAfter { |
| 569 | t.Fatalf("second abort from %s wrote the record: %v", c.state, gerr) |
| 570 | } |
| 571 | if err != nil || again.Move.State != OfflineReverting || again.Region != "" { |
| 572 | t.Fatalf("second abort from %s: %v %+v", c.state, err, again) |
| 573 | } |
| 574 | // The way back to rest is finish. |
| 575 | done, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineFinish(offID, now.Add(time.Minute))) |
| 576 | if err != nil || done.Move != nil || done.LastMove == nil || done.LastMove.Outcome != "aborted" || done.LastMove.ID != offID || done.Region != "" { |
| 577 | t.Fatalf("finish after abort from %s: %v %+v", c.state, err, done) |
| 578 | } |
| 579 | } |
| 580 | |
| 581 | // Refused from open and finishing, and nothing changes. |
| 582 | for walked, state := range map[int]string{6: OfflineOpen, 7: OfflineFinishing} { |
| 583 | s := release.NewDirStore(t.TempDir()) |
| 584 | rec := offlineRecord(t, s) |
| 585 | offlineWalk(t, s, rec, steps[:walked]) |
| 586 | before := offlineGet(t, s, rec) |
| 587 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineAbort(offID)); !errors.Is(err, ErrMoveConflict) { |
| 588 | t.Fatalf("abort from %s: %v", state, err) |
| 589 | } |
| 590 | if after := offlineGet(t, s, rec); !reflect.DeepEqual(after, before) { |
| 591 | t.Fatalf("abort from %s changed the record", state) |
| 592 | } |
| 593 | } |
| 594 | } |
| 595 | |
| 596 | func TestOfflineFinishFromBothAndTheRegionGuard(t *testing.T) { |
| 597 | ctx := context.Background() |
| 598 | now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| 599 | steps := offlineSteps(now) // open at now+30m, finish at now+1h |
| 600 | |
| 601 | // Done: offline time runs from the request to the open. |
| 602 | s := release.NewDirStore(t.TempDir()) |
| 603 | rec := offlineRecord(t, s) |
| 604 | got := offlineWalk(t, s, rec, steps) |
| 605 | if got.LastMove.Outcome != "done" { |
| 606 | t.Fatalf("outcome %q", got.LastMove.Outcome) |
| 607 | } |
| 608 | if got.LastMove.OfflineS != 1800 || got.LastMove.At != now.Add(time.Hour) { |
| 609 | t.Fatalf("offline_s %v at %v, want 1800 (request to open) at the finish", got.LastMove.OfflineS, got.LastMove.At) |
| 610 | } |
| 611 | |
| 612 | // finish refuses a region that is not where the outcome says. |
| 613 | s = release.NewDirStore(t.TempDir()) |
| 614 | rec = offlineRecord(t, s) |
| 615 | offlineWalk(t, s, rec, steps[:7]) |
| 616 | if _, err := updateRecord(ctx, s, rec.Account, rec.App, rec.Host, func(r *Record) error { r.Region = "us"; return nil }); err != nil { |
| 617 | t.Fatal(err) |
| 618 | } |
| 619 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineFinish(offID, now)); !errors.Is(err, ErrMoveConflict) { |
| 620 | t.Fatalf("done finish with the wrong region: %v", err) |
| 621 | } |
| 622 | s = release.NewDirStore(t.TempDir()) |
| 623 | rec = offlineRecord(t, s) |
| 624 | offlineWalk(t, s, rec, steps[:5]) |
| 625 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineAbort(offID)); err != nil { |
| 626 | t.Fatal(err) |
| 627 | } |
| 628 | if _, err := updateRecord(ctx, s, rec.Account, rec.App, rec.Host, func(r *Record) error { r.Region = "uk"; return nil }); err != nil { |
| 629 | t.Fatal(err) |
| 630 | } |
| 631 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineFinish(offID, now)); !errors.Is(err, ErrMoveConflict) { |
| 632 | t.Fatalf("aborted finish with the target's region: %v", err) |
| 633 | } |
| 634 | } |
| 635 | |
| 636 | // A record written before this change must parse and re-encode byte for byte. |
| 637 | // The bytes are written out by hand, in the order the struct has always |
| 638 | // encoded them: a record at rest, one with a legacy journal, and one with a |
| 639 | // Phase 1 journal carrying the source baselines. |
| 640 | func TestOfflineFieldsAreAdditiveOnTheWire(t *testing.T) { |
| 641 | const atRest = `{"record_seq":4,"acq":"acq_0123456789abcdef","schema":1,"host":"a.go.example.com","slug":"s","app":"app","account":"bab","pool":"main","region":"ie","channel":"stable","created_at":"2026-09-01T08:00:00Z","created_by":"ops@example.com","last_move":{"id":"mv_abcdefghijklmnopqrstuvwxyz","outcome":"done","at":"2026-10-06T10:00:00Z","offline_s":61.5},"hibernate":true,"environment":"shop","steering":"nearest"}` |
| 642 | const legacy = `{"schema":1,"host":"a.go.example.com","slug":"s","app":"app","account":"bab","region":"ie","channel":"stable","created_at":"2026-09-01T08:00:00Z","move":{"to":"uk","state":"ready","requested_at":"2026-10-06T09:00:00Z","ready_at":"2026-10-06T09:01:00Z","updated_at":"2026-10-06T09:01:00Z"}}` |
| 643 | const phase1 = `{"record_seq":9,"schema":1,"host":"a.go.example.com","slug":"s","app":"app","account":"bab","region":"ie","channel":"stable","created_at":"2026-09-01T08:00:00Z","move":{"id":"mv_abcdefghijklmnopqrstuvwxyz","incarnation":"abababababababababababababababababababababababababababababababab","from":"ie","to":"uk","cut":"auto","state":"cutting","requested_by":"ops@example.com","requested_at":"2026-10-06T09:00:00Z","updated_at":"2026-10-06T09:05:00Z","cut_phase":"stopped","cut_at":"2026-10-06T09:04:00Z","cut_budget_s":600,"source":{"replication":"cold","next_due":99,"final_txid":42,"db_sha256":"abc","files":{"plain":"p","sealed":"s","bytes":7,"entries":2},"timers_sha256":"t","evidence":"e"}}}` |
| 644 | for name, raw := range map[string]string{"at rest": atRest, "legacy move": legacy, "phase 1 move": phase1} { |
| 645 | rec, err := DecodeRecord([]byte(raw)) |
| 646 | if err != nil { |
| 647 | t.Fatalf("%s: %v", name, err) |
| 648 | } |
| 649 | if rec.Move.Offline() { |
| 650 | t.Fatalf("%s: a move written before offline moves reads as offline", name) |
| 651 | } |
| 652 | if rec.Move != nil && (rec.Move.Kind != "" || rec.Move.Sealer != "" || rec.Move.Boxes != nil) { |
| 653 | t.Fatalf("%s: the new fields are not empty: %+v", name, rec.Move) |
| 654 | } |
| 655 | if rec.Move != nil && rec.Move.Source != nil && (rec.Move.Source.Release != "" || rec.Move.Source.EnvDigest != "") { |
| 656 | t.Fatalf("%s: the new source fields are not empty", name) |
| 657 | } |
| 658 | out, err := json.Marshal(rec) |
| 659 | if err != nil { |
| 660 | t.Fatal(err) |
| 661 | } |
| 662 | if string(out) != raw { |
| 663 | t.Errorf("%s did not round-trip byte for byte:\n got %s\nwant %s", name, out, raw) |
| 664 | } |
| 665 | } |
| 666 | |
| 667 | // And an offline journal round-trips too, with its fields in the journal. |
| 668 | const offline = `{"schema":1,"host":"a.go.example.com","slug":"s","app":"app","account":"bab","region":"uk","channel":"stable","created_at":"2026-09-01T08:00:00Z","move":{"id":"mv_abcdefghijklmnopqrstuvwxyz","to":"uk","state":"offline-landing","requested_by":"ops@example.com","requested_at":"2026-10-21T09:00:00Z","updated_at":"2026-10-21T09:05:00Z","source":{"final_txid":42,"release":"v0.28.4","env_digest":"d1"},"kind":"offline","sealer":"box-a","boxes":["box-a","box-b"]}}` |
| 669 | rec, err := DecodeRecord([]byte(offline)) |
| 670 | if err != nil { |
| 671 | t.Fatal(err) |
| 672 | } |
| 673 | if !rec.Move.Offline() || rec.Move.Sealer != "box-a" || len(rec.Move.Boxes) != 2 || rec.Move.Source.Release != "v0.28.4" { |
| 674 | t.Fatalf("offline journal: %+v", rec.Move) |
| 675 | } |
| 676 | if out, _ := json.Marshal(rec); string(out) != offline { |
| 677 | t.Errorf("offline journal did not round-trip:\n got %s\nwant %s", out, offline) |
| 678 | } |
| 679 | } |
| 680 | |
| 681 | // Kinds never cross: no legacy and no Phase 1 transition accepts an offline |
| 682 | // move, in any of its states, and none changes the record. |
| 683 | func TestEveryLegacyAndPhase1TransitionRefusesAnOfflineMove(t *testing.T) { |
| 684 | now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| 685 | steps := offlineSteps(now) |
| 686 | // An empty incarnation and the offline move's own id: the guard that would |
| 687 | // match if the kind were not checked. |
| 688 | g := MoveGuard{ID: offID} |
| 689 | gInc := MoveGuard{ID: offID, Incarnation: strings.Repeat("ab", 32)} |
| 690 | in := MoveInputs{} |
| 691 | refused := map[string]func() func(*Record) error{ |
| 692 | "MoveRequest": func() func(*Record) error { return MoveRequest("us", now) }, |
| 693 | "MoveStartTransition": func() func(*Record) error { |
| 694 | return MoveStartTransition(MoveStart{ID: otherOff, Incarnation: strings.Repeat("ab", 32), From: "", To: "us", Cut: "auto", Now: now}) |
| 695 | }, |
| 696 | "MoveAbort": func() func(*Record) error { return MoveAbort() }, |
| 697 | "MoveMarkCutting": func() func(*Record) error { return MoveMarkCutting() }, |
| 698 | "MoveComplete": func() func(*Record) error { return MoveComplete() }, |
| 699 | "MoveAbortComplete": func() func(*Record) error { return MoveAbortComplete() }, |
| 700 | "MovePromoteReady": func() func(*Record) error { return MovePromoteReady(now) }, |
| 701 | "MoveGuardedPromoteReady": func() func(*Record) error { return MoveGuardedPromoteReady(g, now, in) }, |
| 702 | "MoveGuardedCut": func() func(*Record) error { return MoveGuardedCut(g) }, |
| 703 | "MoveGuardedRepin": func() func(*Record) error { return MoveGuardedRepin(g, in) }, |
| 704 | "MoveGuardedAbort": func() func(*Record) error { return MoveGuardedAbort(g) }, |
| 705 | "MoveGuardedAbortComplete": func() func(*Record) error { return MoveGuardedAbortComplete(g, now) }, |
| 706 | "MoveAdmit": func() func(*Record) error { return MoveAdmit(g, now, 600, now.Add(time.Hour), 3) }, |
| 707 | "MoveBackToReady": func() func(*Record) error { return MoveBackToReady(g) }, |
| 708 | "MoveRecordSource": func() func(*Record) error { return MoveRecordSource(g, MoveSource{}) }, |
| 709 | "MoveFlip": func() func(*Record) error { return MoveFlip(g, now, now.Add(time.Hour)) }, |
| 710 | "MoveCommit": func() func(*Record) error { return MoveCommit(g, now) }, |
| 711 | "MoveClose": func() func(*Record) error { return MoveClose(g, now) }, |
| 712 | "MoveRevert": func() func(*Record) error { return MoveRevert(g, "r") }, |
| 713 | "MoveMarkStuck": func() func(*Record) error { return MoveMarkStuck(g, StuckBaselineMismatch) }, |
| 714 | "MoveRetry": func() func(*Record) error { return MoveRetry(g) }, |
| 715 | "MoveRollBack": func() func(*Record) error { return MoveRollBack(g, "rb_abcdefghijklmnopqrstuvwxyz", "r", now) }, |
| 716 | "guarded, real incarnation": func() func(*Record) error { return MoveGuardedAbort(gInc) }, |
| 717 | } |
| 718 | // The guard itself refuses, whatever the transition would say after it: |
| 719 | // this pins MoveGuard.check, not each transition's own state check. |
| 720 | for _, st := range []string{OfflineStopping, OfflineLanding, OfflineOpen, OfflineReverting} { |
| 721 | offRec := Record{Move: &MoveState{Kind: MoveKindOffline, ID: offID, State: st}} |
| 722 | err := g.check(&offRec) |
| 723 | if !errors.Is(err, ErrMoveConflict) || !strings.Contains(err.Error(), refusedOfflineKind) { |
| 724 | t.Errorf("MoveGuard.check on an offline move in %s: %v", st, err) |
| 725 | } |
| 726 | } |
| 727 | // Walk one record through every offline state and try everything on it. |
| 728 | s := release.NewDirStore(t.TempDir()) |
| 729 | rec := offlineRecord(t, s) |
| 730 | check := func(label string) { |
| 731 | t.Helper() |
| 732 | before := offlineGet(t, s, rec) |
| 733 | for name, mk := range refused { |
| 734 | if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, mk()); !errors.Is(err, ErrMoveConflict) { |
| 735 | t.Errorf("%s: %s: err %v, want ErrMoveConflict", label, name, err) |
| 736 | } |
| 737 | } |
| 738 | if after := offlineGet(t, s, rec); !reflect.DeepEqual(after, before) { |
| 739 | t.Errorf("%s: a refused transition changed the record: %+v -> %+v", label, before.Move, after.Move) |
| 740 | } |
| 741 | } |
| 742 | for _, st := range steps[:len(steps)-1] { |
| 743 | offlineWalk(t, s, rec, []offlineStep{st}) |
| 744 | check(st.name) |
| 745 | } |
| 746 | // The reverting state, reached by abort from sealed. |
| 747 | s = release.NewDirStore(t.TempDir()) |
| 748 | rec = offlineRecord(t, s) |
| 749 | offlineWalk(t, s, rec, steps[:3]) |
| 750 | if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, OfflineAbort(offID)); err != nil { |
| 751 | t.Fatal(err) |
| 752 | } |
| 753 | check("reverting") |
| 754 | } |
| 755 | |
| 756 | // And the other way: no offline transition accepts a legacy or Phase 1 move. |
| 757 | func TestOfflineTransitionsRefuseOtherKinds(t *testing.T) { |
| 758 | now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| 759 | for name, start := range map[string]func(*Record) error{ |
| 760 | "legacy": MoveRequest("uk", now), |
| 761 | "phase1": MoveStartTransition(MoveStart{ID: offID, Incarnation: strings.Repeat("ab", 32), From: "", To: "uk", Cut: "auto", Now: now}), |
| 762 | } { |
| 763 | s := release.NewDirStore(t.TempDir()) |
| 764 | rec := offlineRecord(t, s) |
| 765 | if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, start); err != nil { |
| 766 | t.Fatal(err) |
| 767 | } |
| 768 | before := offlineGet(t, s, rec) |
| 769 | for _, st := range offlineSteps(now)[1:] { |
| 770 | if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, st.mut()); !errors.Is(err, ErrMoveConflict) { |
| 771 | t.Errorf("%s move: offline %s: %v", name, st.name, err) |
| 772 | } |
| 773 | } |
| 774 | if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, OfflineAbort(offID)); !errors.Is(err, ErrMoveConflict) { |
| 775 | t.Errorf("%s move: offline abort: %v", name, err) |
| 776 | } |
| 777 | if after := offlineGet(t, s, rec); !reflect.DeepEqual(after, before) { |
| 778 | t.Errorf("%s move: an offline transition changed it", name) |
| 779 | } |
| 780 | } |
| 781 | } |
| 782 | |
| 783 | func TestMoveStateOfflineIsNilSafe(t *testing.T) { |
| 784 | var m *MoveState |
| 785 | if m.Offline() { |
| 786 | t.Fatal("nil move is offline") |
| 787 | } |
| 788 | if (&MoveState{}).Offline() || (&MoveState{Kind: "other"}).Offline() || !(&MoveState{Kind: MoveKindOffline}).Offline() { |
| 789 | t.Fatal("Offline reads the wrong field") |
| 790 | } |
| 791 | } |
| 792 | |
| 793 | // The refusals an operator can read are the recorded sentences (copy rows |
| 794 | // MV-O34 to MV-O43 and the existing MV-N rows), not free text, and none has an |
| 795 | // em dash. A door maps them; this holds the provision side to its rows. |
| 796 | func TestOfflineRefusalsAreTheRecordedSentences(t *testing.T) { |
| 797 | now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| 798 | steps := offlineSteps(now) |
| 799 | ctx := context.Background() |
| 800 | refuse := func(t *testing.T, s release.Store, rec Record, mut func(*Record) error, want string) { |
| 801 | t.Helper() |
| 802 | _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, mut) |
| 803 | if !errors.Is(err, ErrMoveConflict) { |
| 804 | t.Fatalf("err %v, want ErrMoveConflict", err) |
| 805 | } |
| 806 | if got := strings.TrimPrefix(err.Error(), ErrMoveConflict.Error()+": "); got != want { |
| 807 | t.Errorf("refusal %q, want %q", got, want) |
| 808 | } |
| 809 | if strings.ContainsRune(err.Error(), '\u2014') { |
| 810 | t.Errorf("em dash in %q", err) |
| 811 | } |
| 812 | } |
| 813 | fresh := func(t *testing.T, walked int) (release.Store, Record) { |
| 814 | s := release.NewDirStore(t.TempDir()) |
| 815 | rec := offlineRecord(t, s) |
| 816 | offlineWalk(t, s, rec, steps[:walked]) |
| 817 | return s, rec |
| 818 | } |
| 819 | |
| 820 | s, rec := fresh(t, 0) |
| 821 | refuse(t, s, rec, OfflineSeal(offID, offSealer, offBoxes), refusedNoMove) |
| 822 | refuse(t, s, rec, OfflineStart(offID, "us", "uk", "ops", now), "the instance is in home, not us") |
| 823 | refuse(t, s, rec, OfflineStart(offID, "", "", "ops", now), "the instance is already in home") |
| 824 | refuse(t, s, rec, OfflineStart("mv_short", "", "uk", "ops", now), refusedMalformedStart) |
| 825 | |
| 826 | s, rec = fresh(t, 1) |
| 827 | refuse(t, s, rec, OfflineSeal(otherOff, offSealer, offBoxes), refusedOtherOfflineMove) |
| 828 | refuse(t, s, rec, OfflineFlip(offID), "this offline move is offline-stopping. this step follows offline-sealed") |
| 829 | refuse(t, s, rec, OfflineSeal(offID, "", offBoxes), refusedOfflineSeal) |
| 830 | refuse(t, s, rec, OfflineFinish(offID, now), "this offline move is offline-stopping. it finishes from finishing or reverting") |
| 831 | |
| 832 | s, rec = fresh(t, 2) |
| 833 | refuse(t, s, rec, OfflineSealedBy(offID, "box-b", MoveSource{}), "this offline move is sealed by box-a") |
| 834 | |
| 835 | s, rec = fresh(t, 6) |
| 836 | refuse(t, s, rec, OfflineAbort(offID), "this offline move is offline-open. it cannot be aborted once it is open") |
| 837 | |
| 838 | s, rec = fresh(t, 3) |
| 839 | if _, err := updateRecord(ctx, s, rec.Account, rec.App, rec.Host, func(r *Record) error { r.Region = "us"; return nil }); err != nil { |
| 840 | t.Fatal(err) |
| 841 | } |
| 842 | refuse(t, s, rec, OfflineFlip(offID), "the record's region is us, which this step does not expect. the instance changed under the move") |
| 843 | |
| 844 | // A legacy move meets an offline step. |
| 845 | s = release.NewDirStore(t.TempDir()) |
| 846 | rec = offlineRecord(t, s) |
| 847 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, MoveRequest("uk", now)); err != nil { |
| 848 | t.Fatal(err) |
| 849 | } |
| 850 | refuse(t, s, rec, OfflineSeal(offID, offSealer, offBoxes), refusedNotOffline) |
| 851 | |
| 852 | // And the other way: the Phase 1 guard on an offline move. |
| 853 | s, rec = fresh(t, 1) |
| 854 | refuse(t, s, rec, MoveGuardedCut(MoveGuard{ID: offID}), refusedOfflineKind) |
| 855 | } |
| 856 | |
amadan