| 1 | package provision |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "reflect" |
| 8 | "testing" |
| 9 | "time" |
| 10 | |
| 11 | "github.com/carlosframework/platform/internal/release" |
| 12 | ) |
| 13 | |
| 14 | // The pre-copy (offline-staging): OfflineStage creates the move with the |
| 15 | // instance still serving, OfflineStartStaged takes it to stopping, and abort |
| 16 | // from staging ends a move that was never offline. |
| 17 | |
| 18 | func TestOfflineStageThenStartThenTheWholeMove(t *testing.T) { |
| 19 | ctx := context.Background() |
| 20 | now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| 21 | s := release.NewDirStore(t.TempDir()) |
| 22 | rec := offlineRecord(t, s) |
| 23 | |
| 24 | got, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStage(offID, "", "uk", "ops@example.com", stageAt, now)) |
| 25 | if err != nil { |
| 26 | t.Fatal(err) |
| 27 | } |
| 28 | m := got.Move |
| 29 | if !m.Offline() || m.State != OfflineStaging || m.ID != offID || m.To != "uk" || got.Region != "" || |
| 30 | m.StagedAt == nil || !m.StagedAt.Equal(now) || !m.RequestedAt.Equal(now) || m.Stager != "box-a" || m.StageBucket != "data-uk" || m.StageRegion != "eu-west-2" { |
| 31 | t.Fatalf("staged record: %+v region %q", m, got.Region) |
| 32 | } |
| 33 | // Staging is a move in flight: no second move starts beside it. |
| 34 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStart(otherOff, "", "uk", "ops@example.com", now)); !errors.Is(err, ErrMoveConflict) { |
| 35 | t.Fatalf("a start beside a staged move: %v", err) |
| 36 | } |
| 37 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStage(otherOff, "", "us", "ops@example.com", stageAt, now)); !errors.Is(err, ErrMoveConflict) { |
| 38 | t.Fatalf("a stage beside a staged move: %v", err) |
| 39 | } |
| 40 | |
| 41 | start := now.Add(10 * time.Minute) |
| 42 | got, err = SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStartStaged(offID, "other@example.com", start)) |
| 43 | if err != nil { |
| 44 | t.Fatal(err) |
| 45 | } |
| 46 | if got.Move.State != OfflineStopping || got.Move.RequestedBy != "other@example.com" || !got.Move.RequestedAt.Equal(start) || !got.Move.StagedAt.Equal(now) || got.Move.Stager != "box-a" { |
| 47 | t.Fatalf("started: %+v", got.Move) |
| 48 | } |
| 49 | // The rest of the move is the ordinary path; offline time counts from |
| 50 | // the start, not the stage: start at +10m, open at +30m. |
| 51 | got = offlineWalk(t, s, rec, offlineSteps(now)[1:]) |
| 52 | if got.Move != nil || got.LastMove.Outcome != OfflineOutcomeDone || got.LastMove.OfflineS != 1200 { |
| 53 | t.Fatalf("finished: %+v", got.LastMove) |
| 54 | } |
| 55 | } |
| 56 | |
| 57 | func TestOfflineStartStagedRefusals(t *testing.T) { |
| 58 | ctx := context.Background() |
| 59 | now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| 60 | |
| 61 | // From every state but staging, and with another id. |
| 62 | steps := offlineSteps(now) |
| 63 | for walked := 1; walked <= 7; walked++ { |
| 64 | s := release.NewDirStore(t.TempDir()) |
| 65 | rec := offlineRecord(t, s) |
| 66 | offlineWalk(t, s, rec, steps[:walked]) |
| 67 | before := offlineGet(t, s, rec) |
| 68 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStartStaged(offID, "ops@example.com", now)); !errors.Is(err, ErrMoveConflict) { |
| 69 | t.Fatalf("start-staged from %s: %v", before.Move.State, err) |
| 70 | } |
| 71 | if after := offlineGet(t, s, rec); !reflect.DeepEqual(after, before) { |
| 72 | t.Fatalf("a refused start-staged from %s changed the record", before.Move.State) |
| 73 | } |
| 74 | } |
| 75 | s := release.NewDirStore(t.TempDir()) |
| 76 | rec := offlineRecord(t, s) |
| 77 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStage(offID, "", "uk", "ops@example.com", stageAt, now)); err != nil { |
| 78 | t.Fatal(err) |
| 79 | } |
| 80 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStartStaged(otherOff, "ops@example.com", now)); !errors.Is(err, ErrMoveConflict) { |
| 81 | t.Fatalf("start-staged with another id: %v", err) |
| 82 | } |
| 83 | // A region written under the staged move. |
| 84 | if _, err := updateRecord(ctx, s, rec.Account, rec.App, rec.Host, func(r *Record) error { r.Region = "us"; return nil }); err != nil { |
| 85 | t.Fatal(err) |
| 86 | } |
| 87 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStartStaged(offID, "ops@example.com", now)); !errors.Is(err, ErrMoveConflict) { |
| 88 | t.Fatalf("start-staged over a moved region: %v", err) |
| 89 | } |
| 90 | |
| 91 | // No other step leaves staging: seal through finish all refuse. |
| 92 | s = release.NewDirStore(t.TempDir()) |
| 93 | rec = offlineRecord(t, s) |
| 94 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStage(offID, "", "uk", "ops@example.com", stageAt, now)); err != nil { |
| 95 | t.Fatal(err) |
| 96 | } |
| 97 | before := offlineGet(t, s, rec) |
| 98 | for _, st := range steps[1:] { |
| 99 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, st.mut()); !errors.Is(err, ErrMoveConflict) { |
| 100 | t.Errorf("%s from staging: %v", st.name, err) |
| 101 | } |
| 102 | } |
| 103 | if after := offlineGet(t, s, rec); !reflect.DeepEqual(after, before) { |
| 104 | t.Fatal("a refused step changed the staged record") |
| 105 | } |
| 106 | } |
| 107 | |
| 108 | // A start in the same instant as the stage still lands after it, so the |
| 109 | // finish can tell the two apart. |
| 110 | func TestOfflineStartStagedIsAlwaysAfterTheStage(t *testing.T) { |
| 111 | ctx := context.Background() |
| 112 | now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| 113 | s := release.NewDirStore(t.TempDir()) |
| 114 | rec := offlineRecord(t, s) |
| 115 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStage(offID, "", "uk", "ops@example.com", stageAt, now)); err != nil { |
| 116 | t.Fatal(err) |
| 117 | } |
| 118 | got, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStartStaged(offID, "ops@example.com", now.Add(-time.Second))) |
| 119 | if err != nil { |
| 120 | t.Fatal(err) |
| 121 | } |
| 122 | if !got.Move.RequestedAt.After(*got.Move.StagedAt) || !got.Move.StartedOffline() { |
| 123 | t.Fatalf("start %v is not after the stage %v", got.Move.RequestedAt, got.Move.StagedAt) |
| 124 | } |
| 125 | } |
| 126 | |
| 127 | // Abort from staging: reverting with the region untouched, then a finish |
| 128 | // that records the move aborted and never offline. Aborted after its start, |
| 129 | // a staged move counts its offline time from the start. |
| 130 | func TestOfflineAbortFromStaging(t *testing.T) { |
| 131 | ctx := context.Background() |
| 132 | now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| 133 | s := release.NewDirStore(t.TempDir()) |
| 134 | rec := offlineRecord(t, s) |
| 135 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStage(offID, "", "uk", "ops@example.com", stageAt, now)); err != nil { |
| 136 | t.Fatal(err) |
| 137 | } |
| 138 | got, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineAbort(offID)) |
| 139 | if err != nil || got.Move.State != OfflineReverting || got.Region != "" { |
| 140 | t.Fatalf("abort from staging: %v %+v", err, got) |
| 141 | } |
| 142 | done, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineFinish(offID, now.Add(time.Hour))) |
| 143 | if err != nil || done.Move != nil || done.LastMove.Outcome != OfflineOutcomeAborted || done.LastMove.OfflineS != 0 { |
| 144 | t.Fatalf("finish after abort from staging: %v %+v", err, done.LastMove) |
| 145 | } |
| 146 | |
| 147 | s = release.NewDirStore(t.TempDir()) |
| 148 | rec = offlineRecord(t, s) |
| 149 | for _, mut := range []func(*Record) error{ |
| 150 | OfflineStage(offID, "", "uk", "ops@example.com", stageAt, now), |
| 151 | OfflineStartStaged(offID, "ops@example.com", now.Add(20*time.Minute)), |
| 152 | OfflineAbort(offID), |
| 153 | } { |
| 154 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, mut); err != nil { |
| 155 | t.Fatal(err) |
| 156 | } |
| 157 | } |
| 158 | done, err = SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineFinish(offID, now.Add(30*time.Minute))) |
| 159 | if err != nil || done.LastMove.Outcome != OfflineOutcomeAborted || done.LastMove.OfflineS != 600 { |
| 160 | t.Fatalf("finish after a started staged move: %v %+v", err, done.LastMove) |
| 161 | } |
| 162 | } |
| 163 | |
| 164 | // staged_at is additive like the other offline fields: a staged journal |
| 165 | // round-trips, and every earlier shape still does (TestOfflineFieldsAreAdditiveOnTheWire). |
| 166 | func TestOfflineStagedAtIsAdditiveOnTheWire(t *testing.T) { |
| 167 | const staged = `{"schema":1,"host":"a.go.example.com","slug":"s","app":"app","account":"bab","region":"ie","channel":"stable","created_at":"2026-09-01T08:00:00Z","move":{"id":"mv_abcdefghijklmnopqrstuvwxyz","from":"ie","to":"uk","state":"offline-staging","requested_by":"ops@example.com","requested_at":"2026-10-21T09:00:00Z","updated_at":"2026-10-21T09:00:00Z","kind":"offline","staged_at":"2026-10-21T09:00:00Z","stager":"box-a","stage_bucket":"data-uk","stage_region":"eu-west-2"}}` |
| 168 | rec, err := DecodeRecord([]byte(staged)) |
| 169 | if err != nil { |
| 170 | t.Fatal(err) |
| 171 | } |
| 172 | if rec.Move.State != OfflineStaging || rec.Move.StagedAt == nil || rec.Move.StartedOffline() || rec.Move.Stager != "box-a" { |
| 173 | t.Fatalf("staged journal: %+v", rec.Move) |
| 174 | } |
| 175 | if out, _ := json.Marshal(rec); string(out) != staged { |
| 176 | t.Errorf("staged journal did not round-trip:\n got %s\nwant %s", out, staged) |
| 177 | } |
| 178 | } |
| 179 | |
| 180 | // A second stage is OfflineRestage: staged_at moves forward, never back, |
| 181 | // requested_at stays, and the move still reads as never started. Nothing |
| 182 | // but staging admits it. |
| 183 | func TestOfflineRestage(t *testing.T) { |
| 184 | ctx := context.Background() |
| 185 | now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC) |
| 186 | s := release.NewDirStore(t.TempDir()) |
| 187 | rec := offlineRecord(t, s) |
| 188 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineRestage(offID, now)); !errors.Is(err, ErrMoveConflict) { |
| 189 | t.Fatalf("restage at rest: %v", err) |
| 190 | } |
| 191 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStage(offID, "", "uk", "ops@example.com", stageAt, now)); err != nil { |
| 192 | t.Fatal(err) |
| 193 | } |
| 194 | got, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineRestage(offID, now.Add(5*time.Minute))) |
| 195 | if err != nil || !got.Move.StagedAt.Equal(now.Add(5*time.Minute)) || !got.Move.RequestedAt.Equal(now) || got.Move.StartedOffline() || got.Move.State != OfflineStaging || got.Move.Stager != "box-a" { |
| 196 | t.Fatalf("restage: %v %+v", err, got.Move) |
| 197 | } |
| 198 | // A clock behind the last request still moves it forward. |
| 199 | got, err = SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineRestage(offID, now)) |
| 200 | if err != nil || !got.Move.StagedAt.After(now.Add(5*time.Minute)) { |
| 201 | t.Fatalf("restage with an earlier clock: %v %+v", err, got.Move) |
| 202 | } |
| 203 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineRestage(otherOff, now)); !errors.Is(err, ErrMoveConflict) { |
| 204 | t.Fatalf("restage with another id: %v", err) |
| 205 | } |
| 206 | // The start after a restage is still after the last request. |
| 207 | got, err = SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStartStaged(offID, "ops@example.com", now.Add(time.Minute))) |
| 208 | if err != nil || !got.Move.StartedOffline() || got.Move.State != OfflineStopping { |
| 209 | t.Fatalf("start after restage: %v %+v", err, got.Move) |
| 210 | } |
| 211 | if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineRestage(offID, now)); !errors.Is(err, ErrMoveConflict) { |
| 212 | t.Fatalf("restage after the start: %v", err) |
| 213 | } |
| 214 | } |
| 215 | |
| 216 | // stageAt is the stager and staging destination the tests' stage door names. |
| 217 | var stageAt = StageTarget{Stager: "box-a", Bucket: "data-uk", Region: "eu-west-2"} |
| 218 | |
| 219 | // The stager is pinned at the stage: a stage that names none is malformed, |
| 220 | // and so is one with no staging bucket. |
| 221 | func TestOfflineStageNamesItsStager(t *testing.T) { |
| 222 | s := release.NewDirStore(t.TempDir()) |
| 223 | rec := offlineRecord(t, s) |
| 224 | before := offlineGet(t, s, rec) |
| 225 | if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, OfflineStage(offID, "", "uk", "ops@example.com", StageTarget{Bucket: "data-uk"}, time.Now())); !errors.Is(err, ErrMoveConflict) { |
| 226 | t.Fatalf("a stage with no stager: %v", err) |
| 227 | } |
| 228 | if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, OfflineStage(offID, "", "uk", "ops@example.com", StageTarget{Stager: "box-a"}, time.Now())); !errors.Is(err, ErrMoveConflict) { |
| 229 | t.Fatalf("a stage with no staging bucket: %v", err) |
| 230 | } |
| 231 | if after := offlineGet(t, s, rec); !reflect.DeepEqual(after, before) { |
| 232 | t.Fatal("a refused stage changed the record") |
| 233 | } |
| 234 | } |
| 235 | |
amadan