| 1 | package provision |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "strings" |
| 8 | "testing" |
| 9 | "time" |
| 10 | |
| 11 | "github.com/carlosframework/platform/internal/release" |
| 12 | "github.com/carlosframework/platform/internal/seeds" |
| 13 | ) |
| 14 | |
| 15 | func runTestRecord(t *testing.T, s release.Store) Record { |
| 16 | t.Helper() |
| 17 | rec := Record{Schema: 1, Host: "a.go.example.com", Slug: "s", App: "app", Account: "bab", Region: "ie", Channel: "stable", CreatedAt: time.Now().UTC()} |
| 18 | if err := CreateRecord(context.Background(), s, rec); err != nil { |
| 19 | t.Fatal(err) |
| 20 | } |
| 21 | return rec |
| 22 | } |
| 23 | |
| 24 | func setRun(t *testing.T, s release.Store, rec Record, mut func(*Record) error) (*Record, error) { |
| 25 | t.Helper() |
| 26 | return SetRecordRun(context.Background(), s, rec.Account, rec.App, rec.Host, mut) |
| 27 | } |
| 28 | |
| 29 | func TestRunAdmitPickFinishClear(t *testing.T) { |
| 30 | s := release.NewDirStore(t.TempDir()) |
| 31 | rec := runTestRecord(t, s) |
| 32 | at := time.Now().Add(time.Minute) |
| 33 | got, err := setRun(t, s, rec, RunAdmit(RunAdmission{ID: "run_a", By: "alice@keymail.dev", Deadline: at})) |
| 34 | if err != nil || got.ActiveRun.State != RunQueued || got.ActiveRun.By != "alice@keymail.dev" || !got.ActiveRun.Open() { |
| 35 | t.Fatalf("admit: %+v, %v", got, err) |
| 36 | } |
| 37 | if _, err := setRun(t, s, rec, RunAdmit(RunAdmission{ID: "run_b", Deadline: at})); !errors.Is(err, ErrRunOpen) { |
| 38 | t.Fatalf("second admit: %v, want ErrRunOpen", err) |
| 39 | } |
| 40 | var open *RunOpenError |
| 41 | if _, err := setRun(t, s, rec, RunAdmit(RunAdmission{ID: "run_b", Deadline: at})); !errors.As(err, &open) || open.Run.ID != "run_a" || open.Run.By != "alice@keymail.dev" { |
| 42 | t.Fatalf("the refusal must name the open run and its caller: %v", err) |
| 43 | } |
| 44 | if got, err = setRun(t, s, rec, RunPick("run_a", "box-1", "boot-1", at.Add(time.Hour))); err != nil || got.ActiveRun.State != RunPicked || got.ActiveRun.Box != "box-1" { |
| 45 | t.Fatalf("pick: %+v, %v", got, err) |
| 46 | } |
| 47 | if _, err := setRun(t, s, rec, RunPick("run_a", "box-2", "boot-2", at)); !errors.Is(err, ErrRunConflict) { |
| 48 | t.Fatalf("a second pick: %v, want ErrRunConflict", err) |
| 49 | } |
| 50 | if _, err := setRun(t, s, rec, RunClear("run_a")); !errors.Is(err, ErrRunConflict) { |
| 51 | t.Fatalf("clearing an open run: %v, want ErrRunConflict", err) |
| 52 | } |
| 53 | if got, err = setRun(t, s, rec, RunFinish("run_a")); err != nil || got.ActiveRun.State != RunDone || got.ActiveRun.Open() { |
| 54 | t.Fatalf("finish: %+v, %v", got, err) |
| 55 | } |
| 56 | if got, err = setRun(t, s, rec, RunClear("run_a")); err != nil || got.ActiveRun != nil { |
| 57 | t.Fatalf("clear: %+v, %v", got, err) |
| 58 | } |
| 59 | } |
| 60 | |
| 61 | func TestRunCancelBeforeAndAfterPickup(t *testing.T) { |
| 62 | s := release.NewDirStore(t.TempDir()) |
| 63 | rec := runTestRecord(t, s) |
| 64 | at := time.Now().Add(time.Minute) |
| 65 | if _, err := setRun(t, s, rec, RunAdmit(RunAdmission{ID: "run_a", Deadline: at})); err != nil { |
| 66 | t.Fatal(err) |
| 67 | } |
| 68 | got, err := setRun(t, s, rec, RunCancelTransition("run_a")) |
| 69 | if err != nil || got.ActiveRun.State != RunCancelled || got.ActiveRun.Open() { |
| 70 | t.Fatalf("queued cancel: %+v, %v", got, err) |
| 71 | } |
| 72 | if _, err := setRun(t, s, rec, RunPick("run_a", "box-1", "boot-1", at)); !errors.Is(err, ErrRunConflict) { |
| 73 | t.Fatal("a cancelled run was picked (codex R2-6)") |
| 74 | } |
| 75 | if _, err := setRun(t, s, rec, RunAdmit(RunAdmission{ID: "run_b", Deadline: at})); err != nil { |
| 76 | t.Fatalf("a terminal run must not block the next: %v", err) |
| 77 | } |
| 78 | if _, err := setRun(t, s, rec, RunPick("run_b", "box-1", "boot-1", at)); err != nil { |
| 79 | t.Fatal(err) |
| 80 | } |
| 81 | if got, err = setRun(t, s, rec, RunCancelTransition("run_b")); err != nil || got.ActiveRun.State != RunCancel || !got.ActiveRun.Open() { |
| 82 | t.Fatalf("picked cancel: %+v, %v", got, err) |
| 83 | } |
| 84 | if got, err = setRun(t, s, rec, RunCancelTransition("run_b")); err != nil || got.ActiveRun.State != RunCancel { |
| 85 | t.Fatalf("a second cancel is idempotent: %+v, %v", got, err) |
| 86 | } |
| 87 | if got, err = setRun(t, s, rec, RunFinish("run_b")); err != nil || got.ActiveRun.State != RunDone { |
| 88 | t.Fatalf("finish after cancel: %+v, %v", got, err) |
| 89 | } |
| 90 | } |
| 91 | |
| 92 | func TestRunExpireOnlyFromQueued(t *testing.T) { |
| 93 | s := release.NewDirStore(t.TempDir()) |
| 94 | rec := runTestRecord(t, s) |
| 95 | at := time.Now() |
| 96 | if _, err := setRun(t, s, rec, RunAdmit(RunAdmission{ID: "run_a", Deadline: at})); err != nil { |
| 97 | t.Fatal(err) |
| 98 | } |
| 99 | if got, err := setRun(t, s, rec, RunExpire("run_a")); err != nil || got.ActiveRun.State != RunExpired { |
| 100 | t.Fatalf("expire: %+v, %v", got, err) |
| 101 | } |
| 102 | if _, err := setRun(t, s, rec, RunPick("run_a", "late-box", "boot", at)); !errors.Is(err, ErrRunConflict) { |
| 103 | t.Fatal("an expired run was picked by a late box (codex R2-6)") |
| 104 | } |
| 105 | if _, err := setRun(t, s, rec, RunAdmit(RunAdmission{ID: "run_b", Deadline: at})); err != nil { |
| 106 | t.Fatal(err) |
| 107 | } |
| 108 | if _, err := setRun(t, s, rec, RunPick("run_b", "box-1", "boot-1", at)); err != nil { |
| 109 | t.Fatal(err) |
| 110 | } |
| 111 | if _, err := setRun(t, s, rec, RunExpire("run_b")); !errors.Is(err, ErrRunConflict) { |
| 112 | t.Fatal("a picked run was expired: the late pickup must win") |
| 113 | } |
| 114 | } |
| 115 | |
| 116 | func TestRunAdmitRefusesAMoveAndAnUncommittedImport(t *testing.T) { |
| 117 | s := release.NewDirStore(t.TempDir()) |
| 118 | rec := runTestRecord(t, s) |
| 119 | at := time.Now() |
| 120 | if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, MoveRequest("uk", at)); err != nil { |
| 121 | t.Fatal(err) |
| 122 | } |
| 123 | if _, err := setRun(t, s, rec, RunAdmit(RunAdmission{ID: "run_a", Deadline: at})); !errors.Is(err, ErrRunMoving) { |
| 124 | t.Fatalf("admit while moving: %v, want ErrRunMoving", err) |
| 125 | } |
| 126 | // An import walks its path by the seed transitions; admission is tried |
| 127 | // at each state on the way. |
| 128 | s2 := release.NewDirStore(t.TempDir()) |
| 129 | seedID, err := seeds.NewID() |
| 130 | if err != nil { |
| 131 | t.Fatal(err) |
| 132 | } |
| 133 | imp := Record{Schema: 1, Host: "a.go.example.com", Slug: "s", App: "app", Account: "bab", Region: "ie", Channel: "stable", CreatedAt: time.Now().UTC(), |
| 134 | Seed: &SeedSource{ID: seedID, Kind: seeds.KindImport, State: seeds.StateAwaitingUpload}} |
| 135 | if err := CreateRecord(context.Background(), s2, imp); err != nil { |
| 136 | t.Fatal(err) |
| 137 | } |
| 138 | for _, step := range []struct { |
| 139 | mut func(*Record) error |
| 140 | state string |
| 141 | refused bool |
| 142 | }{ |
| 143 | {SeedUploaded(seedID), seeds.StateUploaded, true}, |
| 144 | {SeedStopped(seedID), seeds.StateReady, true}, |
| 145 | {SeedSwapped(seedID), seeds.StateSwapped, true}, |
| 146 | {SeedPlace(seedID), seeds.StatePlaced, true}, |
| 147 | {SeedCommit(seedID), seeds.StateCommitted, false}, |
| 148 | } { |
| 149 | if _, err := SetRecordSeed(context.Background(), s2, imp.Account, imp.App, imp.Host, step.mut); err != nil { |
| 150 | t.Fatalf("to %s: %v", step.state, err) |
| 151 | } |
| 152 | _, err := setRun(t, s2, imp, RunAdmit(RunAdmission{ID: "run_" + step.state, Deadline: at})) |
| 153 | if step.refused != errors.Is(err, ErrRunImport) { |
| 154 | t.Errorf("import %s: err %v, refused want %v (codex R2-2)", step.state, err, step.refused) |
| 155 | } |
| 156 | } |
| 157 | } |
| 158 | |
| 159 | // A move request is refused while a run is open, in both move transitions, |
| 160 | // and the refusal is a move conflict whose text is RUN-C43 (codex 1, R1-1). |
| 161 | func TestMoveTransitionsRefuseWhileARunIsOpen(t *testing.T) { |
| 162 | s := release.NewDirStore(t.TempDir()) |
| 163 | rec := runTestRecord(t, s) |
| 164 | if _, err := setRun(t, s, rec, RunAdmit(RunAdmission{ID: "run_a", Deadline: time.Now()})); err != nil { |
| 165 | t.Fatal(err) |
| 166 | } |
| 167 | inc := strings.Repeat("a", 64) |
| 168 | for name, mut := range map[string]func(*Record) error{ |
| 169 | "legacy": MoveRequest("uk", time.Now()), |
| 170 | "phase 1": MoveStartTransition(MoveStart{ID: "mv_" + strings.Repeat("a", 26), Incarnation: inc, From: "ie", To: "uk", Cut: "auto", Now: time.Now()}), |
| 171 | } { |
| 172 | _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, mut) |
| 173 | var mro *MoveRunOpenError |
| 174 | if !errors.Is(err, ErrMoveConflict) || !errors.As(err, &mro) || !strings.Contains(err.Error(), "a run is open on this instance: run_a") { |
| 175 | t.Errorf("%s: %v", name, err) |
| 176 | } |
| 177 | } |
| 178 | if _, err := setRun(t, s, rec, RunExpire("run_a")); err != nil { |
| 179 | t.Fatal(err) |
| 180 | } |
| 181 | if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, MoveRequest("uk", time.Now())); err != nil { |
| 182 | t.Fatalf("a terminal run must not block a move: %v", err) |
| 183 | } |
| 184 | } |
| 185 | |
| 186 | // Ordinary record writers carry the run through unchanged, as they carry |
| 187 | // Maintenance (run spec §5.1). |
| 188 | func TestOrdinaryWritersCarryActiveRun(t *testing.T) { |
| 189 | s := release.NewDirStore(t.TempDir()) |
| 190 | rec := runTestRecord(t, s) |
| 191 | at := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC) |
| 192 | if _, err := setRun(t, s, rec, RunAdmit(RunAdmission{ID: "run_a", By: "alice@keymail.dev", Deadline: at})); err != nil { |
| 193 | t.Fatal(err) |
| 194 | } |
| 195 | runJSON := func() string { |
| 196 | raw, _, err := s.Get(context.Background(), RecordKey(rec.Account, rec.App, rec.Host)) |
| 197 | if err != nil { |
| 198 | t.Fatal(err) |
| 199 | } |
| 200 | var m map[string]json.RawMessage |
| 201 | if err := json.Unmarshal(raw, &m); err != nil { |
| 202 | t.Fatal(err) |
| 203 | } |
| 204 | return string(m["run"]) |
| 205 | } |
| 206 | want := runJSON() |
| 207 | if !strings.Contains(want, `"state":"queued"`) { |
| 208 | t.Fatalf("run = %s", want) |
| 209 | } |
| 210 | if err := SetRecordRegion(context.Background(), s, rec.Account, rec.App, rec.Host, "uk"); err != nil { |
| 211 | t.Fatal(err) |
| 212 | } |
| 213 | if got := runJSON(); got != want { |
| 214 | t.Fatalf("SetRecordRegion: run %s, want %s", got, want) |
| 215 | } |
| 216 | cur, etag, err := GetRecordETag(context.Background(), s, rec.Account, rec.App, rec.Host) |
| 217 | if err != nil { |
| 218 | t.Fatal(err) |
| 219 | } |
| 220 | cur.Channel = "beta" |
| 221 | if err := UpdateRecord(context.Background(), s, *cur, etag); err != nil { |
| 222 | t.Fatal(err) |
| 223 | } |
| 224 | if got := runJSON(); got != want { |
| 225 | t.Fatalf("UpdateRecord: run %s, want %s", got, want) |
| 226 | } |
| 227 | } |
| 228 | |
| 229 | // A terminal run's id is not admitted again, whatever its state: a box |
| 230 | // retrying RunPick for it must not find it queued (codex R2-6). |
| 231 | func TestRunAdmitRefusesAReusedID(t *testing.T) { |
| 232 | s := release.NewDirStore(t.TempDir()) |
| 233 | rec := runTestRecord(t, s) |
| 234 | at := time.Now().Add(time.Minute) |
| 235 | if _, err := setRun(t, s, rec, RunAdmit(RunAdmission{ID: "run_a", Deadline: at})); err != nil { |
| 236 | t.Fatal(err) |
| 237 | } |
| 238 | if _, err := setRun(t, s, rec, RunCancelTransition("run_a")); err != nil { |
| 239 | t.Fatal(err) |
| 240 | } |
| 241 | if _, err := setRun(t, s, rec, RunAdmit(RunAdmission{ID: "run_a", Deadline: at})); !errors.Is(err, ErrRunConflict) { |
| 242 | t.Fatalf("re-admitting a cancelled run's id: %v, want ErrRunConflict", err) |
| 243 | } |
| 244 | got, _, err := GetRecordETag(context.Background(), s, rec.Account, rec.App, rec.Host) |
| 245 | if err != nil || got.ActiveRun.State != RunCancelled { |
| 246 | t.Fatalf("the cancelled run must be untouched: %+v, %v", got.ActiveRun, err) |
| 247 | } |
| 248 | if _, err := setRun(t, s, rec, RunAdmit(RunAdmission{ID: "run_b", Deadline: at})); err != nil { |
| 249 | t.Fatalf("a new id replaces the terminal run: %v", err) |
| 250 | } |
| 251 | } |
| 252 | |
| 253 | func TestRunPickRefusesMalformedInput(t *testing.T) { |
| 254 | s := release.NewDirStore(t.TempDir()) |
| 255 | rec := runTestRecord(t, s) |
| 256 | at := time.Now().Add(time.Minute) |
| 257 | if _, err := setRun(t, s, rec, RunAdmit(RunAdmission{ID: "run_a", Deadline: at})); err != nil { |
| 258 | t.Fatal(err) |
| 259 | } |
| 260 | for name, mut := range map[string]func(*Record) error{ |
| 261 | "no box": RunPick("run_a", "", "boot-1", at), |
| 262 | "no boot": RunPick("run_a", "box-1", "", at), |
| 263 | "no deadline": RunPick("run_a", "box-1", "boot-1", time.Time{}), |
| 264 | } { |
| 265 | if _, err := setRun(t, s, rec, mut); !errors.Is(err, ErrRunConflict) { |
| 266 | t.Errorf("%s: %v, want ErrRunConflict", name, err) |
| 267 | } |
| 268 | } |
| 269 | got, _, err := GetRecordETag(context.Background(), s, rec.Account, rec.App, rec.Host) |
| 270 | if err != nil || got.ActiveRun.State != RunQueued { |
| 271 | t.Fatalf("a refused pick must leave the run queued: %+v, %v", got.ActiveRun, err) |
| 272 | } |
| 273 | } |
| 274 | |