| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "net/http" |
| 7 | "os" |
| 8 | "path/filepath" |
| 9 | "strings" |
| 10 | "testing" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/hostclaim" |
| 14 | "github.com/carlosframework/platform/internal/provision" |
| 15 | "github.com/carlosframework/platform/internal/registry" |
| 16 | "github.com/carlosframework/platform/internal/release" |
| 17 | "github.com/carlosframework/platform/internal/schedule" |
| 18 | ) |
| 19 | |
| 20 | // statusDeliverer is a schedule.Deliverer answering one status to every tick. |
| 21 | type statusDeliverer int |
| 22 | |
| 23 | func (s statusDeliverer) Deliver(context.Context, registry.Route, *http.Request) (int, error) { |
| 24 | return int(s), nil |
| 25 | } |
| 26 | |
| 27 | const ( |
| 28 | mhHost = "a.go.example.com" |
| 29 | mhID = "mv_abcdefghijklmnopqrstuvwxyz" |
| 30 | mhRbID = "rb_abcdefghijklmnopqrstuvwxyz" |
| 31 | ) |
| 32 | |
| 33 | // mhRecord is a Phase 1 record mid-move from ie to uk, in state with the |
| 34 | // given phases; region is the record's region at that state. |
| 35 | func mhRecord(region, state, cutPhase string) provision.Record { |
| 36 | return provision.Record{Schema: 1, Host: mhHost, App: "app", Account: "bab", Region: region, Channel: "stable", |
| 37 | Move: &provision.MoveState{ID: mhID, From: "ie", To: "uk", State: state, CutPhase: cutPhase, |
| 38 | Rollback: &provision.MoveRollback{ID: mhRbID}}} |
| 39 | } |
| 40 | |
| 41 | // mhClaims is a claim reader answering kind with participants ie-box and |
| 42 | // uk-box for mhHost; err makes every read fail. |
| 43 | func mhClaims(kind string, err error) func(context.Context, release.Store, string) (hostclaim.Observed, error) { |
| 44 | return func(_ context.Context, _ release.Store, host string) (hostclaim.Observed, error) { |
| 45 | if err != nil { |
| 46 | return hostclaim.Observed{}, err |
| 47 | } |
| 48 | return hostclaim.Observed{Host: host, Claim: &hostclaim.Claim{Host: host, |
| 49 | Op: hostclaim.Op{Kind: kind, ID: mhID, Participants: []string{"ie-box", "uk-box"}}}}, nil |
| 50 | } |
| 51 | } |
| 52 | |
| 53 | func mhView(box, region, kind string, err error, recs ...provision.Record) *movesHoldView { |
| 54 | v := newMovesHoldView(box, region, mhClaims(kind, err), nil) |
| 55 | v.refresh(context.Background(), nil, recs) |
| 56 | return v |
| 57 | } |
| 58 | |
| 59 | // hold's table (moves spec §5.2, §9.6, §19.1): the target holds its row |
| 60 | // landing after the flip, the source moving from stopping, restoring in a |
| 61 | // revert or a rollback, and the target releasing in a rollback. A box the |
| 62 | // claim does not name, and any state with nothing held, answers no hold. |
| 63 | func TestMovesHoldViewHold(t *testing.T) { |
| 64 | cases := []struct { |
| 65 | name string |
| 66 | box, region, kind string |
| 67 | rec provision.Record |
| 68 | wantStatus, wantOp string |
| 69 | wantOK bool |
| 70 | }{ |
| 71 | {"source preparing", "ie-box", "ie", hostclaim.KindMoving, mhRecord("ie", provision.MovePreparing, ""), "", "", false}, |
| 72 | {"source cut requested", "ie-box", "ie", hostclaim.KindMoving, mhRecord("ie", provision.MoveCutting, provision.CutRequested), "", "", false}, |
| 73 | {"source cut admitted", "ie-box", "ie", hostclaim.KindMoving, mhRecord("ie", provision.MoveCutting, provision.CutAdmitted), "", "", false}, |
| 74 | {"source stopping", "ie-box", "ie", hostclaim.KindMoving, mhRecord("ie", provision.MoveCutting, provision.CutStopping), "moving", mhID, true}, |
| 75 | {"source archived", "ie-box", "ie", hostclaim.KindMoving, mhRecord("ie", provision.MoveCutting, provision.CutArchived), "moving", mhID, true}, |
| 76 | {"source after the flip", "ie-box", "ie", hostclaim.KindLanding, mhRecord("uk", provision.MoveLanding, provision.CutFlipped), "moving", mhID, true}, |
| 77 | {"target after the flip", "uk-box", "uk", hostclaim.KindLanding, mhRecord("uk", provision.MoveLanding, provision.CutFlipped), "landing", mhID, true}, |
| 78 | {"target committed", "uk-box", "uk", hostclaim.KindLanding, mhRecord("uk", provision.MoveCommitted, provision.CutFlipped), "", "", false}, |
| 79 | {"source reverting", "ie-box", "ie", hostclaim.KindReverting, mhRecord("ie", provision.MoveReverting, provision.CutStopped), "restoring", mhID, true}, |
| 80 | {"source aborting a revert", "ie-box", "ie", hostclaim.KindReverting, mhRecord("ie", provision.MoveAborting, ""), "restoring", mhID, true}, |
| 81 | {"source aborting before the cut", "ie-box", "ie", hostclaim.KindMoving, mhRecord("ie", provision.MoveAborting, ""), "", "", false}, |
| 82 | {"source rolling back", "ie-box", "ie", hostclaim.KindRestoring, mhRecord("ie", provision.MoveRollingBack, provision.CutFlipped), "restoring", mhRbID, true}, |
| 83 | {"target rolling back", "uk-box", "uk", hostclaim.KindRestoring, mhRecord("ie", provision.MoveRollingBack, provision.CutFlipped), "releasing", mhRbID, true}, |
| 84 | {"target stuck", "uk-box", "uk", hostclaim.KindRestoring, mhRecord("ie", provision.MoveStuck, provision.CutFlipped), "releasing", mhRbID, true}, |
| 85 | {"not a participant", "uk-box-2", "uk", hostclaim.KindLanding, mhRecord("uk", provision.MoveLanding, provision.CutFlipped), "", "", false}, |
| 86 | } |
| 87 | for _, c := range cases { |
| 88 | t.Run(c.name, func(t *testing.T) { |
| 89 | v := mhView(c.box, c.region, c.kind, nil, c.rec) |
| 90 | status, op, ok := v.hold(c.rec) |
| 91 | if status != c.wantStatus || op != c.wantOp || ok != c.wantOK { |
| 92 | t.Fatalf("hold = (%q, %q, %v), want (%q, %q, %v)", status, op, ok, c.wantStatus, c.wantOp, c.wantOK) |
| 93 | } |
| 94 | }) |
| 95 | } |
| 96 | // No move, or a legacy one: nothing held. |
| 97 | v := mhView("uk-box", "uk", hostclaim.KindLanding, nil) |
| 98 | if _, _, ok := v.hold(provision.Record{Host: mhHost}); ok { |
| 99 | t.Fatal("a record at rest is held") |
| 100 | } |
| 101 | if _, _, ok := v.hold(provision.Record{Host: mhHost, Move: &provision.MoveState{To: "uk", State: provision.MoveCutting}}); ok { |
| 102 | t.Fatal("a legacy move is held") |
| 103 | } |
| 104 | } |
| 105 | |
| 106 | // Without a claim answer for the host (the view was built before the move, |
| 107 | // or its claim would not read) hold falls back to the record's regions, and |
| 108 | // never answers "no hold" for a state that holds: the row is never minted |
| 109 | // ready. A rollback with no rollback id holds under the move id. |
| 110 | func TestMovesHoldViewHoldFailsClosedWithoutTheClaim(t *testing.T) { |
| 111 | stale := mhView("uk-box", "uk", hostclaim.KindLanding, nil) // built from a list with no move in it |
| 112 | rec := mhRecord("uk", provision.MoveLanding, provision.CutFlipped) |
| 113 | if status, op, ok := stale.hold(rec); status != "landing" || op != mhID || !ok { |
| 114 | t.Fatalf("stale view: hold = (%q, %q, %v), want landing by the move", status, op, ok) |
| 115 | } |
| 116 | unread := mhView("ie-box", "ie", "", errors.New("store down"), mhRecord("ie", provision.MoveCutting, provision.CutStopping)) |
| 117 | if status, _, ok := unread.hold(mhRecord("ie", provision.MoveCutting, provision.CutStopping)); status != "moving" || !ok { |
| 118 | t.Fatalf("unreadable claim: hold = (%q, %v), want moving", status, ok) |
| 119 | } |
| 120 | noRb := mhRecord("ie", provision.MoveRollingBack, provision.CutFlipped) |
| 121 | noRb.Move.Rollback = nil |
| 122 | if status, op, ok := stale.hold(noRb); status != "releasing" || op != mhID || !ok { |
| 123 | t.Fatalf("rollback with no id: hold = (%q, %q, %v), want releasing by the move", status, op, ok) |
| 124 | } |
| 125 | never := newMovesHoldView("uk-box", "uk", mhClaims(hostclaim.KindLanding, nil), nil) // never built |
| 126 | if status, _, ok := never.hold(rec); status != "landing" || !ok { |
| 127 | t.Fatalf("unbuilt view: hold = (%q, %v), want landing", status, ok) |
| 128 | } |
| 129 | } |
| 130 | |
| 131 | // keep and frozen: a move-kind op freezes the host, and keeps it on a box |
| 132 | // the op names as a participant. A claim that is serving does neither. Fail |
| 133 | // closed: before the first build, and for a host whose claim would not read, |
| 134 | // both answer true. |
| 135 | func TestMovesHoldViewKeepAndFrozen(t *testing.T) { |
| 136 | rec := mhRecord("uk", provision.MoveLanding, provision.CutFlipped) |
| 137 | cases := []struct { |
| 138 | name, box, kind string |
| 139 | err error |
| 140 | keep, frozen bool |
| 141 | }{ |
| 142 | {"moving, source", "ie-box", hostclaim.KindMoving, nil, true, true}, |
| 143 | {"landing, source", "ie-box", hostclaim.KindLanding, nil, true, true}, |
| 144 | {"landing, target", "uk-box", hostclaim.KindLanding, nil, true, true}, |
| 145 | {"reverting, target", "uk-box", hostclaim.KindReverting, nil, true, true}, |
| 146 | {"restoring, target", "uk-box", hostclaim.KindRestoring, nil, true, true}, |
| 147 | {"landing, not a participant", "uk-box-2", hostclaim.KindLanding, nil, false, true}, |
| 148 | {"serving", "ie-box", hostclaim.KindServing, nil, false, false}, |
| 149 | {"relocating is region HA's", "ie-box", hostclaim.KindRelocating, nil, false, false}, |
| 150 | {"unreadable claim", "ie-box", "", errors.New("store down"), true, true}, |
| 151 | } |
| 152 | for _, c := range cases { |
| 153 | t.Run(c.name, func(t *testing.T) { |
| 154 | v := mhView(c.box, "", c.kind, c.err, rec) |
| 155 | if got := v.keep(mhHost); got != c.keep { |
| 156 | t.Errorf("keep = %v, want %v", got, c.keep) |
| 157 | } |
| 158 | if got := v.frozen(mhHost); got != c.frozen { |
| 159 | t.Errorf("frozen = %v, want %v", got, c.frozen) |
| 160 | } |
| 161 | if v.keep("other.go.example.com") || v.frozen("other.go.example.com") { |
| 162 | t.Error("a host with no move is kept or frozen") |
| 163 | } |
| 164 | }) |
| 165 | } |
| 166 | never := newMovesHoldView("ie-box", "ie", mhClaims(hostclaim.KindServing, nil), nil) |
| 167 | if !never.keep("any.go.example.com") || !never.frozen("any.go.example.com") { |
| 168 | t.Fatal("an unbuilt view does not fail closed") |
| 169 | } |
| 170 | // A later build that cannot list keeps the last answer: refresh is only |
| 171 | // called with a list, so a failed list is no refresh at all. |
| 172 | built := mhView("ie-box", "ie", hostclaim.KindMoving, nil, mhRecord("ie", provision.MoveCutting, provision.CutStopping)) |
| 173 | if !built.keep(mhHost) { |
| 174 | t.Fatal("the participant is not kept") |
| 175 | } |
| 176 | built.refresh(context.Background(), nil, nil) |
| 177 | if built.keep(mhHost) || built.frozen(mhHost) { |
| 178 | t.Fatal("a host whose move ended is still kept or frozen") |
| 179 | } |
| 180 | } |
| 181 | |
| 182 | // The view against a real claim in a store, through hostclaim.Read: the |
| 183 | // door's moving claim names ie-box and uk-box. |
| 184 | func TestMovesHoldViewReadsTheClaim(t *testing.T) { |
| 185 | ctx := context.Background() |
| 186 | s := release.NewDirStore(t.TempDir()) |
| 187 | created := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) |
| 188 | inc, err := hostclaim.Incarnation("bab", "app", mhHost, created) |
| 189 | if err != nil { |
| 190 | t.Fatal(err) |
| 191 | } |
| 192 | obs, err := hostclaim.Update(ctx, s, hostclaim.Observed{Host: mhHost, Absent: true}, func(c *hostclaim.Claim) error { |
| 193 | c.Holder = &hostclaim.Holder{Family: "instance", Key: provision.RecordKey("bab", "app", mhHost), Account: "bab", App: "app", |
| 194 | Incarnation: inc, CreatedAt: created, Location: "instances/" + mhHost + "/" + inc, Generation: "cur", Stamp: 1} |
| 195 | c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "ie-box"} |
| 196 | c.Touched = hostclaim.Touched{Complete: true, Regions: []string{"ie"}, Boxes: []string{"ie-box"}} |
| 197 | return nil |
| 198 | }) |
| 199 | if err != nil { |
| 200 | t.Fatal(err) |
| 201 | } |
| 202 | if _, err := hostclaim.Update(ctx, s, obs, hostclaim.BeginMove(mhID, "op", "ie-box", "uk-box", "uk", created)); err != nil { |
| 203 | t.Fatal(err) |
| 204 | } |
| 205 | rec := mhRecord("ie", provision.MoveCutting, provision.CutStopping) |
| 206 | ie := newMovesHoldView("ie-box", "ie", hostclaim.Read, nil) |
| 207 | ie.refresh(ctx, s, []provision.Record{rec}) |
| 208 | if !ie.keep(mhHost) || !ie.frozen(mhHost) { |
| 209 | t.Fatal("the source is not kept and frozen under the door's moving claim") |
| 210 | } |
| 211 | if status, op, ok := ie.hold(rec); status != "moving" || op != mhID || !ok { |
| 212 | t.Fatalf("hold = (%q, %q, %v), want moving by the move", status, op, ok) |
| 213 | } |
| 214 | other := newMovesHoldView("ie-box-2", "ie", hostclaim.Read, nil) |
| 215 | other.refresh(ctx, s, []provision.Record{rec}) |
| 216 | if other.keep(mhHost) { |
| 217 | t.Fatal("a box the claim does not name is kept") |
| 218 | } |
| 219 | } |
| 220 | |
| 221 | // The drain reaches the schedule wiring's gate once armScheduleHolds has |
| 222 | // run, and is a no-op before (no scheduled work armed). |
| 223 | func TestMovesHoldViewDrainsTheScheduleGate(t *testing.T) { |
| 224 | v := newMovesHoldView("ie-box", "ie", mhClaims(hostclaim.KindMoving, nil), nil) |
| 225 | if err := v.drainTimers(context.Background(), mhHost); err != nil { |
| 226 | t.Fatalf("drain with no schedule armed = %v", err) |
| 227 | } |
| 228 | sd := &scheduleDeps{runner: &schedule.Runner{}, conv: &schedule.Converger{}} |
| 229 | armScheduleHolds(sd, v) |
| 230 | if sd.runner.Gate == nil || sd.conv.Gate != sd.runner.Gate || sd.conv.Frozen == nil { |
| 231 | t.Fatalf("the schedule halves do not share one gate and the freeze: %+v %+v", sd.runner, sd.conv) |
| 232 | } |
| 233 | if v.gate.Load() != sd.runner.Gate { |
| 234 | t.Fatal("the view does not drain the runner's gate") |
| 235 | } |
| 236 | if err := v.drainTimers(context.Background(), mhHost); err != nil { |
| 237 | t.Fatalf("drain = %v", err) |
| 238 | } |
| 239 | } |
| 240 | |
| 241 | // The switch off (moves.Enabled false) is main: no hold view, so reconcile |
| 242 | // gets no Hold, KeepParticipant or HoldWritten, and the schedule halves no |
| 243 | // gate and no freeze. |
| 244 | func TestMovesHoldsAreMainWithTheSwitchOff(t *testing.T) { |
| 245 | prev := armedMovesWorker.Load() |
| 246 | t.Cleanup(func() { armedMovesWorker.Store(prev) }) |
| 247 | off := &movesWorker{cfg: moveCfg{Enabled: false}, holds: newMovesHoldView("ie-box", "ie", hostclaim.Read, nil)} |
| 248 | armedMovesWorker.Store(off) |
| 249 | if armedHoldView() != nil { |
| 250 | t.Fatal("a hold view with the switch off") |
| 251 | } |
| 252 | cfg := withMovesHolds(provision.ReconcileConfig{}) |
| 253 | if cfg.Hold != nil || cfg.KeepParticipant != nil || cfg.HoldWritten != nil { |
| 254 | t.Fatal("reconcile hooks with the switch off") |
| 255 | } |
| 256 | |
| 257 | // The schedule halves, wired as the agent wires them with the switch |
| 258 | // off, against a row whose status is a hold: the runner delivers its |
| 259 | // due timer and records the outcome, and the converger converges its |
| 260 | // declaration, exactly as main does (fix round 1: behaviour, not the |
| 261 | // absence of hooks). |
| 262 | ctx := context.Background() |
| 263 | reg, err := registry.Open(filepath.Join(t.TempDir(), "r.db")) |
| 264 | if err != nil { |
| 265 | t.Fatal(err) |
| 266 | } |
| 267 | t.Cleanup(func() { reg.Close() }) |
| 268 | const host = "held.bab.oncarlos.com" |
| 269 | if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", Account: "bab", App: "app", Status: "ready"}); err != nil { |
| 270 | t.Fatal(err) |
| 271 | } |
| 272 | if err := reg.UpsertTimer(registry.Timer{Host: host, Name: "remind", Kind: schedule.KindOnce, Spec: "at 1", Path: "/r", NextAt: 1, DueAt: 1}); err != nil { |
| 273 | t.Fatal(err) |
| 274 | } |
| 275 | if err := reg.SetHold(host, "moving", mhID); err != nil { |
| 276 | t.Fatal(err) |
| 277 | } |
| 278 | now := time.Date(2026, 8, 23, 13, 7, 0, 0, time.UTC) |
| 279 | st := release.NewDirStore(t.TempDir()) |
| 280 | if _, err := schedule.UpsertSchedule(ctx, st, "bab", "app", schedule.Schedule{Name: "sync", Every: "6h", Path: "/jobs/sync"}, now); err != nil { |
| 281 | t.Fatal(err) |
| 282 | } |
| 283 | runner := &schedule.Runner{Reg: reg, Deliver: statusDeliverer(503), Now: func() time.Time { return now }, Timeout: time.Minute, |
| 284 | Box: "ie-box", Build: "test", Token: func(string) (string, error) { return "tok", nil }} |
| 285 | conv := schedule.NewConverger(reg, st, "ie-box", nil) |
| 286 | conv.Now = func() time.Time { return now } |
| 287 | sd := &scheduleDeps{runner: runner, conv: conv} |
| 288 | armScheduleHolds(sd, armedHoldView()) |
| 289 | if runner.Gate != nil || conv.Gate != nil || conv.Frozen != nil { |
| 290 | t.Fatal("schedule hooks with the switch off") |
| 291 | } |
| 292 | runner.Tick(ctx) |
| 293 | // Tick launches the delivery and returns: wait for the record, then |
| 294 | // for the host's goroutine to let go. |
| 295 | var tm *registry.Timer |
| 296 | for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(time.Millisecond) { |
| 297 | if tm, _ = reg.GetTimer(host, "remind"); tm != nil && tm.Attempts == 1 && !runner.Busy(host) { |
| 298 | break |
| 299 | } |
| 300 | } |
| 301 | if tm == nil || tm.Attempts != 1 || tm.LastStatus != "503" { |
| 302 | t.Fatalf("with the switch off a held row's due timer was not delivered and recorded as main does: %+v", tm) |
| 303 | } |
| 304 | if err := conv.Pass(ctx); err != nil { |
| 305 | t.Fatal(err) |
| 306 | } |
| 307 | if _, err := reg.GetTimer(host, "sync"); err != nil { |
| 308 | t.Fatalf("with the switch off the held host's declaration did not converge: %v", err) |
| 309 | } |
| 310 | |
| 311 | armedMovesWorker.Store(nil) |
| 312 | if cfg := withMovesHolds(provision.ReconcileConfig{}); cfg.Hold != nil { |
| 313 | t.Fatal("reconcile hooks with no worker armed") |
| 314 | } |
| 315 | |
| 316 | on := &movesWorker{cfg: moveCfg{Enabled: true}, holds: newMovesHoldView("ie-box", "ie", hostclaim.Read, func(string) {})} |
| 317 | armedMovesWorker.Store(on) |
| 318 | cfg = withMovesHolds(provision.ReconcileConfig{}) |
| 319 | if cfg.Hold == nil || cfg.KeepParticipant == nil || cfg.HoldWritten == nil { |
| 320 | t.Fatal("no reconcile hooks with the switch on") |
| 321 | } |
| 322 | } |
| 323 | |
| 324 | // Ruling J3 (fix round 1): the hand-over of a move's hold to its rollback |
| 325 | // touches the stage-requested marker when it lands, and only then. |
| 326 | func TestHandHoldTouchesTheStageMarkerOnlyWhenItLands(t *testing.T) { |
| 327 | const rb = "rb_abcdefghijklmnopqrstuvwxyz" |
| 328 | reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 329 | if err != nil { |
| 330 | t.Fatal(err) |
| 331 | } |
| 332 | defer reg.Close() |
| 333 | if err := reg.Upsert(registry.Route{Host: p1Host, Kind: "instance", App: "app", Account: "bab", Status: "ready", Provisioned: true}); err != nil { |
| 334 | t.Fatal(err) |
| 335 | } |
| 336 | if err := reg.SetHold(p1Host, "landing", p1ID); err != nil { |
| 337 | t.Fatal(err) |
| 338 | } |
| 339 | marker := func(dd string) bool { |
| 340 | _, err := os.Stat(filepath.Join(dd, stageRequestMarker)) |
| 341 | return err == nil |
| 342 | } |
| 343 | |
| 344 | // A refusal: the row is held by p1ID, not p1ID2. Nothing moves, no marker. |
| 345 | refused := t.TempDir() |
| 346 | if ok, err := handHold(reg, holdStageRequester(refused))(p1Host, "releasing", rb, p1ID2, nil); err != nil || ok || marker(refused) { |
| 347 | t.Fatalf("refused hand-over: %v %v, marker %v", ok, err, marker(refused)) |
| 348 | } |
| 349 | if rt, _ := reg.Get(p1Host); rt.Status != "landing" || rt.HoldOp != p1ID { |
| 350 | t.Fatalf("row changed by a refusal: %+v", rt) |
| 351 | } |
| 352 | // An absent row: no marker. |
| 353 | if ok, err := handHold(reg, holdStageRequester(refused))("absent.example.com", "releasing", rb, p1ID, nil); err != nil || ok || marker(refused) { |
| 354 | t.Fatalf("absent row: %v %v, marker %v", ok, err, marker(refused)) |
| 355 | } |
| 356 | |
| 357 | // The hand-over: the row is the rollback's, and the marker is written. |
| 358 | landed := t.TempDir() |
| 359 | if ok, err := handHold(reg, holdStageRequester(landed))(p1Host, "releasing", rb, p1ID, nil); err != nil || !ok || !marker(landed) { |
| 360 | t.Fatalf("hand-over: %v %v, marker %v", ok, err, marker(landed)) |
| 361 | } |
| 362 | if rt, _ := reg.Get(p1Host); rt.Status != "releasing" || rt.HoldOp != rb { |
| 363 | t.Fatalf("after the hand-over: %+v", rt) |
| 364 | } |
| 365 | |
| 366 | // A replay finds the row already the rollback's: done, nothing written. |
| 367 | var touched []string |
| 368 | if ok, err := handHold(reg, func(h string) { touched = append(touched, h) })(p1Host, "releasing", rb, p1ID, nil); err != nil || !ok || len(touched) != 0 { |
| 369 | t.Fatalf("replay: %v %v, marker touched %d times", ok, err, len(touched)) |
| 370 | } |
| 371 | } |
| 372 | |
| 373 | // Task 4 fix round 1, ruling 1: the hand-over's caller validates its own |
| 374 | // record under the hold lock, after the row is re-read and before the hold |
| 375 | // moves. A validation that refuses moves nothing and writes no marker; one |
| 376 | // that passes is asked once, and the hold moves. The lock is the wrapper's |
| 377 | // alone: the callback runs inside it and must not take it again. |
| 378 | func TestHandHoldValidatesUnderTheLock(t *testing.T) { |
| 379 | const rb = "rb_abcdefghijklmnopqrstuvwxyz" |
| 380 | reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 381 | if err != nil { |
| 382 | t.Fatal(err) |
| 383 | } |
| 384 | defer reg.Close() |
| 385 | if err := reg.Upsert(registry.Route{Host: p1Host, Kind: "instance", App: "app", Account: "bab", Status: "ready", Provisioned: true}); err != nil { |
| 386 | t.Fatal(err) |
| 387 | } |
| 388 | if err := reg.SetHold(p1Host, "landing", p1ID); err != nil { |
| 389 | t.Fatal(err) |
| 390 | } |
| 391 | changed := errors.New("the record is no longer this rollback's release") |
| 392 | var seen []string |
| 393 | refused := t.TempDir() |
| 394 | ok, err := handHold(reg, holdStageRequester(refused))(p1Host, "releasing", rb, p1ID, func() error { |
| 395 | rt, _ := reg.Get(p1Host) |
| 396 | seen = append(seen, rt.Status+"/"+rt.HoldOp) |
| 397 | return changed |
| 398 | }) |
| 399 | if !errors.Is(err, changed) || ok { |
| 400 | t.Fatalf("a refused validation: %v %v", ok, err) |
| 401 | } |
| 402 | if _, err := os.Stat(filepath.Join(refused, stageRequestMarker)); err == nil { |
| 403 | t.Fatal("a refused validation wrote the marker") |
| 404 | } |
| 405 | if rt, _ := reg.Get(p1Host); rt.Status != "landing" || rt.HoldOp != p1ID { |
| 406 | t.Fatalf("the hold moved under a refused validation: %+v", rt) |
| 407 | } |
| 408 | landed := t.TempDir() |
| 409 | if ok, err := handHold(reg, holdStageRequester(landed))(p1Host, "releasing", rb, p1ID, func() error { |
| 410 | rt, _ := reg.Get(p1Host) |
| 411 | seen = append(seen, rt.Status+"/"+rt.HoldOp) |
| 412 | return nil |
| 413 | }); err != nil || !ok { |
| 414 | t.Fatalf("a passing validation: %v %v", ok, err) |
| 415 | } |
| 416 | if rt, _ := reg.Get(p1Host); rt.Status != "releasing" || rt.HoldOp != rb { |
| 417 | t.Fatalf("after the hand-over: %+v", rt) |
| 418 | } |
| 419 | if strings.Join(seen, " ") != "landing/"+p1ID+" landing/"+p1ID { |
| 420 | t.Fatalf("validations saw %v, want the row before the hand-over, once each", seen) |
| 421 | } |
| 422 | } |
| 423 | |
amadan