amadan

carlos / platform Public

console: tests for per-box reservations, the reclaim rule and bound operator admission

TestReserveWritesEveryParticipantAndTheProjection, TestReserveRefusesAHeld-
Participant, TestOwnRetryNeedsTheCompleteSelection, TestSentinelReservation-
ConflictsBothWays, TestDisjointBoxesReserveAtOnceWithTheSwitch,
TestWithoutTheSwitchOneReservationAtATime, TestLegacyLockBodyIsNormalised,
TestMalformedEntryIsUnreadable, TestReclaimOnlyOnEvidence (Review Focus 2),
TestReleaseRemovesOnlyItsEntriesAndDeletesWhenEmpty,
TestOldConsoleAlgorithmsAgainstTheNewBody, TestOldConsoleReleaseErasesATable,
TestSwitchOffRecoveryReacquiresItsOwnReservation,
TestOperatorDoorsUseReservations.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
claude e42fdadbc35e7c921550283c455a728c347ec87e parent f6e695c
1 file changed, +1012 −0
  • internal/console/offlinereserve_test.go +1012 −0
diff --git a/internal/console/offlinereserve_test.go b/internal/console/offlinereserve_test.go
new file mode 100644
index 000000000..d77cfe9b7
--- /dev/null
+++ b/internal/console/offlinereserve_test.go
@@ -0,0 +1,1012 @@
+package console
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log"
+ "net/http"
+ "os"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/moves"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// Per-box reservations (self-serve moves plan 2 Task 3; spec §7.1):
+// active.json as a table keyed by box id with a legacy projection and the
+// sentinel, reserveBoxes' six steps in order, releaseBoxes, the reclaim rule
+// (Review Focus 2) and the operator doors' predecessor-bound admission.
+
+const (
+ betaHost = "beta.shop.example.com"
+ mvA = "mv_aaaaaaaaaaaaaaaaaaaaaaaaaa"
+ mvB = "mv_bbbbbbbbbbbbbbbbbbbbbbbbbb"
+ mvC = "mv_cccccccccccccccccccccccccc"
+)
+
+var (
+ refBusyAcme = &offlineRefusal{status: http.StatusConflict, msg: offlineHost + " is moving offline. move one instance at a time"}
+ refBusyBeta = &offlineRefusal{status: http.StatusConflict, msg: betaHost + " is moving offline. move one instance at a time"}
+ refMismatch = &offlineRefusal{status: http.StatusConflict, msg: "the reservation does not match this move's participants"}
+ refUnread = &offlineRefusal{status: http.StatusBadGateway, msg: "could not read the instance"}
+)
+
+// selOf is a self-serve-shaped selection: every participant pinned with an
+// instance id, the last box the target.
+func selOf(boxes ...string) offlineSelection {
+ var sel offlineSelection
+ for i, b := range boxes {
+ p := provision.Participant{BoxID: b, Label: b, InstanceID: "i-" + b}
+ if i == len(boxes)-1 {
+ sel.Target = p
+ } else {
+ sel.Source = append(sel.Source, p)
+ }
+ }
+ return sel
+}
+
+// sentinelSel is an operator selection: no instance ids, so the sentinel.
+func sentinelSel() offlineSelection {
+ return offlineSelection{Source: []provision.Participant{{BoxID: "ie-box", Label: "ie-box"}},
+ Target: provision.Participant{BoxID: "uk-box", Label: "uk-box"}, Sentinel: true}
+}
+
+func mres(id, host string, age time.Duration) moves.OfflineReservation {
+ return moves.OfflineReservation{Kind: moves.OfflineKindMove, ID: id, Host: host, Account: "bab", App: "shop",
+ ReservedAt: time.Now().UTC().Add(-age).Truncate(time.Second)}
+}
+
+// newReserveWorld is the offline world with a second instance, beta.
+func newReserveWorld(t *testing.T) *offlineWorld {
+ t.Helper()
+ w := newOfflineWorld(t, oneEach...)
+ w.create(betaHost)
+ return w
+}
+
+func (w *offlineWorld) create(host string) {
+ w.t.Helper()
+ if c, _, out := rawReq(w.t, http.MethodPost, instancesURL(w.fx, "bab", "shop"), w.token, `{"host":"`+host+`","region":"ie"}`); c != http.StatusCreated {
+ w.t.Fatalf("create %s = %d %s", host, c, out)
+ }
+}
+
+func (w *offlineWorld) reserve(sel offlineSelection, r moves.OfflineReservation) *offlineRefusal {
+ return w.fx.srv.reserveBoxes(context.Background(), sel, r)
+}
+
+func sameRefusal(got, want *offlineRefusal) bool {
+ if got == nil || want == nil {
+ return got == want
+ }
+ return *got == *want
+}
+
+func (w *offlineWorld) wantReserve(sel offlineSelection, r moves.OfflineReservation, want *offlineRefusal) {
+ w.t.Helper()
+ if got := w.reserve(sel, r); !sameRefusal(got, want) {
+ w.t.Fatalf("reserve %s = %+v, want %+v", r.ID, got, want)
+ }
+}
+
+// activeRaw is active.json's bytes, "" when absent.
+func (w *offlineWorld) activeRaw() string {
+ w.t.Helper()
+ raw, _, err := w.fx.rel.Get(context.Background(), moves.OfflineActiveKey)
+ if errors.Is(err, os.ErrNotExist) {
+ return ""
+ }
+ if err != nil {
+ w.t.Fatal(err)
+ }
+ return string(raw)
+}
+
+// table is active.json's reservations (nil when absent).
+func (w *offlineWorld) table() map[string]moves.OfflineReservation {
+ w.t.Helper()
+ raw := w.activeRaw()
+ if raw == "" {
+ return nil
+ }
+ tb, err := moves.ParseOfflineActive([]byte(raw))
+ if err != nil {
+ w.t.Fatal(err)
+ }
+ return tb
+}
+
+func (w *offlineWorld) putTable(tb map[string]moves.OfflineReservation) {
+ w.t.Helper()
+ body, err := json.Marshal(moves.OfflineActiveBody(tb))
+ if err != nil {
+ w.t.Fatal(err)
+ }
+ offlinePut(w.t, w.fx.rel, moves.OfflineActiveKey, body)
+}
+
+// backdate makes every reservation older than the grace period.
+func (w *offlineWorld) backdate() {
+ w.t.Helper()
+ tb := w.table()
+ for k, e := range tb {
+ e.ReservedAt = e.ReservedAt.Add(-time.Hour)
+ tb[k] = e
+ }
+ w.putTable(tb)
+}
+
+func (w *offlineWorld) recordRaw(host string) string {
+ w.t.Helper()
+ raw, _, err := w.fx.rel.Get(context.Background(), provision.RecordKey("bab", "shop", host))
+ if err != nil {
+ w.t.Fatal(err)
+ }
+ return string(raw)
+}
+
+// queueOn puts a queued move with id on host's record, as the job's queue
+// write would.
+func (w *offlineWorld) queueOn(host, id string) {
+ w.t.Helper()
+ rec, err := provision.GetRecord(context.Background(), w.fx.rel, "bab", "shop", host)
+ if err != nil {
+ w.t.Fatal(err)
+ }
+ pin := provision.Participants{Source: []provision.Participant{{BoxID: "ie-box", Label: "ie-box"}}, Target: provision.Participant{BoxID: "uk-box", Label: "uk-box"}}
+ if _, err := provision.SetRecordMove(context.Background(), w.fx.rel, "bab", "shop", host,
+ provision.OfflineQueue(id, "ie", "uk", "owner@example.com", provision.StageTarget{Bucket: "data-uk"}, pin,
+ provision.SelfServeMove{ID: "ss_" + id, Incarnation: rec.Incarnation}, provision.ProposeStamp(rec, time.Now()), provision.Receipt{Nonce: "n-" + id})); err != nil {
+ w.t.Fatal(err)
+ }
+}
+
+// fakeRows is the self-serve rows' evidence (Task 4's tables): a move id
+// maps to whether its row is terminal and released; an absent id has no
+// row. issued names hosts with a request issued. err fails every read.
+type fakeRows struct {
+ terminal map[string]bool
+ issued map[string]bool
+ err error
+}
+
+func (f fakeRows) MoveTerminalOrAbsent(id string) (bool, error) {
+ if f.err != nil {
+ return false, f.err
+ }
+ t, ok := f.terminal[id]
+ return !ok || t, nil
+}
+
+func (f fakeRows) HostHasIssuedRequest(acct, app, host string) (bool, error) {
+ if f.err != nil {
+ return false, f.err
+ }
+ return f.issued[host], nil
+}
+
+// fakeCleanup is the owning box's status evidence for a cleanup (plan 4).
+type fakeCleanup struct {
+ settled bool
+ err error
+}
+
+func (f fakeCleanup) CleanupSettled(ctx context.Context, boxID, id string) (bool, error) {
+ return f.settled, f.err
+}
+
+// ---- the table ------------------------------------------------------------
+
+func TestReserveWritesEveryParticipantAndTheProjection(t *testing.T) {
+ w := newReserveWorld(t)
+ w.fx.srv.cfg.SelfServeMoves = "offline"
+ r := mres(mvA, offlineHost, 0)
+ r.ExpectSeq, r.ExpectIncarnation = 4, 2
+ w.wantReserve(selOf("ie-1", "ie-2", "uk-1"), r, nil)
+ var body moves.OfflineActive
+ if err := json.Unmarshal([]byte(w.activeRaw()), &body); err != nil {
+ t.Fatal(err)
+ }
+ if body.V != moves.OfflineActiveVersion || len(body.Reservations) != 3 {
+ t.Fatalf("body = %+v", body)
+ }
+ for _, k := range []string{"ie-1", "ie-2", "uk-1"} {
+ if body.Reservations[k] != r {
+ t.Fatalf("%s = %+v, want %+v", k, body.Reservations[k], r)
+ }
+ }
+ if body.Host != offlineHost || body.Op != mvA || body.Account != "bab" || body.App != "shop" || !body.At.Equal(r.ReservedAt) {
+ t.Fatalf("projection = %+v", body)
+ }
+ // A reservation stamps reserved_at when the caller leaves it zero.
+ r2 := mres(mvB, betaHost, 0)
+ r2.ReservedAt = time.Time{}
+ w.wantReserve(selOf("ie-3", "uk-3"), r2, nil)
+ if got := w.table()["uk-3"].ReservedAt; got.IsZero() || time.Since(got) > time.Minute {
+ t.Fatalf("reserved_at = %v", got)
+ }
+ // The projection stays the oldest move.
+ if a, _ := w.lock(); a.Op != mvA {
+ t.Fatalf("projection after a second move = %+v", a)
+ }
+}
+
+func TestReserveRefusesAHeldParticipant(t *testing.T) {
+ w := newReserveWorld(t)
+ w.fx.srv.cfg.SelfServeMoves = "offline"
+ w.wantReserve(selOf("ie-1", "uk-1"), mres(mvA, offlineHost, 0), nil)
+ before := w.activeRaw()
+ w.wantReserve(selOf("ie-1", "uk-2"), mres(mvB, betaHost, 0), refBusyAcme)
+ w.wantReserve(selOf("ie-2", "uk-1"), mres(mvB, betaHost, 0), refBusyAcme)
+ if w.activeRaw() != before {
+ t.Fatal("a refused reservation changed active.json")
+ }
+}
+
+func TestOwnRetryNeedsTheCompleteSelection(t *testing.T) {
+ t.Run("exact keeps reserved_at", func(t *testing.T) {
+ w := newReserveWorld(t)
+ w.fx.srv.cfg.SelfServeMoves = "offline"
+ w.wantReserve(selOf("ie-1", "uk-1"), mres(mvA, offlineHost, 0), nil)
+ w.backdate()
+ before := w.activeRaw()
+ w.wantReserve(selOf("ie-1", "uk-1"), mres(mvA, offlineHost, 0), nil)
+ if w.activeRaw() != before {
+ t.Fatal("an exact retry rewrote active.json")
+ }
+ })
+ t.Run("partial with free keys completes in one write", func(t *testing.T) {
+ w := newReserveWorld(t)
+ w.fx.srv.cfg.SelfServeMoves = "offline"
+ own := mres(mvA, offlineHost, time.Hour)
+ w.putTable(map[string]moves.OfflineReservation{"ie-1": own, "ie-9": mres(mvB, betaHost, 0)})
+ lt := &ltStore{Store: w.fx.rel}
+ w.fx.srv.cfg.Releases = lt
+ w.wantReserve(selOf("ie-1", "uk-1"), mres(mvA, offlineHost, 0), nil)
+ tb := w.table()
+ if len(tb) != 3 || tb["ie-1"] != own || tb["uk-1"] != own || tb["ie-9"].ID != mvB {
+ t.Fatalf("table = %+v", tb)
+ }
+ if puts := countOps(lt.opsOf(moves.OfflineActiveKey), "put"); puts != 1 {
+ t.Fatalf("%d writes of active.json, want one", puts)
+ }
+ })
+ t.Run("partial with a held key is busy", func(t *testing.T) {
+ w := newReserveWorld(t)
+ w.fx.srv.cfg.SelfServeMoves = "offline"
+ w.putTable(map[string]moves.OfflineReservation{"ie-1": mres(mvA, offlineHost, 0), "uk-1": mres(mvB, betaHost, 0)})
+ before := w.activeRaw()
+ w.wantReserve(selOf("ie-1", "uk-1"), mres(mvA, offlineHost, 0), refBusyBeta)
+ if w.activeRaw() != before {
+ t.Fatal("a busy partial retry changed active.json")
+ }
+ })
+ t.Run("a changed selection is the mismatch", func(t *testing.T) {
+ w := newReserveWorld(t)
+ w.fx.srv.cfg.SelfServeMoves = "offline"
+ w.wantReserve(selOf("ie-1", "uk-1"), mres(mvA, offlineHost, 0), nil)
+ before := w.activeRaw()
+ w.wantReserve(selOf("ie-1", "uk-2"), mres(mvA, offlineHost, 0), refMismatch)
+ w.wantReserve(selOf("ie-1"), mres(mvA, offlineHost, 0), refMismatch)
+ w.wantReserve(sentinelSel(), mres(mvA, offlineHost, 0), refMismatch)
+ if w.activeRaw() != before {
+ t.Fatal("a mismatch changed active.json")
+ }
+ })
+ t.Run("a malformed unrelated entry is found before the retry is accepted", func(t *testing.T) {
+ w := newReserveWorld(t)
+ w.fx.srv.cfg.SelfServeMoves = "offline"
+ bad := mres(mvB, betaHost, 0)
+ bad.Kind = "lease"
+ w.putTable(map[string]moves.OfflineReservation{"ie-1": mres(mvA, offlineHost, 0), "uk-1": mres(mvA, offlineHost, 0), "ie-9": bad})
+ before := w.activeRaw()
+ w.wantReserve(selOf("ie-1", "uk-1"), mres(mvA, offlineHost, 0), refUnread)
+ if w.activeRaw() != before {
+ t.Fatal("an unreadable table was rewritten")
+ }
+ })
+}
+
+func countOps(ops []string, op string) int {
+ n := 0
+ for _, o := range ops {
+ if o == op {
+ n++
+ }
+ }
+ return n
+}
+
+func TestSentinelReservationConflictsBothWays(t *testing.T) {
+ w := newReserveWorld(t)
+ w.fx.srv.cfg.SelfServeMoves = "offline"
+ w.wantReserve(sentinelSel(), mres(mvA, offlineHost, 0), nil)
+ if tb := w.table(); len(tb) != 1 || tb[moves.OfflineSentinel].ID != mvA {
+ t.Fatalf("a sentinel selection wrote %+v", tb)
+ }
+ w.wantReserve(selOf("ie-7", "uk-7"), mres(mvB, betaHost, 0), refBusyAcme)
+ w.wantReserve(sentinelSel(), mres(mvB, betaHost, 0), refBusyAcme)
+ if err := w.fx.srv.releaseBoxes(context.Background(), moves.OfflineKindMove, mvA); err != nil {
+ t.Fatal(err)
+ }
+ w.wantReserve(selOf("ie-7", "uk-7"), mres(mvB, betaHost, 0), nil)
+ w.wantReserve(sentinelSel(), mres(mvA, offlineHost, 0), refBusyBeta)
+ // A participant with no instance id makes the selection the sentinel
+ // even when the flag is not set.
+ partial := selOf("ie-8", "uk-8")
+ partial.Target.InstanceID = ""
+ if keys := partial.reservationKeys(); len(keys) != 1 || keys[0] != moves.OfflineSentinel {
+ t.Fatalf("keys of a partly pinned selection = %v", keys)
+ }
+ w.wantReserve(partial, mres(mvC, offlineHost, 0), refBusyBeta)
+}
+
+func TestDisjointBoxesReserveAtOnceWithTheSwitch(t *testing.T) {
+ w := newReserveWorld(t)
+ w.fx.srv.cfg.SelfServeMoves = "offline"
+ var wg sync.WaitGroup
+ got := make([]*offlineRefusal, 2)
+ gate := make(chan struct{})
+ for i, c := range []struct {
+ sel offlineSelection
+ r moves.OfflineReservation
+ }{{selOf("ie-1", "uk-1"), mres(mvA, offlineHost, 0)}, {selOf("ie-2", "uk-2"), mres(mvB, betaHost, 0)}} {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ <-gate
+ got[i] = w.reserve(c.sel, c.r)
+ }()
+ }
+ close(gate)
+ wg.Wait()
+ if got[0] != nil || got[1] != nil {
+ t.Fatalf("disjoint reservations = %+v %+v", got[0], got[1])
+ }
+ if tb := w.table(); len(tb) != 4 || tb["ie-1"].ID != mvA || tb["uk-2"].ID != mvB {
+ t.Fatalf("table = %+v", tb)
+ }
+}
+
+func TestWithoutTheSwitchOneReservationAtATime(t *testing.T) {
+ w := newReserveWorld(t)
+ w.wantReserve(selOf("ie-1", "uk-1"), mres(mvA, offlineHost, 0), nil)
+ w.wantReserve(selOf("ie-2", "uk-2"), mres(mvB, betaHost, 0), refBusyAcme)
+ if err := w.fx.srv.releaseBoxes(context.Background(), moves.OfflineKindMove, mvA); err != nil {
+ t.Fatal(err)
+ }
+ w.wantReserve(selOf("ie-2", "uk-2"), mres(mvB, betaHost, 0), nil)
+}
+
+func TestLegacyLockBodyIsNormalised(t *testing.T) {
+ legacy := func(w *offlineWorld, host, op string, age time.Duration) {
+ body, _ := json.Marshal(moves.OfflineActive{Host: host, Op: op, Account: "bab", App: "shop", At: time.Now().UTC().Add(-age)})
+ offlinePut(w.t, w.fx.rel, moves.OfflineActiveKey, body)
+ }
+ w := newReserveWorld(t)
+ w.fx.srv.cfg.SelfServeMoves = "offline"
+ // A young legacy lock reserves every box.
+ legacy(w, betaHost, mvB, 0)
+ before := w.activeRaw()
+ w.wantReserve(selOf("ie-1", "uk-1"), mres(mvA, offlineHost, 0), refBusyBeta)
+ w.wantReserve(sentinelSel(), mres(mvA, offlineHost, 0), refBusyBeta)
+ // Its own move's retry is the sentinel's exact retry: accepted, unwritten.
+ w.wantReserve(sentinelSel(), mres(mvB, betaHost, 0), nil)
+ if w.activeRaw() != before {
+ t.Fatal("an exact retry of a legacy lock rewrote it")
+ }
+ // An old legacy lock whose record carries no move is reclaimed, and the
+ // new write is the table form.
+ legacy(w, betaHost, mvB, time.Hour)
+ w.wantReserve(selOf("ie-1", "uk-1"), mres(mvA, offlineHost, 0), nil)
+ var body moves.OfflineActive
+ if err := json.Unmarshal([]byte(w.activeRaw()), &body); err != nil {
+ t.Fatal(err)
+ }
+ if body.V != moves.OfflineActiveVersion || len(body.Reservations) != 2 || body.Reservations["ie-1"].ID != mvA || body.Op != mvA {
+ t.Fatalf("normalised body = %+v", body)
+ }
+ // A legacy lock is released by its move.
+ legacy(w, betaHost, mvB, 0)
+ if err := w.fx.srv.releaseBoxes(context.Background(), moves.OfflineKindMove, mvB); err != nil {
+ t.Fatal(err)
+ }
+ if w.activeRaw() != "" {
+ t.Fatalf("a released legacy lock left %s", w.activeRaw())
+ }
+}
+
+func TestMalformedEntryIsUnreadable(t *testing.T) {
+ good := `{"kind":"move","id":"` + mvB + `","host":"` + betaHost + `","account":"bab","app":"shop","reserved_at":"2026-01-01T00:00:00Z"}`
+ for name, body := range map[string]string{
+ "not json": `not json`,
+ "unknown kind": `{"v":1,"reservations":{"ie-1":{"kind":"lease","id":"x","host":"h","reserved_at":"2026-01-01T00:00:00Z"},"uk-1":` + good + `}}`,
+ "empty id": `{"v":1,"reservations":{"ie-1":{"kind":"move","host":"h","reserved_at":"2026-01-01T00:00:00Z"}}}`,
+ "empty host": `{"v":1,"reservations":{"ie-1":{"kind":"move","id":"x","reserved_at":"2026-01-01T00:00:00Z"}}}`,
+ "zero time": `{"v":1,"reservations":{"ie-1":{"kind":"move","id":"x","host":"h"}}}`,
+ "id-less sentinel": `{"v":1,"reservations":{"*":{"kind":"move","host":"` + betaHost + `","reserved_at":"2026-01-01T00:00:00Z"}}}`,
+ "unknown version": `{"v":3,"reservations":{"uk-1":` + good + `}}`,
+ "empty legacy": `{}`,
+ } {
+ t.Run(name, func(t *testing.T) {
+ w := newReserveWorld(t)
+ w.fx.srv.cfg.SelfServeMoves = "offline"
+ offlinePut(t, w.fx.rel, moves.OfflineActiveKey, []byte(body))
+ beta := w.recordRaw(betaHost)
+ w.wantReserve(selOf("ie-5", "uk-5"), mres(mvA, offlineHost, 0), refUnread)
+ w.wantReserve(sentinelSel(), mres(mvA, offlineHost, 0), refUnread)
+ if w.activeRaw() != body || w.recordRaw(betaHost) != beta {
+ t.Fatal("a malformed table was rewritten or a record fenced")
+ }
+ if err := w.fx.srv.releaseBoxes(context.Background(), moves.OfflineKindMove, mvB); err == nil || w.activeRaw() != body {
+ t.Fatalf("release of a malformed table = %v", err)
+ }
+ })
+ }
+}
+
+// ---- the reclaim rule (Review Focus 2) ------------------------------------
+
+func TestReclaimOnlyOnEvidence(t *testing.T) {
+ ctx := context.Background()
+ acmeKey := provision.RecordKey("bab", "shop", offlineHost)
+ // held is beta's reserver taking ie-1/uk-1 against acme's reservation
+ // on them (old unless said otherwise).
+ held := func(t *testing.T, age time.Duration, kind string) *offlineWorld {
+ w := newReserveWorld(t)
+ w.fx.srv.cfg.SelfServeMoves = "offline"
+ e := mres(mvA, offlineHost, age)
+ e.Kind = kind
+ w.putTable(map[string]moves.OfflineReservation{"ie-1": e, "uk-1": e})
+ return w
+ }
+ reclaimed := func(w *offlineWorld) {
+ w.t.Helper()
+ w.wantReserve(selOf("ie-1", "uk-1"), mres(mvB, betaHost, 0), nil)
+ if tb := w.table(); len(tb) != 2 || tb["ie-1"].ID != mvB || tb["uk-1"].ID != mvB {
+ w.t.Fatalf("after the reclaim: %+v", tb)
+ }
+ }
+ kept := func(w *offlineWorld) {
+ w.t.Helper()
+ before := w.activeRaw()
+ w.wantReserve(selOf("ie-1", "uk-1"), mres(mvB, betaHost, 0), refBusyAcme)
+ if w.activeRaw() != before {
+ w.t.Fatal("a kept reservation was rewritten")
+ }
+ }
+ fencedRecord := func(w *offlineWorld) bool {
+ w.t.Helper()
+ rec, err := provision.GetRecord(ctx, w.fx.rel, "bab", "shop", offlineHost)
+ if err != nil {
+ w.t.Fatal(err)
+ }
+ return rec.RecordSeq > 0
+ }
+
+ t.Run("operator reservation: fenced and no move is reclaimed", func(t *testing.T) {
+ w := held(t, time.Hour, moves.OfflineKindMove)
+ reclaimed(w)
+ if !fencedRecord(w) {
+ t.Fatal("reclaimed without fencing the holder's record")
+ }
+ })
+ t.Run("absent record is reclaimed", func(t *testing.T) {
+ w := held(t, time.Hour, moves.OfflineKindMove)
+ tb := w.table()
+ for k, e := range tb {
+ e.Host = "gone.shop.example.com"
+ tb[k] = e
+ }
+ w.putTable(tb)
+ reclaimed(w)
+ })
+ t.Run("young is kept without a fence", func(t *testing.T) {
+ w := held(t, time.Second, moves.OfflineKindMove)
+ before := w.recordRaw(offlineHost)
+ kept(w)
+ if w.recordRaw(offlineHost) != before {
+ t.Fatal("a young reservation's record was fenced")
+ }
+ })
+ t.Run("record carrying the move is kept without a fence", func(t *testing.T) {
+ w := held(t, time.Hour, moves.OfflineKindMove)
+ w.queueOn(offlineHost, mvA)
+ before := w.recordRaw(offlineHost)
+ kept(w)
+ if w.recordRaw(offlineHost) != before {
+ t.Fatal("a live move's record was fenced")
+ }
+ })
+ t.Run("a queue commit between the negative read and the fence is seen", func(t *testing.T) {
+ w := held(t, time.Hour, moves.OfflineKindMove)
+ lt := &ltStore{Store: w.fx.rel}
+ w.fx.srv.cfg.Releases = lt
+ var once sync.Once
+ lt.setAfter(func(op, key string, err error) {
+ if op == "get" && key == acmeKey {
+ once.Do(func() { w.queueOn(offlineHost, mvA) })
+ }
+ })
+ kept(w)
+ rec := w.record()
+ if rec.Move == nil || rec.Move.ID != mvA || rec.RecordSeq == 0 {
+ t.Fatalf("the delayed queue %+v (seq %d) was not kept and fenced", rec.Move, rec.RecordSeq)
+ }
+ })
+ t.Run("a store read failure is kept", func(t *testing.T) {
+ w := held(t, time.Hour, moves.OfflineKindMove)
+ lt := &ltStore{Store: w.fx.rel}
+ w.fx.srv.cfg.Releases = lt
+ lt.setHook(failNth("get", acmeKey, 1, true))
+ kept(w)
+ })
+ t.Run("a fence error is kept", func(t *testing.T) {
+ w := held(t, time.Hour, moves.OfflineKindMove)
+ lt := &ltStore{Store: w.fx.rel}
+ w.fx.srv.cfg.Releases = lt
+ before := w.recordRaw(offlineHost)
+ lt.setHook(failNth("put", acmeKey, 1, true))
+ kept(w)
+ if w.recordRaw(offlineHost) != before {
+ t.Fatal("the record changed")
+ }
+ })
+ t.Run("a corrupt record is kept", func(t *testing.T) {
+ w := held(t, time.Hour, moves.OfflineKindMove)
+ offlinePut(t, w.fx.rel, acmeKey, []byte("{not a record"))
+ kept(w)
+ })
+ t.Run("a pending lifetime operation on the host is kept", func(t *testing.T) {
+ w := held(t, time.Hour, moves.OfflineKindMove)
+ if err := w.fx.st.BeginRetirement(Retirement{OpID: "other", Kind: retireOrphan, Account: "bab", App: "shop", Host: offlineHost, Incarnation: 1}); err != nil {
+ t.Fatal(err)
+ }
+ before := w.recordRaw(offlineHost)
+ kept(w)
+ if w.recordRaw(offlineHost) != before {
+ t.Fatal("fenced without admission")
+ }
+ })
+ t.Run("self-serve rows: (c) and (d)", func(t *testing.T) {
+ for _, c := range []struct {
+ name string
+ rows fakeRows
+ ok bool
+ }{
+ {"no row", fakeRows{}, true},
+ {"terminal and released", fakeRows{terminal: map[string]bool{mvA: true}}, true},
+ {"not terminal", fakeRows{terminal: map[string]bool{mvA: false}}, false},
+ {"a request issued for the host", fakeRows{issued: map[string]bool{offlineHost: true}}, false},
+ {"a SQL failure", fakeRows{err: errors.New("database is locked")}, false},
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ w := held(t, time.Hour, moves.OfflineKindMove)
+ w.fx.srv.selfServeRows = c.rows
+ if c.ok {
+ reclaimed(w)
+ } else {
+ kept(w)
+ }
+ })
+ }
+ })
+ t.Run("cleanup", func(t *testing.T) {
+ for _, c := range []struct {
+ name string
+ ev cleanupReportEvidence
+ ok bool
+ }{
+ {"no report evidence yet", nil, false},
+ {"settled", fakeCleanup{settled: true}, true},
+ {"incomplete status", fakeCleanup{}, false},
+ {"a read failure", fakeCleanup{settled: true, err: errors.New("status read")}, false},
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ w := held(t, time.Hour, moves.OfflineKindCleanup)
+ w.fx.srv.cleanupReports = c.ev
+ if c.ok {
+ reclaimed(w)
+ } else {
+ kept(w)
+ }
+ })
+ }
+ })
+ t.Run("a partial own reservation does not reclaim a live holder", func(t *testing.T) {
+ w := newReserveWorld(t)
+ w.fx.srv.cfg.SelfServeMoves = "offline"
+ w.queueOn(offlineHost, mvA)
+ w.putTable(map[string]moves.OfflineReservation{"ie-1": mres(mvB, betaHost, time.Hour), "uk-1": mres(mvA, offlineHost, time.Hour)})
+ w.wantReserve(selOf("ie-1", "uk-1"), mres(mvB, betaHost, 0), refBusyAcme)
+ // Once the holder's record no longer carries its move, the missing
+ // key is reclaimed and the retry completes.
+ w.edit(provision.OfflineAbortAt(mvA, time.Now(), provision.Receipt{}))
+ w.wantReserve(selOf("ie-1", "uk-1"), mres(mvB, betaHost, 0), nil)
+ if tb := w.table(); len(tb) != 2 || tb["uk-1"].ID != mvB {
+ t.Fatalf("table = %+v", tb)
+ }
+ })
+ t.Run("switch off with two disjoint reservations and a cleanup-only obligation", func(t *testing.T) {
+ w := newReserveWorld(t)
+ w.putTable(map[string]moves.OfflineReservation{
+ "ie-1": mres(mvA, offlineHost, time.Hour), "uk-1": mres(mvA, offlineHost, time.Hour),
+ "ie-2": mres(mvB, betaHost, time.Hour), "uk-2": mres(mvB, betaHost, time.Hour),
+ "ie-4": {Kind: moves.OfflineKindCleanup, ID: "rt_1", Host: "old.shop.example.com", ReservedAt: time.Now().Add(-time.Hour)},
+ })
+ w.queueOn(offlineHost, mvA)
+ w.queueOn(betaHost, mvB)
+ before := w.activeRaw()
+ w.wantReserve(selOf("ie-1", "uk-1"), mres(mvA, offlineHost, 0), nil)
+ w.wantReserve(selOf("ie-2", "uk-2"), mres(mvB, betaHost, 0), nil)
+ if w.activeRaw() != before {
+ t.Fatal("recovery rewrote active.json")
+ }
+ // A new operation is refused; the cleanup is kept.
+ w.wantReserve(selOf("ie-6", "uk-6"), mres(mvC, "gamma.shop.example.com", 0), refBusyAcme)
+ // Recovery releases its own and leaves the cleanup.
+ for _, id := range []string{mvA, mvB} {
+ if err := w.fx.srv.releaseBoxes(ctx, moves.OfflineKindMove, id); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if tb := w.table(); len(tb) != 1 || tb["ie-4"].Kind != moves.OfflineKindCleanup {
+ t.Fatalf("after recovery: %+v", tb)
+ }
+ })
+ t.Run("an operator door paused between reserving and its record write", func(t *testing.T) {
+ w := newReserveWorld(t)
+ lt := &ltStore{Store: w.fx.rel}
+ w.fx.srv.cfg.Releases = lt
+ gate, hook := pauseNth(t, "put", acmeKey, 1)
+ lt.setHook(hook)
+ done := make(chan [2]any, 1)
+ go func() {
+ code, out := w.req(http.MethodPost, "", `{"region":"uk"}`)
+ done <- [2]any{code, out}
+ }()
+ <-gate.entered
+ paused := w.table()[moves.OfflineSentinel]
+ if paused.ID == "" || paused.Host != offlineHost {
+ t.Fatalf("the paused door's reservation = %+v", paused)
+ }
+ w.backdate()
+ // Another operation is admitted: it fences acme's record and
+ // reclaims the paused door's reservation.
+ if v, code, out := w.startOther(betaHost); code != http.StatusAccepted {
+ t.Fatalf("beta start = %d %s", code, out)
+ } else {
+ w.id = v.ID
+ }
+ gate.open()
+ got := <-done
+ if got[0] != http.StatusConflict || errorText(got[1].(string)) != "the instance changed while the move was being started. try again" {
+ t.Fatalf("the paused door answered %v %s", got[0], got[1])
+ }
+ if rec := w.record(); rec.Move != nil {
+ t.Fatalf("the paused door started %+v", rec.Move)
+ }
+ if tb := w.table(); len(tb) != 1 || tb[moves.OfflineSentinel].ID != w.id || tb[moves.OfflineSentinel].Host != betaHost {
+ t.Fatalf("the admitted operation's reservation = %+v", tb)
+ }
+ })
+ t.Run("a door paused across a delete and recreate at the same sequence", func(t *testing.T) {
+ w := newReserveWorld(t)
+ lt := &ltStore{Store: w.fx.rel}
+ w.fx.srv.cfg.Releases = lt
+ gate, hook := pauseNth(t, "put", moves.OfflineActiveKey, 1)
+ lt.setHook(hook)
+ done := make(chan [2]any, 1)
+ go func() {
+ code, out := w.req(http.MethodPost, "", `{"region":"uk"}`)
+ done <- [2]any{code, out}
+ }()
+ <-gate.entered
+ // The host is recreated at a new incarnation whose record happens
+ // to be at the sequence the door observed.
+ old := w.record()
+ var body map[string]any
+ if err := json.Unmarshal([]byte(w.recordRaw(offlineHost)), &body); err != nil {
+ t.Fatal(err)
+ }
+ body["incarnation"] = old.Incarnation + 7
+ raw, _ := json.Marshal(body)
+ offlinePut(t, w.fx.rel, acmeKey, raw)
+ gate.open()
+ got := <-done
+ if got[0] != http.StatusConflict || errorText(got[1].(string)) != "the instance changed while the move was being started. try again" {
+ t.Fatalf("the paused door answered %v %s", got[0], got[1])
+ }
+ if w.recordRaw(offlineHost) != string(raw) {
+ t.Fatal("the paused door wrote the recreated record")
+ }
+ if w.activeRaw() != "" {
+ t.Fatalf("the refused door kept its reservation: %s", w.activeRaw())
+ }
+ })
+}
+
+// ---- release ----------------------------------------------------------------
+
+func TestReleaseRemovesOnlyItsEntriesAndDeletesWhenEmpty(t *testing.T) {
+ ctx := context.Background()
+ w := newReserveWorld(t)
+ cleanupB := moves.OfflineReservation{Kind: moves.OfflineKindCleanup, ID: mvB, Host: "old.shop.example.com", ReservedAt: time.Now().Add(-2 * time.Hour).UTC().Truncate(time.Second)}
+ w.putTable(map[string]moves.OfflineReservation{
+ "ie-1": mres(mvA, offlineHost, time.Minute), "uk-1": mres(mvA, offlineHost, time.Minute),
+ "ie-2": mres(mvB, betaHost, 0), "ie-3": cleanupB,
+ })
+ if err := w.fx.srv.releaseBoxes(ctx, moves.OfflineKindMove, mvA); err != nil {
+ t.Fatal(err)
+ }
+ var body moves.OfflineActive
+ if err := json.Unmarshal([]byte(w.activeRaw()), &body); err != nil {
+ t.Fatal(err)
+ }
+ if len(body.Reservations) != 2 || body.Reservations["ie-2"].ID != mvB || body.Reservations["ie-3"] != cleanupB || body.Host != betaHost || body.Op != mvB {
+ t.Fatalf("after releasing A: %+v", body)
+ }
+ // The kind is part of the identity: the cleanup with B's id stays.
+ if err := w.fx.srv.releaseBoxes(ctx, moves.OfflineKindMove, mvB); err != nil {
+ t.Fatal(err)
+ }
+ if tb := w.table(); len(tb) != 1 || tb["ie-3"] != cleanupB {
+ t.Fatalf("after releasing B's move: %+v", tb)
+ }
+ before := w.activeRaw()
+ if err := w.fx.srv.releaseBoxes(ctx, moves.OfflineKindMove, mvC); err != nil || w.activeRaw() != before {
+ t.Fatalf("releasing what is not there = %v", err)
+ }
+ if err := w.fx.srv.releaseBoxes(ctx, moves.OfflineKindCleanup, mvB); err != nil {
+ t.Fatal(err)
+ }
+ if w.activeRaw() != "" {
+ t.Fatalf("an empty table was left: %s", w.activeRaw())
+ }
+ // Absence is success.
+ if err := w.fx.srv.releaseBoxes(ctx, moves.OfflineKindMove, mvA); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// ---- old consoles (the rollout note) -------------------------------------
+
+// oldTakeLock and oldReleaseLock are main's offlineTakeLock and
+// offlineReleaseLock (before Task 3), verbatim but for the receiver: the
+// algorithms an old console runs against whatever body it finds.
+func oldTakeLock(ctx context.Context, s *Server, host, id, acct, app string) *offlineRefusal {
+ unreadable := &offlineRefusal{status: http.StatusBadGateway, msg: "could not read the instance"}
+ body, err := json.Marshal(moves.OfflineActive{Host: host, Op: id, Account: acct, App: app, At: time.Now().UTC()})
+ if err != nil {
+ return unreadable
+ }
+ _, err = s.cfg.Releases.Put(ctx, moves.OfflineActiveKey, body, "")
+ if err == nil {
+ return nil
+ }
+ if !errors.Is(err, release.ErrCASConflict) {
+ return unreadable
+ }
+ raw, etag, err := s.cfg.Releases.Get(ctx, moves.OfflineActiveKey)
+ var held moves.OfflineActive
+ if err != nil || json.Unmarshal(raw, &held) != nil || held.Host == "" {
+ return unreadable
+ }
+ busy := &offlineRefusal{status: http.StatusConflict, msg: fmt.Sprintf("%s is moving offline. move one instance at a time", held.Host)}
+ if time.Since(held.At) < offlineLockGrace {
+ return busy
+ }
+ other, _, err := s.readRecord(ctx, held.Account, held.App, held.Host)
+ switch {
+ case errors.Is(err, os.ErrNotExist):
+ case err != nil:
+ return unreadable
+ case other.Move.Offline() && other.Move.ID == held.Op:
+ return busy
+ }
+ if _, err := s.cfg.Releases.Put(ctx, moves.OfflineActiveKey, body, etag); err != nil {
+ if errors.Is(err, release.ErrCASConflict) {
+ return busy
+ }
+ return unreadable
+ }
+ return nil
+}
+
+func oldReleaseLock(ctx context.Context, s *Server, host, id string) {
+ raw, etag, err := s.cfg.Releases.Get(ctx, moves.OfflineActiveKey)
+ if errors.Is(err, os.ErrNotExist) {
+ return
+ }
+ var held moves.OfflineActive
+ if err != nil || json.Unmarshal(raw, &held) != nil {
+ return
+ }
+ if held.Host != host || held.Op != id {
+ return
+ }
+ if err := release.DeleteIfMatch(ctx, s.cfg.Releases, moves.OfflineActiveKey, etag); err != nil {
+ log.Printf("old console release: %v", err)
+ }
+}
+
+func TestOldConsoleAlgorithmsAgainstTheNewBody(t *testing.T) {
+ ctx := context.Background()
+ w := newReserveWorld(t)
+ w.fx.srv.cfg.SelfServeMoves = "offline"
+ w.wantReserve(selOf("ie-1", "uk-1"), mres(mvA, offlineHost, 0), nil)
+ w.wantReserve(selOf("ie-2", "uk-2"), mres(mvB, betaHost, 0), nil)
+ before := w.activeRaw()
+ // An old console's start reads the projection as a young lock.
+ if got := oldTakeLock(ctx, w.fx.srv, "gamma.shop.example.com", mvC, "bab", "shop"); !sameRefusal(got, refBusyAcme) {
+ t.Fatalf("an old start against a young table = %+v", got)
+ }
+ // Old, it checks the projection's record: a live move is still busy.
+ w.queueOn(offlineHost, mvA)
+ w.backdate()
+ before = w.activeRaw()
+ if got := oldTakeLock(ctx, w.fx.srv, "gamma.shop.example.com", mvC, "bab", "shop"); !sameRefusal(got, refBusyAcme) {
+ t.Fatalf("an old start against an old table with a live move = %+v", got)
+ }
+ // An old release of a move the projection does not name leaves it.
+ oldReleaseLock(ctx, w.fx.srv, betaHost, mvB)
+ if w.activeRaw() != before {
+ t.Fatal("an old release of another move changed the table")
+ }
+ // A new console still reads what an old one writes: a legacy lock is
+ // the sentinel.
+ offlinePut(t, w.fx.rel, moves.OfflineActiveKey, nil)
+ if err := w.fx.rel.Delete(ctx, moves.OfflineActiveKey); err != nil {
+ t.Fatal(err)
+ }
+ if got := oldTakeLock(ctx, w.fx.srv, betaHost, mvB, "bab", "shop"); got != nil {
+ t.Fatalf("an old start on an empty bucket = %+v", got)
+ }
+ w.wantReserve(selOf("ie-1", "uk-1"), mres(mvA, offlineHost, 0), refBusyBeta)
+}
+
+// The hazard the rollout note names: an old console's release deletes the
+// whole object when the projection names its own move, erasing every other
+// operation's reservation. So no old console is reintroduced while a table
+// it did not create exists.
+func TestOldConsoleReleaseErasesATable(t *testing.T) {
+ ctx := context.Background()
+ w := newReserveWorld(t)
+ w.fx.srv.cfg.SelfServeMoves = "offline"
+ w.wantReserve(selOf("ie-1", "uk-1"), mres(mvA, offlineHost, time.Minute), nil)
+ w.wantReserve(selOf("ie-2", "uk-2"), mres(mvB, betaHost, 0), nil)
+ oldReleaseLock(ctx, w.fx.srv, offlineHost, mvA)
+ if w.activeRaw() != "" {
+ t.Fatalf("the hazard did not reproduce: %s", w.activeRaw())
+ }
+}
+
+func TestSwitchOffRecoveryReacquiresItsOwnReservation(t *testing.T) {
+ w := newReserveWorld(t)
+ w.fx.srv.cfg.SelfServeMoves = "offline"
+ w.wantReserve(selOf("ie-1", "uk-1"), mres(mvA, offlineHost, 0), nil)
+ w.wantReserve(selOf("ie-2", "uk-2"), mres(mvB, betaHost, 0), nil)
+ w.fx.srv.cfg.SelfServeMoves = ""
+ before := w.activeRaw()
+ w.wantReserve(selOf("ie-1", "uk-1"), mres(mvA, offlineHost, 0), nil)
+ w.wantReserve(selOf("ie-2", "uk-2"), mres(mvB, betaHost, 0), nil)
+ if w.activeRaw() != before {
+ t.Fatal("an own retry rewrote the table")
+ }
+ w.wantReserve(selOf("ie-3", "uk-3"), mres(mvC, "gamma.shop.example.com", 0), refBusyAcme)
+}
+
+// ---- the operator doors -----------------------------------------------------
+
+func TestOperatorDoorsUseReservations(t *testing.T) {
+ changed := "the instance changed while the move was being started. try again"
+ t.Run("start reserves the sentinel bound to its observation and finish releases", func(t *testing.T) {
+ w := newReserveWorld(t)
+ rec := w.record()
+ w.start()
+ var body moves.OfflineActive
+ if err := json.Unmarshal([]byte(w.activeRaw()), &body); err != nil {
+ t.Fatal(err)
+ }
+ e := body.Reservations[moves.OfflineSentinel]
+ if body.V != moves.OfflineActiveVersion || len(body.Reservations) != 1 || e.Kind != moves.OfflineKindMove || e.ID != w.id ||
+ e.Host != offlineHost || e.Account != "bab" || e.App != "shop" || e.ExpectSeq != rec.RecordSeq || e.ExpectIncarnation != rec.Incarnation {
+ t.Fatalf("start reserved %+v", body)
+ }
+ if body.Host != offlineHost || body.Op != w.id {
+ t.Fatalf("projection = %+v", body)
+ }
+ if _, code, out := w.startOther(betaHost); code != http.StatusConflict || errorText(out) != offlineHost+" is moving offline. move one instance at a time" {
+ t.Fatalf("beta start = %d %s", code, out)
+ }
+ w.ok("/abort", "", provision.OfflineReverting)
+ w.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseReverted, nil)
+ w.ok("/finish", "", "none")
+ if w.activeRaw() != "" {
+ t.Fatalf("finish left %s", w.activeRaw())
+ }
+ })
+ t.Run("stage reserves and the staged start keeps it", func(t *testing.T) {
+ w := newReserveWorld(t)
+ v := w.ok("/stage", `{"region":"uk"}`, provision.OfflineStaging)
+ w.id = v.ID
+ if tb := w.table(); len(tb) != 1 || tb[moves.OfflineSentinel].ID != v.ID {
+ t.Fatalf("stage reserved %+v", tb)
+ }
+ w.backdate()
+ before := w.activeRaw()
+ w.ok("/stage", `{"region":"uk"}`, provision.OfflineStaging)
+ w.ok("", `{"region":"uk"}`, provision.OfflineStopping)
+ if w.activeRaw() != before {
+ t.Fatal("the staged move's doors rewrote its reservation")
+ }
+ })
+ t.Run("a record changed after the observation refuses and releases", func(t *testing.T) {
+ w := newReserveWorld(t)
+ lt := &ltStore{Store: w.fx.rel}
+ w.fx.srv.cfg.Releases = lt
+ key := provision.RecordKey("bab", "shop", offlineHost)
+ var once sync.Once
+ lt.setHook(func(op, k string) error {
+ if op == "put" && k == moves.OfflineActiveKey {
+ once.Do(func() { sequenceRecord(t, w.fx.rel, key) })
+ }
+ return nil
+ })
+ code, out := w.req(http.MethodPost, "", `{"region":"uk"}`)
+ if code != http.StatusConflict || errorText(out) != changed {
+ t.Fatalf("start = %d %s", code, out)
+ }
+ if rec := w.record(); rec.Move != nil {
+ t.Fatalf("a refused start wrote %+v", rec.Move)
+ }
+ if w.activeRaw() != "" {
+ t.Fatalf("a refused start kept its reservation: %s", w.activeRaw())
+ }
+ })
+ t.Run("a staged start on a changed record refuses and keeps the stage's reservation", func(t *testing.T) {
+ w := newReserveWorld(t)
+ w.ok("/stage", `{"region":"uk"}`, provision.OfflineStaging)
+ lt := &ltStore{Store: w.fx.rel}
+ w.fx.srv.cfg.Releases = lt
+ key := provision.RecordKey("bab", "shop", offlineHost)
+ var once sync.Once
+ lt.setHook(func(op, k string) error {
+ if op == "get" && k == moves.OfflineActiveKey {
+ once.Do(func() { sequenceRecord(t, w.fx.rel, key) })
+ }
+ return nil
+ })
+ before := w.activeRaw()
+ code, out := w.req(http.MethodPost, "", `{"region":"uk"}`)
+ if code != http.StatusConflict || errorText(out) != changed {
+ t.Fatalf("staged start = %d %s", code, out)
+ }
+ if rec := w.record(); rec.Move == nil || rec.Move.State != provision.OfflineStaging {
+ t.Fatalf("after the refusal: %+v", rec.Move)
+ }
+ if w.activeRaw() != before {
+ t.Fatal("a refused staged start changed the stage's reservation")
+ }
+ })
+}
+
+// sequenceRecord rewrites a record's exact bytes with the record sequence
+// on, as a fence or any sequenced writer does: record_seq advances.
+func sequenceRecord(t *testing.T, rel release.Store, key string) {
+ t.Helper()
+ ctx := provision.WithRecordSequence(context.Background())
+ raw, etag, err := rel.Get(ctx, key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, _, err := provision.WriteRecord(ctx, rel, key, raw, raw, etag); err != nil {
+ t.Fatal(err)
+ }
+}