carlos / platform Public

console: operator selections reserve the sentinel outside self-serve mode

Fix round 1, Important 1. Pinned from id-bearing reports, an operator
selection reserved box keys, so a start whose report crossed the
ten-minute line since the stage, or a restage after the source box set
changed, met its own reservation as a mismatch (MV-O194) where main
proceeds. Outside self-serve mode every operator selection is now the
sentinel; the participants still carry the instances they pinned.

The golden replay is regenerated from the base tree (cf3dd10f) with every
box's report naming its box id and instance, and gains the cases the old
fixture could not see: reports gone stale between the stage and the start
(both ways), a source box gone before a restage, and the stager gone then
back. At 05fd2d36 the new golden fails with MV-O194 on four answers.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

claude 2 days ago 9d508769f0ccc4d098733e009c00aa8c1b9859b5 parent 05fd2d3

3 files changed, +95 −1

internal/console/apiinstancemove_offline.go +6 −1
internal/console/apiinstancemove_offline_snapshot_test.go +80 −0
internal/console/testdata/operator-door-answers.golden +9 −0

diff --git a/internal/console/apiinstancemove_offline.go b/internal/console/apiinstancemove_offline.go

index 217907e8b..cbb79d862 100644

--- a/internal/console/apiinstancemove_offline.go

+++ b/internal/console/apiinstancemove_offline.go

@@ -656,7 +656,12 @@ func (s *Server) pinParticipants(ctx context.Context, sel *offlineSelection, req

}

sel.Target = p

}

- sel.Sentinel = false

+ // Outside self-serve mode an operator selection reserves the sentinel

+ // whatever it pinned: one operation at a time across the deployment, as

+ // before the table, so a report that crosses the freshness line between

+ // a stage and its start, or a source box set that changes before a

+ // restage, never turns the move's own retry into a mismatch.

+ sel.Sentinel = !requireIDs && !s.selfServeMode()

for _, p := range append(append([]provision.Participant(nil), sel.Source...), sel.Target) {

if p.InstanceID == "" {

sel.Sentinel = true

diff --git a/internal/console/apiinstancemove_offline_snapshot_test.go b/internal/console/apiinstancemove_offline_snapshot_test.go

index 09204bc08..3dd0cf859 100644

--- a/internal/console/apiinstancemove_offline_snapshot_test.go

+++ b/internal/console/apiinstancemove_offline_snapshot_test.go

@@ -8,15 +8,20 @@ import (

"path/filepath"

"strings"

"testing"

+ "time"

"github.com/carlosframework/platform/internal/hostops"

"github.com/carlosframework/platform/internal/provision"

+ "github.com/carlosframework/platform/internal/status"

)

// The operator start and stage doors' answers, replayed from main (self-serve

// moves plan 2 Task 5). The golden file was written by this test run against

// the tree before Task 5 split the doors' checks (base cf3dd10f), with

// CARLOS_OPERATOR_SNAPSHOT_WRITE=1; it is never rewritten after the split.

+// Every box's status report names its box id and instance (snapshotReports),

+// so the replay sees what pinning reads; the reports' age is a case of its

+// own (a report that goes stale between a stage and its start).

// Every refusal of both doors is here, each with the good body and with a

// malformed one, so a class refusal combined with a malformed body keeps its

// order; so are the successes on boxes whose status reports carry no metrics

@@ -100,6 +105,22 @@ func operatorSnapshotCases() []operatorSnapshotCase {

}

+// snapshotReports rewrites each box's status report to name its box id and

+// instance, at age, keeping what the fixture's report said about claims. A

+// box published with no metrics keeps none.

+func snapshotReports(t *testing.T, w *offlineWorld, age time.Duration, boxes ...offlineBox) {

+ t.Helper()

+ for _, b := range boxes {

+ sb := status.Box{Label: b.Box, GeneratedAt: time.Now().Add(-age)}

+ if !b.NoMetrics {

+ sb.Metrics = &status.BoxMetrics{HostClaimEnabled: b.ClaimsOn, BoxID: b.Box, InstanceID: "i-" + b.Box}

+ }

+ if err := status.WriteBox(t.Context(), w.fx.rel, sb); err != nil {

+ t.Fatal(err)

+ }

// operatorAnswer is one answer as the snapshot keeps it: the status and the

// error sentence, or for a 202 the view's stable fields.

func operatorAnswer(code int, out string) string {

@@ -130,6 +151,7 @@ func operatorSnapshot(t *testing.T) []string {

for _, b := range bodies {

t.Run(c.name+"/"+door.name+"/"+b.name, func(t *testing.T) {

w := newOfflineWorld(t, c.boxes...)

+ snapshotReports(t, w, 0, c.boxes...)

if c.setup != nil {

c.setup(t, w)

}

@@ -147,6 +169,7 @@ func operatorSnapshot(t *testing.T) []string {

// and the target's changes after the stage.

staged := func(t *testing.T, boxes ...offlineBox) *offlineWorld {

w := newOfflineWorld(t, boxes...)

+ snapshotReports(t, w, 0, boxes...)

if code, out := w.req(http.MethodPost, "/stage", `{"region":"uk"}`); code != http.StatusAccepted {

t.Fatalf("stage = %d %s", code, out)

}

@@ -174,6 +197,7 @@ func operatorSnapshot(t *testing.T) []string {

}

w.fleet(ieOff("ie-other"))

+ snapshotReports(t, w, 0, ieOff("ie-other"))

code, out := w.req(http.MethodGet, "", "")

note("staged, the stager gone", "status", code, out)

code, out = w.req(http.MethodPost, "", `{"region":"uk"}`)

@@ -184,6 +208,7 @@ func operatorSnapshot(t *testing.T) []string {

t.Run("staged, the target's bucket changed", func(t *testing.T) {

w := staged(t, oneEach...)

w.fleet(func() offlineBox { b := ukOff("uk-box"); b.Bucket = "data-uk-2"; return b }())

+ snapshotReports(t, w, 0, ukOff("uk-box"))

code, out := w.req(http.MethodGet, "", "")

note("staged, the target's bucket changed", "status", code, out)

code, out = w.req(http.MethodPost, "", `{"region":"uk"}`)

@@ -191,6 +216,61 @@ func operatorSnapshot(t *testing.T) []string {

code, out = w.req(http.MethodPost, "/stage", `{"region":"uk"}`)

note("staged, the target's bucket changed", "restage", code, out)

})

+ // The reports cross the ten-minute freshness line between the stage and

+ // the start, each way.

+ t.Run("staged, the reports went stale", func(t *testing.T) {

+ w := staged(t, oneEach...)

+ snapshotReports(t, w, 11*time.Minute, oneEach...)

+ code, out := w.req(http.MethodPost, "/stage", `{"region":"uk"}`)

+ note("staged, the reports went stale", "restage", code, out)

+ code, out = w.req(http.MethodPost, "", `{"region":"uk"}`)

+ note("staged, the reports went stale", "start", code, out)

+ })

+ t.Run("staged on stale reports, fresh at the start", func(t *testing.T) {

+ w := newOfflineWorld(t, oneEach...)

+ snapshotReports(t, w, 11*time.Minute, oneEach...)

+ code, out := w.req(http.MethodPost, "/stage", `{"region":"uk"}`)

+ note("staged on stale reports, fresh at the start", "stage", code, out)

+ snapshotReports(t, w, 0, oneEach...)

+ code, out = w.req(http.MethodPost, "/stage", `{"region":"uk"}`)

+ note("staged on stale reports, fresh at the start", "restage", code, out)

+ code, out = w.req(http.MethodPost, "", `{"region":"uk"}`)

+ note("staged on stale reports, fresh at the start", "start", code, out)

+ })

+ // The source box set changes before a restage: a box that is not the

+ // stager goes, then the stager itself.

+ t.Run("staged, a source box gone", func(t *testing.T) {

+ w := staged(t, ieOff("ie-a"), ieOff("ie-b"), ukOff("uk-box"))

+ gone := "ie-a"

+ if w.record().Move.Stager == gone {

+ gone = "ie-b"

+ }

+ for _, key := range []string{edgesFilePrefix + gone + ".json", "status/boxes/" + gone + ".json"} {

+ if err := w.fx.rel.Delete(t.Context(), key); err != nil {

+ t.Fatal(err)

+ }

+ code, out := w.req(http.MethodPost, "/stage", `{"region":"uk"}`)

+ note("staged, a source box gone", "restage", code, out)

+ code, out = w.req(http.MethodPost, "", `{"region":"uk"}`)

+ note("staged, a source box gone", "start", code, out)

+ })

+ t.Run("staged, the stager gone, then restage and start", func(t *testing.T) {

+ w := staged(t, oneEach...)

+ for _, key := range []string{edgesFilePrefix + "ie-box.json", "status/boxes/ie-box.json"} {

+ if err := w.fx.rel.Delete(t.Context(), key); err != nil {

+ t.Fatal(err)

+ }

+ w.fleet(ieOff("ie-other"))

+ snapshotReports(t, w, 0, ieOff("ie-other"))

+ code, out := w.req(http.MethodPost, "/stage", `{"region":"uk"}`)

+ note("staged, the stager gone, then restage and start", "restage", code, out)

+ w.fleet(ieOff("ie-box"))

+ snapshotReports(t, w, 0, ieOff("ie-box"))

+ code, out = w.req(http.MethodPost, "", `{"region":"uk"}`)

+ note("staged, the stager gone, then restage and start", "start with the stager back", code, out)

+ })

t.Run("staged, a claim since", func(t *testing.T) {

w := staged(t, oneEach...)

offlineClaimFor(t, w.fx, offlineHost, "ie-box", "ie")

diff --git a/internal/console/testdata/operator-door-answers.golden b/internal/console/testdata/operator-door-answers.golden

index aa8daaf27..4dad7fc07 100644

--- a/internal/console/testdata/operator-door-answers.golden

+++ b/internal/console/testdata/operator-door-answers.golden

@@ -209,6 +209,15 @@ staged, the stager gone | restage | 202 state=offline-staging from=ie to=uk stag

staged, the target's bucket changed | status | 200 state=offline-staging from=ie to=uk stager=ie-box next= waiting="uk now publishes a different data bucket or region from what was recorded when acme.shop.example.com was staged. abort the move and stage it again" can_abort=true never_offline=true

staged, the target's bucket changed | start | 409 "uk now publishes a different data bucket or region from what was recorded when acme.shop.example.com was staged. abort the move and stage it again"

staged, the target's bucket changed | restage | 409 "uk now publishes a different data bucket or region from what was recorded when acme.shop.example.com was staged. abort the move and stage it again"

+staged, the reports went stale | restage | 202 state=offline-staging from=ie to=uk stager=ie-box next=start waiting="" can_abort=true never_offline=true

+staged, the reports went stale | start | 202 state=offline-stopping from=ie to=uk stager=ie-box next= waiting="Waiting for ie-box in ie to confirm it has stopped acme.shop.example.com. acme.shop.example.com is offline." can_abort=true never_offline=false

+staged on stale reports, fresh at the start | stage | 202 state=offline-staging from=ie to=uk stager=ie-box next=start waiting="" can_abort=true never_offline=true

+staged on stale reports, fresh at the start | restage | 202 state=offline-staging from=ie to=uk stager=ie-box next=start waiting="" can_abort=true never_offline=true

+staged on stale reports, fresh at the start | start | 202 state=offline-stopping from=ie to=uk stager=ie-box next= waiting="Waiting for ie-box in ie to confirm it has stopped acme.shop.example.com. acme.shop.example.com is offline." can_abort=true never_offline=false

+staged, a source box gone | restage | 202 state=offline-staging from=ie to=uk stager=ie-a next=start waiting="" can_abort=true never_offline=true

+staged, a source box gone | start | 202 state=offline-stopping from=ie to=uk stager=ie-a next= waiting="Waiting for ie-a in ie to confirm it has stopped acme.shop.example.com. acme.shop.example.com is offline." can_abort=true never_offline=false

+staged, the stager gone, then restage and start | restage | 202 state=offline-staging from=ie to=uk stager=ie-box next= waiting="ie-box, which staged acme.shop.example.com, no longer publishes an edges file in ie. bring it back or abort the move" can_abort=true never_offline=true

+staged, the stager gone, then restage and start | start with the stager back | 202 state=offline-stopping from=ie to=uk stager=ie-box next= waiting="Waiting for ie-box in ie to confirm it has stopped acme.shop.example.com. acme.shop.example.com is offline." can_abort=true never_offline=false

staged, a claim since | status | 200 state=offline-staging from=ie to=uk stager=ie-box next=start waiting="" can_abort=true never_offline=true

staged, a claim since | start | 409 "acme.shop.example.com has a host claim. offline moves are only for hosts without one"

staged, a claim since | restage | 409 "acme.shop.example.com has a host claim. offline moves are only for hosts without one"