amadan

carlos / platform Public

console: move resolve --retry reruns the recovery probe (§7.10)

The resolve door's {"retry": true} writes provision.MoveRetry under the guard
from the record: 409 unless the move is stuck, the flat view on success. Under
the host claim it answers HC5-N1 before any write (ruling J17). The box side
(verification again, never bypassed; the probe's whole budget; the install
from its journal) is held by movesretry_test.go.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
claude 2ef84004d8739c3e19520543f3739d4dd8eea0d2 parent d60a9f2
4 files changed, +525 −8
  • internal/console/apiinstancemove.go +27 −5
  • internal/console/apiinstancemove_claim_test.go +40 −0
  • internal/console/apiinstancemove_test.go +197 −3
  • internal/host/movesretry_test.go +261 −0
diff --git a/internal/console/apiinstancemove.go b/internal/console/apiinstancemove.go
index ac15c5fdf..ac38b9e06 100644
--- a/internal/console/apiinstancemove.go
+++ b/internal/console/apiinstancemove.go
@@ -844,18 +844,40 @@ func (s *Server) handleAPIInstanceMoveAbort(w http.ResponseWriter, r *http.Reque
}
// handleAPIInstanceMoveResolve is the operator's door out of a stuck move
-// (M-C11). This plan gives it only its gate; Plan 4 Task 10 and Plan 6
-// Task 2 give it its body. With the switch off it is the 404 the route
-// answered before it existed.
+// (M-C11). Its body is {"retry": true} (Plan 4 Task 7, §7.8, §7.10): the
+// record write is provision.MoveRetry under the guard from the record, which
+// answers 409 unless the move is stuck. The retry clears the stuck reason
+// only; the box then resumes at the phase the stuck rollback recorded
+// (a verification runs again and is never bypassed, a probe gets its nine
+// attempts again, a revert resumes its rev_phase). The answer is the flat
+// view: the stuck reason is the CLI's to say (M-C11), read from the record.
+// Any other body is "nothing to resolve yet" until Plan 4 Task 10 and Plan 6
+// Task 2 give the door the rest. With the switch off it is the 404 the
+// route answered before it existed.
func (s *Server) handleAPIInstanceMoveResolve(w http.ResponseWriter, r *http.Request) {
if !s.cfg.MovesEnabled {
http.NotFound(w, r)
return
}
- if _, _, _, _, _, ok := s.moveOperatorGate(w, r); !ok {
+ acct, app, host, rec, _, ok := s.moveOperatorGate(w, r)
+ if !ok {
+ return
+ }
+ var req struct {
+ Retry bool `json:"retry"`
+ }
+ if !readJSON(w, r, &req) {
+ return
+ }
+ if !req.Retry {
+ jsonError(w, http.StatusConflict, "nothing to resolve yet")
return
}
- jsonError(w, http.StatusConflict, "nothing to resolve yet")
+ // The record write is a console move write, so on a host-claim console
+ // moveTransitionSaying refuses it with HC5-N1 before any write (ruling
+ // J17). Claim Plan 10's console move grant (platform-1e accepted J17)
+ // covers resolve --retry and lifts that refusal.
+ s.moveTransitionP1(w, r, acct, app, host, provision.MoveRetry(moveGuardOf(rec)), http.StatusAccepted)
}
// registerMovesPhase1Routes adds the routes that exist only with the switch
diff --git a/internal/console/apiinstancemove_claim_test.go b/internal/console/apiinstancemove_claim_test.go
index 2abc27a58..57a15c244 100644
--- a/internal/console/apiinstancemove_claim_test.go
+++ b/internal/console/apiinstancemove_claim_test.go
@@ -62,6 +62,46 @@ func TestMoveDoorPhase1RefusedUnderHostClaim(t *testing.T) {
}
}
+// Ruling J17 (Plan 4 Task 7): under the host claim the resolve door's
+// {"retry": true} record write answers HC5-N1 like the other console move
+// writes, before any write, even on a stuck move. Claim Plan 10 adds the
+// console move grant that covers resolve --retry.
+func TestResolveRetryRefusedUnderHostClaim(t *testing.T) {
+ fx, token := operatorInstanceFixture(t, "code-inst-move-claim-retry")
+ m := titoManifest()
+ m.Regions = nil
+ m.Pool = ""
+ enableInstances(t, fx, "titogo", m)
+ u := instancesURL(fx, "bab", "titogo")
+ if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated {
+ t.Fatalf("create = %d %s", code, out)
+ }
+ stuckMove(t, fx, func(m *provision.MoveState) {
+ m.State, m.Rollback = provision.MoveRollingBack, &provision.MoveRollback{ID: "rb_abcdefghijklmnopqrstuvwxyz", SourcePhase: provision.RbSourceVerified}
+ }, provision.StuckBaselineMismatch)
+ ctx := context.Background()
+ rec, err := provision.GetRecord(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io")
+ if err != nil {
+ t.Fatal(err)
+ }
+ before, _ := json.Marshal(rec.Move)
+ fx.srv.cfg.HostClaim = true
+ code, _, out := rawReq(t, http.MethodPost, u+"/acme.go.tito.io/move/resolve", token, `{"retry":true}`)
+ if code != http.StatusConflict || errorText(out) != claimMoveUnsupportedMsg {
+ t.Fatalf("resolve --retry under the host claim = %d %s", code, out)
+ }
+ rec, err = provision.GetRecord(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if after, _ := json.Marshal(rec.Move); string(after) != string(before) {
+ t.Fatalf("a refused retry wrote the record:\n%s\n%s", before, after)
+ }
+ if rec.Move.State != provision.MoveStuck {
+ t.Fatalf("the move is %s, want stuck", rec.Move.State)
+ }
+}
+
// claimMoveAppID is the canonical app id armClaimMoves binds titogo to.
const claimMoveAppID = "titogo-canonical"
diff --git a/internal/console/apiinstancemove_test.go b/internal/console/apiinstancemove_test.go
index cf552824b..03a792870 100644
--- a/internal/console/apiinstancemove_test.go
+++ b/internal/console/apiinstancemove_test.go
@@ -559,12 +559,17 @@ func TestMoveMutationsAreOperatorOnly(t *testing.T) {
if claimBytes(t, fx, "acme.go.tito.io") != "" {
t.Fatal("a refused mutation wrote a claim")
}
- // An operator reaches the resolve door, which has nothing to do yet.
+ // An operator reaches the resolve door. A retry with no move in flight
+ // is the guarded transition's own 409; a body that asks for nothing the
+ // door does yet is "nothing to resolve yet".
if err := fx.st.SetOperator("alice@keymail.dev", true); err != nil {
t.Fatal(err)
}
- if code, _, out := rawReq(t, http.MethodPost, moveURL+"/resolve", token, `{"retry":true}`); code != http.StatusConflict || !strings.Contains(out, "nothing to resolve yet") {
- t.Fatalf("resolve as an operator = %d %s", code, out)
+ if code, _, out := rawReq(t, http.MethodPost, moveURL+"/resolve", token, `{"retry":true}`); code != http.StatusConflict || errorText(out) != "no journaled move in flight" {
+ t.Fatalf("resolve --retry as an operator with no move = %d %s", code, out)
+ }
+ if code, _, out := rawReq(t, http.MethodPost, moveURL+"/resolve", token, `{}`); code != http.StatusConflict || errorText(out) != "nothing to resolve yet" {
+ t.Fatalf("resolve with nothing asked as an operator = %d %s", code, out)
}
}
@@ -1451,3 +1456,192 @@ func TestPhase1ConflictPassesPlan4Refusals(t *testing.T) {
}
}
}
+
+// stuckMove is a move journal stuck in the given way, as the box's own
+// transitions leave one (MoveMarkStuck), for the resolve door to act on.
+func stuckMove(t *testing.T, fx *consoleFixture, setup func(*provision.MoveState), reason string) provision.MoveGuard {
+ t.Helper()
+ g := provision.MoveGuard{ID: "mv_abcdefghijklmnopqrstuvwxyz", Incarnation: strings.Repeat("ab", 32)}
+ ctx := context.Background()
+ if _, err := provision.SetRecordMove(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io", func(rec *provision.Record) error {
+ rec.Move = &provision.MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", Cut: "auto", RequestedAt: time.Now().UTC()}
+ setup(rec.Move)
+ return nil
+ }); err != nil {
+ t.Fatal(err)
+ }
+ if reason != "" {
+ if _, err := provision.SetRecordMove(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io", provision.MoveMarkStuck(g, reason)); err != nil {
+ t.Fatal(err)
+ }
+ }
+ return g
+}
+
+// resolveRetryBody is the resolve door's retry request.
+const resolveRetryBody = `{"retry":true}`
+
+// TestResolveRetryFromStuckOnly (Plan 4 Task 7, §7.10): the resolve door's
+// {"retry": true} runs provision.MoveRetry under the guard from the record,
+// and answers 409 unless the move is stuck. The retry clears the stuck
+// reason only; the box resumes at the phase StuckAt names. The answer is
+// the flat view, which carries no reason.
+func TestResolveRetryFromStuckOnly(t *testing.T) {
+ fx, token := operatorInstanceFixture(t, "code-inst-move-retry")
+ m := titoManifest()
+ m.Regions = nil
+ m.Pool = ""
+ enableInstances(t, fx, "titogo", m)
+ u := instancesURL(fx, "bab", "titogo")
+ ctx := context.Background()
+ if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated {
+ t.Fatalf("create = %d %s", code, out)
+ }
+ resolveURL := u + "/acme.go.tito.io/move/resolve"
+ const rb = "rb_abcdefghijklmnopqrstuvwxyz"
+ read := func() *provision.Record {
+ t.Helper()
+ rec, err := provision.GetRecord(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io")
+ if err != nil {
+ t.Fatal(err)
+ }
+ return rec
+ }
+ moveBytes := func() string {
+ t.Helper()
+ b, err := json.Marshal(read().Move)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return string(b)
+ }
+ clear := func() {
+ t.Helper()
+ if _, err := provision.SetRecordMove(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io", func(rec *provision.Record) error { rec.Move = nil; return nil }); err != nil {
+ t.Fatal(err)
+ }
+ }
+ serial := func() string {
+ t.Helper()
+ b, _, err := fx.rel.Get(ctx, "control/serial")
+ if err != nil {
+ return ""
+ }
+ return string(b)
+ }
+
+ // No move in flight: the guarded transition's own 409, nothing written.
+ if code, _, out := rawReq(t, http.MethodPost, resolveURL, token, resolveRetryBody); code != http.StatusConflict || errorText(out) != "no journaled move in flight" {
+ t.Fatalf("retry with no move = %d %s", code, out)
+ }
+
+ // A move that is not stuck is refused with the recorded sentence and
+ // writes nothing, in every state a move can be in.
+ for _, c := range []struct {
+ name string
+ setup func(*provision.MoveState)
+ }{
+ {"preparing", func(m *provision.MoveState) { m.State = provision.MovePreparing }},
+ {"ready", func(m *provision.MoveState) { m.State = provision.MoveReady }},
+ {"cutting", func(m *provision.MoveState) { m.State, m.CutPhase = provision.MoveCutting, provision.CutAdmitted }},
+ {"aborting", func(m *provision.MoveState) { m.State = provision.MoveAborting }},
+ {"landing", func(m *provision.MoveState) { m.State = provision.MoveLanding }},
+ {"committed", func(m *provision.MoveState) { m.State = provision.MoveCommitted }},
+ {"rolling back", func(m *provision.MoveState) {
+ m.State, m.Rollback = provision.MoveRollingBack, &provision.MoveRollback{ID: rb, SourcePhase: provision.RbSourceVerified}
+ }},
+ {"reverting", func(m *provision.MoveState) { m.State = provision.MoveReverting }},
+ } {
+ stuckMove(t, fx, c.setup, "")
+ before, ser := moveBytes(), serial()
+ if code, _, out := rawReq(t, http.MethodPost, resolveURL, token, resolveRetryBody); code != http.StatusConflict || errorText(out) != provision.RefusedNotStuck {
+ t.Fatalf("retry of a move that is %s = %d %s, want %q", c.name, code, out, provision.RefusedNotStuck)
+ }
+ if moveBytes() != before || serial() != ser {
+ t.Fatalf("a refused retry of a move that is %s wrote", c.name)
+ }
+ clear()
+ }
+
+ // A body that asks for no retry changes nothing, and a stuck move stays
+ // stuck. A body that is not JSON is the family's 400.
+ stuckMove(t, fx, func(m *provision.MoveState) {
+ m.State, m.Rollback = provision.MoveRollingBack, &provision.MoveRollback{ID: rb, Reason: provision.RollbackLandingDeadline, TargetPhase: provision.RbTargetCleaned, SourcePhase: provision.RbSourceVerified}
+ }, provision.StuckBaselineMismatch)
+ before := moveBytes()
+ for _, body := range []string{`{}`, `{"retry":false}`} {
+ if code, _, out := rawReq(t, http.MethodPost, resolveURL, token, body); code != http.StatusConflict || errorText(out) != "nothing to resolve yet" {
+ t.Fatalf("resolve %s = %d %s", body, code, out)
+ }
+ }
+ if code, _, out := rawReq(t, http.MethodPost, resolveURL, token, `{"retry":`); code != http.StatusBadRequest {
+ t.Fatalf("resolve with a broken body = %d %s", code, out)
+ }
+ if moveBytes() != before {
+ t.Fatal("a resolve that asked for no retry wrote")
+ }
+ clear()
+
+ // Stuck rollbacks: each returns to the source phase StuckAt names, with
+ // the stuck reason cleared and every other field kept.
+ for _, c := range []struct {
+ name, reason, at string
+ }{
+ {"a baseline mismatch", provision.StuckBaselineMismatch, provision.RbSourceVerified},
+ {"a recovery probe out of attempts", provision.StuckRecoveryProbeAttempts, provision.RbSourceProbing},
+ } {
+ stuckMove(t, fx, func(m *provision.MoveState) {
+ m.State, m.Reason = provision.MoveRollingBack, provision.RollbackLandingDeadline
+ m.Rollback = &provision.MoveRollback{ID: rb, Reason: provision.RollbackLandingDeadline, TargetPhase: provision.RbTargetCleaned, SourcePhase: c.at}
+ }, c.reason)
+ stuck := read().Move
+ if stuck.State != provision.MoveStuck || stuck.StuckReason != c.reason || stuck.Rollback.StuckAt != c.at {
+ t.Fatalf("%s: setup %+v %+v", c.name, stuck, stuck.Rollback)
+ }
+ ser := serial()
+ code, _, out := rawReq(t, http.MethodPost, resolveURL, token, resolveRetryBody)
+ if code != http.StatusAccepted {
+ t.Fatalf("retry after %s = %d %s", c.name, code, out)
+ }
+ var view map[string]any
+ if err := json.Unmarshal([]byte(out), &view); err != nil {
+ t.Fatal(err)
+ }
+ if len(view) != 3 || view["state"] != provision.MoveRollingBack || view["target"] != "uk" || view["updated_at"] == nil {
+ t.Fatalf("retry after %s answered %s, want the flat view in rolling_back", c.name, out)
+ }
+ got := read().Move
+ want := *stuck
+ want.State, want.StuckReason, want.UpdatedAt = provision.MoveRollingBack, "", got.UpdatedAt
+ rbWant := *stuck.Rollback
+ rbWant.SourcePhase, rbWant.StuckAt = c.at, ""
+ want.Rollback = &rbWant
+ if !reflect.DeepEqual(*got, want) {
+ t.Fatalf("retry after %s:\n got %+v / %+v\nwant %+v / %+v", c.name, *got, *got.Rollback, want, rbWant)
+ }
+ if serial() == ser {
+ t.Fatalf("retry after %s did not bump the serial, so the box would not notice", c.name)
+ }
+ // Not stuck any more: a second retry is refused.
+ if code, _, out := rawReq(t, http.MethodPost, resolveURL, token, resolveRetryBody); code != http.StatusConflict || errorText(out) != provision.RefusedNotStuck {
+ t.Fatalf("second retry after %s = %d %s", c.name, code, out)
+ }
+ clear()
+ }
+
+ // A stuck revert has no rollback to return to: it goes back to
+ // reverting, and keeps the cause the revert recorded.
+ stuckMove(t, fx, func(m *provision.MoveState) {
+ m.State, m.Reason = provision.MoveReverting, provision.RevertCutDeadline
+ }, provision.StuckRevertProbeBudget)
+ if got := read().Move; got.State != provision.MoveStuck || got.StuckReason != provision.StuckRevertProbeBudget {
+ t.Fatalf("revert setup %+v", got)
+ }
+ code, _, out := rawReq(t, http.MethodPost, resolveURL, token, resolveRetryBody)
+ if code != http.StatusAccepted || !strings.Contains(out, `"state":"reverting"`) {
+ t.Fatalf("retry of a stuck revert = %d %s", code, out)
+ }
+ if got := read().Move; got.State != provision.MoveReverting || got.StuckReason != "" || got.Reason != provision.RevertCutDeadline || got.Rollback != nil {
+ t.Fatalf("a retried revert %+v", got)
+ }
+}
diff --git a/internal/host/movesretry_test.go b/internal/host/movesretry_test.go
new file mode 100644
index 000000000..57cb51d41
--- /dev/null
+++ b/internal/host/movesretry_test.go
@@ -0,0 +1,261 @@
+package host
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+ "reflect"
+ "strings"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/provision"
+)
+
+// The box's side of `move resolve --retry` (moves spec §7.8 stuck, §7.10;
+// Plan 4 Task 7). The console door writes provision.MoveRetry and nothing
+// else (h.retry is the same transition through the same record writer); the
+// box then resumes the source's recovery at the phase the stuck rollback
+// recorded as StuckAt, with the stuck reason gone and the rollback's own
+// reason kept. A verification runs again and is never bypassed (INV-4); a
+// probe that ran out of attempts is called again with the whole budget;
+// nothing earlier than StuckAt runs again. The fixture is Task 5's.
+
+// verifications counts how many times the source ran its baseline check.
+func (h *recoverHarness) verifications() int {
+ n := 0
+ for _, p := range h.rcHits {
+ if p == "verify" {
+ n++
+ }
+ }
+ return n
+}
+
+// retried asserts what MoveRetry leaves for the box: rolling back at the
+// phase the stuck mark named, no stuck reason, no stuck-at, the cause kept.
+func (h *recoverHarness) retried(t *testing.T, at, cause string) {
+ t.Helper()
+ m := h.read(t, p1Host).Move
+ if m.State != provision.MoveRollingBack || m.StuckReason != "" || m.Rollback == nil || m.Rollback.SourcePhase != at || m.Rollback.StuckAt != "" {
+ t.Fatalf("after the retry: %+v %+v, want rolling_back at %s", m, m.Rollback, at)
+ }
+ if m.Rollback.Reason != cause {
+ t.Fatalf("the retry changed the rollback's cause: %q, want %q", m.Rollback.Reason, cause)
+ }
+}
+
+// TestRetryResumesTheReturnsVerification (J20, INV-4): a clean return whose
+// files no longer match the source's stopped baseline is stuck at verified.
+// A retry returns the record to verified, and the box verifies again, once
+// per retry; unrepaired it is stuck again with the same sentence and nothing
+// after the check has run; repaired it recovers. The retry never skips the
+// check, and never runs the hand-over, the claim write or anything before
+// verified a second time.
+func TestRetryResumesTheReturnsVerification(t *testing.T) {
+ h := newRecoverHarness(t, false)
+ rb := h.released(t)
+ cause := h.read(t, p1Host).Move.Rollback.Reason
+ upload := filepath.Join(h.sdir, p1Host, "upload.txt")
+ if err := os.WriteFile(upload, []byte("changed on the source"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ stuck := func(verified int) {
+ t.Helper()
+ m := h.read(t, p1Host).Move
+ if m.State != provision.MoveStuck || m.StuckReason != provision.StuckBaselineMismatch || m.Rollback.StuckAt != provision.RbSourceVerified {
+ t.Fatalf("not stuck at verified: %+v %+v", m, m.Rollback)
+ }
+ if n := h.verifications(); n != verified {
+ t.Fatalf("verified %d times, want %d (%v)", n, verified, h.rcHits)
+ }
+ if len(h.probes) != 0 {
+ t.Fatalf("probed past a mismatch: %+v", h.probes)
+ }
+ }
+ h.srcPass(t)
+ stuck(1)
+ handed := len(h.srcHanded)
+ if handed != 1 {
+ t.Fatalf("hand-overs before the retry: %v", h.srcHanded)
+ }
+ // Stuck is not driven on: a pass between the stuck mark and the retry
+ // verifies nothing.
+ h.srcPass(t)
+ stuck(1)
+
+ // The retry alone verifies nothing: it only moves the record.
+ h.retry(t)
+ h.retried(t, provision.RbSourceVerified, cause)
+ if n := h.verifications(); n != 1 {
+ t.Fatalf("the retry verified by itself: %d", n)
+ }
+ h.srcPass(t)
+ stuck(2)
+ h.retry(t)
+ h.srcPass(t)
+ stuck(3)
+
+ // The operator repairs the source; one more retry reaches opened.
+ if err := os.WriteFile(upload, []byte("hello"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ h.retry(t)
+ h.retried(t, provision.RbSourceVerified, cause)
+ h.srcPass(t)
+ got := h.opened(t)
+ if got.ID != rb.ID || got.StuckAt != "" || h.read(t, p1Host).Move.StuckReason != "" {
+ t.Fatalf("opened: %+v", got)
+ }
+ if n := h.verifications(); n != 4 {
+ t.Fatalf("verified %d times, want 4 (one per attempt)", n)
+ }
+ if len(h.srcHanded) != handed {
+ t.Fatalf("the retry handed the row over again: %v", h.srcHanded)
+ }
+ if len(h.probes) != 1 {
+ t.Fatalf("probes: %d, want the one after the repair", len(h.probes))
+ }
+}
+
+// TestRetryGivesTheProbeItsWholeBudgetAgain (J20): a recovery probe that ran
+// out of attempts is stuck at probing. A retry resumes at probing, so the
+// box calls the probe again with the full nine attempts (the counter starts
+// again at 0) and runs nothing before it: no verification, no install, no
+// claim step. The probe keeps its own count (Task 9 builds it on claim
+// Plan 10's attempts); the box's journal holds none to reset.
+func TestRetryGivesTheProbeItsWholeBudgetAgain(t *testing.T) {
+ h := newRecoverHarness(t, false)
+ h.released(t)
+ h.probe = func(recoveryProbeRequest) (recoveryProbeOutcome, error) { return recoveryProbeExhausted, nil }
+ h.srcPass(t)
+ cause := h.read(t, p1Host).Move.Rollback.Reason
+ stuck := func(probes int) {
+ t.Helper()
+ m := h.read(t, p1Host).Move
+ if m.State != provision.MoveStuck || m.StuckReason != provision.StuckRecoveryProbeAttempts || m.Rollback.StuckAt != provision.RbSourceProbing {
+ t.Fatalf("not stuck at probing: %+v %+v", m, m.Rollback)
+ }
+ if len(h.probes) != probes {
+ t.Fatalf("probe calls: %d, want %d", len(h.probes), probes)
+ }
+ }
+ stuck(1)
+ h.srcPass(t)
+ stuck(1) // stuck is not driven on
+ // The source's files change under the stuck probe, as a live app's would.
+ if err := os.WriteFile(filepath.Join(h.sdir, p1Host, "upload.txt"), []byte("written by the recovered app"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+
+ h.retry(t)
+ h.retried(t, provision.RbSourceProbing, cause)
+ h.srcPass(t)
+ stuck(2)
+ h.retry(t)
+ h.probe = nil
+ h.srcPass(t)
+ h.opened(t)
+ if len(h.probes) != 3 {
+ t.Fatalf("probe calls: %d, want 3", len(h.probes))
+ }
+ for i, req := range h.probes {
+ if req.Attempts != 9 {
+ t.Fatalf("probe call %d carries a budget of %d, want the whole nine", i, req.Attempts)
+ }
+ if !req.Clean || req.Coverage != recoveryCoverageClean || req.SourceGeneration != "cur" {
+ t.Fatalf("probe call %d: %+v", i, req)
+ }
+ }
+ if n := h.verifications(); n != 1 {
+ t.Fatalf("verified %d times, want 1: a retry at probing never verifies again (INV-2)", n)
+ }
+ if len(h.srcHanded) != 1 {
+ t.Fatalf("hand-overs: %v", h.srcHanded)
+ }
+}
+
+// TestRetryResumesADirtyInstallAtLocal: a dirty return's install that found
+// the source's files occupied is stuck at local (the three install reasons
+// are legal at local and configured). A retry resumes the shared installer
+// at local from its journal, not from the start: the pre-move park, run
+// once, is not run again, and once the operator has cleared the files the
+// recovery reaches opened.
+func TestRetryResumesADirtyInstallAtLocal(t *testing.T) {
+ h := newRecoverHarness(t, true)
+ h.released(t)
+ sentinel := filepath.Join(h.sdir, p1Host, "sentinel.txt")
+ h.landFault = func(p string) error {
+ if p == "after-"+landStepRetired {
+ if err := os.MkdirAll(filepath.Dir(sentinel), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ return os.WriteFile(sentinel, []byte("not the return's"), 0o644)
+ }
+ return nil
+ }
+ h.srcPass(t)
+ cause := h.read(t, p1Host).Move.Rollback.Reason
+ m := h.read(t, p1Host).Move
+ if m.State != provision.MoveStuck || m.StuckReason != provision.StuckReturnFilesOccupied || m.Rollback.StuckAt != provision.RbSourceLocal {
+ t.Fatalf("not stuck at local: %+v %+v", m, m.Rollback)
+ }
+ if len(h.probes) != 0 {
+ t.Fatalf("probed past a refused install: %+v", h.probes)
+ }
+ // Stuck, retried with the sentinel still there: stuck again, the same
+ // sentence, the sentinel untouched.
+ h.retry(t)
+ h.retried(t, provision.RbSourceLocal, cause)
+ h.srcPass(t)
+ if m := h.read(t, p1Host).Move; m.State != provision.MoveStuck || m.StuckReason != provision.StuckReturnFilesOccupied || m.Rollback.StuckAt != provision.RbSourceLocal {
+ t.Fatalf("an unrepaired retry: %+v %+v", m, m.Rollback)
+ }
+ if body, err := os.ReadFile(sentinel); err != nil || string(body) != "not the return's" {
+ t.Fatalf("the sentinel: %q %v", body, err)
+ }
+ // Repaired: the operator removes what is not the return's.
+ if err := os.Remove(sentinel); err != nil {
+ t.Fatal(err)
+ }
+ h.landFault = nil
+ h.retry(t)
+ h.srcPass(t)
+ rb := h.opened(t)
+ if body, err := os.ReadFile(provision.DefaultDBPath(h.sdir, p1Host) + ".gen"); err != nil || strings.TrimSpace(string(body)) != rb.ID {
+ t.Fatalf("the installed generation marker: %q %v", body, err)
+ }
+ if n := h.verifications(); n != 0 {
+ t.Fatalf("a dirty return verified the baseline %d times; it installs instead", n)
+ }
+ if len(h.probes) != 1 || h.probes[0].Clean || h.probes[0].Coverage != recoveryCoverageSealed || h.probes[0].Attempts != recoveryProbeAttempts {
+ t.Fatalf("probes: %+v", h.probes)
+ }
+}
+
+// TestRetryLeavesTheClaimAndTheRowAlone: the retry is a record write. It
+// does not touch the host claim or the registry row; the box's next pass
+// does what the phase asks.
+func TestRetryLeavesTheClaimAndTheRowAlone(t *testing.T) {
+ h := newRecoverHarness(t, false)
+ h.released(t)
+ if err := os.WriteFile(filepath.Join(h.sdir, p1Host, "upload.txt"), []byte("changed"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ h.srcPass(t)
+ claim, row := h.claim(t), h.srcRoute
+ h.retry(t)
+ after := h.claim(t)
+ if after.Op.Phase != hostclaim.PhaseClaimed || after.Op.Phase != claim.Op.Phase || after.Op.Exec != claim.Op.Exec {
+ t.Fatalf("the retry changed the claim: %+v -> %+v", claim.Op, after.Op)
+ }
+ if !reflect.DeepEqual(h.srcRoute, row) {
+ t.Fatalf("the retry changed the row: %+v -> %+v", row, h.srcRoute)
+ }
+ // A second retry, now that the move is not stuck, is the provision
+ // refusal the door speaks as its 409.
+ g := provision.MoveGuard{ID: p1ID, Incarnation: h.inc}
+ if _, err := provision.SetRecordMove(context.Background(), h.s, "bab", "app", p1Host, provision.MoveRetry(g)); err == nil || !strings.Contains(err.Error(), provision.RefusedNotStuck) {
+ t.Fatalf("a retry of a move that is not stuck: %v", err)
+ }
+}