amadan
console: a row about to be finished resolves an unconfirmed request by the fence first
The reconciliation branches that finish a row (the record names the move's outcome, carries no move, or carries another move) no longer wait on an unconfirmed request whose write can never be replayed there: it is fenced and confirmed or voided, and the row is decided again next tick. The internal/console envelope is re-pinned (PROPOSED-self-serve-moves-2).
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
3 files changed,
+38
−7
internal/console/selfserve_job.go+14 −6internal/e2e/selfserve/job_reconcile_test.go+23 −0internal/provision/testdata/plan14-root-reads-self-serve-moves-2.tsv+1 −1
CI failed — run details
CI log
=== gate === .amadan/test-gate.sh amadan: job timed out after 30m0s amadan: job failed at step gate — its output is above, not at the tail of this log
diff --git a/internal/console/selfserve_job.go b/internal/console/selfserve_job.go| index 744a55211..6006ac08e 100644 |
| --- a/internal/console/selfserve_job.go |
| +++ b/internal/console/selfserve_job.go |
| @@ -327,10 +327,18 @@ func (s *Server) selfServeReconcile(ctx context.Context, t *leaseToken, row *Sel |
| return true, s.selfServeFinalise(ctx, t, row, provision.OfflineOutcomeAborted, msgSelfServeRecreated, "") |
| } |
| mv := rec.Move |
| + // A row about to be finished is never finished with a request |
| + // outstanding: an unconfirmed one is resolved by the fence first (its |
| + // write may otherwise land later, or never be replayable on this |
| + // record), and the row is decided again next tick. |
| + unresolved := func() bool { |
| + out, conf := selfServeOutstanding(row) |
| + return out && !conf |
| + } |
| switch { |
| case mv == nil && rec.LastMove != nil && rec.LastMove.ID == row.ID: |
| - if out, conf := selfServeOutstanding(row); out && !conf { |
| - return true, nil // resolved first |
| + if unresolved() { |
| + return true, s.selfServeResolveFenced(ctx, t, row) |
| } |
| return true, s.selfServeFinalise(ctx, t, row, rec.LastMove.Outcome, s.selfServeOutcomeReason(row, rec.LastMove.Outcome), "") |
| case mv == nil: |
| @@ -338,13 +346,13 @@ func (s *Server) selfServeReconcile(ctx context.Context, t *leaseToken, row *Sel |
| case ssRequested, ssReserving: |
| return s.selfServeQueueFence(ctx, t, row, rec, now) |
| } |
| - if out, conf := selfServeOutstanding(row); out && !conf { |
| - return true, nil |
| + if unresolved() { |
| + return true, s.selfServeResolveFenced(ctx, t, row) |
| } |
| return true, s.selfServeFinalise(ctx, t, row, provision.OfflineOutcomeAborted, msgSelfServeCouldNotStart, "") |
| case mv.ID != row.ID: |
| - if out, conf := selfServeOutstanding(row); out && !conf { |
| - return true, nil |
| + if unresolved() { |
| + return true, s.selfServeResolveFenced(ctx, t, row) |
| } |
| return true, s.selfServeFinalise(ctx, t, row, provision.OfflineOutcomeAborted, msgSelfServeAnotherMove, "") |
| } |
diff --git a/internal/e2e/selfserve/job_reconcile_test.go b/internal/e2e/selfserve/job_reconcile_test.go| index cc5c4f937..de5d2e604 100644 |
| --- a/internal/e2e/selfserve/job_reconcile_test.go |
| +++ b/internal/e2e/selfserve/job_reconcile_test.go |
| @@ -201,6 +201,29 @@ WHERE id = ?`, time.Now().UnixNano(), rec.Incarnation, rec.RecordSeq, time.Now() |
| t.Fatalf("the record was touched: %+v (seq %d, was %d)", after.Move, after.RecordSeq, before.RecordSeq) |
| } |
| }) |
| + t.Run("an unconfirmed queue on a record carrying another move", func(t *testing.T) { |
| + // The queue was issued against a record another move had already |
| + // taken (a move that landed between the job's checks and its |
| + // read): its write can never land and is never replayable, so it |
| + // is fenced and voided before the row is finished. |
| + w := newJobWorld(t) |
| + row := w.admit() |
| + other := moveID() |
| + w.edit(provision.OfflineStage(other, "", host.E2ETargetRegion, "op@example.com", |
| + provision.StageTarget{Stager: host.E2ESourceBox, Bucket: "data-uk", Region: "eu-west-2"}, time.Now(), "")) |
| + rec := w.record() |
| + w.setPhase(row.ID, "reserving", "") |
| + w.sql(`UPDATE self_serve_moves SET request_issued = ?, request_kind = 'queue', request_phase = 'reserving', request_nonce = 'rq_stuckqueue', |
| + request_expect_inc = ?, request_expect_seq = ?, request_proposed = ?, request_confirmed = 0, request_consumed = 0, request_voided = 0, request_deadline = 0 |
| +WHERE id = ?`, time.Now().UnixNano(), rec.Incarnation, rec.RecordSeq, time.Now().UnixNano(), row.ID) |
| + r := w.finished(row.ID, 6) |
| + if r.OutcomeReason != "another move is in flight" || w.hits["queue:fence"] == 0 { |
| + t.Fatalf("finished %s, fences %d", describe(r), w.hits["queue:fence"]) |
| + } |
| + if after := w.record(); after.Move == nil || after.Move.ID != other { |
| + t.Fatalf("the other move was touched: %+v", after.Move) |
| + } |
| + }) |
| t.Run("the record is gone", func(t *testing.T) { |
| w := newJobWorld(t) |
| row := w.admit() |
diff --git a/internal/provision/testdata/plan14-root-reads-self-serve-moves-2.tsv b/internal/provision/testdata/plan14-root-reads-self-serve-moves-2.tsv| index fa2736277..7a8c86d7e 100644 |
| --- a/internal/provision/testdata/plan14-root-reads-self-serve-moves-2.tsv |
| +++ b/internal/provision/testdata/plan14-root-reads-self-serve-moves-2.tsv |
| @@ -23,7 +23,7 @@ internal/console/templates/settings.html external legacy accepted-legacy accepte |
| internal/provision/offline_move.go offlineBegin legacy accepted-legacy accepted-legacy;rev3=PROPOSED-self-serve-moves-2;basis=root-input (J4, as OfflineStart); the console (cmd/console serve under console.service, user carlos-console): self-serve moves 1 Task 7's body (OfflineStart's, the state a parameter; a direct start advances move_seq and stamps move.seq, refusing ErrLifetimeExhausted at MaxInt64 before any write), and it now also creates the self-serve move in offline-queued for OfflineQueue (queued_at, staged_at and precheck_at at the caller's stamp, move.seq set to the record's move_seq, no advance) and records return_of; a stage and a queue hold nothing; its one site is still rec.ActiveRun.Open(), a method on the record's run state (RunState.Open), not a file read; self-serve moves 2 Task 2 file:Open@208 0a9b432ddf2f39a977c0651e377258b35c1d05635e9979583eeac6253d6c0ec8 |
| @coverage internal/provision 42588b9f64803b41c4463efb31e69fda0fc66573efd5d6a2d6e709682eeca385 |
| # Task 3 (per-box reservations with a legacy projection and a sentinel, the fenced reclaim and the operator doors' bound admission), rev3=PROPOSED-self-serve-moves-2: no root read, class or task is added and no accepted-legacy unit is re-pinned, so the seal does not move. The new units (moves.ParseOfflineActive, moves.OfflineActiveBody, the console's offlinereserve.go, the doors' offlineApplyBound and offlineAdvanceBound) read no file: their reads are the bucket's control/offline-moves/active.json and the instance record through the judged readers and fenceRecord. Envelopes re-pinned in place (Task 2's rows replaced): internal/console (offlinereserve.go, the start and stage doors, the Server's two evidence fields) and internal/moves (the reservation table in offline.go). |
| -@coverage internal/console 02de4303cae08253409c954948edd85be6ac28e3526ed1739df2611e2ac59c49 |
| +@coverage internal/console 15d866f96bd8172d527b9eb40c99c327deab70034c207506072d1ea7440ce631 |
| @coverage internal/moves 6a68b01ef865d9356bb19971f4cd4a61f7694769b63cee1d8234f615d5e9c8ba |
| # Task 4 (the journal, the events table, the console lease and the pass skeleton), rev3=PROPOSED-self-serve-moves-2: no root read, class or task is added. The console SQLite journal (selfserve_store.go) adds twenty-two units, each frozen accepted-legacy with a reviewed basis (console SQL as carlos-console, no bucket or root action), and the lease's console name adds one crypto/rand read (Server.selfServeConsoleID); serve (in place: the paging arm takes the store, the pass is started), NewServer (the row evidence and the recovery-mode scan) and Store.migrate (migrateSelfServe) are re-pinned. The seal moves. The lease and the pass read control/offline-moves/console.json and active.json from the bucket, not a file. Envelopes re-pinned in place: cmd/console, internal/console and internal/provision (WithWriteGuard in WriteRecord). |
| @legacy a8dd51a494ab3cd2ad371ded2850b6d78492569729527ea82cd8dd0f2b418a78 internal/console/selfserve_lease.go Server.selfServeConsoleID legacy accepted-legacy accepted-legacy;rev3=PROPOSED-self-serve-moves-2;basis=console.service user carlos-console, J4 root-input: reads crypto/rand once per process for the random console name the self-serve moves lease carries; randomness grants no root authority file:Read@110 3303b18ec5c77ccaf662064801f94276bfe947c31378cc1ecb8c4fe509d5d4d1 |