amadan
moves: the console pass takes the target's fence one step a pass (Task 5, Q8)
The recovery hook reads the target's termination fence for a landing claim before its commit or a restoring claim: the claim's consumed fence once it carries one (ConsumedMoveTargetFence), otherwise the bucket's control/box-fences/<target enrollment>.json. decideMoveRecovery takes one ReleaseFencedTarget step through the MoveRecoveryGrant (the grant-only release token is minted inside it): after the flip, ahead of the claim's rollback half and the deadline's rollback, since the grant writes both halves. Once the claim has consumed the fence and the record carries it as both LossRef and Evidence, the pass stops calling it. A dirty target's fence comes after the claim's rollback half (the grant takes neither half). The grant verifies the fence and refuses it with ErrPending; the pass pages once (incident, durable rate limit) that an operator must declare the target lost, and leaves it for J4. The console logs the fence's writes as its own: the record transitions (moves.TransitionOf, matched by record_seq) and the claim's rollback. Census: one new authority site, movesRecovery's g.ReleaseFencedTarget, in testdata/claim-authority-sites.txt (ReleaseFencedTarget was already a selected name). Copy: MV-N450 and MV-N451, pending rows, listed in movesCopyIDs and in the scan's page functions. MV-N452/453 dropped (loss-timer pages are the claim's).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
7 files changed,
+497
−15
approved_moves_copy_test.go+6 −2docs/superpowers/plans/2026-10-06-moves-phase1-copy.md+9 −0internal/console/claim_authority_census_test.go+4 −0internal/console/movesrecovery.go+178 −10internal/console/movesrecovery_test.go+291 −0internal/console/server.go+8 −3internal/console/testdata/claim-authority-sites.txt+1 −0
diff --git a/approved_moves_copy_test.go b/approved_moves_copy_test.go| index 6e858ff38..c8a733d2d 100644 |
| --- a/approved_moves_copy_test.go |
| +++ b/approved_moves_copy_test.go |
| @@ -86,6 +86,7 @@ var movesCopyIDs = []string{ |
| "MV-N380", "MV-N381", |
| "MV-N385", "MV-N386", "MV-N387", "MV-N389", "MV-N390", |
| "MV-N440", "MV-N441", "MV-N442", "MV-N443", |
| + "MV-N450", "MV-N451", |
| } |
| // copyRow is one row of the record. The File cell is the file the sentence is |
| @@ -1397,8 +1398,11 @@ var movesReturnedSentences = map[string]map[string]bool{ |
| // The box's move pages (Plan 6 Task 5, MV-N385 to MV-N390) and M-C20, |
| // the operators' sentence for a stray process (the stray page's body). |
| "internal/host/movespaging.go": {"rolledBackPage": true, "strayPage": true, "strayLine": true, "startRefusedPage": true}, |
| - // The console's move pass pages (Plan 5 Task 7, MV-N325 to MV-N334). |
| - "internal/console/movesrecovery.go": {"moveWaitingPage": true, "movePreparingPageText": true, "moveOrphanPage": true, "moveCloseOverduePage": true, "moveAbortOverduePage": true}, |
| + // The console's move pass pages (Plan 5 Task 7, MV-N325 to MV-N334), and |
| + // the dirty fenced target's page (moves after claim Plan 12, Task 5, |
| + // MV-N450 and MV-N451). |
| + "internal/console/movesrecovery.go": {"moveWaitingPage": true, "movePreparingPageText": true, "moveOrphanPage": true, "moveCloseOverduePage": true, "moveAbortOverduePage": true, |
| + "moveTargetFencedDirtyPage": true}, |
| } |
| // movesLiteralFuncs are functions whose every string literal is a sentence |
diff --git a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md| index 7bee097d0..810daa96f 100644 |
| --- a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md |
| +++ b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md |
| @@ -416,3 +416,12 @@ Paul's ruling of 2026-10-11 (N1.3 in the Plan 7 preflight): the source refuses t |
| | MV-N441 | internal/provision/provision.go | the target box cannot land the move or roll it back yet | approved | |
| | MV-N442 | internal/provision/provision.go | the source box cannot recover the move if the cut fails | approved | |
| | MV-N443 | move.go | The cut did not start: <reason>. | approved | |
| + |
| +## Moves after claim Plan 12 |
| + |
| +This branch's rows take MV-N450 to MV-N499 (Ruling F13: each a pending row in the commit of the task that writes it). Task 5's two are the console move pass's page for a dirty fenced target (ruling Q8): a target box terminated after it released with writes the source does not have, whose fence the recovery grant refuses, so the pass leaves it for an operator's declaration (J4) and pages once. `<to>` and `<from>` are the move's regions as a sentence names them (moves.RegionText). MV-N452 and MV-N453 are not used: loss-timer pages are the claim's (platform-1e, 2026-10-12). |
| + |
| +| id | file | text | status | |
| +|---|---|---|---| |
| +| MV-N450 | internal/console/movesrecovery.go | carlos move target terminated before returning its writes: <host> | pending | |
| +| MV-N451 | internal/console/movesrecovery.go | <box> in <to> was terminated after it wrote data it had not returned to <from>. the move <id> of <host> cannot finish on its own. run carlos move resolve --host <host> --declare-lost <box> to move <host> back to <from>. | pending | |
diff --git a/internal/console/claim_authority_census_test.go b/internal/console/claim_authority_census_test.go| index ee731d684..0a04cf7a7 100644 |
| --- a/internal/console/claim_authority_census_test.go |
| +++ b/internal/console/claim_authority_census_test.go |
| @@ -29,6 +29,10 @@ func claimAuthoritySites(t *testing.T) []string { |
| // The console's move pass (moves Plan 5 Task 7): every write of the |
| // move-recovery grant, and the lost target's stamp. |
| "ResolveStart": true, "Convert": true, "Claim": true, "Record": true, "StampLostMoveTargetParked": true, |
| + // The pass's fence action (moves after claim Plan 12, Task 5, Q8) |
| + // calls ReleaseFencedTarget, selected above, on the same grant: one |
| + // step of the target's fence per observation, with the grant-only |
| + // release token minted inside it. |
| // The recovery pass's move log writer (Plan 6 Task 3, fix round 3 |
| // ruling 1): minted inside RunRecovery for the console's outbox |
| // drainer; one create-only entry per call, under the entry's own |
diff --git a/internal/console/movesrecovery.go b/internal/console/movesrecovery.go| index 2b60f6496..fa85b55af 100644 |
| --- a/internal/console/movesrecovery.go |
| +++ b/internal/console/movesrecovery.go |
| @@ -48,7 +48,8 @@ import ( |
| // after its request is aborted through ConsoleMove(ConsoleMoveAbort); |
| // the source's own abort (movesabort.go) takes it from there. |
| // 4. The claim half the record decided (§7.5, Review Focus 5): flip, |
| -// commit or rollback. Anything else is left alone. |
| +// commit or rollback, with the target's fence between the flip and the |
| +// rest (below). Anything else is left alone. |
| // 5. The deadline: a landing past the earlier of its cut and landing |
| // deadlines is rolled back by the record (J11, J9's recorded reason). |
| // 6. A lost target released with nothing of its box or region owed: |
| @@ -70,11 +71,34 @@ import ( |
| // (ruling M4); after a box crashed between the halves, only this pass |
| // writes it, because no box visits a record without a move. |
| // |
| +// The target's fence (moves after claim Plan 12, Task 5, ruling Q8). A |
| +// landing claim before its commit, or a restoring claim, whose target box was |
| +// terminated has its regional worker's termination fence at |
| +// control/box-fences/<target enrollment>.json. The grant's |
| +// ReleaseFencedTarget takes one step of it per call, record first: on a |
| +// landing it rolls the record back with the target lost under the fence |
| +// (RollbackTargetFenced), or on a rollback marks the target lost; then the |
| +// claim's release, which rolls a landing claim back and consumes the fence in |
| +// one write; then the record's loss commit (released, the fence as its |
| +// evidence). The fence comes after the flip (the grant refuses a moving |
| +// claim) and ahead of the claim's rollback half and the deadline's rollback, |
| +// because the grant writes both halves itself. Once the claim carries the |
| +// consumed fence, the reference is the claim's (ConsumedMoveTargetFence), |
| +// never the bucket's; and once the record carries it as both its loss and its |
| +// evidence, the pass stops calling it. A dirty target (one that released with |
| +// writes the source does not have) cannot be finished from a dead box: the |
| +// grant refuses it after it verified the fence, the pass pages once that an |
| +// operator must declare the target lost, and leaves it for J4. Its fence |
| +// comes after the claim's rollback half, which the grant does not take for |
| +// it. The console logs the fence's record writes, and its claim rollback, as |
| +// its own move writes. |
| +// |
| // Pages (§13) go through alerting.Page on the sinks cmd/console arms, best |
| // effort: a rollback waiting past its release deadline (an incident, again |
| // only a full hour after the last attempt), a move preparing for over an |
| // hour, an orphan past the seal (an incident), an overdue close the console |
| -// wrote, and an overdue abort the console leaves to its source (each once). |
| +// wrote, an overdue abort the console leaves to its source, and a dirty |
| +// fenced target left for J4 (an incident) (each once). |
| // The rate limit is durable: the last attempt's time is kept per finding in |
| // the console's store (move_pages, fix round 1 ruling 3), written whether or |
| // not delivery worked, so a restarted console keeps the spacing. |
| @@ -103,7 +127,10 @@ const ( |
| moveActClaim moveRecoveryAction = "claim" |
| moveActRecord moveRecoveryAction = "record" |
| moveActStampLost moveRecoveryAction = "stamp-lost" |
| - moveActClose moveRecoveryAction = "close" |
| + // moveActFence is one step of the target's termination fence, through |
| + // the grant's ReleaseFencedTarget (Q8). |
| + moveActFence moveRecoveryAction = "fence" |
| + moveActClose moveRecoveryAction = "close" |
| // moveActAbortOverdue writes nothing: an abort due to close, which only |
| // its source may close, is checked for being overdue and paged. |
| moveActAbortOverdue moveRecoveryAction = "abort-overdue" |
| @@ -117,17 +144,25 @@ const ( |
| movePageOrphan movePageKind = "orphan" |
| movePageCloseOverdue movePageKind = "close-overdue" |
| movePageAbortOverdue movePageKind = "abort-overdue" |
| + // movePageTargetFencedDirty is a dirty target whose fence the grant |
| + // refused: left for J4 (Q8). |
| + movePageTargetFencedDirty movePageKind = "target-fenced-dirty" |
| ) |
| // moveRecoveryInput is what one observation saw. Start is the op's move-start |
| // effect, read only for a moving claim with no record move; Fence is the |
| -// source's termination fence, read only for a moving claim. |
| +// source's termination fence, read only for a moving claim. TargetFence is |
| +// the target's (Q8), read only for a landing claim before its commit or a |
| +// restoring claim: the claim's consumed fence when it carries one |
| +// (TargetFenceConsumed), otherwise the bucket's. |
| type moveRecoveryInput struct { |
| - Now time.Time |
| - Claim *hostclaim.Claim |
| - Rec provision.Record |
| - Start *hostops.Effect |
| - Fence *hostclaim.FenceRef |
| + Now time.Time |
| + Claim *hostclaim.Claim |
| + Rec provision.Record |
| + Start *hostops.Effect |
| + Fence *hostclaim.FenceRef |
| + TargetFence *hostclaim.FenceRef |
| + TargetFenceConsumed bool |
| } |
| // moveRecoveryDecision is at most one action, with the grant action it |
| @@ -183,6 +218,7 @@ func decideMoveRecovery(in moveRecoveryInput) moveRecoveryDecision { |
| d.Pages = append(d.Pages, movePagePreparing) |
| } |
| owed := owedUnderOp(c, op.ID, nil) |
| + fence, fenceDirty := targetFenceStep(in) |
| switch { |
| case op.Kind == hostclaim.KindMoving && in.Fence != nil && (m.State == provision.MovePreparing || m.State == provision.MoveReady || m.State == provision.MoveCutting || m.State == provision.MoveRelocating): |
| @@ -191,12 +227,19 @@ func decideMoveRecovery(in moveRecoveryInput) moveRecoveryDecision { |
| d.Action = moveActAbort |
| case m.State == provision.MoveLanding && m.FlippedAt != nil && rec.Region == m.To && sealed: |
| d.Action, d.Claim = moveActClaim, hostops.RecoveryMoveFlip |
| + case fence && !fenceDirty: |
| + d.Action = moveActFence |
| case m.State == provision.MoveCommitted && m.CommittedAt != nil && m.FlippedAt != nil && rec.Region == m.To && op.Kind == hostclaim.KindLanding && op.Phase == hostclaim.PhaseLandingProbed: |
| d.Action, d.Claim = moveActClaim, hostops.RecoveryMoveCommit |
| case m.State == provision.MoveRollingBack && rb != nil && rec.Region == m.From && (op.Kind == hostclaim.KindLanding && op.Phase != hostclaim.PhaseCommitted || sealed): |
| d.Action, d.Claim = moveActClaim, hostops.RecoveryMoveRollback |
| case m.State == provision.MoveLanding && op.Kind == hostclaim.KindLanding && op.Phase != hostclaim.PhaseCommitted && moveDeadlinePassed(m, in.Now): |
| d.Action, d.Record = moveActRecord, hostops.RecoveryMoveDeadlineRollback |
| + case fence: |
| + // A dirty target: after the claim's rollback half, which the grant |
| + // does not take for it. The grant verifies the fence and refuses it; |
| + // the pass pages (movesRecovery). |
| + d.Action = moveActFence |
| case m.State == provision.MoveRollingBack && rb != nil && rb.TargetPhase == provision.RbTargetReleased && !rb.Dirty && rb.TargetParkedAt == nil && |
| hostclaim.LostMoveTargetParked(c, park.PlacementRegion(m.To)): |
| d.Action = moveActStampLost |
| @@ -213,6 +256,53 @@ func decideMoveRecovery(in moveRecoveryInput) moveRecoveryDecision { |
| return d |
| } |
| +// targetFenceStep reports whether the target's fence (in.TargetFence) has a |
| +// step left for the grant's ReleaseFencedTarget, and whether the step is a |
| +// dirty target's, which the grant refuses and the pass leaves for J4 (Q8). |
| +// The record's regions are compared as the grant compares them, code with |
| +// code. |
| +// |
| +// Consumed (the claim's release is done), the one step left is the record's |
| +// loss commit: the target lost under this fence. Once the record carries the |
| +// fence as its loss and its evidence, or the target released on its own |
| +// evidence, the fence is spent. Not yet consumed, a step is one the grant |
| +// would take: a landing's rollback; a rollback whose target still holds |
| +// something the source needs (releasing, returning, or released dirty) is |
| +// marked lost; the target lost under this fence, the claim's release; or a |
| +// target that released clean while the claim still waits on it |
| +// (restoring/releasing under the target), the claim's release. A target |
| +// lost under another loss (J4's declaration) is not the fence's. |
| +func targetFenceStep(in moveRecoveryInput) (step, dirty bool) { |
| + c, rec, ref := in.Claim, in.Rec, in.TargetFence |
| + if ref == nil || c == nil || rec.Move == nil { |
| + return false, false |
| + } |
| + op, m := c.Op, rec.Move |
| + rb := m.Rollback |
| + if in.TargetFenceConsumed { |
| + return op.Kind == hostclaim.KindRestoring && m.State == provision.MoveRollingBack && rb != nil && rec.Region == m.From && |
| + rb.TargetPhase == provision.RbTargetLost && rb.LossRef == ref.Key && !rb.Dirty, false |
| + } |
| + if !(op.Kind == hostclaim.KindLanding && op.Phase != hostclaim.PhaseCommitted) && op.Kind != hostclaim.KindRestoring { |
| + return false, false |
| + } |
| + if m.State == provision.MoveLanding { |
| + return op.Kind == hostclaim.KindLanding && m.FlippedAt != nil && rec.Region == m.To, false |
| + } |
| + if m.State != provision.MoveRollingBack || rb == nil || rec.Region != m.From { |
| + return false, false |
| + } |
| + switch rb.TargetPhase { |
| + case provision.RbTargetReleasing, provision.RbTargetReturning: |
| + step = true |
| + case provision.RbTargetReleased: |
| + step = rb.Dirty || op.Kind == hostclaim.KindRestoring && op.Phase == hostclaim.PhaseReleasing && op.Exec == op.Intent["target"] |
| + case provision.RbTargetLost: |
| + step = rb.LossRef == ref.Key |
| + } |
| + return step, step && rb.Dirty |
| +} |
| + |
| // moveDeadlinePassed is the grant's deadline rule: the earlier of the cut and |
| // landing deadlines (J11, provision.LandingDeadlineOf, the box's own rule) is |
| // behind now. |
| @@ -284,6 +374,14 @@ func (s *Server) movesRecovery(ctx context.Context, g *hostops.MoveRecoveryGrant |
| return err |
| } |
| } |
| + if ours && (c.Op.Kind == hostclaim.KindLanding && c.Op.Phase != hostclaim.PhaseCommitted || c.Op.Kind == hostclaim.KindRestoring) { |
| + // Once consumed, the fence is the claim's, never the bucket's (Q8). |
| + if ref, ok := hostclaim.ConsumedMoveTargetFence(c); ok { |
| + in.TargetFence, in.TargetFenceConsumed = &ref, true |
| + } else if in.TargetFence, err = s.bucketFence(ctx, c.Op.Intent["target_enrollment"]); err != nil { |
| + return err |
| + } |
| + } |
| d := decideMoveRecovery(in) |
| for _, kind := range d.Pages { |
| s.pageMove(ctx, kind, o, c, rec, now) |
| @@ -352,6 +450,23 @@ func (s *Server) movesRecovery(ctx context.Context, g *hostops.MoveRecoveryGrant |
| return nil |
| case moveActStampLost: |
| return g.StampLostMoveTargetParked(ctx) |
| + case moveActFence: |
| + if f := s.moveFenceCall; f != nil { |
| + f(*in.TargetFence) |
| + } |
| + err := g.ReleaseFencedTarget(ctx, *in.TargetFence) |
| + if errors.Is(err, hostops.ErrPending) && rec.Move.Rollback != nil && rec.Move.Rollback.Dirty { |
| + // The grant refuses a dirty target only after it verified the |
| + // fence, so the page rests on a fence the grant accepted. J4, |
| + // the operator's declaration, takes it from here. |
| + s.pageMove(ctx, movePageTargetFencedDirty, o, c, rec, now) |
| + return nil |
| + } |
| + if err != nil { |
| + return err |
| + } |
| + s.logFenceStep(ctx, o, c, rec, now) |
| + return nil |
| case moveActClose: |
| if !s.moveCloseOverdue(o, d.Record, now) { |
| return nil |
| @@ -378,7 +493,13 @@ func (s *Server) movesRecovery(ctx context.Context, g *hostops.MoveRecoveryGrant |
| // the reference to control/box-fences/<source enrollment>.json and its exact |
| // bytes. Convert verifies it; nothing here trusts it. |
| func (s *Server) sourceFence(ctx context.Context, c *hostclaim.Claim) (*hostclaim.FenceRef, error) { |
| - enrollment := c.Op.Intent["source_enrollment"] |
| + return s.bucketFence(ctx, c.Op.Intent["source_enrollment"]) |
| +} |
| + |
| +// bucketFence is the termination fence of the box enrolled as enrollment, |
| +// when one exists in the bucket: its reference and its exact bytes' digest. |
| +// The grant verifies it; nothing here trusts it. |
| +func (s *Server) bucketFence(ctx context.Context, enrollment string) (*hostclaim.FenceRef, error) { |
| if enrollment == "" { |
| return nil, nil |
| } |
| @@ -394,6 +515,39 @@ func (s *Server) sourceFence(ctx context.Context, c *hostclaim.Claim) (*hostclai |
| return &hostclaim.FenceRef{Enrollment: enrollment, Key: key, SHA256: hex.EncodeToString(sum[:])}, nil |
| } |
| +// logFenceStep queues the log entry of the write one fence step made, as the |
| +// console's own move write (fix round 3, ruling 1). A record write is the one |
| +// whose sequence follows the record the pass acted on, named by |
| +// moves.TransitionOf: the rollback (RollbackTargetFenced), the target marked |
| +// lost, or the loss committed. A step from a landing claim that wrote no |
| +// record was the claim's release, which rolled the claim back |
| +// (claim_rollback). A record that moved on twice, or cannot be read, loses |
| +// the entry with a journal line: the write itself stands. |
| +func (s *Server) logFenceStep(ctx context.Context, o hostops.ConsoleMoveObservation, c *hostclaim.Claim, rec provision.Record, now time.Time) { |
| + r, err := hostops.ReadRecord(ctx, s.cfg.Releases, c.Holder.Key) |
| + if err != nil { |
| + log.Printf("console: moves: %s: the log of move %s's fence step is lost: the record cannot be read: %v", o.Host, o.OpID, err) |
| + return |
| + } |
| + after, err := provision.DecodeRecord(r.Body) |
| + if err != nil { |
| + log.Printf("console: moves: %s: the log of move %s's fence step is lost: %v", o.Host, o.OpID, err) |
| + return |
| + } |
| + switch { |
| + case after.RecordSeq == rec.RecordSeq+1: |
| + if t, ok := moves.TransitionOf(rec, after); ok { |
| + s.queueMoveLog(o, t, now) |
| + } |
| + case after.RecordSeq == rec.RecordSeq: |
| + if c.Op.Kind == hostclaim.KindLanding { |
| + s.queueMoveLog(o, claimTransition(o, rec, "claim_rollback", "claim-rollback"), now) |
| + } |
| + default: |
| + log.Printf("console: moves: %s: the log of move %s's fence step is lost: the record moved on past it", o.Host, o.OpID) |
| + } |
| +} |
| + |
| // moveDueSince is when the pass first saw this close due, remembering now |
| // if it had not. The memory is the process's: a restarted console waits the |
| // grace again, which only delays the console's close or page. Entries a day |
| @@ -465,6 +619,9 @@ func (s *Server) pageMove(ctx context.Context, kind movePageKind, o hostops.Cons |
| subject, body = moveCloseOverduePage(o.Host, o.OpID) |
| case movePageAbortOverdue: |
| subject, body = moveAbortOverduePage(o.Host, o.OpID, rec.Move.To, c.Op.Intent["source"]) |
| + case movePageTargetFencedDirty: |
| + level = alerting.LevelIncident |
| + subject, body = moveTargetFencedDirtyPage(o.Host, o.OpID, c.Op.Intent["target"], moves.RegionText(rec.Move.To), moves.RegionText(rec.Move.From)) |
| default: |
| return |
| } |
| @@ -533,6 +690,17 @@ func moveAbortOverduePage(host, id, to, box string) (subject, body string) { |
| return subject, body |
| } |
| +// moveTargetFencedDirtyPage is a target terminated before it returned the |
| +// writes it made, whose fence the grant refuses (Q8, left for J4): the |
| +// subject (MV-N450) and the body (MV-N451). to and from are as a sentence |
| +// names them (moves.RegionText). |
| +func moveTargetFencedDirtyPage(host, id, box, to, from string) (subject, body string) { |
| + subject = fmt.Sprintf("carlos move target terminated before returning its writes: %s", host) |
| + body = fmt.Sprintf("%s in %s was terminated after it wrote data it had not returned to %s. the move %s of %s cannot finish on its own. run carlos move resolve --host %s --declare-lost %s to move %s back to %s.", |
| + box, to, from, id, host, host, box, host, from) |
| + return subject, body |
| +} |
| + |
| // The move log of the console's own move writes (Plan 6 Task 3; fix round 3, |
| // ruling 1): the closes, the landing deadline's rollback, the six-hour abort, |
| // and the claim's rollback and release. Each entry is named as a box's write |
diff --git a/internal/console/movesrecovery_test.go b/internal/console/movesrecovery_test.go| index 81d298b36..9a43e1ab5 100644 |
| --- a/internal/console/movesrecovery_test.go |
| +++ b/internal/console/movesrecovery_test.go |
| @@ -1477,3 +1477,294 @@ func TestDestroyedLocalOnlyLogsNoMoveEvent(t *testing.T) { |
| }) |
| } |
| } |
| + |
| +// ---- the target fence (moves after claim Plan 12, Task 5, ruling Q8) ---- |
| + |
| +// mvTargetFence is a target fence reference, on values. |
| +var mvTargetFence = hostclaim.FenceRef{Enrollment: strings.Repeat("b", 32), Key: "control/box-fences/" + strings.Repeat("b", 32) + ".json", SHA256: strings.Repeat("c", 64)} |
| + |
| +// TestRecoveryPassTargetFenceDecision: on values, when the target's fence has |
| +// a step left for the grant, and where it sits in the order (Q8): after the |
| +// flip, ahead of the claim's rollback half and the deadline's rollback, and |
| +// never once the record carries the consumed fence as its evidence. A dirty |
| +// target's fence comes after the rollback half, since the grant takes |
| +// neither half of it. |
| +func TestRecoveryPassTargetFenceDecision(t *testing.T) { |
| + ref := mvTargetFence |
| + past := mvT0.Add(-time.Hour) |
| + target := func(phase string, dirty bool) func(*provision.Record) { |
| + return func(r *provision.Record) { r.Move.Rollback.TargetPhase, r.Move.Rollback.Dirty = phase, dirty } |
| + } |
| + lost := func(key string) func(*provision.Record) { |
| + return func(r *provision.Record) { |
| + r.Move.Rollback.TargetPhase, r.Move.Rollback.LossRef = provision.RbTargetLost, key |
| + } |
| + } |
| + committedLoss := func(r *provision.Record) { |
| + r.Move.Rollback.TargetPhase, r.Move.Rollback.LossRef, r.Move.Rollback.Evidence = provision.RbTargetReleased, ref.Key, ref.Key |
| + } |
| + restoring := func(phase, exec string) *hostclaim.Claim { |
| + return mvClaim(hostclaim.KindRestoring, phase, func(c *hostclaim.Claim) { c.Op.Exec = exec }) |
| + } |
| + for _, c := range []struct { |
| + name string |
| + claim *hostclaim.Claim |
| + rec provision.Record |
| + consumed bool |
| + action moveRecoveryAction |
| + half hostops.MoveRecoveryClaimAction |
| + }{ |
| + {"moving/sealed, record landing: the flip, never the fence", mvClaim(hostclaim.KindMoving, hostclaim.PhaseSealed), mvRecord(provision.MoveLanding), false, moveActClaim, hostops.RecoveryMoveFlip}, |
| + {"a landing: the fence rolls the record back", mvClaim(hostclaim.KindLanding, "probing"), mvRecord(provision.MoveLanding), false, moveActFence, ""}, |
| + {"a landing past its deadline: the fence, not the deadline's rollback", mvClaim(hostclaim.KindLanding, "probing"), |
| + mvRecord(provision.MoveLanding, func(r *provision.Record) { r.Move.LandingDeadline = &past }), false, moveActFence, ""}, |
| + {"rolling back, the claim landing: the fence, not the rollback half", mvClaim(hostclaim.KindLanding, "probing"), mvRecord(provision.MoveRollingBack), false, moveActFence, ""}, |
| + {"rolling back, the target lost under this fence: the fence (its claim half)", mvClaim(hostclaim.KindLanding, "probing"), mvRecord(provision.MoveRollingBack, lost(ref.Key)), false, moveActFence, ""}, |
| + {"rolling back, the target lost under another loss (J4): not the fence", restoring(hostclaim.PhaseReleased, "none"), mvRecord(provision.MoveRollingBack, lost("control/host-loss/other")), false, moveActNone, ""}, |
| + {"released clean, the claim restoring/releasing under the target: the fence", restoring(hostclaim.PhaseReleasing, "uk-box"), mvRecord(provision.MoveRollingBack, target(provision.RbTargetReleased, false)), false, moveActFence, ""}, |
| + {"released clean, the claim past releasing: not the fence", restoring(hostclaim.PhaseClaimed, "ie-box"), mvRecord(provision.MoveRollingBack, target(provision.RbTargetReleased, false)), false, moveActNone, ""}, |
| + {"cleaned: nothing left for the fence", restoring(hostclaim.PhaseOpened, "ie-box"), mvRecord(provision.MoveRollingBack, target(provision.RbTargetCleaned, false)), false, moveActNone, ""}, |
| + {"dirty, the claim landing: the rollback half first", mvClaim(hostclaim.KindLanding, "probing"), mvRecord(provision.MoveRollingBack, target(provision.RbTargetReleased, true)), false, moveActClaim, hostops.RecoveryMoveRollback}, |
| + {"dirty, the claim restoring: the fence (which leaves it to J4)", restoring(hostclaim.PhaseReleasing, "uk-box"), mvRecord(provision.MoveRollingBack, target(provision.RbTargetReleased, true)), false, moveActFence, ""}, |
| + {"a committed landing: never the fence", mvClaim(hostclaim.KindLanding, hostclaim.PhaseCommitted), mvRecord(provision.MoveLanding), false, moveActNone, ""}, |
| + {"consumed, the loss not yet on the record: the fence", restoring(hostclaim.PhaseReleased, "none"), mvRecord(provision.MoveRollingBack, lost(ref.Key)), true, moveActFence, ""}, |
| + {"consumed, LossRef and Evidence the fence: stop", restoring(hostclaim.PhaseReleased, "none"), mvRecord(provision.MoveRollingBack, committedLoss), true, moveActNone, ""}, |
| + {"consumed, the target released on its own evidence: stop", restoring(hostclaim.PhaseReleased, "none"), mvRecord(provision.MoveRollingBack, target(provision.RbTargetReleased, false)), true, moveActNone, ""}, |
| + } { |
| + t.Run(c.name, func(t *testing.T) { |
| + got := decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: c.claim, Rec: c.rec, TargetFence: &ref, TargetFenceConsumed: c.consumed}) |
| + if got.Action != c.action || got.Claim != c.half || got.Record != "" { |
| + t.Fatalf("decided %+v, want %s %s", got, c.action, c.half) |
| + } |
| + }) |
| + } |
| + // With no fence, a landing past its deadline is the deadline's rollback, |
| + // as before. |
| + d := decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindLanding, "probing"), |
| + Rec: mvRecord(provision.MoveLanding, func(r *provision.Record) { r.Move.LandingDeadline = &past })}) |
| + if d.Action != moveActRecord || d.Record != hostops.RecoveryMoveDeadlineRollback { |
| + t.Fatalf("no fence, past the deadline: %+v", d) |
| + } |
| +} |
| + |
| +// fencedTargetWorld is the landing world (j4LandingMove) with its target, |
| +// uk-box, terminated: the regional worker's fence for it in the bucket, and |
| +// the pass's fence authority binding it to uk. calls lists the fences the |
| +// pass's fence calls named. |
| +func fencedTargetWorld(t *testing.T, code string) (*movePassWorld, hostclaim.FenceRef, func() []hostclaim.FenceRef) { |
| + t.Helper() |
| + w := landingMovePass(t, code) |
| + w.auth = oneBoxFenceAuthority{c7MoveAuthority{region: "uk"}} |
| + ref := terminatedSourceFence(t, w.fx, "uk-box", "ie-box") |
| + if c := w.claim(t); c.Op.Intent["target_enrollment"] != ref.Enrollment { |
| + t.Fatalf("the fence is not the target's: %s, claim %s", ref.Enrollment, c.Op.Intent["target_enrollment"]) |
| + } |
| + var mu sync.Mutex |
| + var seen []hostclaim.FenceRef |
| + w.fx.srv.moveFenceCall = func(r hostclaim.FenceRef) { mu.Lock(); seen = append(seen, r); mu.Unlock() } |
| + return w, ref, func() []hostclaim.FenceRef { |
| + mu.Lock() |
| + defer mu.Unlock() |
| + return append([]hostclaim.FenceRef(nil), seen...) |
| + } |
| +} |
| + |
| +// TestRecoveryPassReleasesAFencedTargetInThreePasses (Q8): a landing whose |
| +// target was terminated. The grant takes one step a pass, record first: the |
| +// record's rollback with the target lost under the fence, then the claim's |
| +// release that consumes it, then the record's loss commit (released, the |
| +// fence as its evidence). Each pass writes one object; the fourth writes |
| +// nothing. The console logs its writes: the rollback, the claim's rollback |
| +// and the target's release. |
| +func TestRecoveryPassReleasesAFencedTargetInThreePasses(t *testing.T) { |
| + w, ref, calls := fencedTargetWorld(t, "code-mvpass-fence") |
| + claim := claimBytes(t, w.fx, j4Host) |
| + |
| + w.pass(t) |
| + rec := w.record(t) |
| + rb := rec.Move.Rollback |
| + if rec.Move.State != provision.MoveRollingBack || rec.Region != "ie" || rb == nil || rb.Reason != provision.RollbackTargetFenced || rb.TargetPhase != provision.RbTargetLost || rb.LossRef != ref.Key { |
| + t.Fatalf("the first pass: region %s %+v %+v", rec.Region, rec.Move, rb) |
| + } |
| + if claimBytes(t, w.fx, j4Host) != claim { |
| + t.Fatal("the record's pass wrote the claim too") |
| + } |
| + |
| + w.pass(t) |
| + c := w.claim(t) |
| + if got, ok := hostclaim.ConsumedMoveTargetFence(c); !ok || got != ref || c.Op.Kind != hostclaim.KindRestoring { |
| + t.Fatalf("the second pass did not consume the fence: %+v %v %+v", got, ok, c.Op) |
| + } |
| + if after := w.record(t); after.RecordSeq != rec.RecordSeq { |
| + t.Fatalf("the claim's pass wrote the record: %d after %d", after.RecordSeq, rec.RecordSeq) |
| + } |
| + |
| + claim = claimBytes(t, w.fx, j4Host) |
| + w.pass(t) |
| + rec = w.record(t) |
| + rb = rec.Move.Rollback |
| + if rb.TargetPhase != provision.RbTargetReleased || rb.Dirty || rb.Evidence != ref.Key || rb.LossRef != ref.Key { |
| + t.Fatalf("the third pass did not commit the loss on the record: %+v", rb) |
| + } |
| + if claimBytes(t, w.fx, j4Host) != claim { |
| + t.Fatal("the record's loss commit wrote the claim") |
| + } |
| + |
| + w.pass(t) |
| + if after := w.record(t); after.RecordSeq != rec.RecordSeq || claimBytes(t, w.fx, j4Host) != claim { |
| + t.Fatalf("the fourth pass wrote: record %d after %d", after.RecordSeq, rec.RecordSeq) |
| + } |
| + for i, r := range calls() { |
| + if r != ref { |
| + t.Fatalf("call %d named %+v, not the target's fence %+v", i, r, ref) |
| + } |
| + } |
| + log := moveLogOf(t, w) |
| + if ev := log["rolling_back"]; ev.Actor != "console" || ev.Fields["reason"] != provision.RollbackTargetFenced { |
| + t.Fatalf("the fenced rollback's entry: %+v in %+v", ev, log) |
| + } |
| + if ev := log["rolling_back-target-released"]; ev.Actor != "console" || ev.Event != provision.RbTargetReleased { |
| + t.Fatalf("the loss commit's entry: %+v in %+v", ev, log) |
| + } |
| + if ev := log["claim-rollback"]; ev.Actor != "console" || ev.Event != "claim_rollback" { |
| + t.Fatalf("the claim release's rollback entry: %+v in %+v", ev, log) |
| + } |
| + if p := w.paged(t); len(p) != 0 { |
| + t.Fatalf("a clean fenced target paged: %+v", p) |
| + } |
| +} |
| + |
| +// TestRecoveryPassFenceBeatsTheDeadlineRollback (Q8): a landing past its |
| +// deadline whose target was terminated is rolled back by the fence (the |
| +// grant does both halves), not by the deadline. |
| +func TestRecoveryPassFenceBeatsTheDeadlineRollback(t *testing.T) { |
| + w, ref, _ := fencedTargetWorld(t, "code-mvpass-fence-deadline") |
| + past := time.Now().UTC().Add(-time.Minute) |
| + j17SetRecord(t, w.fx, func(r *provision.Record) { r.Move.LandingDeadline = &past }) |
| + w.pass(t) |
| + rb := w.record(t).Move.Rollback |
| + if rb == nil || rb.Reason != provision.RollbackTargetFenced || rb.TargetPhase != provision.RbTargetLost || rb.LossRef != ref.Key { |
| + t.Fatalf("the deadline won: %+v", rb) |
| + } |
| +} |
| + |
| +// TestRecoveryPassFlipsBeforeTheFence (Q8): with the claim still moving/sealed |
| +// and the record landing, the pass writes the flip and does not call the |
| +// fence (the grant refuses a moving claim). The fence follows on the next |
| +// pass. |
| +func TestRecoveryPassFlipsBeforeTheFence(t *testing.T) { |
| + d := decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindMoving, hostclaim.PhaseSealed), Rec: mvRecord(provision.MoveLanding), TargetFence: &mvTargetFence}) |
| + if d.Action != moveActClaim || d.Claim != hostops.RecoveryMoveFlip { |
| + t.Fatalf("decided %+v, want the flip", d) |
| + } |
| + w, ref, calls := fencedTargetWorld(t, "code-mvpass-fence-flip") |
| + w.setClaim(t, func(c *hostclaim.Claim) { |
| + c.Op.Kind, c.Op.Phase, c.Op.Exec = hostclaim.KindMoving, hostclaim.PhaseSealed, "" |
| + c.Op.Participants = []string{"ie-box", "uk-box"} |
| + }) |
| + w.pass(t) |
| + if c := w.claim(t); c.Op.Kind != hostclaim.KindLanding { |
| + t.Fatalf("not flipped: %+v", c.Op) |
| + } |
| + if n := len(calls()); n != 0 { |
| + t.Fatalf("the flip's pass called the fence %d times", n) |
| + } |
| + if m := w.record(t).Move; m.State != provision.MoveLanding { |
| + t.Fatalf("the flip's pass rolled the record back: %+v", m) |
| + } |
| + w.pass(t) |
| + if rb := w.record(t).Move.Rollback; rb == nil || rb.LossRef != ref.Key { |
| + t.Fatalf("the fence did not follow the flip: %+v", rb) |
| + } |
| +} |
| + |
| +// TestRecoveryPassLeavesADirtyFencedTargetToJ4AndPagesOnce (Q8): a target |
| +// terminated after it released dirty holds writes the source does not have. |
| +// The grant refuses it (no return lane can be built from a dead box), so the |
| +// pass writes nothing for the fence, pages once that an operator must declare |
| +// the target lost, and leaves it for J4. |
| +func TestRecoveryPassLeavesADirtyFencedTargetToJ4AndPagesOnce(t *testing.T) { |
| + w, _, calls := fencedTargetWorld(t, "code-mvpass-fence-dirty") |
| + rbID, at := newRollbackID(t), time.Now().UTC().Add(-time.Minute) |
| + deadline := at.Add(time.Hour) |
| + j17SetRecord(t, w.fx, func(r *provision.Record) { |
| + r.Region = "ie" |
| + r.Move.State = provision.MoveRollingBack |
| + r.Move.Rollback = &provision.MoveRollback{ID: rbID, RequestedAt: &at, Reason: provision.RollbackLandingDeadline, TargetPhase: provision.RbTargetReleased, Dirty: true, |
| + SourcePhase: provision.RbSourceWaiting, ReleaseDeadline: &deadline} |
| + }) |
| + w.pass(t) // the claim's rollback half, which the fence does not take for a dirty target |
| + if c := w.claim(t); c.Op.Kind != hostclaim.KindRestoring || c.Op.Intent["rollback"] != rbID { |
| + t.Fatalf("the rollback half: %+v", c.Op) |
| + } |
| + if n := len(calls()); n != 0 { |
| + t.Fatalf("the fence was called before the rollback half: %d", n) |
| + } |
| + claim, rec := claimBytes(t, w.fx, j4Host), w.record(t) |
| + w.pass(t) |
| + w.pass(t) |
| + w.advance(2 * time.Hour) |
| + w.pass(t) |
| + if n := len(calls()); n == 0 { |
| + t.Fatal("the fence was never tried: its verification is the page's evidence") |
| + } |
| + if after := w.record(t); after.RecordSeq != rec.RecordSeq || claimBytes(t, w.fx, j4Host) != claim { |
| + t.Fatalf("a dirty fenced target was written: %+v", after.Move.Rollback) |
| + } |
| + p := w.paged(t) |
| + if len(p) != 1 { |
| + t.Fatalf("pages %+v, want one", p) |
| + } |
| + subject, body := moveTargetFencedDirtyPage(j4Host, w.id, "uk-box", "uk", "ie") |
| + if p[0].Level != alerting.LevelIncident || p[0].Subject != subject || p[0].Body != body { |
| + t.Fatalf("the page: %+v", p[0]) |
| + } |
| + if !strings.Contains(body, "--declare-lost uk-box") || !strings.Contains(body, j4Host) { |
| + t.Fatalf("the page does not say how to declare the target lost: %q", body) |
| + } |
| +} |
| + |
| +// TestRecoveryPassStopsCallingTheFenceOnceConsumed (Q8): once the claim |
| +// carries the consumed fence and the record its loss (LossRef and Evidence |
| +// the fence's key), the pass never calls the fence again. |
| +func TestRecoveryPassStopsCallingTheFenceOnceConsumed(t *testing.T) { |
| + w, ref, calls := fencedTargetWorld(t, "code-mvpass-fence-stop") |
| + for range 3 { |
| + w.pass(t) |
| + } |
| + if rb := w.record(t).Move.Rollback; rb == nil || rb.LossRef != ref.Key || rb.Evidence != ref.Key { |
| + t.Fatalf("not at the stop condition: %+v", rb) |
| + } |
| + n := len(calls()) |
| + if n != 3 { |
| + t.Fatalf("%d fence calls for three steps", n) |
| + } |
| + for range 3 { |
| + w.pass(t) |
| + } |
| + if got := len(calls()); got != n { |
| + t.Fatalf("the fence was called %d more times after its loss was committed", got-n) |
| + } |
| +} |
| + |
| +// TestRecoveryPassReadsTheConsumedFenceNotTheBucket (Q8): after the claim |
| +// consumed the fence, the pass takes the reference from the claim, never the |
| +// bucket: the fence object deleted, the record's loss commit still follows. |
| +func TestRecoveryPassReadsTheConsumedFenceNotTheBucket(t *testing.T) { |
| + w, ref, calls := fencedTargetWorld(t, "code-mvpass-fence-consumed") |
| + w.pass(t) |
| + w.pass(t) |
| + if _, ok := hostclaim.ConsumedMoveTargetFence(w.claim(t)); !ok { |
| + t.Fatal("the fence was not consumed in two passes") |
| + } |
| + if err := w.fx.rel.(*deletableStore).Delete(context.Background(), ref.Key); err != nil { |
| + t.Fatal(err) |
| + } |
| + w.pass(t) |
| + if rb := w.record(t).Move.Rollback; rb == nil || rb.TargetPhase != provision.RbTargetReleased || rb.Evidence != ref.Key { |
| + t.Fatalf("the loss commit did not follow the deleted fence: %+v", rb) |
| + } |
| + if got := calls(); len(got) != 3 || got[2] != ref { |
| + t.Fatalf("the third call: %+v", got) |
| + } |
| +} |
diff --git a/internal/console/server.go b/internal/console/server.go| index 7a60ea992..913213a57 100644 |
| --- a/internal/console/server.go |
| +++ b/internal/console/server.go |
| @@ -39,6 +39,7 @@ import ( |
| "github.com/carlosframework/platform/internal/economics" |
| "github.com/carlosframework/platform/internal/edgesfile" |
| "github.com/carlosframework/platform/internal/geo" |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| "github.com/carlosframework/platform/internal/hostops" |
| "github.com/carlosframework/platform/internal/infraview" |
| "github.com/carlosframework/platform/internal/lease" |
| @@ -841,9 +842,13 @@ type Server struct { |
| // unconfirmed declared_lost row, after it read the claim (a TEST SEAM: |
| // a declaration that commits in between, movesrecovery.go). |
| moveLogSettleHook func() |
| - dbHandleKey []byte |
| - cfg Config |
| - store *Store |
| + // moveFenceCall, when set, runs as the pass calls the grant's |
| + // ReleaseFencedTarget, with the fence it names (a TEST SEAM: which |
| + // passes call the target's fence, movesrecovery.go). |
| + moveFenceCall func(hostclaim.FenceRef) |
| + dbHandleKey []byte |
| + cfg Config |
| + store *Store |
| // appRegionOf and beforeClaimInsert are TEST SEAMS (nil in |
| // production): a store failure on the app-region read (C20), and a |
| // competing insert landing after the region was resolved. |
diff --git a/internal/console/testdata/claim-authority-sites.txt b/internal/console/testdata/claim-authority-sites.txt| index 9e1fbf855..e3db6a4f8 100644 |
| --- a/internal/console/testdata/claim-authority-sites.txt |
| +++ b/internal/console/testdata/claim-authority-sites.txt |
| @@ -116,6 +116,7 @@ movesrecovery.go|movesRecovery|g.Claim(ctx, d.Claim) |
| movesrecovery.go|movesRecovery|g.Convert(ctx, *in.Fence) |
| movesrecovery.go|movesRecovery|g.Record(ctx, d.Record) |
| movesrecovery.go|movesRecovery|g.Record(ctx, d.Record) |
| +movesrecovery.go|movesRecovery|g.ReleaseFencedTarget(ctx, *in.TargetFence) |
| movesrecovery.go|movesRecovery|g.ResolveStart(ctx) |
| movesrecovery.go|movesRecovery|g.StampLostMoveTargetParked(ctx) |
| scanpass.go|storeLister|objstore.S3ClientFor(ctx, p.Region, p.Endpoint) |