amadan

carlos / platform Public

host: with the claim and region HA on and moves' files provider bound, every box replays its region's relocations one pass per host and converts a fenced committed landing, and with claim mode alone it finishes only the completions serving on itself, through main's own discharge, before its first serving wake hands over from the probe's receipt; the journal sweep never runs beside a journal write; edgesfile: the live placement read the console and the pass share

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Paul Campbell pushed by claude 210ce9d5e0fa0ddf27ce788ef12c936d1ae6ae40 parent 8f23012
30 files changed, +2775 −258
  • internal/console/apiinstances.go +1 −1
  • internal/console/boxchoice.go +3 −20
  • internal/console/boxselect.go +1 −1
  • internal/console/claim_runtime.go +3 −0
  • internal/console/claim_runtime_chooser_test.go +21 −0
  • internal/console/hostclaimexecutor.go +5 −4
  • internal/console/instancesplacement.go +19 −208
  • internal/console/regioncandidates.go +1 −1
  • internal/console/regioncandidates_test.go +2 −2
  • internal/console/regiondefault.go +1 −1
  • internal/edgesfile/snapshot.go +261 −0
  • internal/edgesfile/snapshot_test.go +75 −0
  • internal/host/agent.go +18 −0
  • internal/host/claimparticipation_relocation.go +9 −0
  • internal/host/claimrelocation.go +11 −0
  • internal/host/relocatepass.go +585 −0
  • internal/host/relocatepass_carries_test.go +332 −0
  • internal/host/relocatepass_test.go +518 −0
  • internal/host/relocationattempts.go +1 −4
  • internal/host/relocationfinish_unix_test.go +280 −0
  • internal/host/relocationfirstwake_unix_test.go +159 −0
  • internal/host/relocationflush.go +3 −5
  • internal/host/relocationinstall.go +1 −4
  • internal/host/relocationlanding.go +40 −0
  • internal/host/relocationlanding_test.go +240 −0
  • internal/host/relocationowners.go +1 −4
  • internal/host/relocationsweep.go +27 −3
  • internal/hostrelocate/pin_versioned_test.go +66 −0
  • internal/hostrelocate/worker.go +57 −0
  • internal/hostrelocate/worker_bounds_test.go +34 −0
diff --git a/internal/console/apiinstances.go b/internal/console/apiinstances.go
index 06218c128..fbbcab189 100644
--- a/internal/console/apiinstances.go
+++ b/internal/console/apiinstances.go
@@ -872,7 +872,7 @@ func (s *Server) handleAPIInstanceCreate(w http.ResponseWriter, r *http.Request)
// (C28, the same answer as a failed listing), not C16: the region
// may well be fine, and a box that could not be read is the one
// that may not be (re-review r1). The home needs no evidence.
- if region != "" && !snap.complete() {
+ if region != "" && !snap.EvidenceComplete() {
log.Printf("console: api instances create %s/%s: incomplete edges or snapshot read, region %s unconfirmed", acct.Sqid, app, region)
jsonError(w, http.StatusServiceUnavailable, msgAppRegionUnconfirmed(s.homeRegionOut(region)))
return
diff --git a/internal/console/boxchoice.go b/internal/console/boxchoice.go
index 5c15f0771..86d681505 100644
--- a/internal/console/boxchoice.go
+++ b/internal/console/boxchoice.go
@@ -15,30 +15,13 @@ type (
// eligibleBoxes is boxchoice.Eligible on console's edges snapshot. The
// snapshot is complete only when both its files and its liveness are
-// (edgesSnapshot.complete).
+// (edgesfile.Snapshot.EvidenceComplete).
func eligibleBoxes(snap edgesSnapshot, roster map[string]rosterBox, placement, awsRegion, homeName, expectedFP string, lost map[string]bool) []boxCandidate {
return boxchoice.Eligible(boxSnapshot(snap), roster, placement, awsRegion, homeName, expectedFP, lost)
}
-// boxSnapshot carries every field the predicate reads. A field dropped here
-// would silently read as absent, so console's eligibility tests run through
-// this conversion.
-func boxSnapshot(snap edgesSnapshot) boxchoice.Snapshot {
- out := boxchoice.Snapshot{Complete: snap.complete(), Boxes: make([]boxchoice.Box, 0, len(snap.Files))}
- for _, f := range snap.Files {
- b := boxchoice.Box{
- Box: f.Box, BoxID: f.BoxID, EIP: f.EIP, Region: f.Region, Placement: f.Placement,
- HibernationConfigured: f.HibernationConfigured, ServesSharedPool: f.ServesSharedPool,
- AcceptsInbound: f.AcceptsInbound, SecretsKeyFP: f.SecretsKeyFP, Capabilities: f.Capabilities,
- Draining: f.Draining, Fresh: f.Fresh,
- }
- if f.Headroom != nil {
- b.Headroom = &boxchoice.Headroom{MemFreeMiB: f.Headroom.MemFreeMiB, DiskFreeMiB: f.Headroom.DiskFreeMiB, Awake: f.Headroom.Awake}
- }
- out.Boxes = append(out.Boxes, b)
- }
- return out
-}
+// boxSnapshot is edgesfile.Snapshot.BoxChoice.
+func boxSnapshot(snap edgesSnapshot) boxchoice.Snapshot { return snap.BoxChoice() }
// chooseBox is boxchoice.Choose.
func chooseBox(host string, cands []boxCandidate, running map[string]boxLoad, app string) (boxCandidate, bool) {
diff --git a/internal/console/boxselect.go b/internal/console/boxselect.go
index 46d49e2b7..605ff0a2e 100644
--- a/internal/console/boxselect.go
+++ b/internal/console/boxselect.go
@@ -74,7 +74,7 @@ var errEdgesIncomplete = errors.New("console: the fleet's edges files could not
// a box that stopped reporting may come back.
func (s *Server) selectMoveBoxes(ctx context.Context, host, to, from, app string) (moveSelection, error) {
snap, err := s.liveEdges(ctx)
- if err != nil || !snap.complete() {
+ if err != nil || !snap.EvidenceComplete() {
return moveSelection{}, errEdgesIncomplete
}
floor, err := moves.ReadFloor(ctx, s.cfg.Releases)
diff --git a/internal/console/claim_runtime.go b/internal/console/claim_runtime.go
index 6a41d1e10..7c8deb6c0 100644
--- a/internal/console/claim_runtime.go
+++ b/internal/console/claim_runtime.go
@@ -71,6 +71,9 @@ func (s *Server) ConfigureClaimRuntime(ctx context.Context) error {
}
a := &hostops.ProductionAuthority{Store: data, Control: control, Directory: d, Now: time.Now, ConsoleID: s.cfg.HostConsoleID, Enrollment: self.Enrollment, Namespace: self.Namespaces[0], Signer: release.RootSigner(s.cfg.Key)}
a.Backfill = &hostops.Backfill{Store: data, Directory: d}
+ // Region HA §5.2 (L8): create's executor from the chooser, only with
+ // Config.RegionHA on; nil keeps today's create path.
+ a.Executors = s.executorChooser(d)
a.Evidence = hostactors.ExactEvidence{PublicKey: s.cfg.ReleasePub, Store: data, Boxes: d.PrivateBoxReport, Self: s.localHostReport}
c := hostops.Controller{Store: data, Control: control, Authority: a, Now: time.Now, NewID: claimOperationID}
owner := hostops.RegistrationOwner{Console: s.cfg.HostConsoleID, Enrollment: self.Enrollment, Controller: claimOperationID()}
diff --git a/internal/console/claim_runtime_chooser_test.go b/internal/console/claim_runtime_chooser_test.go
new file mode 100644
index 000000000..d8e6dad28
--- /dev/null
+++ b/internal/console/claim_runtime_chooser_test.go
@@ -0,0 +1,21 @@
+package console
+
+import "testing"
+
+// Region HA §5.2 (L8), stage B Task 10: ConfigureClaimRuntime gives the
+// create door the executor chooser only with RegionHA on. Off, create is
+// today's (no Executors).
+func TestCreateDoorUsesTheChooserOnlyWithRegionHAOn(t *testing.T) {
+ off := newClaimCase(t)
+ if off.srv.cfg.ClaimAuthority.Executors != nil {
+ t.Fatal("a chooser with region HA off: create would change")
+ }
+ on := newClaimCaseWith(t, func(c *Config) { c.RegionHA = true })
+ x := on.srv.cfg.ClaimAuthority.Executors
+ if x == nil || x.Edges == nil || x.Loads == nil || x.Now == nil || x.HomeName != on.srv.cfg.HomeRegion || x.SecretsKeyFP != on.srv.expectedSecretsFP() {
+ t.Fatalf("chooser with region HA on: %+v", x)
+ }
+ if x.Directory.Store == nil {
+ t.Fatal("the chooser reads no roster")
+ }
+}
diff --git a/internal/console/hostclaimexecutor.go b/internal/console/hostclaimexecutor.go
index b0bf172a0..8c2dd834e 100644
--- a/internal/console/hostclaimexecutor.go
+++ b/internal/console/hostclaimexecutor.go
@@ -10,10 +10,11 @@ import (
)
// executorChooser is create's executor choice (region HA §5.2, L8) over
-// console's live edges read and the claim listing's loads. The create
-// door's hostops.ProductionAuthority uses it as Executors once that door
-// lands; nothing calls it before then. With Config.RegionHA off it is nil,
-// and the create path is today's (spec §8 step 2: dormant).
+// console's live edges read and the claim listing's loads.
+// ConfigureClaimRuntime binds it as the create door's
+// hostops.ProductionAuthority.Executors (stage B Task 10). With
+// Config.RegionHA off it is nil, and the create path is today's (spec §8
+// step 2: dormant).
func (s *Server) executorChooser(d hostactors.Directory) *hostops.ExecutorChooser {
if !s.cfg.RegionHA {
return nil
diff --git a/internal/console/instancesplacement.go b/internal/console/instancesplacement.go
index 711005ee9..394e55b5d 100644
--- a/internal/console/instancesplacement.go
+++ b/internal/console/instancesplacement.go
@@ -2,17 +2,19 @@ package console
import (
"context"
- "encoding/json"
"log"
"sort"
- "strings"
"sync"
"time"
+ "github.com/carlosframework/platform/internal/edgesfile"
"github.com/carlosframework/platform/internal/release"
- "github.com/carlosframework/platform/internal/status"
)
+// livenessSaid throttles the console's own incomplete-evidence diagnostic
+// to once per process (the liveness join's are edgesfile's).
+var livenessSaid sync.Map
+
// edgesFilePrefix is where every box publishes what it serves — one object
// per box, control/edges/<box>.json, written by internal/host's edges-file
// pass and read by every edge's home map. This is the third reader of that
@@ -30,160 +32,10 @@ type servedBy struct {
Region string `json:"region,omitempty"`
}
-// edgesFilePlacementView is the reader's half of the edges-file schema this
-// join needs: version, box, region, and the host set. Roles are not
-// consulted — a host a box names under ANY role is served there in some
-// capacity, and "where is this instance" is the question, not "who is its
-// write home" (that is the home map's question, internal/edge).
-type edgesFilePlacementView struct {
- V int `json:"v"`
- Box string `json:"box"`
- EIP string `json:"eip"`
- Region string `json:"region"`
- Placement string `json:"placement"`
- // DataBucket is the box's CARLOS_HIBERNATE_BUCKET, verbatim
- // (internal/host/edgesfile.go writes it from that env and omits the key
- // entirely when it is unset). It carries the move lane's actual
- // coordinates, and doubles as positive hosting evidence from a writer
- // too old to publish HibernationConfigured.
- DataBucket string `json:"data_bucket"`
- // HibernationConfigured is the box's own answer to "is my hibernation
- // activator wired" (internal/host/edgesfile.go). A POINTER because
- // absent and false are different facts and the difference is the whole
- // point: absent is a writer that predates the field, whose capability is
- // UNKNOWN, while false is a box that published the field and said no.
- // Reading absence as no is what made an earlier version withdraw healthy
- // regions mid-roll — see placements.
- HibernationConfigured *bool `json:"hibernation_configured"`
- // ServesSharedPool and AcceptsInbound: POINTERS, absent = a writer
- // predating them (unknown). SecretsKeyFP: absent = no key, or a writer
- // predating it; AcceptsInbound's presence tells the two apart
- // (nearest-region spec §6.1, R9).
- ServesSharedPool *bool `json:"serves_shared_pool"`
- AcceptsInbound *bool `json:"accepts_inbound"`
- SecretsKeyFP string `json:"secrets_key_fp"`
- // Box evidence (region HA spec 1 §5.1). All absent on a writer that
- // predates it: BoxID "", Capabilities nil, Draining false, Headroom nil.
- BoxID string `json:"box_id"`
- Capabilities []string `json:"capabilities"`
- Draining bool `json:"draining"`
- Headroom *struct {
- MemFreeMiB int64 `json:"mem_free_mib"`
- DiskFreeMiB int64 `json:"disk_free_mib"`
- Awake int `json:"awake"`
- } `json:"headroom"`
- Hosts map[string]json.RawMessage `json:"hosts"`
- // The moves protocol (moves spec §11), read only by the Phase 1 move
- // door's floor check (selectMoveBoxes); Task 13 adds the writer's side.
- // MovesProtocol is the protocol the box speaks (absent: 0).
- // MovesFloorMinRelease is the published floor's min_release the box
- // has armed locally. PrevRelease is its carlos.prev's release, "none"
- // when it keeps none, and PrevAllowedFor the floor's min_release that
- // carlos.prev was judged at or above.
- MovesProtocol int `json:"moves_protocol"`
- MovesFloorMinRelease string `json:"moves_floor_min_release"`
- PrevRelease string `json:"prev_release"`
- PrevAllowedFor string `json:"prev_allowed_for"`
- // Fresh is not on the wire: readLiveEdgesSnapshot sets it when the
- // box's own status snapshot is still fresh (status.BoxStale). An edges
- // file is not liveness, see readLiveEdgesSnapshot.
- Fresh bool `json:"-"`
-}
-
-// livenessSaid throttles readLiveEdgesSnapshot's two diagnostics to once
-// per process, the way noPointSaid does for a code with no point.
-var livenessSaid sync.Map
-
-// readLiveEdgesSnapshot is readEdgesSnapshot joined with box liveness, for
-// AUTOMATIC eligibility (regionCandidates). An edges file is not evidence
-// that its box is alive: nothing deletes a retired box's file, and the
-// file's own Generation moves only when the file is rewritten, so a
-// healthy box's is arbitrarily old (internal/edgesfile.EdgeIPs's doc
-// comment has the whole argument). The status snapshot is written every
-// tick, so status.BoxStale is a real liveness signal, and this is the same
-// join EdgeIPs makes, over the same period.
-//
-// A box with no fresh snapshot, and every box when the snapshots cannot be
-// loaded, is simply not Fresh. The fold then counts it for nothing, so a
-// deployment whose status store is split from its deployment bucket
-// (CARLOS_STATUS_BUCKET, which the console does not read) degrades to
-// home-only guesses. That is safe but invisible, so both cases are said
-// once: the load failure, and files found with no snapshot matching any.
-// The explicit door's placements stay on the generous file-only read; a
-// person asked for that region and can see it refused later.
-func readLiveEdgesSnapshot(ctx context.Context, s release.Store, now time.Time) (edgesSnapshot, error) {
- snap, err := readEdgesSnapshot(ctx, s)
- if err != nil {
- return snap, err
- }
- if len(snap.Files) == 0 {
- snap.BoxesComplete = true // nothing to be incomplete about
- return snap, nil
- }
- // Completeness is carried separately from staleness: LoadBoxesComplete
- // returns what LoadBoxes would, plus whether every listed snapshot was
- // actually read. A skipped read leaves that box's liveness UNKNOWN,
- // and unknown is never "stale".
- boxes, boxesComplete, err := status.LoadBoxesComplete(ctx, s)
- if err != nil {
- if _, said := livenessSaid.LoadOrStore("load", true); !said {
- log.Printf("console: could not load box snapshots (%v), so no box outside the home counts for automatic region placement until they load", err)
- }
- return snap, nil
- }
- snap.BoxesComplete = boxesComplete
- // known is every box with a status object at all; live is the fresh
- // subset. The join key is the status snapshot's Label against the edges
- // file's own box field, the pair edgesfile.EdgeIPs joins on; the object
- // key names play no part.
- known, live := map[string]bool{}, map[string]bool{}
- for _, b := range boxes {
- known[b.Label] = true
- if !status.BoxStale(b, now, status.SnapshotPeriod) {
- live[b.Label] = true
- }
- }
- published := make(map[string]bool, len(snap.Files))
- joined := 0
- for i := range snap.Files {
- published[snap.Files[i].Box] = true
- snap.Files[i].Fresh = live[snap.Files[i].Box]
- if snap.Files[i].Fresh {
- joined++
- }
- // Ruling (B) reversed, re-review r2, on production evidence: every
- // box writes both objects and a retired box leaves both behind, so
- // an ABSENT snapshot is unknown liveness (incomplete evidence,
- // whole-request) and a STALE one is the retired box (not fresh,
- // counts for neither side, evidence complete).
- if !known[snap.Files[i].Box] {
- snap.BoxesComplete = false
- if _, said := livenessSaid.LoadOrStore("nostatus", true); !said {
- log.Printf("console: box %q has an edges file under %s but no status snapshot at all, so no region outside the home is automatically eligible until it reports; said once", snap.Files[i].Box, edgesFilePrefix)
- }
- }
- }
- // Ruling (A), re-review r2: a FRESH snapshot whose box has no edges
- // file is incomplete evidence, whole-request, like a failed read.
- // Reconcile admits a box without its edges file (its publication may
- // simply have failed), so a live box the fold cannot see could still
- // take the record. A stale snapshot with no file is a retired box and
- // is ignored.
- for label := range live {
- if !published[label] {
- snap.BoxesComplete = false
- if _, said := livenessSaid.LoadOrStore("unmatched", true); !said {
- log.Printf("console: box %q has a fresh status snapshot but no edges file under %s, so no region outside the home is automatically eligible until it publishes one; said once", label, edgesFilePrefix)
- }
- }
- }
- if joined == 0 {
- if _, said := livenessSaid.LoadOrStore("none", true); !said {
- log.Printf("console: %d edges file(s) under %s but no fresh box snapshot matched any of them, so automatic region placement sees the home alone (are edges and status in different buckets? see CARLOS_STATUS_BUCKET/CARLOS_STATUS_PREFIX)", len(snap.Files), edgesFilePrefix)
- }
- }
- return snap, nil
-}
+// edgesFilePlacementView is the edges-file placement view, shared with
+// the box agent's relocation chooser through internal/edgesfile (region HA
+// stage B Task 10): one read for every automatic-eligibility caller.
+type edgesFilePlacementView = edgesfile.PlacementView
// loadEdgesJoin folds every readable edges file into host → the boxes that
// serve it, box-sorted so two consoles answer identically. Best-effort in
@@ -223,59 +75,18 @@ func readEdgesFiles(ctx context.Context, s release.Store) []edgesFilePlacementVi
return snap.Files
}
-// edgesSnapshot is one read of control/edges/. Complete is false when any
-// listed file could not be read or parsed: positive facts about the boxes
-// that WERE read still hold, but nothing may be concluded about "every
-// box" (R9's NoKeyProven, codex plan-R2 1).
-type edgesSnapshot struct {
- Files []edgesFilePlacementView
- Complete bool
- // BoxesComplete: every listed status snapshot was read, and every box
- // on either side has its counterpart, so each file's Fresh is a KNOWN
- // answer. Set only by readLiveEdgesSnapshot; a plain readEdgesSnapshot
- // carries no liveness evidence at all. Unknown liveness is not stale
- // (re-review r1 hole 2, r2 rulings (A) and (B) reversed): a snapshot
- // that could not be read, a fresh snapshot with no edges file, or an
- // edges file with no snapshot at all each make the evidence incomplete.
- BoxesComplete bool
-}
-
-// complete is the evidence bar for AUTOMATIC non-home eligibility: every
-// listed edges file read and parsed, and every listed status snapshot
-// read. Short of it a failing sibling may simply be missing from the fold,
-// so nothing non-home is eligible for the request (ruling, re-review r1).
-func (s edgesSnapshot) complete() bool { return s.Complete && s.BoxesComplete }
+// edgesSnapshot is one read of control/edges/ (edgesfile.Snapshot).
+type edgesSnapshot = edgesfile.Snapshot
-// readEdgesSnapshot REPORTS a failed listing instead of answering empty:
-// the door must tell "could not read" (503, C28) from "no box publishes
-// this code" (codex R2-2). A single unreadable or malformed file is
-// skipped, as before, and marks the snapshot incomplete.
+// readEdgesSnapshot is edgesfile.ReadSnapshot.
func readEdgesSnapshot(ctx context.Context, s release.Store) (edgesSnapshot, error) {
- keys, err := s.List(ctx, edgesFilePrefix)
- if err != nil {
- return edgesSnapshot{}, err
- }
- snap := edgesSnapshot{Complete: true}
- for _, key := range keys {
- name, ok := strings.CutPrefix(key, edgesFilePrefix)
- if !ok || !strings.HasSuffix(name, ".json") || strings.Contains(name, "/") {
- continue
- }
- body, _, err := s.Get(ctx, key)
- if err != nil {
- log.Printf("console: api instances: could not read %s for the placement join: %v", key, err)
- snap.Complete = false
- continue
- }
- var f edgesFilePlacementView
- if err := json.Unmarshal(body, &f); err != nil || f.V < 1 || f.Box == "" {
- log.Printf("console: api instances: skipping %s in the placement join: not a versioned edges file", key)
- snap.Complete = false
- continue
- }
- snap.Files = append(snap.Files, f)
- }
- return snap, nil
+ return edgesfile.ReadSnapshot(ctx, s)
+}
+
+// readLiveEdgesSnapshot is edgesfile.ReadLiveSnapshot: the edges files
+// joined with box liveness, for automatic eligibility.
+func readLiveEdgesSnapshot(ctx context.Context, s release.Store, now time.Time) (edgesSnapshot, error) {
+ return edgesfile.ReadLiveSnapshot(ctx, s, now)
}
// placementCapability is what one placement code's boxes can do, folded
diff --git a/internal/console/regioncandidates.go b/internal/console/regioncandidates.go
index c7bf3bf14..4eec3cad0 100644
--- a/internal/console/regioncandidates.go
+++ b/internal/console/regioncandidates.go
@@ -164,7 +164,7 @@ func regionCandidates(snap edgesSnapshot, homeName, expectedFP string) []regionC
// failing sibling may be the box that is missing, and the keyed box
// that WAS read would speak for the region alone. Whole-request scope,
// deliberately: simpler, and stricter than per region.
- evidence := snap.complete()
+ evidence := snap.EvidenceComplete()
if !evidence && len(by) > 1 {
if _, said := livenessSaid.LoadOrStore("incomplete", true); !said {
log.Printf("console: an incomplete read of the edges files or box snapshots (edges complete %v, snapshots complete %v), so no region outside the home is automatically eligible this request; said once", snap.Complete, snap.BoxesComplete)
diff --git a/internal/console/regioncandidates_test.go b/internal/console/regioncandidates_test.go
index afa0bdf6f..d2f36a16f 100644
--- a/internal/console/regioncandidates_test.go
+++ b/internal/console/regioncandidates_test.go
@@ -197,7 +197,7 @@ func TestEdgesFileWithoutAnyStatusSnapshotIsIncompleteEvidence(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if snap.complete() {
+ if snap.EvidenceComplete() {
t.Fatalf("a keyless edges file with NO status object left the evidence complete (edges %v, boxes %v)", snap.Complete, snap.BoxesComplete)
}
if c, _ := findCandidate(regionCandidates(snap, "ie", testKeyFP), "uk"); c.Eligible {
@@ -213,7 +213,7 @@ func TestEdgesFileWithoutAnyStatusSnapshotIsIncompleteEvidence(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if !snap.complete() {
+ if !snap.EvidenceComplete() {
t.Fatalf("a stale snapshot beside its edges file made the evidence incomplete (edges %v, boxes %v); a retired box is complete evidence", snap.Complete, snap.BoxesComplete)
}
if c, _ := findCandidate(regionCandidates(snap, "ie", testKeyFP), "uk"); !c.Eligible {
diff --git a/internal/console/regiondefault.go b/internal/console/regiondefault.go
index 5233d0bcb..053ea246d 100644
--- a/internal/console/regiondefault.go
+++ b/internal/console/regiondefault.go
@@ -78,7 +78,7 @@ func (s *Server) appDefault(ctx context.Context, acct *Account, app string) appD
// Nothing is stated about it, and no reason either: the placement
// WAS read (controller ruling PF7).
return v
- case ferr != nil, stored != "" && !snap.complete():
+ case ferr != nil, stored != "" && !snap.EvidenceComplete():
// A failed listing, or (for a non-home region) a listing with an
// edges file or a box snapshot unread: the evidence is not there
// to say the region can take an instance, and the create door
diff --git a/internal/edgesfile/snapshot.go b/internal/edgesfile/snapshot.go
new file mode 100644
index 000000000..cffde957d
--- /dev/null
+++ b/internal/edgesfile/snapshot.go
@@ -0,0 +1,261 @@
+package edgesfile
+
+import (
+ "context"
+ "encoding/json"
+ "log"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/carlosframework/platform/internal/boxchoice"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/status"
+)
+
+// The deployment-wide placement read: every box's edges file joined with
+// its liveness. The console's automatic-eligibility doors and the box
+// agent's relocation chooser (region HA stage B Task 10) read through it,
+// so a relocation's target and a create's executor come from the same
+// evidence. Moved here unchanged from the console (instancesplacement.go,
+// boxchoice.go).
+
+// PlacementView is the reader's half of the edges-file schema this
+// join needs: version, box, region, and the host set. Roles are not
+// consulted — a host a box names under ANY role is served there in some
+// capacity, and "where is this instance" is the question, not "who is its
+// write home" (that is the home map's question, internal/edge).
+type PlacementView struct {
+ V int `json:"v"`
+ Box string `json:"box"`
+ EIP string `json:"eip"`
+ Region string `json:"region"`
+ Placement string `json:"placement"`
+ // DataBucket is the box's CARLOS_HIBERNATE_BUCKET, verbatim
+ // (internal/host/edgesfile.go writes it from that env and omits the key
+ // entirely when it is unset). It carries the move lane's actual
+ // coordinates, and doubles as positive hosting evidence from a writer
+ // too old to publish HibernationConfigured.
+ DataBucket string `json:"data_bucket"`
+ // HibernationConfigured is the box's own answer to "is my hibernation
+ // activator wired" (internal/host/edgesfile.go). A POINTER because
+ // absent and false are different facts and the difference is the whole
+ // point: absent is a writer that predates the field, whose capability is
+ // UNKNOWN, while false is a box that published the field and said no.
+ // Reading absence as no is what made an earlier version withdraw healthy
+ // regions mid-roll — see placements.
+ HibernationConfigured *bool `json:"hibernation_configured"`
+ // ServesSharedPool and AcceptsInbound: POINTERS, absent = a writer
+ // predating them (unknown). SecretsKeyFP: absent = no key, or a writer
+ // predating it; AcceptsInbound's presence tells the two apart
+ // (nearest-region spec §6.1, R9).
+ ServesSharedPool *bool `json:"serves_shared_pool"`
+ AcceptsInbound *bool `json:"accepts_inbound"`
+ SecretsKeyFP string `json:"secrets_key_fp"`
+ // Box evidence (region HA spec 1 §5.1). All absent on a writer that
+ // predates it: BoxID "", Capabilities nil, Draining false, Headroom nil.
+ BoxID string `json:"box_id"`
+ Capabilities []string `json:"capabilities"`
+ Draining bool `json:"draining"`
+ Headroom *struct {
+ MemFreeMiB int64 `json:"mem_free_mib"`
+ DiskFreeMiB int64 `json:"disk_free_mib"`
+ Awake int `json:"awake"`
+ } `json:"headroom"`
+ Hosts map[string]json.RawMessage `json:"hosts"`
+ // The moves protocol (moves spec §11), read only by the Phase 1 move
+ // door's floor check (selectMoveBoxes); Task 13 adds the writer's side.
+ // MovesProtocol is the protocol the box speaks (absent: 0).
+ // MovesFloorMinRelease is the published floor's min_release the box
+ // has armed locally. PrevRelease is its carlos.prev's release, "none"
+ // when it keeps none, and PrevAllowedFor the floor's min_release that
+ // carlos.prev was judged at or above.
+ MovesProtocol int `json:"moves_protocol"`
+ MovesFloorMinRelease string `json:"moves_floor_min_release"`
+ PrevRelease string `json:"prev_release"`
+ PrevAllowedFor string `json:"prev_allowed_for"`
+ // Fresh is not on the wire: ReadLiveSnapshot sets it when the
+ // box's own status snapshot is still fresh (status.BoxStale). An edges
+ // file is not liveness, see ReadLiveSnapshot.
+ Fresh bool `json:"-"`
+}
+
+// livenessSaid throttles ReadLiveSnapshot's two diagnostics to once
+// per process, the way noPointSaid does for a code with no point.
+var livenessSaid sync.Map
+
+// ReadLiveSnapshot is ReadSnapshot joined with box liveness, for
+// AUTOMATIC eligibility (regionCandidates). An edges file is not evidence
+// that its box is alive: nothing deletes a retired box's file, and the
+// file's own Generation moves only when the file is rewritten, so a
+// healthy box's is arbitrarily old (internal/edgesfile.EdgeIPs's doc
+// comment has the whole argument). The status snapshot is written every
+// tick, so status.BoxStale is a real liveness signal, and this is the same
+// join EdgeIPs makes, over the same period.
+//
+// A box with no fresh snapshot, and every box when the snapshots cannot be
+// loaded, is simply not Fresh. The fold then counts it for nothing, so a
+// deployment whose status store is split from its deployment bucket
+// (CARLOS_STATUS_BUCKET, which the console does not read) degrades to
+// home-only guesses. That is safe but invisible, so both cases are said
+// once: the load failure, and files found with no snapshot matching any.
+// The explicit door's placements stay on the generous file-only read; a
+// person asked for that region and can see it refused later.
+func ReadLiveSnapshot(ctx context.Context, s release.Store, now time.Time) (Snapshot, error) {
+ snap, err := ReadSnapshot(ctx, s)
+ if err != nil {
+ return snap, err
+ }
+ if len(snap.Files) == 0 {
+ snap.BoxesComplete = true // nothing to be incomplete about
+ return snap, nil
+ }
+ // Completeness is carried separately from staleness: LoadBoxesComplete
+ // returns what LoadBoxes would, plus whether every listed snapshot was
+ // actually read. A skipped read leaves that box's liveness UNKNOWN,
+ // and unknown is never "stale".
+ boxes, boxesComplete, err := status.LoadBoxesComplete(ctx, s)
+ if err != nil {
+ if _, said := livenessSaid.LoadOrStore("load", true); !said {
+ log.Printf("edges: could not load box snapshots (%v), so no box outside the home counts for automatic region placement until they load", err)
+ }
+ return snap, nil
+ }
+ snap.BoxesComplete = boxesComplete
+ // known is every box with a status object at all; live is the fresh
+ // subset. The join key is the status snapshot's Label against the edges
+ // file's own box field, the pair edgesfile.EdgeIPs joins on; the object
+ // key names play no part.
+ known, live := map[string]bool{}, map[string]bool{}
+ for _, b := range boxes {
+ known[b.Label] = true
+ if !status.BoxStale(b, now, status.SnapshotPeriod) {
+ live[b.Label] = true
+ }
+ }
+ published := make(map[string]bool, len(snap.Files))
+ joined := 0
+ for i := range snap.Files {
+ published[snap.Files[i].Box] = true
+ snap.Files[i].Fresh = live[snap.Files[i].Box]
+ if snap.Files[i].Fresh {
+ joined++
+ }
+ // Ruling (B) reversed, re-review r2, on production evidence: every
+ // box writes both objects and a retired box leaves both behind, so
+ // an ABSENT snapshot is unknown liveness (incomplete evidence,
+ // whole-request) and a STALE one is the retired box (not fresh,
+ // counts for neither side, evidence complete).
+ if !known[snap.Files[i].Box] {
+ snap.BoxesComplete = false
+ if _, said := livenessSaid.LoadOrStore("nostatus", true); !said {
+ log.Printf("edges: box %q has an edges file under %s but no status snapshot at all, so no region outside the home is automatically eligible until it reports; said once", snap.Files[i].Box, Prefix)
+ }
+ }
+ }
+ // Ruling (A), re-review r2: a FRESH snapshot whose box has no edges
+ // file is incomplete evidence, whole-request, like a failed read.
+ // Reconcile admits a box without its edges file (its publication may
+ // simply have failed), so a live box the fold cannot see could still
+ // take the record. A stale snapshot with no file is a retired box and
+ // is ignored.
+ for label := range live {
+ if !published[label] {
+ snap.BoxesComplete = false
+ if _, said := livenessSaid.LoadOrStore("unmatched", true); !said {
+ log.Printf("edges: box %q has a fresh status snapshot but no edges file under %s, so no region outside the home is automatically eligible until it publishes one; said once", label, Prefix)
+ }
+ }
+ }
+ if joined == 0 {
+ if _, said := livenessSaid.LoadOrStore("none", true); !said {
+ log.Printf("edges: %d edges file(s) under %s but no fresh box snapshot matched any of them, so automatic region placement sees the home alone (are edges and status in different buckets? see CARLOS_STATUS_BUCKET/CARLOS_STATUS_PREFIX)", len(snap.Files), Prefix)
+ }
+ }
+ return snap, nil
+}
+
+// Snapshot is one read of control/edges/. Complete is false when any
+// listed file could not be read or parsed: positive facts about the boxes
+// that WERE read still hold, but nothing may be concluded about "every
+// box" (R9's NoKeyProven, codex plan-R2 1).
+type Snapshot struct {
+ Files []PlacementView
+ Complete bool
+ // BoxesComplete: every listed status snapshot was read, and every box
+ // on either side has its counterpart, so each file's Fresh is a KNOWN
+ // answer. Set only by ReadLiveSnapshot; a plain ReadSnapshot
+ // carries no liveness evidence at all. Unknown liveness is not stale
+ // (re-review r1 hole 2, r2 rulings (A) and (B) reversed): a snapshot
+ // that could not be read, a fresh snapshot with no edges file, or an
+ // edges file with no snapshot at all each make the evidence incomplete.
+ BoxesComplete bool
+}
+
+// EvidenceComplete is the evidence bar for AUTOMATIC non-home eligibility: every
+// listed edges file read and parsed, and every listed status snapshot
+// read. Short of it a failing sibling may simply be missing from the fold,
+// so nothing non-home is eligible for the request (ruling, re-review r1).
+func (s Snapshot) EvidenceComplete() bool { return s.Complete && s.BoxesComplete }
+
+// ReadSnapshot REPORTS a failed listing instead of answering empty:
+// the door must tell "could not read" (503, C28) from "no box publishes
+// this code" (codex R2-2). A single unreadable or malformed file is
+// skipped, as before, and marks the snapshot incomplete.
+func ReadSnapshot(ctx context.Context, s release.Store) (Snapshot, error) {
+ keys, err := s.List(ctx, Prefix)
+ if err != nil {
+ return Snapshot{}, err
+ }
+ snap := Snapshot{Complete: true}
+ for _, key := range keys {
+ name, ok := strings.CutPrefix(key, Prefix)
+ if !ok || !strings.HasSuffix(name, ".json") || strings.Contains(name, "/") {
+ continue
+ }
+ body, _, err := s.Get(ctx, key)
+ if err != nil {
+ log.Printf("edges: could not read %s for the placement join: %v", key, err)
+ snap.Complete = false
+ continue
+ }
+ var f PlacementView
+ if err := json.Unmarshal(body, &f); err != nil || f.V < 1 || f.Box == "" {
+ log.Printf("edges: skipping %s in the placement join: not a versioned edges file", key)
+ snap.Complete = false
+ continue
+ }
+ snap.Files = append(snap.Files, f)
+ }
+ return snap, nil
+}
+
+// BoxChoice is the snapshot as boxchoice's predicate reads it. Every field
+// the predicate reads is carried: a dropped field would silently read as
+// absent, so the console's eligibility tests run through this conversion.
+func (s Snapshot) BoxChoice() boxchoice.Snapshot {
+ out := boxchoice.Snapshot{Complete: s.EvidenceComplete(), Boxes: make([]boxchoice.Box, 0, len(s.Files))}
+ for _, f := range s.Files {
+ b := boxchoice.Box{
+ Box: f.Box, BoxID: f.BoxID, EIP: f.EIP, Region: f.Region, Placement: f.Placement,
+ HibernationConfigured: f.HibernationConfigured, ServesSharedPool: f.ServesSharedPool,
+ AcceptsInbound: f.AcceptsInbound, SecretsKeyFP: f.SecretsKeyFP, Capabilities: f.Capabilities,
+ Draining: f.Draining, Fresh: f.Fresh,
+ }
+ if f.Headroom != nil {
+ b.Headroom = &boxchoice.Headroom{MemFreeMiB: f.Headroom.MemFreeMiB, DiskFreeMiB: f.Headroom.DiskFreeMiB, Awake: f.Headroom.Awake}
+ }
+ out.Boxes = append(out.Boxes, b)
+ }
+ return out
+}
+
+// LiveSnapshot is ReadLiveSnapshot in boxchoice's form: the read
+// hostops.ExecutorChooser's Edges takes.
+func LiveSnapshot(ctx context.Context, s release.Store, now time.Time) (boxchoice.Snapshot, error) {
+ snap, err := ReadLiveSnapshot(ctx, s, now)
+ if err != nil {
+ return boxchoice.Snapshot{}, err
+ }
+ return snap.BoxChoice(), nil
+}
diff --git a/internal/edgesfile/snapshot_test.go b/internal/edgesfile/snapshot_test.go
new file mode 100644
index 000000000..2f60ed8ba
--- /dev/null
+++ b/internal/edgesfile/snapshot_test.go
@@ -0,0 +1,75 @@
+package edgesfile
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/status"
+)
+
+// Stage B Task 10: the console's automatic-eligibility read and the box
+// agent's relocation chooser read the same live snapshot. Every listed
+// edges file is read, each box's liveness is its status snapshot's
+// freshness, a stale snapshot is a retired box (complete evidence, not
+// fresh), and a file with no snapshot at all, or an unreadable file, makes
+// the evidence incomplete. BoxChoice carries every field the predicate
+// reads.
+func TestLiveSnapshotJoinsEdgesFilesWithLiveness(t *testing.T) {
+ ctx := context.Background()
+ now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)
+ s := release.NewDirStore(t.TempDir())
+ put := func(box, body string) {
+ t.Helper()
+ if _, err := s.Put(ctx, Key(box), []byte(body), ""); err != nil {
+ t.Fatal(err)
+ }
+ }
+ snapshot := func(box string, at time.Time) {
+ t.Helper()
+ if err := status.WriteBox(ctx, s, status.Box{Label: box, GeneratedAt: at}); err != nil {
+ t.Fatal(err)
+ }
+ }
+ put("box-a", `{"v":1,"box":"box-a","box_id":"i-a","eip":"203.0.113.1","region":"eu-west-1","placement":"ie","hibernation_configured":true,"secrets_key_fp":"fp","capabilities":["region-ha-1"],"headroom":{"mem_free_mib":512,"disk_free_mib":2048,"awake":3},"hosts":{}}`)
+ put("box-b", `{"v":1,"box":"box-b","eip":"203.0.113.2","region":"eu-west-1","placement":"ie","hosts":{}}`)
+ snapshot("box-a", now.Add(-time.Second))
+ snapshot("box-b", now.Add(-24*time.Hour))
+
+ live, err := ReadLiveSnapshot(ctx, s, now)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !live.EvidenceComplete() || len(live.Files) != 2 {
+ t.Fatalf("snapshot %+v", live)
+ }
+ fresh := map[string]bool{}
+ for _, f := range live.Files {
+ fresh[f.Box] = f.Fresh
+ }
+ if !fresh["box-a"] || fresh["box-b"] {
+ t.Fatalf("liveness %v: box-a fresh, box-b retired", fresh)
+ }
+ choice, err := LiveSnapshot(ctx, s, now)
+ if err != nil || !choice.Complete || len(choice.Boxes) != 2 {
+ t.Fatalf("boxchoice %+v %v", choice, err)
+ }
+ for _, b := range choice.Boxes {
+ if b.Box == "box-a" && (b.BoxID != "i-a" || b.SecretsKeyFP != "fp" || !b.Fresh || b.Headroom == nil || b.Headroom.Awake != 3 || len(b.Capabilities) != 1 || b.HibernationConfigured == nil || !*b.HibernationConfigured) {
+ t.Fatalf("box-a lost a field: %+v", b)
+ }
+ }
+
+ // A box with an edges file and no status snapshot: unknown liveness.
+ put("box-c", `{"v":1,"box":"box-c","region":"eu-west-1","placement":"ie","hosts":{}}`)
+ if live, err = ReadLiveSnapshot(ctx, s, now); err != nil || live.EvidenceComplete() {
+ t.Fatalf("a file with no snapshot left the evidence complete: %+v %v", live, err)
+ }
+ // An unreadable file: incomplete, the others still read.
+ put("box-d", `not json`)
+ plain, err := ReadSnapshot(ctx, s)
+ if err != nil || plain.Complete || len(plain.Files) != 3 {
+ t.Fatalf("an unreadable file: %+v %v", plain, err)
+ }
+}
diff --git a/internal/host/agent.go b/internal/host/agent.go
index 7457ed444..f8eddedc6 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -276,6 +276,21 @@ func runAgent(args []string, buildClaims ClaimFactory) {
log.Fatalf("agent: %v", err)
}
claimCtx = withClaimRuntimeResources(claimCtx, ClaimRuntimeResources{Mode: "agent", Registry: reg, Manager: cgm, DataDir: dd, OptDir: release.OptDir(), UnitsDir: "/etc/systemd/system"})
+ if hostwrite.Enabled(claimEntryEnv(claimCtx)) {
+ // Region HA (stage B Task 10): with CARLOS_REGION_HA on, the
+ // relocation owner source is registered before the claim's owners
+ // open. In claim mode the assembly is opened here, as the claim
+ // factory would open it, so the relocation pass, or with region HA
+ // off the completion pass (ruling 82), binds to the same one.
+ if regionHAEnabled(claimEntryEnv(claimCtx)) {
+ if err := registerRelocationSource(claimCtx, dd); err != nil {
+ log.Fatalf("agent: region HA: %v", err)
+ }
+ }
+ if claimCtx, err = ensureProductionClaimAssembly(claimCtx); err != nil {
+ log.Fatalf("agent: host claims: %v", err)
+ }
+ }
claims, err := newClaimWiring(claimCtx, claimEntryEnv(claimCtx), buildClaims)
if err != nil {
log.Fatalf("agent: host claims: %v", err)
@@ -590,6 +605,9 @@ func runAgent(args []string, buildClaims ClaimFactory) {
}
}()
+ if relocation := startRelocationPass(agentCtx, claimCtx, claims, reg, svc); relocation != nil {
+ log.Printf("agent: relocation pass started (completions only: %v)", relocation.Worker == nil)
+ }
go adoptLoop(reg, svc, sd, claims.Writes)
go fastConvergeLoop(reg, svc, sd, claims.Writes, serialClaims, claims.Providers.Provision)
if logSink != nil {
diff --git a/internal/host/claimparticipation_relocation.go b/internal/host/claimparticipation_relocation.go
index 741057013..0a83a4bdd 100644
--- a/internal/host/claimparticipation_relocation.go
+++ b/internal/host/claimparticipation_relocation.go
@@ -44,6 +44,15 @@ func (p *productionClaimLocal) Acknowledged(ctx context.Context, o hostclaim.Obs
return slot.Controller.Ready(ctx)
}
+// boundAssembly is the assembly Build bound this owner to: the copy whose
+// reconcile adapter, census and guard reader Build configured. Nil before
+// Build. N1's second accessor.
+func (p *productionClaimLocal) boundAssembly() *ClaimAssembly {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ return p.assembly
+}
+
// DischargeCompleted is ruling 66's gate before a completed relocation's
// hold is released: main's own Prepare for the serving observation, which
// closes the participation slot (its cleanup is C10's discharge, first)
diff --git a/internal/host/claimrelocation.go b/internal/host/claimrelocation.go
index 3a418adcf..e14888d8a 100644
--- a/internal/host/claimrelocation.go
+++ b/internal/host/claimrelocation.go
@@ -53,6 +53,17 @@ func openClaimRelocationOwners(ctx context.Context, getenv func(string) string,
if source == nil || getenv == nil || !regionHAEnabled(getenv) {
return nil, nil
}
+ // N5: region HA's source gets main's own serving launch, markers and
+ // C8 provider, inside the region HA gate, before it opens. Its
+ // Identity is main's serving owners' identity check (carry 5): the
+ // owners' own IdentityProvider.Verify, as claimconstruction.go binds
+ // it before every data-bucket request main's serving owners make.
+ if s, ok := source.(*RelocationSource); ok {
+ serving := out.Serving
+ identity := out.Identity
+ serving.Identity = func(ctx context.Context) error { _, err := identity.Verify(ctx); return err }
+ s.SetServingSources(serving)
+ }
owners, refusals, err := source.Open(ctx, store, data, actor)
if err != nil {
return nil, err
diff --git a/internal/host/relocatepass.go b/internal/host/relocatepass.go
new file mode 100644
index 000000000..5ab01a894
--- /dev/null
+++ b/internal/host/relocatepass.go
@@ -0,0 +1,585 @@
+package host
+
+import (
+ "context"
+ "crypto/ed25519"
+ "errors"
+ "log"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/appconfig"
+ "github.com/carlosframework/platform/internal/boxchoice"
+ "github.com/carlosframework/platform/internal/boxvolumes"
+ "github.com/carlosframework/platform/internal/edgesfile"
+ "github.com/carlosframework/platform/internal/genselect"
+ "github.com/carlosframework/platform/internal/hostactors"
+ "github.com/carlosframework/platform/internal/hostbox"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/hostfence"
+ "github.com/carlosframework/platform/internal/hostfloor"
+ "github.com/carlosframework/platform/internal/hostops"
+ "github.com/carlosframework/platform/internal/hostrelocate"
+ "github.com/carlosframework/platform/internal/hostwrite"
+ "github.com/carlosframework/platform/internal/lease"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/termfence"
+)
+
+// RelocationRuntime is one agent's relocation: its worker, participant and
+// effects, built once the claim assembly is bound. It replays relocations;
+// it starts none (Worker.Start is the operator's and the placer's, not
+// this pass's).
+type RelocationRuntime struct {
+ Worker relocationStepper
+ Participant relocationStepper
+ Box string // this box: its alerts, and the completions it finishes
+ // Convert is J3 for a landing claim (convertFencedLanding); Finish is
+ // RelocationEffects.Finish for a completion this box has not yet
+ // marked discharged (ruling 80).
+ Convert func(context.Context, release.Store, hostclaim.Observed) error
+ Finish func(ctx context.Context, host, op string) error
+ // Hosts, when set, is the hosts a pass reads instead of every claim
+ // in the bucket: the completion-only runtime's (ruling 82) are this
+ // box's own registry rows, since a completion this box can finish
+ // always has its row here.
+ Hosts func() ([]string, error)
+ // Self, when Box is empty, answers this box once per pass: the
+ // completion-only runtime has no roster read of its own, so it asks
+ // the bound assembly's verified stop identity. A pass that cannot
+ // answer it finishes nothing (carry 8: a completion is finished only
+ // on the box it serves on).
+ Self func(context.Context) (string, error)
+ DataDir string // the managed data root: relocate/ journals (N9)
+ alerted sync.Map // host\x00cause: one alert per host and cause (rulings 35, 36)
+}
+
+// relocationStepper is hostrelocate.Worker's and Participant's Step.
+type relocationStepper interface {
+ Step(context.Context, string) (hostclaim.Observed, error)
+}
+
+// relocationStores is rulings 50 and 51: the data bucket as a release.Store
+// of its own (production's lease.VersionedStore is not one), and the AWS
+// signing region kept apart from the logical placement region.
+type relocationStores struct {
+ Files release.Store
+ Versions lease.VersionedStore
+ Bucket string
+ Region string // logical: genselect, pins, the worker, the chooser
+ LeaseRegion string // AWS: lease.Acquire, lease.NewVersionedS3
+}
+
+func openRelocationStores(ctx context.Context, self hostactors.Box, dataBucket string, files func(context.Context, string) (release.Store, error), versioned func(context.Context, string, string) (lease.VersionedStore, error)) (relocationStores, error) {
+ if dataBucket == "" || self.Region == "" || self.Cloud.Region == "" || files == nil || versioned == nil {
+ return relocationStores{}, hostwrite.ErrUnavailable
+ }
+ f, err := files(ctx, dataBucket)
+ if err != nil {
+ return relocationStores{}, err
+ }
+ v, err := versioned(ctx, dataBucket, self.Cloud.Region)
+ if err != nil {
+ return relocationStores{}, err
+ }
+ return relocationStores{Files: f, Versions: v, Bucket: dataBucket, Region: self.Region, LeaseRegion: self.Cloud.Region}, nil
+}
+
+// relocationLeases is the worker's lease provider (ruling 51): the replica
+// lease on a holder's location, signed for the cloud region.
+func relocationLeases(s relocationStores) func(context.Context, string) (hostrelocate.Lease, error) {
+ return func(ctx context.Context, location string) (hostrelocate.Lease, error) {
+ held, err := lease.Acquire(ctx, s.Bucket, location, s.LeaseRegion)
+ if err != nil {
+ return nil, err
+ }
+ return held, nil
+ }
+}
+
+// relocationAssemblyConfig fills Task 8a's config from the bound assembly,
+// the stores and main's participation discharge.
+func relocationAssemblyConfig(a *ClaimAssembly, providers ClaimProviders, s relocationStores, reg *registry.Registry, act RelocationActivator, d hostactors.Directory, optDir string, discharge func(context.Context, hostclaim.Observed) error) (RelocationAssemblyConfig, error) {
+ src, ok := providers.Selector.(hostbox.WorkSource)
+ if !ok || a == nil || reg == nil || act == nil || discharge == nil || optDir == "" || a.Guard.DataDir == "" {
+ return RelocationAssemblyConfig{}, hostwrite.ErrUnavailable
+ }
+ dd := a.Guard.DataDir
+ return RelocationAssemblyConfig{Directory: d, Data: s.Files, Versions: s.Versions, Bucket: s.Bucket, Region: s.Region, LeaseRegion: s.LeaseRegion,
+ Registry: reg, DataDir: dd, OptDir: optDir, WorkDir: filepath.Join(dd, "relocate", "build"), ReleaseDir: filepath.Join(dd, "relocate", "release"),
+ Activator: act, Source: src, Discharge: discharge, Now: time.Now}, nil
+}
+
+// relocationRuntimeConfig is relocationAssemblyConfig with what the agent
+// binds beyond it. ReleasePub and Ratchet are the held stage's evidence
+// (carry 4): the agent's release key (CARLOS_RELEASE_PUBKEY, the key the
+// held-stage pass verifies under) and the edge's pointer digest ratchet,
+// pointerRatchet(false, ...), exactly as moves' held stage takes it in the
+// agent (movespass.go, heldReleaseReady); never root's. The secrets key,
+// when the box has one, renders the held config.
+func relocationRuntimeConfig(getenv func(string) string, a *ClaimAssembly, providers ClaimProviders, s relocationStores, reg *registry.Registry, act RelocationActivator, d hostactors.Directory, in relocationRuntimeInputs, discharge func(context.Context, hostclaim.Observed) error) (RelocationAssemblyConfig, error) {
+ if getenv == nil || len(in.Pub) != ed25519.PublicKeySize {
+ return RelocationAssemblyConfig{}, hostwrite.ErrUnavailable
+ }
+ cfg, err := relocationAssemblyConfig(a, providers, s, reg, act, d, in.OptDir, discharge)
+ if err != nil {
+ return RelocationAssemblyConfig{}, err
+ }
+ cfg.ReleasePub = in.Pub
+ cfg.Ratchet = pointerRatchet(false, cfg.OptDir, cfg.DataDir, getenv)
+ if raw := getenv("CARLOS_SECRETS_KEY"); raw != "" {
+ key, err := appconfig.ParsePrivateKey(raw, "CARLOS_SECRETS_KEY")
+ if err != nil {
+ return RelocationAssemblyConfig{}, err
+ }
+ cfg.SecretsKey = key
+ }
+ return cfg, nil
+}
+
+// channelRelease is ruling 30: the host's channel pointer in the control
+// bucket, by the holder's account and app and the record's channel.
+func channelRelease(store release.Store, pub ed25519.PublicKey) func(context.Context, string) (string, error) {
+ return func(ctx context.Context, host string) (string, error) {
+ obs, err := hostclaim.Read(ctx, store, host)
+ if err != nil || obs.Absent || obs.Claim.Holder == nil {
+ return "", errors.Join(err, hostrelocate.ErrRefused)
+ }
+ rec, err := provision.GetRecord(ctx, store, obs.Claim.Holder.Account, obs.Claim.Holder.App, host)
+ if err != nil {
+ return "", err
+ }
+ p, err := release.Channel(ctx, store, pub, obs.Claim.Holder.Account, obs.Claim.Holder.App, rec.Channel)
+ if err != nil {
+ return "", err
+ }
+ return p.Version, nil
+ }
+}
+
+// volumeDischarge is ruling 3 with enrolled_at in the roster: unqualified
+// without it.
+func volumeDischarge(store release.Store) func(context.Context, hostactors.Box, termfence.Record) *hostclaim.LocalDischarge {
+ return func(ctx context.Context, b hostactors.Box, f termfence.Record) *hostclaim.LocalDischarge {
+ if b.EnrolledAt.IsZero() {
+ return &hostclaim.LocalDischarge{Reason: "the roster records no enrollment time"}
+ }
+ at, err := time.Parse(time.RFC3339Nano, f.ObservedAt)
+ if err != nil {
+ return &hostclaim.LocalDischarge{Reason: err.Error()}
+ }
+ records, err := boxvolumes.List(ctx, store, b.Enrollment)
+ if err != nil {
+ return &hostclaim.LocalDischarge{Reason: err.Error()}
+ }
+ v := boxvolumes.Discharge(records, boxvolumes.Entry{Box: b.BoxID, Enrollment: b.Enrollment, Cloud: b.Cloud}, b.EnrolledAt, at)
+ return &hostclaim.LocalDischarge{Qualified: v.Discharge, Reason: v.Reason}
+ }
+}
+
+// relocationRelocatable is Worker.Relocatable over relocationRecord (K24):
+// only a hibernating, path-deriving instance record that mints nested
+// relocates (ruling 45).
+func relocationRelocatable(dataDir string, store release.Store) func(context.Context, hostclaim.Observed) error {
+ return func(ctx context.Context, obs hostclaim.Observed) error {
+ _, err := relocationRecord(ctx, dataDir, store, obs)
+ return err
+ }
+}
+
+// errRelocationNotStarted is the pass's answer to Worker.Start's two
+// providers: this runtime replays relocations and never starts one. The
+// app admission and the census are the starter's (the operator's carlos
+// ops host relocate and spec 3's placer), with their own bindings.
+var errRelocationNotStarted = errors.New("host: the relocation pass starts no relocation")
+
+type relocationStartRefused struct{}
+
+func (relocationStartRefused) Register(context.Context, hostclaim.Observed, hostrelocate.Owner, string) (string, error) {
+ return "", errRelocationNotStarted
+}
+
+func (relocationStartRefused) Recheck(context.Context, string) error { return errRelocationNotStarted }
+
+// relocationRuntimeInputs is what NewRelocationRuntime binds beyond the
+// claim's own: the box's enrollment, the release key, the bucket
+// constructors (production's S3; tests' fakes).
+type relocationRuntimeInputs struct {
+ Enrollment hostfloor.Enrollment
+ Pub ed25519.PublicKey
+ Files func(context.Context, string) (release.Store, error)
+ Versioned func(context.Context, string, string) (lease.VersionedStore, error)
+ OptDir string
+}
+
+// productionRelocationRuntimeInputs reads the box's enrollment and the
+// release key, and opens the data bucket with release.NewS3Store and
+// lease.NewVersionedS3.
+func productionRelocationRuntimeInputs(getenv func(string) string, optDir string) (relocationRuntimeInputs, error) {
+ pub, err := release.ParsePublicKey(getenv("CARLOS_RELEASE_PUBKEY"), "CARLOS_RELEASE_PUBKEY")
+ if err != nil {
+ return relocationRuntimeInputs{}, err
+ }
+ enrollment, err := hostfloor.ReadEnrollment("/etc/carlos/host-actor.json")
+ if err != nil {
+ return relocationRuntimeInputs{}, err
+ }
+ return relocationRuntimeInputs{Enrollment: enrollment, Pub: pub, OptDir: optDir,
+ Files: func(ctx context.Context, bucket string) (release.Store, error) {
+ s, err := release.NewS3Store(ctx, bucket)
+ if err != nil {
+ return nil, err
+ }
+ return s, nil
+ },
+ Versioned: func(ctx context.Context, bucket, region string) (lease.VersionedStore, error) {
+ return lease.NewVersionedS3(ctx, map[string]string{bucket: region})
+ }}, nil
+}
+
+// NewRelocationRuntime builds the agent's relocation runtime from the bound
+// claim assembly. It returns (nil, nil) unless CARLOS_HOST_CLAIM is on.
+// With CARLOS_REGION_HA off, or the relocation owners not bound into the
+// wake's dispatcher (ruling 53: they need moves' C8 provider), it is the
+// completion-only runtime (ruling 82): a relocation that already completed
+// on this box is still finished. Only with both on and the owners bound is
+// it the full pass.
+func NewRelocationRuntime(ctx context.Context, getenv func(string) string, store release.Store, local *productionClaimLocal, providers ClaimProviders, reg *registry.Registry, act RelocationActivator, in relocationRuntimeInputs) (*RelocationRuntime, error) {
+ if !hostwrite.Enabled(getenv) || local == nil {
+ return nil, nil
+ }
+ if !regionHAEnabled(getenv) {
+ return newCompletionRuntime(local, reg, act)
+ }
+ a := local.boundAssembly()
+ if a == nil {
+ return nil, hostwrite.ErrUnavailable
+ }
+ if d, ok := wakeDispatch(a); !ok || d.Relocation == nil {
+ return newCompletionRuntime(local, reg, act)
+ }
+ if a.RetirementAuthority == nil || store == nil || reg == nil || act == nil || len(in.Pub) != ed25519.PublicKeySize || in.Enrollment.Box == "" {
+ return nil, hostwrite.ErrUnavailable
+ }
+ d := a.RetirementAuthority.Directory
+ roster, err := d.Read(ctx)
+ if err != nil {
+ return nil, err
+ }
+ self, ok := roster.Roster.Boxes[in.Enrollment.Box]
+ if !ok || self.Enrollment != in.Enrollment.Enrollment {
+ return nil, hostbox.ErrChanged
+ }
+ stores, err := openRelocationStores(ctx, self, getenv("CARLOS_HIBERNATE_BUCKET"), in.Files, in.Versioned)
+ if err != nil {
+ return nil, err
+ }
+ cfg, err := relocationRuntimeConfig(getenv, a, providers, stores, reg, act, d, in, local.DischargeCompleted)
+ if err != nil {
+ return nil, err
+ }
+ effects := a.RelocationEffects(cfg, local.Acknowledged)
+ chooser := hostrelocate.RegionChooser{Records: store, Namespace: self.Namespace, Region: stores.Region,
+ Executors: hostops.ExecutorChooser{Directory: d, Namespace: self.Namespace, Pool: self.Pool, HomeName: stores.Region, Now: time.Now,
+ Edges: func(ctx context.Context) (boxchoice.Snapshot, error) {
+ return edgesfile.LiveSnapshot(ctx, store, time.Now())
+ },
+ Loads: func(ctx context.Context) (map[string]boxchoice.Load, error) { return hostops.ClaimLoads(ctx, store) },
+ // The fingerprint this box's own edges file publishes: a target
+ // must hold the same secrets key to render the held config.
+ SecretsKeyFP: edgesFileSecretsFP(getenv)}}
+ worker := hostrelocate.Worker{Claims: store, Directory: d,
+ Fences: hostfence.ClaimFenceAuthority{Directory: d, Now: time.Now},
+ Releases: StopReceiptReleases{Receipts: RelocationReleaseReceipts{Store: store, Directory: d}},
+ Admission: relocationStartRefused{},
+ Census: func(context.Context, string, []string) error { return errRelocationNotStarted },
+ Choose: chooser.Choose, Eligible: chooser.Eligible,
+ Relocatable: relocationRelocatable(cfg.DataDir, store),
+ Leases: relocationLeases(stores),
+ Pins: hostrelocate.StorePinner{Claims: store, Data: stores.Files, Bucket: stores.Bucket, Region: stores.Region,
+ SelectDB: func(ctx context.Context, r genselect.Request) (*genselect.Plan, error) {
+ return genselect.Select(ctx, stores.Versions, r)
+ },
+ Release: channelRelease(store, in.Pub)},
+ Prepared: PreparedFromInstallManifest{Claims: store, Files: stores.Files, Data: stores.Versions, Bucket: stores.Bucket},
+ Discharge: volumeDischarge(store),
+ Notify: func(ctx context.Context) { _ = release.BumpSerial(ctx, store, time.Now()) },
+ Self: termfence.WrittenBy{Box: in.Enrollment.Box, Enrollment: in.Enrollment.Enrollment},
+ Region: stores.Region, Controller: newClaimID(), NewID: newClaimID, Now: time.Now}
+ participant := hostrelocate.Participant{Claims: store, Box: in.Enrollment.Box, Enrollment: in.Enrollment.Enrollment, Effects: effects, Now: time.Now}
+ return &RelocationRuntime{Worker: worker, Participant: participant, Box: in.Enrollment.Box, Convert: convertFencedLanding(worker, store), Finish: effects.Finish, DataDir: cfg.DataDir}, nil
+}
+
+// newCompletionRuntime is ruling 82: the smallest runtime that finishes a
+// relocation already completed on this box (discharge, hold release,
+// owned-fence lift, mark; relocationReleaseHold) and does nothing else. It
+// steps no relocation, converts no landing and starts nothing, and it
+// reads only the claims of this box's own registry rows, so a box that
+// never completed a relocation reads its own hosts' claims and writes
+// nothing.
+func newCompletionRuntime(local *productionClaimLocal, reg *registry.Registry, act RelocationActivator) (*RelocationRuntime, error) {
+ a := local.boundAssembly()
+ if a == nil || reg == nil || act == nil || a.Guard.DataDir == "" {
+ return nil, hostwrite.ErrUnavailable
+ }
+ cfg := RelocationAssemblyConfig{Registry: reg, Activator: act, DataDir: a.Guard.DataDir, Discharge: local.DischargeCompleted, Now: time.Now}
+ hosts := func() ([]string, error) {
+ rows, err := reg.List()
+ if err != nil {
+ return nil, err
+ }
+ out := make([]string, 0, len(rows))
+ for _, row := range rows {
+ out = append(out, row.Host)
+ }
+ return out, nil
+ }
+ // Carry 8: the completion-only runtime has no roster read, so this box
+ // is the bound assembly's own verified stop identity, asked each pass.
+ self := func(ctx context.Context) (string, error) {
+ l, err := a.localStopIdentity(ctx)
+ if err != nil {
+ return "", err
+ }
+ return l.Actor.Box, nil
+ }
+ return &RelocationRuntime{Finish: a.relocationReleaseHold(cfg), Hosts: hosts, Self: self, DataDir: a.Guard.DataDir}, nil
+}
+
+// wakeDispatch is the wake's owner dispatcher, as Build bound it.
+func wakeDispatch(a *ClaimAssembly) (ClaimOwnerDispatch, bool) {
+ if a == nil || a.Wake == nil {
+ return ClaimOwnerDispatch{}, false
+ }
+ d, ok := a.Wake.Inputs.(ClaimOwnerDispatch)
+ return d, ok
+}
+
+// Run is one pass every 30 seconds and on a control serial change, each
+// bounded at two minutes.
+func (r *RelocationRuntime) Run(ctx context.Context, store release.Store, serial <-chan struct{}) {
+ tick := time.NewTicker(30 * time.Second)
+ defer tick.Stop()
+ for {
+ pass, cancel := context.WithTimeout(ctx, 2*time.Minute)
+ r.once(pass, store)
+ cancel()
+ select {
+ case <-ctx.Done():
+ return
+ case <-tick.C:
+ case <-serial:
+ }
+ }
+}
+
+// once is one pass. Per host, in this order, so a passed probe completes
+// in one pass (ruling 49): the participant (its probe stop), the worker
+// (the serving CAS), then the completion this box has not yet finished,
+// found from the claim's record (ruling 80). A landing whose target is
+// fenced is converted when its move committed (J3).
+func (r *RelocationRuntime) once(ctx context.Context, store release.Store) {
+ hosts, err := r.hosts(ctx, store)
+ if err != nil {
+ log.Printf("relocate: list hosts: %v", err)
+ return
+ }
+ box := r.Box
+ if box == "" && r.Self != nil {
+ // No answer is no box: this pass finishes nothing (carry 8).
+ if box, err = r.Self(ctx); err != nil {
+ box = ""
+ r.alert("", "this box", err)
+ }
+ }
+ for _, host := range hosts {
+ obs, err := hostclaim.Read(ctx, store, host)
+ if err != nil || obs.Absent {
+ continue
+ }
+ switch obs.Claim.Op.Kind {
+ case hostclaim.KindLanding:
+ if r.Convert != nil {
+ if err := r.Convert(ctx, store, obs); err != nil {
+ r.alert(host, "convert", err)
+ }
+ }
+ case hostclaim.KindRelocating:
+ if r.Participant != nil {
+ if _, err := r.Participant.Step(ctx, host); err != nil && !errors.Is(err, hostrelocate.ErrNotYet) {
+ r.alert(host, r.Box, err)
+ }
+ }
+ if r.Worker != nil {
+ if _, err := r.Worker.Step(ctx, host); err != nil && !errors.Is(err, hostrelocate.ErrNotYet) {
+ r.alert(host, "worker", err)
+ }
+ }
+ }
+ r.finishCompletion(ctx, store, host, box)
+ }
+ // Ruling 61: journals of operations that are over go, once per pass,
+ // after the pass's own steps and never beside a journal write (carry 3:
+ // sweepRelocationJournals holds relocationJournals exclusively).
+ if r.DataDir != "" {
+ if err := sweepRelocationJournals(ctx, r.DataDir, store); err != nil {
+ log.Printf("relocate: sweep journals: %v", err)
+ }
+ }
+}
+
+// hosts is every claim in the bucket, or Hosts when it is set.
+func (r *RelocationRuntime) hosts(ctx context.Context, store release.Store) ([]string, error) {
+ if r.Hosts != nil {
+ return r.Hosts()
+ }
+ keys, err := store.List(ctx, "control/hosts/")
+ if err != nil {
+ return nil, err
+ }
+ var out []string
+ for _, key := range keys {
+ host, ok := strings.CutSuffix(strings.TrimPrefix(key, "control/hosts/"), ".json")
+ if ok && !strings.Contains(host, "/") {
+ out = append(out, host)
+ }
+ }
+ return out, nil
+}
+
+// finishCompletion is ruling 80: a claim serving on this box whose
+// completed relocation names this box and is not yet marked discharged is
+// handed to Finish (the hold release, relocationReleaseHold) on every
+// pass, read afresh after this pass's steps so a serving CAS made just now
+// is finished in the same pass. The work is found from the claim's record,
+// never from the row: a row already released, a lift that never ran or a
+// mark whose CAS failed is retried until the mark is written.
+//
+// A claim that does not serve on this box is skipped before Finish is
+// called (carry 8). A record that cannot be read is alerted and retried,
+// never read as "no completion" (carry 1), and so is every refusal of
+// Finish, a probe-fence owner it cannot find among them (carry 6): the
+// record stays unmarked. ErrChanged from Finish is the claim moving on
+// under the pass: nothing to do now, and never "finished".
+func (r *RelocationRuntime) finishCompletion(ctx context.Context, store release.Store, host, box string) {
+ if r.Finish == nil || box == "" {
+ return
+ }
+ obs, err := hostclaim.Read(ctx, store, host)
+ if err != nil {
+ r.alert(host, "finish", err)
+ return
+ }
+ if obs.Absent || obs.Claim == nil {
+ return
+ }
+ c := obs.Claim
+ if c.Op.Kind != hostclaim.KindServing || c.Op.Exec != box {
+ return
+ }
+ done, ok, err := hostclaim.ReadCompletedRelocation(c)
+ if err != nil {
+ r.alert(host, "finish", err)
+ return
+ }
+ if !ok || done.Discharged || done.Stop.Box != box {
+ return
+ }
+ err = r.Finish(ctx, host, done.Op)
+ if errors.Is(err, hostbox.ErrChanged) {
+ return
+ }
+ if err != nil {
+ r.alert(host, "finish", err)
+ }
+}
+
+// convertFencedLanding is J3 through the worker's own authorization
+// (hostrelocate.Worker.ConvertLanding): a landing whose op.exec box has a
+// termination fence becomes a failover relocation when its move committed
+// (CommittedLandingRecords). An uncommitted one is moves' case and answers
+// nil (ErrWrongState); any other refusal, a caller that is not an active
+// worker of the landing's region included, is returned for the pass to
+// alert once.
+func convertFencedLanding(w hostrelocate.Worker, store release.Store) func(context.Context, release.Store, hostclaim.Observed) error {
+ return func(ctx context.Context, _ release.Store, obs hostclaim.Observed) error {
+ _, err := w.ConvertLanding(ctx, obs, CommittedLandingRecords{Records: MovesRecordsFrom(store)})
+ if errors.Is(err, hostclaim.ErrWrongState) {
+ return nil
+ }
+ return err
+ }
+}
+
+func (r *RelocationRuntime) alert(host, who string, err error) {
+ if _, said := r.alerted.LoadOrStore(host+"\x00"+err.Error(), struct{}{}); !said {
+ log.Printf("relocate %s: %s: %v", host, who, err)
+ }
+}
+
+// registerRelocationSource registers region HA's owner source with the
+// roster directory the claim's own owners read (claimproduction.go).
+func registerRelocationSource(claimCtx context.Context, dataDir string) error {
+ store := claimStore(claimCtx)
+ pub, err := release.ParsePublicKey(os.Getenv("CARLOS_RELEASE_PUBKEY"), "CARLOS_RELEASE_PUBKEY")
+ if err != nil || store == nil {
+ return errors.Join(hostwrite.ErrUnavailable, err)
+ }
+ return RegisterProductionClaimRelocationSource(&RelocationSource{DataDir: dataDir,
+ Directory: hostactors.Directory{Store: store, PublicKey: pub, Bucket: os.Getenv("CARLOS_DEPLOYMENT_BUCKET")},
+ FilesStore: func(ctx context.Context, bucket string) (release.Store, error) {
+ s, err := release.NewS3Store(ctx, bucket)
+ if err != nil {
+ return nil, err
+ }
+ return s, nil
+ }})
+}
+
+// startRelocationPass builds the relocation runtime from the claim entry's
+// assembly and starts its loop. With claim mode on it always starts one:
+// the full pass with region HA on and the owners bound, otherwise the
+// completion-only pass (ruling 82). Nil with claim mode off, with no
+// activator, or when not even the completion pass can be built (logged).
+func startRelocationPass(agentCtx, claimCtx context.Context, claims *ClaimWiring, reg *registry.Registry, svc *activator.Service) *RelocationRuntime {
+ getenv := claimEntryEnv(claimCtx)
+ if !hostwrite.Enabled(getenv) || claims == nil || svc == nil {
+ return nil
+ }
+ entry, _ := claimCtx.Value(claimEntryKey{}).(claimEntry)
+ if entry.Assembly == nil {
+ return nil
+ }
+ local, ok := entry.Assembly.Local.(*productionClaimLocal)
+ if !ok {
+ log.Printf("agent: region HA: relocation pass not started: the assembly has no production participation owner")
+ return nil
+ }
+ var runtime *RelocationRuntime
+ var err error
+ if regionHAEnabled(getenv) {
+ var in relocationRuntimeInputs
+ if in, err = productionRelocationRuntimeInputs(os.Getenv, release.OptDir()); err == nil {
+ runtime, err = NewRelocationRuntime(agentCtx, getenv, entry.Store, local, claims.Providers, reg, svc, in)
+ }
+ if err != nil {
+ log.Printf("agent: region HA: relocation pass not started, finishing completions only: %v", err)
+ }
+ }
+ if runtime == nil {
+ // Ruling 82: with claim mode on, a relocation that completed here
+ // is finished whatever CARLOS_REGION_HA says.
+ if runtime, err = newCompletionRuntime(local, reg, svc); err != nil {
+ log.Printf("agent: relocation completion pass not started: %v", err)
+ return nil
+ }
+ }
+ go runtime.Run(agentCtx, entry.Store, nil)
+ return runtime
+}
diff --git a/internal/host/relocatepass_carries_test.go b/internal/host/relocatepass_carries_test.go
new file mode 100644
index 000000000..962afc87d
--- /dev/null
+++ b/internal/host/relocatepass_carries_test.go
@@ -0,0 +1,332 @@
+package host
+
+import (
+ "bytes"
+ "context"
+ "crypto/ed25519"
+ "crypto/rand"
+ "errors"
+ "os"
+ "path/filepath"
+ "slices"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/hostactors"
+ "github.com/carlosframework/platform/internal/hostbox"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/hostfloor"
+ "github.com/carlosframework/platform/internal/hostguard"
+ "github.com/carlosframework/platform/internal/hostops"
+ "github.com/carlosframework/platform/internal/hostrelocate"
+ "github.com/carlosframework/platform/internal/hostwrite"
+ "github.com/carlosframework/platform/internal/lease"
+ "github.com/carlosframework/platform/internal/lease/leasetest"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// finishRecorder is a Finish that records each host it is handed and
+// answers err.
+type finishRecorder struct {
+ calls []string
+ err error
+}
+
+func (f *finishRecorder) finish(_ context.Context, host, op string) error {
+ f.calls = append(f.calls, host+":"+op)
+ return f.err
+}
+
+// Carry 8: the completion pass hands Finish only a claim serving on this
+// box whose completion names this box. The completion-only runtime has no
+// Box; it asks Self each pass, and a pass that cannot answer finishes
+// nothing.
+func TestCompletionPassFinishesOnlyWhatServesOnThisBox(t *testing.T) {
+ ctx := t.Context()
+ store := release.NewDirStore(t.TempDir())
+ passClaim(t, store, "mine.example", hostclaim.Op{})
+ completeOn(t, store, "mine.example", "rel-m", "dst", false)
+ // Serving on spa, its completion naming spa: spa's to finish.
+ passClaim(t, store, "theirs.example", hostclaim.Op{})
+ completeOn(t, store, "theirs.example", "rel-t", "spa", false)
+ // Serving on spa while its record names dst: not serving here, so not
+ // this box's to finish, whatever the record says.
+ passClaim(t, store, "moved.example", hostclaim.Op{})
+ o := completeOn(t, store, "moved.example", "rel-x", "dst", false)
+ if _, err := hostclaim.Update(ctx, store, o, func(c *hostclaim.Claim) error { c.Op.Exec = "spa"; return nil }); err != nil {
+ t.Fatal(err)
+ }
+ // Serving on dst while its record names spa: the completion is spa's.
+ passClaim(t, store, "foreign.example", hostclaim.Op{})
+ o = completeOn(t, store, "foreign.example", "rel-f", "spa", false)
+ if _, err := hostclaim.Update(ctx, store, o, func(c *hostclaim.Claim) error { c.Op.Exec = "dst"; return nil }); err != nil {
+ t.Fatal(err)
+ }
+ f := &finishRecorder{}
+ hosts := func() ([]string, error) {
+ return []string{"mine.example", "theirs.example", "moved.example", "foreign.example"}, nil
+ }
+ r := &RelocationRuntime{Finish: f.finish, Hosts: hosts, Self: func(context.Context) (string, error) { return "dst", nil }}
+ r.once(ctx, store)
+ if !slices.Equal(f.calls, []string{"mine.example:rel-m"}) {
+ t.Fatalf("Finish was handed %v, want only mine.example", f.calls)
+ }
+ // No answer to "which box is this": nothing is finished.
+ f.calls = nil
+ buf := captureRacyLog(t)
+ r = &RelocationRuntime{Finish: f.finish, Hosts: hosts, Self: func(context.Context) (string, error) { return "", errors.New("no verified identity") }}
+ r.once(ctx, store)
+ if len(f.calls) != 0 || !strings.Contains(buf.String(), "no verified identity") {
+ t.Fatalf("finished %v without knowing this box:\n%s", f.calls, buf.String())
+ }
+}
+
+// Carry 8: ErrChanged from the hold release is the claim moving on under
+// the pass: nothing to do, no alert, and asked again next pass, never
+// taken as finished. Every other refusal is alerted once and retried
+// (carry 6: a probe-fence owner Finish cannot find is ErrRefused).
+func TestCompletionPassRetriesARefusedFinishAndIgnoresAChangedClaim(t *testing.T) {
+ ctx := t.Context()
+ store := release.NewDirStore(t.TempDir())
+ passClaim(t, store, "a.example", hostclaim.Op{})
+ completeOn(t, store, "a.example", "rel-a", "dst", false)
+ buf := captureRacyLog(t)
+ f := &finishRecorder{err: hostbox.ErrChanged}
+ r := &RelocationRuntime{Finish: f.finish, Box: "dst"}
+ r.once(ctx, store)
+ r.once(ctx, store)
+ if len(f.calls) != 2 || strings.Contains(buf.String(), "finish") {
+ t.Fatalf("a changed claim: calls %v, log:\n%s", f.calls, buf.String())
+ }
+ f.calls, f.err = nil, errors.Join(hostrelocate.ErrRefused, errors.New("a.example has no probe-fence owner"))
+ r.once(ctx, store)
+ r.once(ctx, store)
+ if len(f.calls) != 2 || strings.Count(buf.String(), "no probe-fence owner") != 1 {
+ t.Fatalf("a refused Finish: calls %v (want a retry each pass), log:\n%s", f.calls, buf.String())
+ }
+ if obs, err := hostclaim.Read(ctx, store, "a.example"); err != nil {
+ t.Fatal(err)
+ } else if done, ok, err := hostclaim.ReadCompletedRelocation(obs.Claim); err != nil || !ok || done.Discharged {
+ t.Fatalf("the record was marked over a refusal: %+v %v %v", done, ok, err)
+ }
+}
+
+// Carry 1: a completion record that cannot be read is alerted and never
+// read as "no completion": Finish is not called over it, and the next pass
+// reads it again.
+func TestCompletionPassAlertsAnUnreadableRecord(t *testing.T) {
+ ctx := t.Context()
+ store := release.NewDirStore(t.TempDir())
+ passClaim(t, store, "a.example", hostclaim.Op{})
+ o := completeOn(t, store, "a.example", "rel-a", "dst", false)
+ if _, err := hostclaim.Update(ctx, store, o, func(c *hostclaim.Claim) error {
+ c.Op.Journal[hostclaim.JournalCompleted] = "not a record"
+ return nil
+ }); err != nil {
+ t.Fatal(err)
+ }
+ buf := captureRacyLog(t)
+ f := &finishRecorder{}
+ r := &RelocationRuntime{Finish: f.finish, Box: "dst"}
+ r.once(ctx, store)
+ if len(f.calls) != 0 || !strings.Contains(buf.String(), "relocate a.example: finish:") {
+ t.Fatalf("an unreadable record: calls %v, log:\n%s", f.calls, buf.String())
+ }
+}
+
+// Carry 3: the sweep never runs beside a relocation journal write. A
+// write in flight (its directory made, its temporary not yet renamed)
+// holds relocationJournals shared; the pass's sweep waits for it, then
+// keeps the journal it committed. Without the lock the sweep removes the
+// directory as a write's leftover under the write.
+func TestRelocatePassSweepWaitsForAJournalWrite(t *testing.T) {
+ ctx := t.Context()
+ store := release.NewDirStore(t.TempDir())
+ passClaim(t, store, "a.example", relocatingOp("rel-a"))
+ data := t.TempDir()
+ dir := filepath.Join(data, "relocate", "ops", "rel-a")
+ relocationJournals.RLock()
+ locked := true
+ defer func() {
+ if locked {
+ relocationJournals.RUnlock()
+ }
+ }()
+ if err := os.MkdirAll(dir, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ temp := filepath.Join(dir, ".attempts.json.123")
+ if err := os.WriteFile(temp, []byte("{"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ r := &RelocationRuntime{Box: "dst", DataDir: data}
+ done := make(chan struct{})
+ go func() { r.once(ctx, store); close(done) }()
+ select {
+ case <-done:
+ t.Fatal("the sweep ran beside a journal write")
+ case <-time.After(300 * time.Millisecond):
+ }
+ // The write lands: its journal committed, its temporary gone.
+ if err := os.WriteFile(filepath.Join(dir, "attempts.json"), []byte(`{"v":1,"host":"a.example","op":"rel-a"}`), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Remove(temp); err != nil {
+ t.Fatal(err)
+ }
+ relocationJournals.RUnlock()
+ locked = false
+ <-done
+ if _, err := os.Stat(filepath.Join(dir, "attempts.json")); err != nil {
+ t.Fatalf("the running op's journal was swept: %v", err)
+ }
+}
+
+// Carry 3, the writers' half: every relocation journal write this process
+// makes outside the pass's own steps holds relocationJournals shared, so
+// it waits while a sweep holds it.
+func TestRelocationJournalWritesWaitForTheSweep(t *testing.T) {
+ data := t.TempDir()
+ fence, err := relocationProbeFencePath(data, "rel-a")
+ if err != nil {
+ t.Fatal(err)
+ }
+ positions := ReleasePositionsFile{Dir: filepath.Join(data, "relocate", "release")}
+ for name, write := range map[string]func() error{
+ "attempts": func() error {
+ return relocationAttemptJournal{V: 1, Host: "a.example", Op: "rel-a", Opened: map[string]time.Time{}, Sessions: map[string]int{}, Starts: map[string]relocationStart{}}.write(data)
+ },
+ "probe fence": func() error { _, err := relocationProbeFenceOwner(fence, "a.example", 7); return err },
+ "positions": func() error {
+ return positions.Record(t.Context(), ReleasePosition{Request: StopRequest{Host: "a.example", OperationID: "rel-a"}, PID: 1, ProcessStart: "1"})
+ },
+ } {
+ t.Run(name, func(t *testing.T) {
+ relocationJournals.Lock()
+ done := make(chan error, 1)
+ go func() { done <- write() }()
+ select {
+ case err := <-done:
+ relocationJournals.Unlock()
+ t.Fatalf("wrote beside the sweep: %v", err)
+ case <-time.After(300 * time.Millisecond):
+ }
+ relocationJournals.Unlock()
+ if err := <-done; err != nil {
+ t.Fatal(err)
+ }
+ })
+ }
+}
+
+// Carry 4: the full runtime's config carries the agent's held-stage
+// evidence: the release key it was given and the edge's pointer digest
+// ratchet (pointerRatchet(false, ...), never root's), with the box's
+// secrets key when it has one. No release key, or a secrets key that does
+// not parse, refuses.
+func TestRelocationRuntimeConfigBindsTheAgentsHeldStageEvidence(t *testing.T) {
+ pub, _, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ a := &ClaimAssembly{}
+ a.Guard.DataDir = t.TempDir()
+ opt := t.TempDir()
+ env := map[string]string{}
+ getenv := func(k string) string { return env[k] }
+ s := relocationStores{Files: release.NewDirStore(t.TempDir()), Versions: leasetest.NewVersionedBucket(), Bucket: "data-ie", Region: "ie", LeaseRegion: "eu-west-1"}
+ discharge := func(context.Context, hostclaim.Observed) error { return nil }
+ build := func(in relocationRuntimeInputs) (RelocationAssemblyConfig, error) {
+ return relocationRuntimeConfig(getenv, a, ClaimProviders{Selector: hostbox.WorkSource{Box: "dst", Region: "ie"}}, s, &registry.Registry{}, &relocationProbeActivator{}, hostactors.Directory{}, in, discharge)
+ }
+ cfg, err := build(relocationRuntimeInputs{Pub: pub, OptDir: opt})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(cfg.ReleasePub, pub) || cfg.Ratchet == nil || cfg.Ratchet != pointerRatchet(false, opt, a.Guard.DataDir, getenv) || cfg.Ratchet == pointerRatchet(true, opt, a.Guard.DataDir, getenv) || cfg.SecretsKey != nil {
+ t.Fatalf("held-stage evidence: pub %v ratchet %p secrets %v", cfg.ReleasePub, cfg.Ratchet, cfg.SecretsKey)
+ }
+ if _, err := build(relocationRuntimeInputs{OptDir: opt}); !errors.Is(err, hostwrite.ErrUnavailable) {
+ t.Fatalf("no release key: %v", err)
+ }
+ env["CARLOS_SECRETS_KEY"] = "not a key"
+ if _, err := build(relocationRuntimeInputs{Pub: pub, OptDir: opt}); err == nil {
+ t.Fatal("a secrets key that does not parse was accepted")
+ }
+}
+
+// With claim mode and region HA on and the relocation owners bound into
+// the wake's dispatcher, NewRelocationRuntime builds the full pass: this
+// box's worker in its roster region, which starts nothing (its Admission
+// and Census refuse), the participant, the J3 conversion and the hold
+// release. A roster entry that is not this box's enrollment, no release
+// key, or no data bucket refuses rather than build a pass over a guess.
+func TestRelocationRuntimeBuildsTheFullPassWithTheOwnersBound(t *testing.T) {
+ ctx := t.Context()
+ w := newLandingWorld(t, true, nil)
+ reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer reg.Close()
+ data := t.TempDir()
+ a := &ClaimAssembly{Claims: w.store, Guard: hostguard.Guard{DataDir: data}, RetirementAuthority: &hostops.ProductionAuthority{Store: w.store, Directory: w.directory},
+ Wake: &ClaimWakeAssembly{Inputs: ClaimOwnerDispatch{Relocation: &ClaimKindOwners{}}}}
+ local := &productionClaimLocal{assembly: a}
+ env := map[string]string{"CARLOS_HOST_CLAIM": "1", "CARLOS_REGION_HA": "1", "CARLOS_HIBERNATE_BUCKET": "data-ie"}
+ getenv := func(k string) string { return env[k] }
+ pub, _, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ in := relocationRuntimeInputs{Enrollment: hostfloor.Enrollment{Box: "wrk", Enrollment: w.enrollment["wrk"]}, Pub: pub, OptDir: t.TempDir(),
+ Files: func(context.Context, string) (release.Store, error) { return release.NewDirStore(t.TempDir()), nil },
+ Versioned: func(context.Context, string, string) (lease.VersionedStore, error) {
+ return leasetest.NewVersionedBucket(), nil
+ }}
+ providers := ClaimProviders{Selector: hostbox.WorkSource{Box: "wrk", Region: "ie"}}
+ build := func(in relocationRuntimeInputs) (*RelocationRuntime, error) {
+ return NewRelocationRuntime(ctx, getenv, w.store, local, providers, reg, &relocationProbeActivator{}, in)
+ }
+ r, err := build(in)
+ if err != nil {
+ t.Fatal(err)
+ }
+ worker, ok := r.Worker.(hostrelocate.Worker)
+ if !ok || r.Participant == nil || r.Convert == nil || r.Finish == nil || r.Box != "wrk" || r.DataDir != data || r.Hosts != nil {
+ t.Fatalf("not the full pass: %+v", r)
+ }
+ if worker.Self.Box != "wrk" || worker.Self.Enrollment != w.enrollment["wrk"] || worker.Region != "ie" {
+ t.Fatalf("the worker is not this box in its roster region: %+v %q", worker.Self, worker.Region)
+ }
+ if _, err := worker.Admission.Register(ctx, hostclaim.Observed{}, hostrelocate.Owner{}, ""); !errors.Is(err, errRelocationNotStarted) {
+ t.Fatalf("the pass's admission registered a start: %v", err)
+ }
+ if err := worker.Census(ctx, "wrk", nil); !errors.Is(err, errRelocationNotStarted) {
+ t.Fatalf("the pass's census passed a start: %v", err)
+ }
+ for name, change := range map[string]func(*relocationRuntimeInputs){
+ "another enrollment": func(in *relocationRuntimeInputs) { in.Enrollment.Enrollment = strings.Repeat("f", 32) },
+ "not in the roster": func(in *relocationRuntimeInputs) { in.Enrollment.Box = "gone" },
+ "no release key": func(in *relocationRuntimeInputs) { in.Pub = nil },
+ } {
+ bad := in
+ change(&bad)
+ if r, err := build(bad); err == nil || r != nil {
+ t.Fatalf("%s: built %+v", name, r)
+ }
+ }
+ // Region HA off: the completion-only runtime, though the owners are
+ // bound (ruling 82; the dormant constraint).
+ env["CARLOS_REGION_HA"] = ""
+ if r, err := build(in); err != nil || r == nil || r.Worker != nil || r.Participant != nil || r.Convert != nil || r.Finish == nil {
+ t.Fatalf("region HA off: %+v %v", r, err)
+ }
+ env["CARLOS_REGION_HA"] = "1"
+ delete(env, "CARLOS_HIBERNATE_BUCKET")
+ if r, err := build(in); !errors.Is(err, hostwrite.ErrUnavailable) || r != nil {
+ t.Fatalf("no data bucket: %+v %v", r, err)
+ }
+}
diff --git a/internal/host/relocatepass_test.go b/internal/host/relocatepass_test.go
new file mode 100644
index 000000000..18862ffdd
--- /dev/null
+++ b/internal/host/relocatepass_test.go
@@ -0,0 +1,518 @@
+package host
+
+import (
+ "context"
+ "crypto/ed25519"
+ "crypto/rand"
+ "encoding/json"
+ "errors"
+ "os"
+ "path/filepath"
+ "reflect"
+ "slices"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/boxvolumes"
+ "github.com/carlosframework/platform/internal/hostactors"
+ "github.com/carlosframework/platform/internal/hostbox"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/hostguard"
+ "github.com/carlosframework/platform/internal/hostrelocate"
+ "github.com/carlosframework/platform/internal/lease"
+ "github.com/carlosframework/platform/internal/lease/leasetest"
+ "github.com/carlosframework/platform/internal/moves"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/termfence"
+ "github.com/superfly/ltx"
+)
+
+// passClaim writes a claim for host into store with op as given; serving
+// claims have no op id.
+func passClaim(t *testing.T, store release.Store, host string, op hostclaim.Op) {
+ t.Helper()
+ ctx := t.Context()
+ created := time.Date(2026, 10, 7, 1, 0, 0, 0, time.UTC)
+ nh := hostclaim.NewHolder{Family: "instance", Key: "apps/bab/jam/instances/" + host + ".json", Account: "bab", App: "jam", Host: host, Region: "ie", RecordRegion: "ie", CreatedAt: created}
+ body := []byte(`{"host":"` + host + `","account":"bab","app":"jam","region":"ie","created_at":"2026-10-07T01:00:00Z"}`)
+ obs, err := hostclaim.Read(ctx, store, host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, tr := range []func(*hostclaim.Claim) error{
+ hostclaim.BeginCreateOn(nh, body, "c1", "console", created, hostclaim.ExecutorSelection{Box: "src", Enrollment: strings.Repeat("b", 32)}),
+ hostclaim.CompleteCreate("c1", "src"),
+ hostclaim.AdmitWriter(hostclaim.Admission{Box: "src", Mode: "cold", NewGeneration: "g1", Region: "ie", KeepStamp: true}),
+ func(c *hostclaim.Claim) error {
+ if op.Kind != "" {
+ c.Op = op
+ }
+ return nil
+ },
+ } {
+ if obs, err = hostclaim.Update(ctx, store, obs, tr); err != nil {
+ t.Fatal(err)
+ }
+ }
+}
+
+func relocatingOp(id string) hostclaim.Op {
+ return hostclaim.Op{Kind: hostclaim.KindRelocating, ID: id, Phase: hostclaim.PhaseInstalling, Exec: "dst", Participants: []string{"src", "dst"},
+ Intent: map[string]string{"region": "ie", "source_box": "src", "target_box": "dst"}}
+}
+
+// stepRecorder is a worker or participant that records each Step.
+type stepRecorder struct {
+ name string
+ order *[]string
+ err error
+}
+
+func (s stepRecorder) Step(_ context.Context, host string) (hostclaim.Observed, error) {
+ *s.order = append(*s.order, s.name+":"+host)
+ return hostclaim.Observed{}, s.err
+}
+
+// Spec §8 step 2 and ruling 82: nothing runs with CARLOS_HOST_CLAIM off.
+// With claim mode on, CARLOS_REGION_HA off, or the relocation owners not
+// bound, the runtime is the completion-only one: it steps nothing,
+// converts nothing and reads only this box's registry rows.
+func TestRelocationRuntimeFollowsTheFlags(t *testing.T) {
+ reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer reg.Close()
+ local := &productionClaimLocal{assembly: &ClaimAssembly{Guard: hostguard.Guard{DataDir: t.TempDir()}}}
+ for _, tc := range []struct {
+ claim, ha string
+ completion bool
+ }{{"", "", false}, {"", "1", false}, {"1", "", true}, {"1", "1", true}} {
+ getenv := func(k string) string {
+ switch k {
+ case "CARLOS_HOST_CLAIM":
+ return tc.claim
+ case "CARLOS_REGION_HA":
+ return tc.ha
+ }
+ return ""
+ }
+ r, err := NewRelocationRuntime(t.Context(), getenv, release.NewDirStore(t.TempDir()), local, ClaimProviders{}, reg, &activator.Service{}, relocationRuntimeInputs{})
+ if err != nil {
+ t.Fatalf("claim %q region HA %q: %v", tc.claim, tc.ha, err)
+ }
+ if !tc.completion {
+ if r != nil {
+ t.Fatalf("claim %q region HA %q: built %+v", tc.claim, tc.ha, r)
+ }
+ continue
+ }
+ // With region HA on, the owners are not bound here (no wake
+ // dispatcher), so this is the completion-only runtime too.
+ if r == nil || r.Worker != nil || r.Participant != nil || r.Convert != nil || r.Finish == nil || r.Hosts == nil {
+ t.Fatalf("claim %q region HA %q: not a completion-only runtime: %+v", tc.claim, tc.ha, r)
+ }
+ }
+}
+
+// Ruling 82: on a box with region HA off that never completed a
+// relocation, the completion pass writes nothing. Its hosts are its own
+// rows; a serving claim with no completion is left alone, and so is one
+// whose completion names another box.
+func TestCompletionPassChangesNothingWithoutACompletion(t *testing.T) {
+ ctx := t.Context()
+ store := release.NewDirStore(t.TempDir())
+ reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer reg.Close()
+ passClaim(t, store, "a.example", hostclaim.Op{})
+ passClaim(t, store, "b.example", hostclaim.Op{})
+ completeOn(t, store, "b.example", "rel-b", "spa", false)
+ passClaim(t, store, "c.example", hostclaim.Op{}) // no row here
+ for _, host := range []string{"a.example", "b.example"} {
+ if err := reg.Upsert(registry.Route{Host: host, Status: "ready", Socket: "/run/" + host + ".sock"}); err != nil {
+ t.Fatal(err)
+ }
+ }
+ before := map[string]string{}
+ for _, host := range []string{"a.example", "b.example", "c.example"} {
+ obs, err := hostclaim.Read(ctx, store, host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ before[host] = obs.ETag
+ }
+ rowsBefore, err := reg.List()
+ if err != nil {
+ t.Fatal(err)
+ }
+ data := t.TempDir()
+ local := &productionClaimLocal{assembly: &ClaimAssembly{Claims: store, Guard: hostguard.Guard{DataDir: data}, stopLocal: func(context.Context) (claimStopLocal, error) {
+ return claimStopLocal{Actor: hostbox.Actor{Box: "dst"}}, nil
+ }}}
+ claimOnly := func(k string) string {
+ if k == "CARLOS_HOST_CLAIM" {
+ return "1"
+ }
+ return ""
+ }
+ svc := &activator.Service{}
+ r, err := NewRelocationRuntime(ctx, claimOnly, store, local, ClaimProviders{}, reg, svc, relocationRuntimeInputs{})
+ if err != nil || r == nil {
+ t.Fatal(err)
+ }
+ r.once(ctx, store)
+ for host, etag := range before {
+ if obs, err := hostclaim.Read(ctx, store, host); err != nil || obs.ETag != etag {
+ t.Fatalf("%s was written: %v", host, err)
+ }
+ }
+ if rows, err := reg.List(); err != nil || !reflect.DeepEqual(rows, rowsBefore) {
+ t.Fatalf("the registry changed: %v", err)
+ }
+ if _, err := os.Stat(filepath.Join(data, "relocate")); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("the pass made journals: %v", err)
+ }
+ if svc.Fenced("a.example") || svc.Fenced("b.example") {
+ t.Fatal("the pass fenced a host")
+ }
+}
+
+// One pass steps every relocating claim (participant, then worker), hands
+// every landing to the J3 conversion, leaves serving claims alone, and
+// sweeps the journals of operations that are over (ruling 61).
+func TestRelocatePassStepsEveryRelocationAndSweeps(t *testing.T) {
+ ctx := t.Context()
+ store := release.NewDirStore(t.TempDir())
+ passClaim(t, store, "a.example", relocatingOp("rel-a"))
+ passClaim(t, store, "b.example", relocatingOp("rel-b"))
+ passClaim(t, store, "c.example", hostclaim.Op{})
+ enrollment := map[string]string{"src": strings.Repeat("b", 32), "dst": strings.Repeat("c", 32)}
+ landingAtProbed(t, store, "d.example", "mv_abcdefghijklmnopqrstuvwxyz", time.Date(2026, 10, 7, 9, 0, 0, 0, time.UTC), enrollment)
+ data := t.TempDir()
+ stale := filepath.Join(data, "relocate", "ops", "rel-gone", "install.json")
+ if err := os.MkdirAll(filepath.Dir(stale), 0o700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(stale, []byte(`{"v":1,"host":"c.example","op":"rel-gone"}`), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ var order, converted []string
+ r := &RelocationRuntime{Worker: stepRecorder{name: "worker", order: &order}, Participant: stepRecorder{name: "participant", order: &order}, Box: "dst", DataDir: data,
+ Convert: func(_ context.Context, _ release.Store, o hostclaim.Observed) error {
+ converted = append(converted, o.Claim.Host)
+ return nil
+ }}
+ r.once(ctx, store)
+ want := []string{"participant:a.example", "worker:a.example", "participant:b.example", "worker:b.example"}
+ if !slices.Equal(order, want) {
+ t.Fatalf("stepped %v, want %v", order, want)
+ }
+ if !slices.Equal(converted, []string{"d.example"}) {
+ t.Fatalf("offered for conversion %v, want only the landing d.example", converted)
+ }
+ if _, err := os.Stat(stale); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("an ended op's journal survived the pass: %v", err)
+ }
+}
+
+// Rulings 35 and 36: a refusal that never clears (a conversion waiting for
+// a target) is logged once per host and cause, never per pass.
+func TestRelocatePassAlertsOnceWhileAConversionWaitsForATarget(t *testing.T) {
+ ctx := t.Context()
+ store := release.NewDirStore(t.TempDir())
+ passClaim(t, store, "a.example", relocatingOp("rel-a"))
+ buf := captureRacyLog(t)
+ var order []string
+ none := errors.New("no eligible box for the converted landing")
+ r := &RelocationRuntime{Worker: stepRecorder{name: "worker", order: &order, err: none}, Participant: stepRecorder{name: "participant", order: &order, err: hostrelocate.ErrNotYet}, Box: "dst"}
+ for range 3 {
+ r.once(ctx, store)
+ }
+ if n := strings.Count(buf.String(), none.Error()); n != 1 {
+ t.Fatalf("logged %d times, want once:\n%s", n, buf.String())
+ }
+ if strings.Contains(buf.String(), hostrelocate.ErrNotYet.Error()) {
+ t.Fatalf("ErrNotYet was alerted:\n%s", buf.String())
+ }
+}
+
+// Ruling 49: one pass completes a passed probe on the target: the
+// participant (its probe stop), then the worker (the serving CAS), then
+// Finish for the row this box still holds relocating.
+func TestRelocatePassStopsCompletesAndReleasesInOnePass(t *testing.T) {
+ ctx := t.Context()
+ store := release.NewDirStore(t.TempDir())
+ passClaim(t, store, "a.example", relocatingOp("rel-a"))
+ // Ruling 80: completions are found from the claim's record. One this
+ // box finished already, and one another box finishes, are left alone.
+ passClaim(t, store, "b.example", hostclaim.Op{})
+ completeOn(t, store, "b.example", "rel-b", "dst", true)
+ passClaim(t, store, "c.example", hostclaim.Op{})
+ completeOn(t, store, "c.example", "rel-c", "spa", false)
+ var order []string
+ r := &RelocationRuntime{Worker: completingWorker{order: &order, store: store}, Participant: stepRecorder{name: "participant", order: &order}, Box: "dst",
+ Finish: func(_ context.Context, host, op string) error {
+ order = append(order, "finish:"+host+":"+op)
+ return nil
+ }}
+ r.once(ctx, store)
+ if want := []string{"participant:a.example", "worker:a.example", "finish:a.example:rel-a"}; !slices.Equal(order, want) {
+ t.Fatalf("order %v, want %v", order, want)
+ }
+}
+
+// completeOn writes host's serving op with op's completed relocation on
+// box, marked discharged or not.
+func completeOn(t *testing.T, store release.Store, host, op, box string, discharged bool) hostclaim.Observed {
+ t.Helper()
+ ctx := t.Context()
+ obs, err := hostclaim.Read(ctx, store, host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ done, _ := json.Marshal(hostclaim.CompletedRelocation{Op: op, Stop: hostclaim.ProbeStop{Box: box, Session: obs.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)},
+ Phase: hostclaim.PhaseRelocatingProbing, Discharged: discharged})
+ obs, err = hostclaim.Update(ctx, store, obs, func(c *hostclaim.Claim) error {
+ c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: box, Participants: []string{box}, Journal: map[string]string{hostclaim.JournalCompleted: string(done)}}
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ return obs
+}
+
+// completingWorker is the worker as the pass sees it when its Step makes
+// the serving CAS: the claim then serves on dst with the completion.
+type completingWorker struct {
+ order *[]string
+ store release.Store
+}
+
+func (w completingWorker) Step(ctx context.Context, host string) (hostclaim.Observed, error) {
+ *w.order = append(*w.order, "worker:"+host)
+ obs, err := hostclaim.Read(ctx, w.store, host)
+ if err != nil {
+ return obs, err
+ }
+ done, _ := json.Marshal(hostclaim.CompletedRelocation{Op: obs.Claim.Op.ID, Stop: hostclaim.ProbeStop{Box: "dst", Session: obs.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)},
+ Phase: hostclaim.PhaseRelocatingProbing})
+ return hostclaim.Update(ctx, w.store, obs, func(c *hostclaim.Claim) error {
+ c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "dst", Participants: []string{"dst"}, Journal: map[string]string{hostclaim.JournalCompleted: string(done)}}
+ return nil
+ })
+}
+
+// J3: the guard admits a conversion only for the move the claim's op
+// names, committed for this incarnation with no rollback; anything else is
+// moves' case (ErrWrongState).
+func TestCommittedLandingRecordsAdmitOnlyACommittedMove(t *testing.T) {
+ ctx := t.Context()
+ inc, err := hostclaim.Incarnation("bab", "jam", "a.example", time.Date(2026, 10, 7, 1, 0, 0, 0, time.UTC))
+ if err != nil {
+ t.Fatal(err)
+ }
+ committedAt := time.Date(2026, 10, 7, 5, 0, 0, 0, time.UTC)
+ flipped := committedAt.Add(-time.Hour)
+ o := hostclaim.Observed{Host: "a.example", Claim: &hostclaim.Claim{Host: "a.example", Holder: &hostclaim.Holder{Account: "bab", App: "jam", Incarnation: inc, Location: "instances/a.example"},
+ Op: hostclaim.Op{Kind: hostclaim.KindLanding, ID: "mv-1", Phase: "probed", Journal: map[string]string{hostclaim.JournalSeal: moves.Seal{V: moves.SealVersion, N: 42, FromDB: true, SrcCovered: true,
+ SealedAt: flipped.Add(-time.Minute), LeaseUntil: flipped, Replica: []moves.SealedObject{{Key: "moves/a.example/mv-1/replica/0009/" + ltx.FormatFilename(1, 42), VersionID: "v1", Size: 100}}}.Encode()}}}}
+ record := func(change func(*provision.MoveState)) MovesRecordReader {
+ return func(context.Context, string, string, string) (*provision.Record, error) {
+ m := &provision.MoveState{ID: "mv-1", Incarnation: inc, From: "uk", To: "ie", State: provision.MoveCommitted, CommittedAt: &committedAt, FlippedAt: &flipped}
+ if change != nil {
+ change(m)
+ }
+ return &provision.Record{Host: "a.example", Account: "bab", App: "jam", Move: m}, nil
+ }
+ }
+ for name, change := range map[string]func(*provision.MoveState){
+ "not committed": func(m *provision.MoveState) { m.State = provision.MoveLanding },
+ "another move": func(m *provision.MoveState) { m.ID = "mv-2" },
+ "another incarnation": func(m *provision.MoveState) { m.Incarnation = "other" },
+ "rolled back": func(m *provision.MoveState) { m.Rollback = &provision.MoveRollback{} },
+ "no commit time": func(m *provision.MoveState) { m.CommittedAt = nil },
+ } {
+ if _, err := (CommittedLandingRecords{Records: record(change)}).CheckLandingCommitted(ctx, o, "mv-1"); !errors.Is(err, hostclaim.ErrWrongState) {
+ t.Fatalf("%s: %v", name, err)
+ }
+ }
+ got, err := (CommittedLandingRecords{Records: record(nil)}).CheckLandingCommitted(ctx, o, "mv-1")
+ if err != nil {
+ t.Fatalf("a committed move: %v", err)
+ }
+ source, staging, err := moves.CommitScopes(o.Claim, &flipped)
+ if err != nil || got.FromRegion != "uk" || got.ToRegion != "ie" || !reflect.DeepEqual(got.SourceScope, source) || !reflect.DeepEqual(got.StagingScope, staging) {
+ t.Fatalf("the commit's regions and scopes: %+v (%v)", got, err)
+ }
+ if _, err := (CommittedLandingRecords{Records: record(nil)}).CheckLandingCommitted(ctx, o, "mv-other"); !errors.Is(err, hostclaim.ErrWrongState) {
+ t.Fatalf("another op id: %v", err)
+ }
+}
+
+// Ruling 30: the release a relocation installs is the host's channel
+// pointer, verified with the release key; an unverifiable pointer refuses.
+func TestRelocationReleaseIsTheChannelPointer(t *testing.T) {
+ ctx := t.Context()
+ store := release.NewDirStore(t.TempDir())
+ pub, priv, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ passClaim(t, store, "a.example", hostclaim.Op{})
+ if err := provision.CreateRecord(ctx, store, provision.Record{Schema: 1, Host: "a.example", Slug: "a", App: "jam", Account: "bab", Region: "ie", Channel: "stable", CreatedAt: time.Date(2026, 10, 7, 1, 0, 0, 0, time.UTC)}); err != nil && !errors.Is(err, release.ErrCASConflict) {
+ t.Fatal(err)
+ }
+ plat := release.AdoptPlatform()
+ for _, v := range []string{"v1.4.0", "v1.5.0"} {
+ m := release.Manifest{App: "jam", Version: v, Kind: "binary", Artifacts: map[string]release.Artifact{plat: {}}}
+ if err := release.Ship(ctx, store, "bab", m, map[string][]byte{plat: []byte("build " + v)}); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := release.Promote(ctx, store, priv, "bab", "jam", "stable", "v1.5.0"); err != nil {
+ t.Fatal(err)
+ }
+ got, err := channelRelease(store, pub)(ctx, "a.example")
+ if err != nil || got != "v1.5.0" {
+ t.Fatalf("release %q %v", got, err)
+ }
+ other, _, _ := ed25519.GenerateKey(rand.Reader)
+ if _, err := channelRelease(store, other)(ctx, "a.example"); err == nil {
+ t.Fatal("an unverifiable pointer answered")
+ }
+ if _, err := channelRelease(store, pub)(ctx, "absent.example"); !errors.Is(err, hostrelocate.ErrRefused) {
+ t.Fatalf("a host with no claim: %v", err)
+ }
+}
+
+// Ruling 3: a terminated box's local state is discharged only with the
+// roster's enrolled_at; without it the discharge is unqualified, with it
+// boxvolumes.Discharge's verdict.
+func TestDischargeIsUnqualifiedWithoutEnrolledAt(t *testing.T) {
+ ctx := t.Context()
+ store := release.NewDirStore(t.TempDir())
+ discharge := volumeDischarge(store)
+ fence := termfence.Record{ObservedAt: time.Date(2026, 10, 7, 9, 0, 0, 0, time.UTC).Format(time.RFC3339Nano)}
+ box := hostactors.Box{BoxID: "src", Enrollment: strings.Repeat("b", 32), Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-src"}}
+ if d := discharge(ctx, box, fence); d == nil || d.Qualified || !strings.Contains(d.Reason, "enrollment time") {
+ t.Fatalf("without enrolled_at: %+v", d)
+ }
+ box.EnrolledAt = time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
+ records, err := boxvolumes.List(ctx, store, box.Enrollment)
+ if err != nil {
+ t.Fatal(err)
+ }
+ at, _ := time.Parse(time.RFC3339Nano, fence.ObservedAt)
+ want := boxvolumes.Discharge(records, boxvolumes.Entry{Box: box.BoxID, Enrollment: box.Enrollment, Cloud: box.Cloud}, box.EnrolledAt, at)
+ if d := discharge(ctx, box, fence); d == nil || d.Qualified != want.Discharge || d.Reason != want.Reason {
+ t.Fatalf("with enrolled_at: %+v, want %+v", d, want)
+ }
+ if d := discharge(ctx, box, termfence.Record{ObservedAt: "not a time"}); d == nil || d.Qualified {
+ t.Fatalf("an unreadable observation: %+v", d)
+ }
+}
+
+// Rulings 50 and 51: the data bucket is opened as a release.Store of its
+// own (production's lease.VersionedStore is never asserted to one), the
+// versioned client and the leases sign for the roster's cloud region, and
+// genselect, pins and the worker use the logical region.
+func TestRelocationRuntimeOpensADataBucketStore(t *testing.T) {
+ ctx := t.Context()
+ files := release.NewDirStore(t.TempDir())
+ var opened, signed string
+ self := hostactors.Box{Region: "ie", Cloud: hostactors.Cloud{Region: "eu-west-1"}}
+ s, err := openRelocationStores(ctx, self, "data-ie",
+ func(_ context.Context, bucket string) (release.Store, error) { opened = bucket; return files, nil },
+ func(_ context.Context, _ string, region string) (lease.VersionedStore, error) {
+ signed = region
+ return leasetest.NewVersionedBucket(), nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if opened != "data-ie" || signed != "eu-west-1" || s.Files != files || s.LeaseRegion != "eu-west-1" || s.Region != "ie" || s.Bucket != "data-ie" {
+ t.Fatalf("stores %+v (opened %q, signed %q)", s, opened, signed)
+ }
+ guard := ClaimAssembly{}
+ guard.Guard.DataDir = "/data"
+ src := hostbox.WorkSource{Box: "dst", Region: "ie"}
+ cfg, err := relocationAssemblyConfig(&guard, ClaimProviders{Selector: src}, s, &registry.Registry{}, &relocationProbeActivator{}, hostactors.Directory{}, "/opt/carlos",
+ func(context.Context, hostclaim.Observed) error { return nil })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if cfg.Data != files || cfg.Versions != s.Versions || cfg.LeaseRegion != "eu-west-1" || cfg.Region != "ie" || cfg.WorkDir != "/data/relocate/build" || cfg.ReleaseDir != "/data/relocate/release" || cfg.OptDir != "/opt/carlos" || cfg.Source.Box != "dst" {
+ t.Fatalf("config %+v", cfg)
+ }
+ if _, err := openRelocationStores(ctx, hostactors.Box{Region: "ie"}, "data-ie", func(context.Context, string) (release.Store, error) { return files, nil },
+ func(context.Context, string, string) (lease.VersionedStore, error) { return nil, nil }); err == nil {
+ t.Fatal("opened with no cloud region to sign for")
+ }
+}
+
+// Finding 11, N5, ruling 53: region HA's owners bind only with moves' C8
+// provider. The production gate hands the registered source main's own
+// serving sources before it opens; with C8 bound the owners are complete
+// and relocating binds; without it, or with region HA off, nothing binds.
+func TestRelocationBindsWithARegisteredMovesSource(t *testing.T) {
+ ctx := t.Context()
+ actor := hostbox.Actor{Box: "dst", Bucket: "data-ie", Region: "ie"}
+ serving := func(files ClaimFilesManifest) ClaimOwnerBindings {
+ var out ClaimOwnerBindings
+ out.Serving = ClaimServingSources{Launch: func(context.Context, registry.Route) (ClaimServingLaunch, error) { return ClaimServingLaunch{}, nil },
+ Markers: plan9D5Markers{leasetest.NewVersionedBucket()}, Files: files}
+ return out
+ }
+ source := func() *RelocationSource {
+ return &RelocationSource{DataDir: t.TempDir(), FilesStore: func(context.Context, string) (release.Store, error) { return release.NewDirStore(t.TempDir()), nil }}
+ }
+ out := serving(&roFiles{})
+ registered := source()
+ closeSource, err := openClaimRelocationOwners(ctx, plan11Env("1", "1"), registered, release.NewDirStore(t.TempDir()), &fakeVersioned{}, actor, &out)
+ if err != nil || closeSource == nil {
+ t.Fatalf("open: %v", err)
+ }
+ // Carry 5: the source's identity check is the owners' own
+ // IdentityProvider.Verify, as main's serving owners bind it. This
+ // one's provider is unconfigured, so it refuses as Verify does.
+ registered.mu.Lock()
+ identity := registered.serving.Identity
+ registered.mu.Unlock()
+ if identity == nil {
+ t.Fatal("the source was handed no identity check")
+ }
+ if _, want := out.Identity.Verify(ctx); want == nil || !errors.Is(identity(ctx), want) {
+ t.Fatalf("the identity check is not the owners' Verify: %v", identity(ctx))
+ }
+ if bound, err := claimRelocationC1(plan11Env("1", "1"), true, out.Relocation); !bound || err != nil || len(out.PolicyRefusals) != 2 {
+ t.Fatalf("with C8: bound %v %v refusals %d", bound, err, len(out.PolicyRefusals))
+ }
+ without := serving(nil)
+ if _, err := openClaimRelocationOwners(ctx, plan11Env("1", "1"), source(), release.NewDirStore(t.TempDir()), &fakeVersioned{}, actor, &without); err != nil || !without.Relocation.zero() {
+ t.Fatalf("without C8: %+v %v", without.Relocation, err)
+ }
+ off := serving(&roFiles{})
+ if closeSource, err := openClaimRelocationOwners(ctx, plan11Env("1", ""), source(), release.NewDirStore(t.TempDir()), &fakeVersioned{}, actor, &off); err != nil || closeSource != nil || !off.Relocation.zero() {
+ t.Fatalf("region HA off: %+v %v", off.Relocation, err)
+ }
+ // Without the bound owners there is no relocation pass, only the
+ // completion-only runtime (ruling 82).
+ reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer reg.Close()
+ local := &productionClaimLocal{assembly: &ClaimAssembly{Wake: &ClaimWakeAssembly{Inputs: ClaimOwnerDispatch{}}, Guard: hostguard.Guard{DataDir: t.TempDir()}}}
+ r, err := NewRelocationRuntime(ctx, plan11Env("1", "1"), release.NewDirStore(t.TempDir()), local, ClaimProviders{}, reg, &activator.Service{}, relocationRuntimeInputs{})
+ if err != nil || r == nil || r.Worker != nil || r.Participant != nil || r.Finish == nil {
+ t.Fatalf("a runtime without the relocation owners: %+v %v", r, err)
+ }
+}
diff --git a/internal/host/relocationattempts.go b/internal/host/relocationattempts.go
index 77fe61b05..563aa2441 100644
--- a/internal/host/relocationattempts.go
+++ b/internal/host/relocationattempts.go
@@ -136,10 +136,7 @@ func (j relocationAttemptJournal) write(dataDir string) error {
if err != nil {
return err
}
- if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
- return err
- }
- return writeFileAtomic(path, raw, 0o600)
+ return writeRelocationJournal(path, raw)
}
// relocationProbeAttempts is region HA's OperationAttempts for a probe
diff --git a/internal/host/relocationfinish_unix_test.go b/internal/host/relocationfinish_unix_test.go
new file mode 100644
index 000000000..fbb7345dc
--- /dev/null
+++ b/internal/host/relocationfinish_unix_test.go
@@ -0,0 +1,280 @@
+//go:build unix
+
+package host
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/hostbox"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/hostguard"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// failingClaimPut is a control bucket whose next fail writes of one claim
+// key fail: a lost or failed claim CAS.
+type failingClaimPut struct {
+ release.Store
+ key string
+ fail int
+}
+
+func (s *failingClaimPut) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) {
+ if key == s.key && s.fail > 0 {
+ s.fail--
+ return "", errors.New("the claim CAS failed")
+ }
+ return s.Store.Put(ctx, key, body, ifMatch)
+}
+
+// finishWorld is a host that serves on dst after relocation rel-1
+// completed there, undischarged: its row held for rel-1, the probe stop's
+// owned fence on the activator, and the pass of dst with the production
+// Finish (relocationReleaseHold).
+type finishWorld struct {
+ t *testing.T
+ store *failingClaimPut
+ reg *registry.Registry
+ act *stopProbeActivator
+ order []string
+ discharged int
+ discharge error
+ runtime *RelocationRuntime
+ dataDir string
+ assembly *ClaimAssembly
+}
+
+const finishHost = "shop.example"
+
+func newFinishWorld(t *testing.T) *finishWorld {
+ t.Helper()
+ ctx := t.Context()
+ w := &finishWorld{t: t, dataDir: t.TempDir()}
+ created := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC)
+ inc, err := hostclaim.Incarnation("acct", "app", finishHost, created)
+ if err != nil {
+ t.Fatal(err)
+ }
+ done, _ := json.Marshal(hostclaim.CompletedRelocation{Op: "rel-1", Stop: hostclaim.ProbeStop{Box: "dst", Session: 12, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)}, Stamp: 9, Phase: hostclaim.PhaseRelocatingProbing, Generation: "P1"})
+ c := &hostclaim.Claim{V: 2, Gen: 31, Host: finishHost,
+ Holder: &hostclaim.Holder{Family: "instance", Key: "apps/acct/app/instances/" + finishHost + ".json", Account: "acct", App: "app", Incarnation: inc, CreatedAt: created,
+ Location: "instances/" + finishHost, Stamp: 9, Session: 12, Generation: "P1", WriterBox: "dst", Admissions: []hostclaim.AdmissionRecord{{Session: 12, Box: "dst", Generation: "P1", Mode: "cold", Region: "ie"}}},
+ Op: hostclaim.Op{Kind: hostclaim.KindServing, Exec: "dst", Participants: []string{"dst"}, Journal: map[string]string{hostclaim.JournalCompleted: string(done)}}}
+ key, _ := hostclaim.Key(finishHost)
+ w.store = &failingClaimPut{Store: release.NewDirStore(t.TempDir()), key: key}
+ raw, _ := json.Marshal(c)
+ if _, err := w.store.Put(ctx, key, raw, ""); err != nil {
+ t.Fatal(err)
+ }
+ if w.reg, err = registry.Open(filepath.Join(t.TempDir(), "registry.db")); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { w.reg.Close() })
+ if err := w.reg.Upsert(registry.Route{Host: finishHost, Status: "relocating", HoldOp: "rel-1", Socket: "/run/shop.sock"}); err != nil {
+ t.Fatal(err)
+ }
+ marker, err := relocationProbeFencePath(w.dataDir, "rel-1")
+ if err != nil {
+ t.Fatal(err)
+ }
+ owner, err := relocationProbeFenceOwner(marker, finishHost, 12)
+ if err != nil {
+ t.Fatal(err)
+ }
+ w.act = newStopProbeActivator(&w.order)
+ fctx, cancel := context.WithTimeout(ctx, time.Second)
+ defer cancel()
+ if _, err := w.act.svc.FenceAndStopAs(fctx, finishHost, 12, owner); err != nil && !errors.Is(err, activator.ErrStopUnproven) {
+ t.Fatal(err)
+ }
+ a := ClaimAssembly{Claims: w.store, Guard: hostguard.Guard{DataDir: w.dataDir}, stopLocal: func(context.Context) (claimStopLocal, error) {
+ return claimStopLocal{Actor: hostbox.Actor{Box: "dst"}}, nil
+ }}
+ w.assembly = &a
+ cfg := RelocationAssemblyConfig{Registry: w.reg, Activator: w.act, DataDir: w.dataDir, Discharge: func(context.Context, hostclaim.Observed) error {
+ w.discharged++
+ return w.discharge
+ }}
+ var steps []string
+ w.runtime = &RelocationRuntime{Worker: stepRecorder{name: "worker", order: &steps}, Participant: stepRecorder{name: "participant", order: &steps}, Box: "dst",
+ Finish: a.relocationReleaseHold(cfg), DataDir: w.dataDir}
+ return w
+}
+
+func (w *finishWorld) pass() { w.runtime.once(w.t.Context(), w.store) }
+
+func (w *finishWorld) held() bool {
+ row, err := w.reg.Get(finishHost)
+ return err == nil && row != nil && row.Status == "relocating" && row.HoldOp == "rel-1"
+}
+
+// marked reads the record, and whether a successor (a move from dst) may
+// start: the completion barrier.
+func (w *finishWorld) marked() (bool, error) {
+ obs, err := hostclaim.Read(w.t.Context(), w.store, finishHost)
+ if err != nil {
+ w.t.Fatal(err)
+ }
+ done, ok, err := hostclaim.ReadCompletedRelocation(obs.Claim)
+ if err != nil {
+ w.t.Fatal(err)
+ }
+ return ok && done.Discharged, hostclaim.StartGuard(obs.Claim, []string{"uk"}, []string{"spa"})
+}
+
+// done asserts the finished state: discharged as many times in all as
+// the passes asked (Task 9 review M2: once the hold is released, a replay
+// discharges again rather than inferring the discharge ran), the hold
+// released, no fence, the completion marked so a successor may start, and
+// the op's journals swept by the same pass.
+func (w *finishWorld) done(discharges int) {
+ w.t.Helper()
+ ok, barrier := w.marked()
+ if w.discharged != discharges || w.held() || w.act.svc.Fenced(finishHost) || !ok || barrier != nil {
+ w.t.Fatalf("discharged %d (want %d) held=%v fenced=%v marked=%v barrier=%v", w.discharged, discharges, w.held(), w.act.svc.Fenced(finishHost), ok, barrier)
+ }
+ if _, err := os.Stat(filepath.Join(w.dataDir, "relocate", "ops", "rel-1")); !errors.Is(err, os.ErrNotExist) {
+ w.t.Fatalf("the finished op's journals survived: %v", err)
+ }
+}
+
+// Astra round 7, finding 1 (ruling 80): a completion is finished from the
+// claim's record on every pass, whatever the row says, and every step
+// resumes after a failure or a crash at any point, through the pass.
+func TestRelocatePassFinishesACompletionFromItsRecord(t *testing.T) {
+ t.Run("in one pass", func(t *testing.T) {
+ w := newFinishWorld(t)
+ w.pass()
+ w.done(1)
+ w.pass()
+ w.done(1)
+ })
+ t.Run("a failed discharge keeps everything, the next pass finishes", func(t *testing.T) {
+ w := newFinishWorld(t)
+ w.discharge = errors.New("the participation slot is still open")
+ w.pass()
+ if ok, barrier := w.marked(); w.discharged != 1 || !w.held() || w.act.svc.FenceGeneration(finishHost) != 12 || ok || !errors.Is(barrier, hostclaim.ErrPendingChore) {
+ t.Fatalf("discharged %d held=%v fence=%d marked=%v barrier=%v", w.discharged, w.held(), w.act.svc.FenceGeneration(finishHost), ok, barrier)
+ }
+ w.discharge = nil
+ w.pass()
+ w.done(2)
+ })
+ t.Run("the mark's CAS fails, the next pass marks", func(t *testing.T) {
+ w := newFinishWorld(t)
+ w.store.fail = 1
+ w.pass()
+ if ok, barrier := w.marked(); w.discharged != 1 || w.held() || w.act.svc.Fenced(finishHost) || ok || !errors.Is(barrier, hostclaim.ErrPendingChore) {
+ t.Fatalf("discharged %d held=%v fenced=%v marked=%v barrier=%v", w.discharged, w.held(), w.act.svc.Fenced(finishHost), ok, barrier)
+ }
+ // The row is no longer held: the record alone brings it back, and
+ // the released-hold branch discharges again (Task 9 review M2).
+ w.pass()
+ w.done(2)
+ })
+ t.Run("the lift never ran after the hold went, the next pass lifts", func(t *testing.T) {
+ w := newFinishWorld(t)
+ // An earlier call discharged and released the hold, then failed.
+ w.discharged = 1
+ if _, err := w.reg.ReleaseHold(finishHost, "rel-1"); err != nil {
+ t.Fatal(err)
+ }
+ w.pass()
+ w.done(2)
+ })
+ t.Run("a restart after the hold release", func(t *testing.T) {
+ w := newFinishWorld(t)
+ w.discharged = 1
+ if _, err := w.reg.ReleaseHold(finishHost, "rel-1"); err != nil {
+ t.Fatal(err)
+ }
+ w.act.svc = &activator.Service{} // the restart: the fence went with the process
+ w.pass()
+ w.done(2)
+ })
+}
+
+// productionLocal is main's participation owner after the restart, bound
+// to the world's assembly: DischargeCompleted is the production discharge
+// (carry 6), its identity a verifying one for dst.
+func (w *finishWorld) productionLocal() *productionClaimLocal {
+ identity := handoverRecoveryIdentity(w.t, hostbox.Actor{Box: "dst", Enrollment: "enr", Region: "ie", Revision: 1})
+ return &productionClaimLocal{Resources: ClaimRuntimeResources{Mode: "agent", DataDir: w.dataDir}, Identity: identity, Store: w.store,
+ slots: map[string]*claimParticipation{}, assembly: w.assembly, lifetime: w.t.Context()}
+}
+
+// Astra round 8 (ruling 82): A to B completes, the mark's CAS fails, and B
+// restarts with CARLOS_REGION_HA off. Claim mode alone builds the
+// completion-only runtime, which steps nothing, and its pass still lifts
+// and marks, so the barrier opens. The restarted runtime's discharge is
+// main's own (productionClaimLocal.DischargeCompleted, carry 6), which
+// finds no slot and no .hostexec record.
+func TestCompletionPassFinishesWithRegionHAOff(t *testing.T) {
+ w := newFinishWorld(t)
+ w.store.fail = 1
+ w.pass()
+ if ok, barrier := w.marked(); ok || w.held() || !errors.Is(barrier, hostclaim.ErrPendingChore) {
+ t.Fatalf("the first pass marked or kept the hold: marked=%v held=%v barrier=%v", ok, w.held(), barrier)
+ }
+ w.act.svc = &activator.Service{} // the restart
+ claimOnly := func(k string) string {
+ if k == "CARLOS_HOST_CLAIM" {
+ return "1"
+ }
+ return ""
+ }
+ r, err := NewRelocationRuntime(t.Context(), claimOnly, w.store, w.productionLocal(), ClaimProviders{}, w.reg, w.act, relocationRuntimeInputs{})
+ if err != nil || r == nil || r.Worker != nil || r.Participant != nil || r.Convert != nil || r.Finish == nil {
+ t.Fatalf("not a completion-only runtime: %+v %v", r, err)
+ }
+ w.runtime = r
+ w.pass()
+ w.done(1)
+}
+
+// Carry 6: a probe-fence owner the hold release cannot find is ErrRefused,
+// which the pass alerts and retries; it never skips the completion and
+// never marks it. Nothing is released while the owner is missing; once it
+// is back, the next pass finishes.
+func TestRelocatePassRetriesAMissingProbeFenceOwner(t *testing.T) {
+ w := newFinishWorld(t)
+ marker, err := relocationProbeFencePath(w.dataDir, "rel-1")
+ if err != nil {
+ t.Fatal(err)
+ }
+ raw, err := os.ReadFile(marker)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Remove(marker); err != nil {
+ t.Fatal(err)
+ }
+ buf := captureRacyLog(t)
+ w.pass()
+ w.pass()
+ if ok, barrier := w.marked(); ok || !w.held() || w.discharged != 0 || w.act.svc.FenceGeneration(finishHost) != 12 || !errors.Is(barrier, hostclaim.ErrPendingChore) {
+ t.Fatalf("over a missing owner: marked=%v held=%v discharged=%d fence=%d barrier=%v", ok, w.held(), w.discharged, w.act.svc.FenceGeneration(finishHost), barrier)
+ }
+ if n := strings.Count(buf.String(), "has no probe-fence owner"); n != 1 {
+ t.Fatalf("alerted %d times, want once:\n%s", n, buf.String())
+ }
+ // An operator puts the owner back. (The sweep has removed the op's
+ // directory meanwhile: with the marker gone it held no journal.)
+ if err := os.MkdirAll(filepath.Dir(marker), 0o700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(marker, raw, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ w.pass()
+ w.done(1)
+}
diff --git a/internal/host/relocationfirstwake_unix_test.go b/internal/host/relocationfirstwake_unix_test.go
new file mode 100644
index 000000000..f81f377d7
--- /dev/null
+++ b/internal/host/relocationfirstwake_unix_test.go
@@ -0,0 +1,159 @@
+//go:build unix
+
+package host
+
+import (
+ "context"
+ "encoding/json"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/hostbox"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/hostguard"
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+// Carry 9 (Task 9 review, ruling 82): a relocation completes on box-1 with
+// region HA on; box-1 restarts with CARLOS_REGION_HA off. One scenario,
+// with no agent restart between its halves, over one control bucket, one
+// guard journal, one registry and one activator:
+//
+// 1. the probe stop: the production probe-stop provider fences the probe
+// session under its persisted owner and root's writer-session stop
+// lands in the guard journal; the serving CAS records the completion;
+// 2. the completion pass: NewRelocationRuntime with claim mode alone
+// builds the completion-only runtime, whose pass discharges through
+// main's own DischargeCompleted, releases the hold, lifts the owned
+// fence and marks the record, so the barrier opens;
+// 3. the first serving wake: the wake runtime (ClaimWakeAssembly.Build)
+// wired as OpenProductionClaimAssembly wires it with moves C1 off
+// (wireProductionAdmission, N6 in front) takes the predecessor stop
+// Prerequisites takes, which re-stops nothing, then admits through
+// the runtime's own Admit, which hands root the probe's receipt as the
+// prior.
+//
+// The production constructor itself cannot run here in agent mode: it
+// dials root at the fixed controlsock.Path, chowns its state directory to
+// the carlos uid and needs a cgroup manager. Checklist item 12 is that run.
+func TestCompletionPassThenFirstServingWakeWithRegionHAOff(t *testing.T) {
+ ctx := t.Context()
+ f := newRecoveryFixture(t)
+ const box, op = "box-1", "rel-1"
+ local := claimStopLocal{Actor: hostbox.Actor{Box: box, Enrollment: "enrollment-1", Revision: 7, Region: "ie"}, BootID: "boot-0", ControllerID: "controller-0", PID: 4000, ProcessStart: "start-0"}
+
+ // The relocating claim at probing on box-1, its probe passed with
+ // session 12 at install stamp 9.
+ // Written whole, as the stop-probe fixture's claim is: the transitions
+ // that lead here are hostrelocate's and are tested there.
+ probe, _ := json.Marshal(hostclaim.Probe{Box: box, Session: 12, At: f.now})
+ created := f.read()
+ c := *created.Claim
+ holder := *c.Holder
+ holder.Stamp, holder.Session, holder.Generation, holder.WriterBox = 9, 12, "P1", box
+ holder.Admissions = []hostclaim.AdmissionRecord{{Session: 12, Box: box, Generation: "P1", Mode: "cold", Region: "ie"}}
+ c.Holder = &holder
+ c.Gen = 30 // the history behind stamp 9 and session 12
+ c.Op = hostclaim.Op{Kind: hostclaim.KindRelocating, ID: op, Phase: hostclaim.PhaseRelocatingProbing, Exec: box, Participants: []string{"src", box},
+ Intent: map[string]string{"region": "ie", "source_box": "src", "target_box": box}, Journal: map[string]string{"probe": string(probe)}}
+ key, _ := hostclaim.Key(recoveryHost)
+ if _, err := f.claims.Put(ctx, key, mustJSON(t, &c), created.ETag); err != nil {
+ t.Fatal(err)
+ }
+ o := f.read()
+ if err := f.guard.Activate(ctx, recoveryHost, box, nil); err != nil {
+ t.Fatal(err)
+ }
+
+ // 1. The probe stop, root's writer-session stop standing as Wake.Stop.
+ h := o.Claim.Holder
+ target := hostguard.FenceTarget{FenceID: "fence-1", Region: "ie", Host: recoveryHost, Incarnation: h.Incarnation, Stamp: 9, Session: 12, Op: op, Phase: hostclaim.PhaseRelocatingProbing,
+ ObligationID: hostguard.WriterStopObligation(12), Box: box, Enrollment: local.Actor.Enrollment, BootID: local.BootID, ControllerID: local.ControllerID, ClaimETag: o.ETag, DirectoryRevision: "7"}
+ at := f.now.Add(time.Minute)
+ receipt := mustMarshalStopReceipt(StopReceipt{V: 1, ID: "r1", Kind: StopReceiptStopped, Region: "ie", ClaimETag: o.ETag, ObligationID: target.ObligationID, FenceID: "fence-1", Host: recoveryHost, Incarnation: h.Incarnation,
+ InstallationStamp: 9, WriterSession: 12, OperationID: op, Phase: hostclaim.PhaseRelocatingProbing, Box: box, Enrollment: local.Actor.Enrollment, BootID: local.BootID, ControllerID: local.ControllerID, DirectoryRevision: "7", EvidenceAt: at})
+ var order []string
+ act := newStopProbeActivator(&order)
+ probeAssembly := ClaimAssembly{Claims: f.claims, Guard: f.guard, stopLocal: func(context.Context) (claimStopLocal, error) { return local, nil },
+ Wake: &ClaimWakeAssembly{Stop: func(ctx context.Context, _ hostclaim.Observed) error {
+ if _, err := f.guard.BeginFence(ctx, target); err != nil {
+ return err
+ }
+ return f.guard.CompleteFence(ctx, target, receipt, at, func(context.Context, hostguard.FenceTarget, []byte, time.Time) error { return nil })
+ }}}
+ stop, err := probeAssembly.relocationStopProbe(RelocationAssemblyConfig{Activator: act, DataDir: f.guard.DataDir})(ctx, o)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !act.svc.Fenced(recoveryHost) {
+ t.Fatal("the probe stop installed no fence")
+ }
+ // The serving CAS, as CompleteRelocation leaves it.
+ done, _ := json.Marshal(hostclaim.CompletedRelocation{Op: op, Stop: stop, Stamp: 9, Phase: hostclaim.PhaseRelocatingProbing, Generation: "P1"})
+ c.Gen = 31
+ c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: box, Participants: []string{box}, Journal: map[string]string{hostclaim.JournalCompleted: string(done)}}
+ if _, err := f.claims.Put(ctx, key, mustJSON(t, &c), o.ETag); err != nil {
+ t.Fatal(err)
+ }
+ reg, err := registry.Open(t.TempDir() + "/registry.db")
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { reg.Close() })
+ if err := reg.Upsert(registry.Route{Host: recoveryHost, Status: "relocating", HoldOp: op, Socket: "/run/shop.sock"}); err != nil {
+ t.Fatal(err)
+ }
+
+ // 2. The completion pass after the restart with region HA off: claim
+ // mode alone, main's own participation owner and discharge.
+ identity := handoverRecoveryIdentity(t, local.Actor)
+ bound := &ClaimAssembly{Claims: f.claims, Guard: f.guard, stopLocal: func(context.Context) (claimStopLocal, error) { return local, nil }}
+ owner := &productionClaimLocal{Resources: ClaimRuntimeResources{Mode: "agent", DataDir: f.guard.DataDir}, Identity: identity, Store: f.claims,
+ slots: map[string]*claimParticipation{}, assembly: bound, lifetime: ctx}
+ claimOnly := func(k string) string {
+ if k == "CARLOS_HOST_CLAIM" {
+ return "1"
+ }
+ return ""
+ }
+ r, err := NewRelocationRuntime(ctx, claimOnly, f.claims, owner, ClaimProviders{}, reg, act, relocationRuntimeInputs{})
+ if err != nil || r == nil || r.Worker != nil || r.Participant != nil || r.Convert != nil {
+ t.Fatalf("not the completion-only runtime: %+v %v", r, err)
+ }
+ r.once(ctx, f.claims)
+ prior := f.read()
+ marked, ok, err := hostclaim.ReadCompletedRelocation(prior.Claim)
+ if err != nil || !ok || !marked.Discharged {
+ t.Fatalf("the completion was not marked: %+v %v %v", marked, ok, err)
+ }
+ if row, err := reg.Get(recoveryHost); err != nil || row == nil || row.HoldOp != "" {
+ t.Fatalf("the hold was not released: %+v %v", row, err)
+ }
+ if act.svc.Fenced(recoveryHost) {
+ t.Fatal("the probe's owned fence was not lifted")
+ }
+ if err := hostclaim.StartGuard(prior.Claim, []string{"uk"}, []string{"spa"}); err != nil {
+ t.Fatalf("the barrier did not open: %v", err)
+ }
+
+ // 3. The first serving wake, N6 in front as the production
+ // constructor wires it with moves C1 off.
+ w := f.controller("boot-1", "controller-1", 4101, "start-1")
+ socket, posted := handoverRoot(t)
+ locals := 0
+ w.wake.AfterAdmission = func(context.Context, hostclaim.Observed, hostclaim.Observed) error { locals++; return nil }
+ wireProductionAdmission(w.wake, box, false, claimCrossBoxHandover{Box: box}, nil,
+ &relocationCompletedHandover{Guard: f.guard, Identity: identity, Socket: socket, Box: box})
+ // Prerequisites' predecessor stop: the probe's receipt is found and
+ // verified; root is asked to stop nothing.
+ if err := w.wake.stopPredecessor(ctx, prior); err != nil || len(f.requests) != 0 {
+ t.Fatalf("the predecessor stop: %v (root asked %d stops)", err, len(f.requests))
+ }
+ admitted := w.admit(t, f)
+ if admitted.Claim.Holder.Session <= 12 || admitted.Claim.Holder.WriterBox != box {
+ t.Fatalf("not a new writer session on %s: %+v", box, admitted.Claim.Holder)
+ }
+ if len(*posted) != 1 || (*posted)[0].Prior != target || (*posted)[0].Next.Session != admitted.Claim.Holder.Session || locals != 0 || len(f.requests) != 0 {
+ t.Fatalf("root was handed %+v (main's local handover %d, stops %d)", *posted, locals, len(f.requests))
+ }
+}
diff --git a/internal/host/relocationflush.go b/internal/host/relocationflush.go
index 5a73cf959..e97e75740 100644
--- a/internal/host/relocationflush.go
+++ b/internal/host/relocationflush.go
@@ -157,16 +157,14 @@ func (f ReleasePositionsFile) Record(_ context.Context, p ReleasePosition) error
if p.PID <= 0 || p.ProcessStart == "" {
return ErrClaimStopUnavailable // a record that cannot name its process could not be told from a restart's
}
- if err := os.MkdirAll(f.Dir, 0o700); err != nil {
- return err
- }
raw, err := json.Marshal(releasePosition{V: 1, ReleasePosition: p})
if err != nil {
return err
}
// Moves' atomic write (moveslandinstall.go): a synced temporary in
- // the same directory, renamed into place, the directory synced.
- return writeFileAtomic(path, raw, 0o600)
+ // the same directory, renamed into place, the directory synced, never
+ // beside the journal sweep (writeRelocationJournal).
+ return writeRelocationJournal(path, raw)
}
// sameStop is the installation and session, not the controller.
diff --git a/internal/host/relocationinstall.go b/internal/host/relocationinstall.go
index 65a84fe29..dfbca99ac 100644
--- a/internal/host/relocationinstall.go
+++ b/internal/host/relocationinstall.go
@@ -595,10 +595,7 @@ func (a ClaimAssembly) relocationInstall(cfg RelocationAssemblyConfig) func(cont
return hostclaim.LocalInstall{}, err
}
if err := work.With(ctx, hostwrite.Identity{}, func() error {
- if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
- return err
- }
- return writeFileAtomic(path, raw, 0o600)
+ return writeRelocationJournal(path, raw)
}); err != nil {
return hostclaim.LocalInstall{}, err
}
diff --git a/internal/host/relocationlanding.go b/internal/host/relocationlanding.go
new file mode 100644
index 000000000..29586032f
--- /dev/null
+++ b/internal/host/relocationlanding.go
@@ -0,0 +1,40 @@
+package host
+
+import (
+ "context"
+
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/moves"
+ "github.com/carlosframework/platform/internal/provision"
+)
+
+// CommittedLandingRecords is hostclaim.CommittedLandingGuard over the
+// authoritative instance record: the move named by the claim's op has
+// committed for this incarnation (provision.MoveCommitted, no rollback),
+// and the regions and park scopes are the commit's (moves.CommitScopes).
+type CommittedLandingRecords struct{ Records MovesRecordReader }
+
+var _ hostclaim.CommittedLandingGuard = CommittedLandingRecords{}
+
+func (g CommittedLandingRecords) CheckLandingCommitted(ctx context.Context, o hostclaim.Observed, opID string) (hostclaim.CommittedLanding, error) {
+ c := o.Claim
+ if g.Records == nil || c == nil || c.Holder == nil || c.Op.ID != opID {
+ return hostclaim.CommittedLanding{}, hostclaim.ErrWrongState
+ }
+ rec, err := g.Records(ctx, c.Holder.Account, c.Holder.App, o.Host)
+ if err != nil {
+ return hostclaim.CommittedLanding{}, err
+ }
+ if rec == nil || rec.Move == nil {
+ return hostclaim.CommittedLanding{}, hostclaim.ErrWrongState
+ }
+ m := rec.Move
+ if m.ID != opID || m.Incarnation != c.Holder.Incarnation || m.State != provision.MoveCommitted || m.Rollback != nil || m.CommittedAt == nil {
+ return hostclaim.CommittedLanding{}, hostclaim.ErrWrongState
+ }
+ source, staging, err := moves.CommitScopes(c, m.FlippedAt)
+ if err != nil {
+ return hostclaim.CommittedLanding{}, err
+ }
+ return hostclaim.CommittedLanding{FromRegion: m.From, ToRegion: m.To, SourceScope: source, StagingScope: staging}, nil
+}
diff --git a/internal/host/relocationlanding_test.go b/internal/host/relocationlanding_test.go
new file mode 100644
index 000000000..291c8c2bf
--- /dev/null
+++ b/internal/host/relocationlanding_test.go
@@ -0,0 +1,240 @@
+package host
+
+import (
+ "context"
+ "crypto/ed25519"
+ "crypto/rand"
+ "encoding/json"
+ "errors"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/hostactors"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/hostfence"
+ "github.com/carlosframework/platform/internal/hostrelocate"
+ "github.com/carlosframework/platform/internal/moves"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/termfence"
+ "github.com/superfly/ltx"
+)
+
+// landingWorld is a landing claim at probed whose target dst is
+// terminated (its fence in the roster's bucket), the authoritative record
+// of its move (committed or still landing) and a signed roster in which
+// wrk, the converting worker, is as change leaves it.
+type landingWorld struct {
+ store release.Store
+ directory hostactors.Directory
+ obs hostclaim.Observed
+ now time.Time
+ enrollment map[string]string
+}
+
+func newLandingWorld(t *testing.T, committed bool, change func(*hostactors.Roster)) landingWorld {
+ t.Helper()
+ ctx := t.Context()
+ const host, id = "shop.example", "mv_abcdefghijklmnopqrstuvwxyz"
+ w := landingWorld{store: release.NewDirStore(t.TempDir()), now: time.Date(2026, 10, 7, 9, 0, 0, 0, time.UTC),
+ enrollment: map[string]string{"src": strings.Repeat("b", 32), "dst": strings.Repeat("c", 32), "spa": strings.Repeat("d", 32), "wrk": strings.Repeat("e", 32)}}
+ pub, key, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ roster := hostactors.Roster{V: 1, DeploymentBucket: "bucket", Revision: 1,
+ Regions: map[string]hostactors.Region{"ie": {Account: "123", AWSRegion: "eu-west-1"}, "uk": {Account: "123", AWSRegion: "eu-west-2"}},
+ Boxes: map[string]hostactors.Box{}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}}
+ for box, enr := range w.enrollment {
+ roster.Boxes[box] = hostactors.Box{Enrollment: enr, Namespace: "platform", BoxID: box, Label: box, Region: "ie", Pool: "default",
+ Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-" + box}, EvidenceSHA256: strings.Repeat("c", 64), State: "active"}
+ }
+ if change != nil {
+ change(&roster)
+ }
+ raw, err := release.SignRootEnvelope(release.RootSigner(key), roster)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := w.store.Put(ctx, hostactors.Key, raw, ""); err != nil {
+ t.Fatal(err)
+ }
+ w.directory = hostactors.Directory{Store: w.store, PublicKey: pub, Bucket: "bucket"}
+ w.obs = landingAtProbed(t, w.store, host, id, w.now, w.enrollment)
+ dst := roster.Boxes["dst"]
+ fence, _ := json.Marshal(termfence.Record{V: 1, Box: "dst", Enrollment: dst.Enrollment, Cloud: dst.Cloud, State: "terminated", ObservedAt: w.now.Add(-time.Minute).Format(time.RFC3339Nano),
+ Via: termfence.ViaDescribe, RequestID: "req", WrittenBy: termfence.WrittenBy{Box: "spa", Enrollment: w.enrollment["spa"]}})
+ if _, err := w.store.Put(ctx, hostfence.FenceKey(dst.Enrollment), fence, ""); err != nil {
+ t.Fatal(err)
+ }
+ created := time.Date(2026, 10, 4, 1, 0, 0, 0, time.UTC)
+ state, committedAt, flipped := provision.MoveLanding, (*time.Time)(nil), w.now.Add(-90*time.Minute)
+ if committed {
+ at := w.now.Add(-30 * time.Minute)
+ state, committedAt = provision.MoveCommitted, &at
+ }
+ record := provision.Record{Schema: 1, Host: host, Slug: "shop", App: "jam", Account: "bab", Region: "ie", Channel: "stable", CreatedAt: created,
+ Move: &provision.MoveState{ID: id, Incarnation: w.obs.Claim.Holder.Incarnation, From: "uk", To: "ie", State: state, CommittedAt: committedAt, FlippedAt: &flipped}}
+ if err := provision.CreateRecord(ctx, w.store, record); err != nil {
+ t.Fatal(err)
+ }
+ return w
+}
+
+// worker is the converting worker: wrk, running as enrollment, in region.
+func (w landingWorld) worker(enrollment, region string) hostrelocate.Worker {
+ now := func() time.Time { return w.now }
+ return hostrelocate.Worker{Claims: w.store, Directory: w.directory, Fences: hostfence.ClaimFenceAuthority{Directory: w.directory, Now: now},
+ Discharge: func(context.Context, hostactors.Box, termfence.Record) *hostclaim.LocalDischarge { return nil },
+ Self: termfence.WrittenBy{Box: "wrk", Enrollment: enrollment}, Region: region, NewID: newClaimID, Now: now}
+}
+
+// J3: the pass converts a landing whose target box is terminated into a
+// failover relocation when the move committed, through
+// CommittedLandingRecords over the authoritative record; a move that has
+// not committed is moves' case, and nothing is written.
+func TestRelocatePassConvertsAFencedCommittedLanding(t *testing.T) {
+ for _, committed := range []bool{true, false} {
+ name := "uncommitted"
+ if committed {
+ name = "committed"
+ }
+ t.Run(name, func(t *testing.T) {
+ ctx := t.Context()
+ w := newLandingWorld(t, committed, nil)
+ if err := convertFencedLanding(w.worker(w.enrollment["wrk"], "ie"), w.store)(ctx, w.store, w.obs); err != nil {
+ t.Fatalf("convert: %v", err)
+ }
+ after, err := hostclaim.Read(ctx, w.store, w.obs.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !committed {
+ if after.ETag != w.obs.ETag || after.Claim.Op.Kind != hostclaim.KindLanding {
+ t.Fatalf("an uncommitted landing was written: %+v", after.Claim.Op)
+ }
+ return
+ }
+ c := after.Claim
+ if c.Op.Kind != hostclaim.KindRelocating || c.Op.Phase != hostclaim.PhaseStarting || c.Op.Intent["source_box"] != "dst" || c.Op.Journal["conversion"] != "committed-landing" {
+ t.Fatalf("not converted: %+v", c.Op)
+ }
+ })
+ }
+}
+
+// Astra round 6, finding 4: the conversion is the worker's, under Step's
+// own checks of the caller. A worker whose roster entry is another
+// enrollment (revoked and re-enrolled), one that is not active, one that
+// is fenced, or one enrolled outside the region it claims, is refused
+// with ErrNotWorker; a worker of another region leaves the landing alone.
+// None of them writes the claim, though the move committed and its target
+// is fenced.
+func TestRelocatePassConversionRefusesAnUnauthorizedWorker(t *testing.T) {
+ type tc struct {
+ name string
+ change func(*hostactors.Roster)
+ enrollment, region string
+ want error
+ }
+ moved := func(state, region string) func(*hostactors.Roster) {
+ return func(r *hostactors.Roster) {
+ b := r.Boxes["wrk"]
+ if state != "" {
+ b.State = state
+ }
+ if region != "" {
+ b.Region = region
+ }
+ r.Boxes["wrk"] = b
+ }
+ }
+ for _, c := range []tc{
+ {name: "a revoked enrollment", enrollment: strings.Repeat("f", 32), region: "ie", want: hostrelocate.ErrNotWorker},
+ {name: "a retired worker", change: moved("retired", ""), region: "ie", want: hostrelocate.ErrNotWorker},
+ {name: "enrolled in another region", change: moved("", "uk"), region: "ie", want: hostrelocate.ErrNotWorker},
+ {name: "a worker of another region", change: moved("", "uk"), region: "uk"},
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ ctx := t.Context()
+ w := newLandingWorld(t, true, c.change)
+ if c.enrollment == "" {
+ c.enrollment = w.enrollment["wrk"]
+ }
+ err := convertFencedLanding(w.worker(c.enrollment, c.region), w.store)(ctx, w.store, w.obs)
+ if c.want == nil && err != nil || c.want != nil && !errors.Is(err, c.want) {
+ t.Fatalf("got %v, want %v", err, c.want)
+ }
+ after, err := hostclaim.Read(ctx, w.store, w.obs.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if after.ETag != w.obs.ETag || after.Claim.Op.Kind != hostclaim.KindLanding {
+ t.Fatalf("an unauthorized worker wrote the claim: %+v", after.Claim.Op)
+ }
+ })
+ }
+ t.Run("a fenced worker", func(t *testing.T) {
+ ctx := t.Context()
+ w := newLandingWorld(t, true, nil)
+ o, err := w.directory.Read(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ wrk := o.Roster.Boxes["wrk"]
+ fence, _ := json.Marshal(termfence.Record{V: 1, Box: "wrk", Enrollment: wrk.Enrollment, Cloud: wrk.Cloud, State: "terminated", ObservedAt: w.now.Add(-time.Minute).Format(time.RFC3339Nano),
+ Via: termfence.ViaDescribe, RequestID: "req", WrittenBy: termfence.WrittenBy{Box: "spa", Enrollment: w.enrollment["spa"]}})
+ if _, err := w.store.Put(ctx, hostfence.FenceKey(wrk.Enrollment), fence, ""); err != nil {
+ t.Fatal(err)
+ }
+ if err := convertFencedLanding(w.worker(w.enrollment["wrk"], "ie"), w.store)(ctx, w.store, w.obs); !errors.Is(err, hostrelocate.ErrNotWorker) {
+ t.Fatalf("got %v", err)
+ }
+ if after, err := hostclaim.Read(ctx, w.store, w.obs.Host); err != nil || after.ETag != w.obs.ETag {
+ t.Fatalf("a fenced worker wrote the claim: %v", err)
+ }
+ })
+}
+
+// landingAtProbed is a landing claim as moves leaves it at probed: the
+// move from src to dst flipped, the target's probe admitted. Its seal is a
+// real one, so moves.CommitScopes reads it.
+func landingAtProbed(t *testing.T, store release.Store, host, id string, now time.Time, enrollment map[string]string) hostclaim.Observed {
+ t.Helper()
+ ctx := t.Context()
+ created := time.Date(2026, 10, 4, 1, 0, 0, 0, time.UTC)
+ nh := hostclaim.NewHolder{Family: "instance", Key: provision.RecordKey("bab", "jam", host), Account: "bab", App: "jam", Host: host, Region: "ie", RecordRegion: "ie", CreatedAt: created}
+ body := []byte(`{"host":"` + host + `","account":"bab","app":"jam","region":"ie","created_at":"2026-10-04T01:00:00Z"}`)
+ o, err := hostclaim.Read(ctx, store, host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ step := func(tr func(*hostclaim.Claim) error) {
+ t.Helper()
+ if o, err = hostclaim.Update(ctx, store, o, tr); err != nil {
+ t.Fatal(err)
+ }
+ }
+ seal := moves.Seal{V: moves.SealVersion, N: 7, FromDB: true, SrcCovered: true, SealedAt: now.Add(-2 * time.Hour), LeaseUntil: now.Add(-time.Hour),
+ Replica: []moves.SealedObject{{Key: "moves/" + host + "/" + id + "/replica/0009/" + ltx.FormatFilename(1, 7), VersionID: "v1", Size: 100}}}
+ step(hostclaim.BeginCreateOn(nh, body, "c1", "console", created, hostclaim.ExecutorSelection{Box: "src", Enrollment: enrollment["src"]}))
+ step(hostclaim.CompleteCreate("c1", "src"))
+ step(hostclaim.AdmitWriter(hostclaim.Admission{Box: "src", Mode: "cold", NewGeneration: "g1", Region: "ie", KeepStamp: true}))
+ step(hostclaim.BeginAdmittedMove(id, "ops", "src", "dst", "ie", now.Add(-3*time.Hour), hostclaim.MoveAdmission{AppID: "app-canonical", AdmissionID: id, SourceEnrollment: enrollment["src"], TargetEnrollment: enrollment["dst"]}))
+ step(hostclaim.ExecToNone(id, "src", "moves/"+host+"/"+id+"/stopped-source.json", hostclaim.JournalEvidenceSource))
+ step(hostclaim.SetJournal(id, map[string]string{hostclaim.JournalSeal: seal.Encode(), hostclaim.JournalCutSession: "1"}))
+ step(hostclaim.SetMovePhase(id, hostclaim.KindMoving, hostclaim.PhaseMoving, hostclaim.PhaseSealed))
+ step(hostclaim.Flip(id, "dst", o.Claim.Holder.Stamp, o.Claim.Holder.Generation))
+ for _, p := range [][2]string{{"checking", "parking"}, {"parking", "building"}, {"building", "published"}, {"published", "local"}, {"local", "configured"}, {"configured", "restored"}, {"restored", "probing"}, {"probing", "probed"}} {
+ step(hostclaim.SetMovePhase(id, hostclaim.KindLanding, p[0], p[1]))
+ }
+ step(func(c *hostclaim.Claim) error {
+ h := c.Holder
+ h.Session++
+ h.Admissions = append(h.Admissions, hostclaim.AdmissionRecord{Region: "ie", PredecessorRegion: "ie", Session: h.Session, Box: "dst", Generation: "gen-landed", Predecessor: h.Generation, Mode: "cold", Coverage: "sealed"})
+ h.Generation, h.WriterBox = "gen-landed", "dst"
+ return nil
+ })
+ return o
+}
diff --git a/internal/host/relocationowners.go b/internal/host/relocationowners.go
index 3ef8cc4b8..28a0229c9 100644
--- a/internal/host/relocationowners.go
+++ b/internal/host/relocationowners.go
@@ -361,10 +361,7 @@ func relocationProbeFenceOwner(path, host string, session uint64) (string, error
if err != nil {
return "", err
}
- if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
- return "", err
- }
- if err := writeFileAtomic(path, raw, 0o600); err != nil {
+ if err := writeRelocationJournal(path, raw); err != nil {
return "", err
}
return rec.Owner, nil
diff --git a/internal/host/relocationsweep.go b/internal/host/relocationsweep.go
index fdbd84ea8..b4ece1790 100644
--- a/internal/host/relocationsweep.go
+++ b/internal/host/relocationsweep.go
@@ -48,10 +48,14 @@ func sayRelocationStray(path string) {
// release/, a build directory left empty. An entry the sweep does not know
// is left in place and logged once.
//
-// The caller runs it once per pass, between the pass's own steps, never
-// beside a relocation journal write: an ops directory and a temporary are
-// only ever a write's leftovers then.
+// It holds relocationJournals exclusively for the whole sweep, so it never
+// runs beside a relocation journal write in this process (carry 3): the
+// pass's own steps write in turn, but the wake's probe attempts write
+// attempts.json whenever a probe runs. An ops directory with no committed
+// journal, and a temporary, are only ever a finished write's leftovers then.
func sweepRelocationJournals(ctx context.Context, dataDir string, claims release.Store) error {
+ relocationJournals.Lock()
+ defer relocationJournals.Unlock()
root := filepath.Join(dataDir, "relocate")
var errs []error
gone := func(host, op string) (bool, error) {
@@ -203,6 +207,26 @@ func sweepRelocationJournals(ctx context.Context, dataDir string, claims release
return errors.Join(errs...)
}
+// relocationJournals serializes the journal sweep against every relocation
+// journal write in this process (carry 3). A writer holds it shared from
+// creating the journal's directory until its atomic rename has landed; the
+// sweep holds it exclusively. Without it the sweep could find an ops
+// directory between the writer's MkdirAll and its rename, holding no
+// committed journal, and remove it under the write.
+var relocationJournals sync.RWMutex
+
+// writeRelocationJournal is every relocation journal write under
+// <DataDir>/relocate: the directory, then moves' writeFileAtomic, under
+// relocationJournals shared.
+func writeRelocationJournal(path string, raw []byte) error {
+ relocationJournals.RLock()
+ defer relocationJournals.RUnlock()
+ if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+ return err
+ }
+ return writeFileAtomic(path, raw, 0o600)
+}
+
// relocationWriteLeftover is a temporary an interrupted atomic write left
// under release/: writeFileAtomic's ".<op>.json.<random>", or a fixed
// "<op>.json.tmp".
diff --git a/internal/hostrelocate/pin_versioned_test.go b/internal/hostrelocate/pin_versioned_test.go
new file mode 100644
index 000000000..5db2ade9e
--- /dev/null
+++ b/internal/hostrelocate/pin_versioned_test.go
@@ -0,0 +1,66 @@
+package hostrelocate
+
+import (
+ "context"
+ "encoding/json"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/genselect"
+ "github.com/carlosframework/platform/internal/hostfiles"
+ "github.com/carlosframework/platform/internal/lease/leasetest"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/superfly/ltx"
+)
+
+// Stage B Task 10's binding: the pass's StorePinner selects the database
+// with genselect.Select over the region's versioned data bucket. The pin
+// names exactly the versions the bucket holds for the holder's generation
+// (its marker's snapshot and the level-0 file after it), the plan's MaxTXID,
+// and the release the pinner's Release answers.
+func TestRelocationPinnerSelectsOverTheVersionedBucket(t *testing.T) {
+ ctx := context.Background()
+ hdr := func(min, max uint64) []byte {
+ b, err := (&ltx.Header{Version: ltx.Version, Flags: ltx.HeaderFlagNoChecksum, PageSize: 4096, MinTXID: ltx.TXID(min), MaxTXID: ltx.TXID(max)}).MarshalBinary()
+ if err != nil {
+ t.Fatal(err)
+ }
+ return b
+ }
+ b := leasetest.NewVersionedBucket()
+ b.AddBucket("data")
+ snap := pinLocation + "/g-g3/0009/" + ltx.FormatFilename(1, 40)
+ snapVersion := b.Put("data", snap, hdr(1, 40))
+ tail := pinLocation + "/g-g3/0000/" + ltx.FormatFilename(41, 41)
+ tailVersion := b.Put("data", tail, hdr(41, 41))
+ marker, _ := json.Marshal(genselect.Marker{Op: "g3", Incarnation: "inc", Host: host, Box: "src", InstalledAt: time.Date(2026, 10, 7, 8, 0, 0, 0, time.UTC),
+ ManifestSHA256: strings.Repeat("a", 64), Pin: genselect.Pin{SnapshotKey: snap, SnapshotVersion: snapVersion, SnapshotTXID: 40, SnapshotCRC64: 1}})
+ b.Put("data", pinLocation+"/generation.json", marker)
+ claims, data := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir())
+ putManifest(t, data, "g3", keyed(hostfiles.Manifest{V: hostfiles.ManifestVersion, Seq: 5, Entries: map[string]hostfiles.Entry{}, Platform: map[string]hostfiles.Entry{}}))
+ p := StorePinner{Claims: claims, Data: data, Release: routeRelease, Bucket: "data", Region: "ie",
+ SelectDB: func(ctx context.Context, r genselect.Request) (*genselect.Plan, error) {
+ return genselect.Select(ctx, b, r)
+ }}
+ obs := pinClaim()
+ obs.Claim.Holder.Incarnation = "inc"
+ pin, err := p.Pin(ctx, obs, time.Date(2026, 10, 7, 9, 0, 0, 0, time.UTC))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if pin.Generation != "g3" || pin.MaxTXID != 41 || pin.Release != "v1.4.2" || pin.FilesSeq != 5 || pin.Empty {
+ t.Fatalf("pin %+v", pin)
+ }
+ obj, err := ReadPin(ctx, claims, pin)
+ if err != nil {
+ t.Fatal(err)
+ }
+ got := map[string]string{}
+ for _, f := range obj.DB.Files {
+ got[f.Key] = f.VersionID
+ }
+ if obj.DB.MaxTXID != 41 || got[snap] != snapVersion || got[tail] != tailVersion || len(got) != 2 {
+ t.Fatalf("pinned files %v (max %d)", got, obj.DB.MaxTXID)
+ }
+}
diff --git a/internal/hostrelocate/worker.go b/internal/hostrelocate/worker.go
index c49786246..97718804a 100644
--- a/internal/hostrelocate/worker.go
+++ b/internal/hostrelocate/worker.go
@@ -20,6 +20,7 @@ import (
"fmt"
"os"
"slices"
+ "strings"
"time"
"github.com/carlosframework/platform/internal/hostactors"
@@ -195,6 +196,55 @@ func (w Worker) fence(ctx context.Context, o hostactors.Observed, box string) (*
return &hostclaim.FenceRef{Enrollment: b.Enrollment, Key: key, SHA256: hex.EncodeToString(sum[:])}, f, nil
}
+// ConvertLanding is J3 for one landing claim, under the same worker
+// authorization as Step (Astra round 6 finding 4): this box must be an
+// active, unfenced worker whose current roster entry is the enrollment it
+// runs as, in the region the landing lands in. A landing whose op.exec
+// box (the move's target) has a termination fence becomes a failover
+// relocation when its move committed (guard); an uncommitted one is
+// moves' case (ErrWrongState). A landing into another region, one with no
+// executor, or one whose target has no fence is left alone with nothing
+// written; a caller that is not such a worker is ErrNotWorker with nothing
+// written.
+func (w Worker) ConvertLanding(ctx context.Context, obs hostclaim.Observed, guard hostclaim.CommittedLandingGuard) (hostclaim.Observed, error) {
+ if w.Claims == nil || w.Directory.Store == nil || w.Fences == nil || w.Discharge == nil || w.NewID == nil || w.Now == nil || w.Self.Box == "" || w.Self.Enrollment == "" || w.Region == "" || guard == nil {
+ return obs, ErrUnsupported
+ }
+ o, err := w.self(ctx)
+ if err != nil {
+ return obs, err
+ }
+ if obs.Absent || obs.Claim == nil {
+ return obs, nil
+ }
+ c := obs.Claim
+ box := c.Op.Exec
+ if c.Op.Kind != hostclaim.KindLanding || c.Op.Intent["to"] != w.Region || box == "" || box == "none" {
+ return obs, nil
+ }
+ ref, record, err := w.fence(ctx, o, box)
+ if err != nil || ref == nil {
+ return obs, err
+ }
+ d := w.Discharge(ctx, o.Roster.Boxes[box], record)
+ return hostclaim.ConvertCommittedLanding(ctx, w.Claims, obs, c.Op.ID, box, *ref, w.Fences, guard, d, w.NewID)
+}
+
+// newestFenceObservation is the latest fence.<box>.observed_at the op
+// journals.
+func newestFenceObservation(c *hostclaim.Claim) (time.Time, bool) {
+ var newest time.Time
+ for k, v := range c.Op.Journal {
+ if !strings.HasPrefix(k, "fence.") || !strings.HasSuffix(k, ".observed_at") {
+ continue
+ }
+ if at, err := time.Parse(time.RFC3339Nano, v); err == nil && at.After(newest) {
+ newest = at
+ }
+ }
+ return newest, !newest.IsZero()
+}
+
// Start writes the start CAS for req (§5.8 "The start CAS", §5.10). It
// refuses unless the claim serves on req.Source; the roster shows the
// starter, the target and (for a drain) the source active, unfenced and
@@ -344,6 +394,13 @@ func (w Worker) Step(ctx context.Context, host string) (hostclaim.Observed, erro
// as an initial failover's target does.
next, err := w.next(ctx, obs)
if err != nil {
+ // J3-3: bounded. A converted landing with no eligible
+ // target is stuck ReleaseDeadline after its target's
+ // fence was observed; before that the chooser's refusal
+ // is returned and the pass alerts once.
+ if at, ok := newestFenceObservation(c); ok && w.Now().Sub(at) > ReleaseDeadline {
+ return w.update(ctx, obs, hostclaim.MarkStuck(id, fmt.Sprintf("no eligible box for the converted landing after %s: %v", ReleaseDeadline, err)))
+ }
return obs, err
}
return w.update(ctx, obs, hostclaim.NameTarget(id, next))
diff --git a/internal/hostrelocate/worker_bounds_test.go b/internal/hostrelocate/worker_bounds_test.go
index 0b4c4bd28..8dc09b658 100644
--- a/internal/hostrelocate/worker_bounds_test.go
+++ b/internal/hostrelocate/worker_bounds_test.go
@@ -181,3 +181,37 @@ func TestWorkerRefusesAnInstalledWithoutItsStart(t *testing.T) {
})
}
}
+
+// J3-3: a converted landing waits for an eligible target in starting, the
+// chooser's refusal returned each pass, and is stuck ReleaseDeadline after
+// its target's fence was observed, never pinned without a target.
+func TestWorkerBoundsAConversionWaitingForATarget(t *testing.T) {
+ f := newFixture(t)
+ converted := f.convertedLanding()
+ var fenced time.Time
+ for k, v := range converted.Claim.Op.Journal {
+ if strings.HasPrefix(k, "fence.") && strings.HasSuffix(k, ".observed_at") {
+ at, err := time.Parse(time.RFC3339Nano, v)
+ if err != nil {
+ t.Fatal(err)
+ }
+ fenced = at
+ }
+ }
+ if fenced.IsZero() {
+ t.Fatalf("the conversion journals no fence observation: %+v", converted.Claim.Op.Journal)
+ }
+ none := errors.New("no eligible box")
+ f.w.Choose = func(context.Context, hostclaim.Observed, []string) (hostclaim.Target, error) {
+ return hostclaim.Target{}, none
+ }
+ f.now = fenced.Add(ReleaseDeadline)
+ if o := f.worker(none); o.Claim.Op.Journal["stuck"] != "" || o.Claim.Op.Intent["target_box"] != "" {
+ t.Fatalf("stuck or named before the bound: %+v", o.Claim.Op)
+ }
+ f.now = f.now.Add(time.Second)
+ o := f.worker(nil)
+ if o.Claim.Op.Journal["stuck"] == "" || o.Claim.Op.Phase != hostclaim.PhaseStarting || o.Claim.Op.Journal["pin"] != "" {
+ t.Fatalf("not stuck for an operator: %+v", o.Claim.Op)
+ }
+}