amadan
host: the passed probe is stopped with root's receipt under an owned fence, its participation is discharged by that receipt before the hold is released and the completion is marked (C10), and the first serving wake hands over from it (N6)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
18 files changed,
+1694
−28
internal/host/claimconstruction.go+5 −1internal/host/claimhandover_crossbox.go+52 −16internal/host/claimhandover_transport.go+12 −2internal/host/claimparticipation.go+11 −3internal/host/claimparticipation_completed.go+59 −0internal/host/claimparticipation_relocation.go+38 −0internal/host/claimparticipation_stale.go+1 −1internal/host/plan11_fix1_relocation_test.go+2 −2internal/host/plan9_production_test.go+2 −2internal/host/relocationassembly.go+7 −1internal/host/relocationdischarge_linux_test.go+168 −0internal/host/relocationdischarge_restart_unix_test.go+174 −0internal/host/relocationhandover.go+105 −0internal/host/relocationhandover_unix_test.go+98 −0internal/host/relocationowners.go+205 −0internal/host/relocationreleasehold_test.go+262 −0internal/host/relocationstopprobe_unix_test.go+293 −0internal/hostbox/reconcile_relocation_target_test.go+200 −0
diff --git a/internal/host/claimconstruction.go b/internal/host/claimconstruction.go| index 8d8c7031d..514de5704 100644 |
| --- a/internal/host/claimconstruction.go |
| +++ b/internal/host/claimconstruction.go |
| @@ -287,7 +287,11 @@ func OpenProductionClaimAssembly(ctx context.Context, store release.Store, r Cla |
| if groupD.Dispatch.Relocation != nil { |
| relocation = &owners.Relocation |
| } |
| - wireClaimAdmission(wake, actor.Box, wake.AfterAdmission, handover, relocation) |
| + var completed *relocationCompletedHandover |
| + if relocation != nil { |
| + completed = &relocationCompletedHandover{Guard: guard, Identity: owners.Identity, Socket: controlsock.Path, Box: actor.Box} |
| + } |
| + wireClaimAdmission(wake, actor.Box, wake.AfterAdmission, handover, relocation, completed) |
| retirement := park.Local{Authorize: func(ctx context.Context, c *hostclaim.Claim, o hostclaim.Obligation) error { |
| return owners.Authorize(ctx, hostclaim.Observed{Host: c.Host, Claim: c}, o) |
| }, Guard: guard, Claims: store, DataDir: r.DataDir, Box: actor.Box} |
diff --git a/internal/host/claimhandover_crossbox.go b/internal/host/claimhandover_crossbox.go| index 3c1cb5e89..e2eba8945 100644 |
| --- a/internal/host/claimhandover_crossbox.go |
| +++ b/internal/host/claimhandover_crossbox.go |
| @@ -140,28 +140,37 @@ func claimPredecessorStop(w *ClaimWakeAssembly, box string, crossBox func(contex |
| // wireClaimAdmission routes the source proof to its operation owner before |
| // lease acquisition and checks the target installation after admission. |
| -func wireClaimAdmission(w *ClaimWakeAssembly, box string, local func(context.Context, hostclaim.Observed, hostclaim.Observed) error, landing claimCrossBoxHandover, relocation *ClaimRelocationOwners) { |
| +func wireClaimAdmission(w *ClaimWakeAssembly, box string, local func(context.Context, hostclaim.Observed, hostclaim.Observed) error, landing claimCrossBoxHandover, relocation *ClaimRelocationOwners, completed *relocationCompletedHandover) { |
| before := landing.Before |
| after := landing.After |
| - if relocation != nil { |
| - before = func(ctx context.Context, prior hostclaim.Observed) error { |
| - if prior.Claim.Op.Kind != hostclaim.KindRelocating { |
| - return landing.Before(ctx, prior) |
| - } |
| - if relocation.BeforeAdmission == nil || prior.Claim.Op.Exec != box || prior.Claim.Op.Phase != "probing" { |
| - return hostbox.ErrChanged |
| + if relocation != nil && completed != nil { |
| + // Region HA N6: the first serving wake after a relocation completed |
| + // on this box hands over from the probe's existing receipt; every |
| + // other serving prior keeps main's stop and local handover. A |
| + // serving prior on this box whose completion record cannot be read |
| + // is refused, never routed to main's stop (absence is never |
| + // evidence): that stop would replace the probe's fence. |
| + completedAfter := completed.After() |
| + w.BeforeAdmission = func(ctx context.Context, prior hostclaim.Observed) error { |
| + if _, ok, err := completed.completedPrior(prior); err != nil { |
| + return err |
| + } else if ok { |
| + return completed.Before(ctx, prior) |
| } |
| - return relocation.BeforeAdmission(ctx, prior) |
| + return claimPredecessorStop(w, box, relocationBefore(landing, relocation, box))(ctx, prior) |
| } |
| - after = func(ctx context.Context, prior, admitted hostclaim.Observed) error { |
| - if prior.Claim.Op.Kind != hostclaim.KindRelocating { |
| - return landing.After(ctx, prior, admitted) |
| - } |
| - if relocation.AfterAdmission == nil || prior.Claim.Op.Exec != box || prior.Claim.Op.Phase != "probing" { |
| - return hostbox.ErrChanged |
| + w.AfterAdmission = func(ctx context.Context, prior, admitted hostclaim.Observed) error { |
| + if _, ok, err := completed.completedPrior(prior); err != nil { |
| + return err |
| + } else if ok { |
| + return completedAfter(ctx, prior, admitted) |
| } |
| - return relocation.AfterAdmission(ctx, prior, admitted) |
| + return claimAdmissionHandover(box, local, relocationAfter(landing, relocation, box))(ctx, prior, admitted) |
| } |
| + return |
| + } |
| + if relocation != nil { |
| + before, after = relocationBefore(landing, relocation, box), relocationAfter(landing, relocation, box) |
| } |
| w.BeforeAdmission = claimPredecessorStop(w, box, before) |
| w.AfterAdmission = claimAdmissionHandover(box, local, after) |
| @@ -173,3 +182,30 @@ func claimRemotePredecessor(prior hostclaim.Observed, box string) bool { |
| } |
| return prior.Claim.Op.Kind == hostclaim.KindRelocating || (prior.Claim.Op.Kind == hostclaim.KindLanding && prior.Claim.Holder.WriterBox != "") |
| } |
| + |
| +// relocationBefore and relocationAfter are wireClaimAdmission's relocating |
| +// routing (Plan 11), unchanged, named so the completed-relocation branch |
| +// (N6) can fall through to them. |
| +func relocationBefore(landing claimCrossBoxHandover, relocation *ClaimRelocationOwners, box string) func(context.Context, hostclaim.Observed) error { |
| + return func(ctx context.Context, prior hostclaim.Observed) error { |
| + if prior.Claim.Op.Kind != hostclaim.KindRelocating { |
| + return landing.Before(ctx, prior) |
| + } |
| + if relocation.BeforeAdmission == nil || prior.Claim.Op.Exec != box || prior.Claim.Op.Phase != "probing" { |
| + return hostbox.ErrChanged |
| + } |
| + return relocation.BeforeAdmission(ctx, prior) |
| + } |
| +} |
| + |
| +func relocationAfter(landing claimCrossBoxHandover, relocation *ClaimRelocationOwners, box string) func(context.Context, hostclaim.Observed, hostclaim.Observed) error { |
| + return func(ctx context.Context, prior, admitted hostclaim.Observed) error { |
| + if prior.Claim.Op.Kind != hostclaim.KindRelocating { |
| + return landing.After(ctx, prior, admitted) |
| + } |
| + if relocation.AfterAdmission == nil || prior.Claim.Op.Exec != box || prior.Claim.Op.Phase != "probing" { |
| + return hostbox.ErrChanged |
| + } |
| + return relocation.AfterAdmission(ctx, prior, admitted) |
| + } |
| +} |
diff --git a/internal/host/claimhandover_transport.go b/internal/host/claimhandover_transport.go| index dc6948c0b..cf0f414cc 100644 |
| --- a/internal/host/claimhandover_transport.go |
| +++ b/internal/host/claimhandover_transport.go |
| @@ -51,12 +51,22 @@ func (h ClaimRootHTTP) serveColdHandover(w http.ResponseWriter, r *http.Request, |
| w.WriteHeader(http.StatusNoContent) |
| } |
| func coldAdmissionHook(guard hostguard.Guard, identity hostbox.IdentityProvider, socket string) func(context.Context, hostclaim.Observed, hostclaim.Observed) error { |
| + return coldHandoverWith(identity, socket, func(ctx context.Context, prior hostclaim.Observed) (hostguard.FenceReceipt, error) { |
| + old := prior.Claim.Holder |
| + return guard.CompletedColdPredecessor(ctx, prior.Host, hostguard.Captured{Incarnation: old.Incarnation, Stamp: old.Stamp}, old.Session, prior.Claim.Op.ID, prior.Claim.Op.Phase) |
| + }) |
| +} |
| + |
| +// coldHandoverWith is coldAdmissionHook with the predecessor's completed |
| +// receipt found by find (extracted unchanged; region HA N6): root's |
| +// ColdHandover.Run validates the exact prior receipt and the successor. |
| +func coldHandoverWith(identity hostbox.IdentityProvider, socket string, find func(context.Context, hostclaim.Observed) (hostguard.FenceReceipt, error)) func(context.Context, hostclaim.Observed, hostclaim.Observed) error { |
| return func(ctx context.Context, prior, admitted hostclaim.Observed) error { |
| if prior.Claim == nil || prior.Claim.Holder == nil || admitted.Claim == nil || admitted.Claim.Holder == nil || prior.Host != admitted.Host { |
| return hostbox.ErrChanged |
| } |
| - old, next := prior.Claim.Holder, admitted.Claim.Holder |
| - receipt, err := guard.CompletedColdPredecessor(ctx, prior.Host, hostguard.Captured{Incarnation: old.Incarnation, Stamp: old.Stamp}, old.Session, prior.Claim.Op.ID, prior.Claim.Op.Phase) |
| + next := admitted.Claim.Holder |
| + receipt, err := find(ctx, prior) |
| if err != nil { |
| return err |
| } |
diff --git a/internal/host/claimparticipation.go b/internal/host/claimparticipation.go| index 285cd3087..dc7272992 100644 |
| --- a/internal/host/claimparticipation.go |
| +++ b/internal/host/claimparticipation.go |
| @@ -148,7 +148,7 @@ func (p *productionClaimLocal) prepareWith(ctx context.Context, actor hostbox.Ac |
| // A previous failed cleanup remains denial until this fresh positive |
| // proof succeeds. Its historical error must not veto CC9 forever. |
| if !old.CleanupSucceeded { |
| - if err = p.cleanup(ctx, old.Captured); err != nil { |
| + if err = p.cleanup(ctx, old.Captured, old.Config.Binding); err != nil { |
| return err |
| } |
| } |
| @@ -206,7 +206,7 @@ func (p *productionClaimLocal) prepareWith(ctx context.Context, actor hostbox.Ac |
| floor, journal := p.updateFloor() |
| slot.Config = execlease.Config{Binding: binding, Store: p.Store, Guard: hostguard.Guard{DataDir: p.Resources.DataDir, LockDir: p.lockDir("hostguard"), Claims: hostguard.StoreReader{Store: p.Store}}, Directory: p.Directory, Clock: execlease.SystemClock{}, UpdateGuardDir: floor, UpdateJournal: journal, LockDir: p.lockDir("hostexec"), |
| CloseAdmission: func() { p.cancelWork(o.Host) }, CancelWork: func() { p.cancelWork(o.Host) }, Cleanup: func(ctx context.Context) error { |
| - err := p.cleanup(ctx, o) |
| + err := p.cleanup(ctx, o, binding) |
| slot.CleanupSucceeded = err == nil |
| return err |
| }} |
| @@ -229,7 +229,7 @@ func (p *productionClaimLocal) cancelWork(host string) { |
| } |
| } |
| } |
| -func (p *productionClaimLocal) cleanup(ctx context.Context, o hostclaim.Observed) error { |
| +func (p *productionClaimLocal) cleanup(ctx context.Context, o hostclaim.Observed, binding execlease.Binding) error { |
| p.mu.Lock() |
| a := p.assembly |
| local := p.stopIdentity |
| @@ -244,6 +244,14 @@ func (p *productionClaimLocal) cleanup(ctx context.Context, o hostclaim.Observed |
| return hostwrite.ErrUnavailable |
| } |
| p.cancelWork(o.Host) |
| + // Claim ask C10 (region HA rulings 56, 66): a relocation that completed |
| + // on this box is discharged by the persisted participation binding and |
| + // its exact completed probe evidence, before any ordinary cleanup, so |
| + // no serving-shaped stop ever replaces the probe's fence. The captured |
| + // observation is not consulted: after a restart it is the current one. |
| + if done, err := a.completedRelocationDischarge(ctx, binding); done || err != nil { |
| + return err |
| + } |
| // Cleanup uses the journal and producer set directly. It must never call |
| // WorkSource.Select: Open is still waiting for this cleanup to finish. |
| has, err := a.Launches.HasHost(ctx, o.Host) |
diff --git a/internal/host/claimparticipation_completed.go b/internal/host/claimparticipation_completed.go| new file mode 100644 |
| index 000000000..65a813f2d |
| --- /dev/null |
| +++ b/internal/host/claimparticipation_completed.go |
| @@ -0,0 +1,59 @@ |
| +package host |
| + |
| +import ( |
| + "context" |
| + |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + execlease "github.com/carlosframework/platform/internal/hostexec" |
| +) |
| + |
| +// completedRelocationDischarge is claim ask C10's CC9 discharge, rev 4 of |
| +// its text with rev 4.2's sentence (region HA rulings 56, 66 and 70; Astra |
| +// rounds 3 and 4). It is keyed by the persisted participation binding (the |
| +// slot's, or after a restart the .hostexec record's), never by a captured |
| +// observation: when the claim now serves on the binding's box, its holder |
| +// is still the stopped probe session the serving op records as completed |
| +// for exactly the binding's op and incarnation, and root's completed |
| +// writer-session receipt for it is in this box's guard journal and |
| +// verifies, the session is stopped with root's evidence and the |
| +// participation is discharged. Nothing is stopped again. It runs first in |
| +// every cleanup path, the slot's close and clearUntracked alike. done |
| +// false with no error falls through to main's cleanup unchanged. Any |
| +// error, a failed claim read or an unreadable completion record included, |
| +// is (true, err), and the caller also stops on any error: the cleanup |
| +// fails, nothing is stopped, and the next pass retries (rev 4.2). Region |
| +// HA never releases the row's hold or the activator fence before this has |
| +// succeeded (ruling 66), so no serving admission can move the holder off |
| +// the probe session first. |
| +func (a ClaimAssembly) completedRelocationDischarge(ctx context.Context, binding execlease.Binding) (done bool, err error) { |
| + if binding.Host == "" || binding.Op == "" || binding.Box == "" { |
| + return false, nil |
| + } |
| + now, err := hostclaim.Read(ctx, a.Claims, binding.Host) |
| + if err != nil { |
| + // Rev 4.2 (Astra round 4): a failed read never falls through to |
| + // ordinary cleanup, whose serving-shaped stop would replace the |
| + // completed probe's active fence (BeginFence) and make the first |
| + // serving wake's handover refuse for good. The cleanup fails and |
| + // is retried on the next pass, with nothing stopped. |
| + return true, err |
| + } |
| + if now.Absent || now.Claim == nil || now.Claim.Holder == nil { |
| + return false, nil |
| + } |
| + h := relocationCompletedHandover{Guard: a.Guard, Box: binding.Box} |
| + record, ok, err := h.completedPrior(now) |
| + if err != nil { |
| + // A completion record that cannot be read is not "no completion" |
| + // (absence is never evidence): the same rule as a failed read. |
| + return true, err |
| + } |
| + if !ok || record.Op != binding.Op || now.Claim.Holder.Incarnation != binding.Incarnation { |
| + return false, nil |
| + } |
| + r, err := h.receipt(ctx, now) |
| + if err != nil { |
| + return true, err |
| + } |
| + return true, verifyRetirementReceipt(ctx, r) |
| +} |
diff --git a/internal/host/claimparticipation_relocation.go b/internal/host/claimparticipation_relocation.go| index 602c45b90..741057013 100644 |
| --- a/internal/host/claimparticipation_relocation.go |
| +++ b/internal/host/claimparticipation_relocation.go |
| @@ -2,6 +2,8 @@ package host |
| import ( |
| "context" |
| + "errors" |
| + "os" |
| "github.com/carlosframework/platform/internal/hostbox" |
| "github.com/carlosframework/platform/internal/hostclaim" |
| @@ -41,3 +43,39 @@ func (p *productionClaimLocal) Acknowledged(ctx context.Context, o hostclaim.Obs |
| } |
| return slot.Controller.Ready(ctx) |
| } |
| + |
| +// DischargeCompleted is ruling 66's gate before a completed relocation's |
| +// hold is released: main's own Prepare for the serving observation, which |
| +// closes the participation slot (its cleanup is C10's discharge, first) |
| +// and clears the .hostexec record, or, after a restart, clears the |
| +// untracked record through clearUntracked (the same discharge, keyed by |
| +// the persisted binding). It succeeds only once neither a slot nor a |
| +// record is left for the host. N1's third accessor; no new authority. |
| +func (p *productionClaimLocal) DischargeCompleted(ctx context.Context, o hostclaim.Observed) error { |
| + if o.Claim == nil || o.Claim.Holder == nil || o.Claim.Op.Kind != hostclaim.KindServing { |
| + return hostbox.ErrChanged |
| + } |
| + actor, err := p.Identity.Verify(ctx) |
| + if err != nil { |
| + return err |
| + } |
| + return p.dischargeCompletedAs(ctx, actor, o) |
| +} |
| + |
| +// dischargeCompletedAs is DischargeCompleted for a verified actor: main's |
| +// prepareWith, then nothing of the participation may be left. |
| +func (p *productionClaimLocal) dischargeCompletedAs(ctx context.Context, actor hostbox.Actor, o hostclaim.Observed) error { |
| + if err := p.prepareWith(ctx, actor, o); err != nil { |
| + return err |
| + } |
| + p.mu.Lock() |
| + slot := p.slots[o.Host] |
| + p.mu.Unlock() |
| + if slot != nil { |
| + return execlease.ErrInactive |
| + } |
| + if _, err := execlease.InspectLocal(p.Resources.DataDir, o.Host); !errors.Is(err, os.ErrNotExist) { |
| + return errors.Join(execlease.ErrInactive, err) |
| + } |
| + return nil |
| +} |
diff --git a/internal/host/claimparticipation_stale.go b/internal/host/claimparticipation_stale.go| index b1ba43d65..1048d47b6 100644 |
| --- a/internal/host/claimparticipation_stale.go |
| +++ b/internal/host/claimparticipation_stale.go |
| @@ -29,7 +29,7 @@ func (p *productionClaimLocal) clearUntracked(ctx context.Context, actor hostbox |
| return execlease.ErrState |
| } |
| if o.Claim != nil && o.Claim.Holder != nil && state.Incarnation == o.Claim.Holder.Incarnation { |
| - if err := p.cleanup(ctx, o); err != nil { |
| + if err := p.cleanup(ctx, o, state.Binding); err != nil { |
| return err |
| } |
| return execlease.ClearStale(ctx, cfg) |
diff --git a/internal/host/plan11_fix1_relocation_test.go b/internal/host/plan11_fix1_relocation_test.go| index da8e5d712..fe33fd1b2 100644 |
| --- a/internal/host/plan11_fix1_relocation_test.go |
| +++ b/internal/host/plan11_fix1_relocation_test.go |
| @@ -26,7 +26,7 @@ func TestPlan11Fix1RelocationAdmissionHooks(t *testing.T) { |
| calls = append(calls, "relocation-after") |
| return nil |
| }} |
| - wireClaimAdmission(w, "target", local, claimCrossBoxHandover{}, &reloc) |
| + wireClaimAdmission(w, "target", local, claimCrossBoxHandover{}, &reloc, nil) |
| o := plan11Relocating("probing") |
| o.Claim.Op.Exec = "target" |
| o.Claim.Holder.WriterBox = writer |
| @@ -117,7 +117,7 @@ func TestPlan11Fix1RelocationComposedAdmission(t *testing.T) { |
| wireClaimAdmission(c.wake, "box-1", func(context.Context, hostclaim.Observed, hostclaim.Observed) error { |
| t.Fatal("remote predecessor reached local handover") |
| return nil |
| - }, claimCrossBoxHandover{}, &owners) |
| + }, claimCrossBoxHandover{}, &owners, nil) |
| prior := f.read() |
| if err := c.wake.stopPredecessor(t.Context(), prior); !errors.Is(err, refusal) { |
| t.Fatalf("release refusal lost: %v", err) |
diff --git a/internal/host/plan9_production_test.go b/internal/host/plan9_production_test.go| index 69c42fdbc..9bc3377e4 100644 |
| --- a/internal/host/plan9_production_test.go |
| +++ b/internal/host/plan9_production_test.go |
| @@ -121,7 +121,7 @@ func TestPlan9ParticipationCleanupDoesNotWaitForItsOpeningSlot(t *testing.T) { |
| p := &productionClaimLocal{assembly: &controller.assembly, slots: map[string]*claimParticipation{recoveryHost: pending}} |
| ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) |
| defer cancel() |
| - if err := p.cleanup(ctx, o); err != nil { |
| + if err := p.cleanup(ctx, o, execlease.Binding{}); err != nil { |
| t.Fatal(err) |
| } |
| select { |
| @@ -169,7 +169,7 @@ func TestPlan9ParticipationCleanupSurvivesRosterWithdrawal(t *testing.T) { |
| controller.assembly.Identity = hostbox.IdentityProvider{} |
| ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) |
| defer cancel() |
| - if err := p.cleanup(ctx, o); err != nil { |
| + if err := p.cleanup(ctx, o, execlease.Binding{}); err != nil { |
| t.Fatalf("withdrawal prevented cleanup: %v", err) |
| } |
| if len(f.receipts()) != 1 { |
diff --git a/internal/host/relocationassembly.go b/internal/host/relocationassembly.go| index 6c8c4594d..861bfa469 100644 |
| --- a/internal/host/relocationassembly.go |
| +++ b/internal/host/relocationassembly.go |
| @@ -70,7 +70,11 @@ type RelocationAssemblyConfig struct { |
| // main: root then authorizes a relocating launch at the admitted |
| // writer's stamp, not only at install_stamp (ruling 58). |
| LaterProbeAttempts bool |
| - Now func() time.Time |
| + // Discharge is main's participation discharge for a completed |
| + // relocation (productionClaimLocal.DischargeCompleted, N1). The hold |
| + // and the activator fence are released only after it (ruling 66). |
| + Discharge func(context.Context, hostclaim.Observed) error |
| + Now func() time.Time |
| // ReleasePub and Ratchet are §4.1's evidence for the held stage |
| // (release.MaterializeHeldUnder): the agent's release key |
| // (CARLOS_RELEASE_PUBKEY) and the edge's pointer digest ratchet, |
| @@ -132,6 +136,8 @@ func (a ClaimAssembly) RelocationEffects(cfg RelocationAssemblyConfig, ack func( |
| VerifyLocal: a.relocationVerifyLocal(cfg), |
| Receipt: a.relocationReceipt(cfg), |
| Admit: a.relocationAdmit(cfg), |
| + StopProbe: a.relocationStopProbe(cfg), |
| + ReleaseHold: a.relocationReleaseHold(cfg), |
| } |
| } |
diff --git a/internal/host/relocationdischarge_linux_test.go b/internal/host/relocationdischarge_linux_test.go| new file mode 100644 |
| index 000000000..e55c3343c |
| --- /dev/null |
| +++ b/internal/host/relocationdischarge_linux_test.go |
| @@ -0,0 +1,168 @@ |
| +//go:build linux |
| + |
| +package host |
| + |
| +import ( |
| + "context" |
| + "crypto/sha256" |
| + "encoding/hex" |
| + "encoding/json" |
| + "strconv" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/hostbox" |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + "github.com/carlosframework/platform/internal/hostguard" |
| +) |
| + |
| +// probeStopped moves the participation box's relocation to a passed and |
| +// stopped probe: dst's session 12 admitted into P1 at the install stamp, |
| +// the guard activated there, and root's writer-session stop of session 12 |
| +// completed with its receipt in the guard journal (ruling 49). It returns |
| +// the probing observation, the stop's fence target and root's receipt. |
| +func (b *participationBox) probeStopped() (hostclaim.Observed, hostguard.FenceTarget, []byte) { |
| + b.t.Helper() |
| + ctx := b.t.Context() |
| + probing := b.update(func(c *hostclaim.Claim) error { |
| + stamp := c.Gen + 1 |
| + c.Holder.Stamp, c.Holder.Session, c.Holder.Generation, c.Holder.WriterBox = stamp, 12, "P1", "dst" |
| + c.Holder.Admissions = append(c.Holder.Admissions, hostclaim.AdmissionRecord{Session: 12, Box: "dst", Generation: "P1", Predecessor: "g1", Mode: "cold", Region: "ie"}) |
| + probe, _ := json.Marshal(hostclaim.Probe{Box: "dst", Session: 12, At: time.Date(2026, 10, 7, 3, 0, 0, 0, time.UTC)}) |
| + c.Op.Phase = hostclaim.PhaseRelocatingProbing |
| + c.Op.Journal = map[string]string{"install_stamp": strconv.FormatUint(stamp, 10), "prepared": "P1", "prepared_opened": "P1", "probe": string(probe)} |
| + return nil |
| + }) |
| + if err := b.guard.Activate(ctx, participationHost, "dst", nil); err != nil { |
| + b.t.Fatal(err) |
| + } |
| + h := probing.Claim.Holder |
| + target := hostguard.FenceTarget{FenceID: "fence-1", Region: "ie", Host: participationHost, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: 12, Op: "rel-1", Phase: hostclaim.PhaseRelocatingProbing, |
| + ObligationID: hostguard.WriterStopObligation(12), Box: "dst", Enrollment: b.actor.Enrollment, BootID: "boot", ControllerID: "ctl", ClaimETag: probing.ETag, DirectoryRevision: "4"} |
| + // The probe's launch is in the box's launch journal, as the wake |
| + // journals it: main's ordinary cleanup would stop that session again. |
| + if err := b.launches.Prelaunch(ctx, hostbox.LaunchInventory{ControllerPID: 4101, ProcessStart: "start-1", Target: target, |
| + Groups: []hostbox.LaunchGroup{{RelativePath: "carlos/" + claimContainmentName(target), Device: 1, Inode: 12}}, NoState: true}); err != nil { |
| + b.t.Fatal(err) |
| + } |
| + if _, err := b.guard.BeginFence(ctx, target); err != nil { |
| + b.t.Fatal(err) |
| + } |
| + at := time.Date(2026, 10, 7, 3, 1, 0, 0, time.UTC) |
| + receipt := mustMarshalStopReceipt(StopReceipt{V: 1, ID: "r1", Kind: StopReceiptStopped, Region: "ie", ClaimETag: probing.ETag, ObligationID: target.ObligationID, FenceID: "fence-1", Host: participationHost, |
| + Incarnation: h.Incarnation, InstallationStamp: h.Stamp, WriterSession: 12, OperationID: "rel-1", Phase: hostclaim.PhaseRelocatingProbing, Box: "dst", Enrollment: b.actor.Enrollment, |
| + BootID: "boot", ControllerID: "ctl", DirectoryRevision: "4", EvidenceAt: at}) |
| + if err := b.guard.CompleteFence(ctx, target, receipt, at, func(context.Context, hostguard.FenceTarget, []byte, time.Time) error { return nil }); err != nil { |
| + b.t.Fatal(err) |
| + } |
| + return probing, target, receipt |
| +} |
| + |
| +// serve is CompleteRelocation's serving CAS as the box sees it: kind |
| +// serving on dst, the holder still the stopped probe session, and the |
| +// serving op recording the stop. |
| +func (b *participationBox) serve(receipt []byte) hostclaim.Observed { |
| + b.t.Helper() |
| + sum := sha256.Sum256(receipt) |
| + return b.update(func(c *hostclaim.Claim) error { |
| + done, _ := json.Marshal(hostclaim.CompletedRelocation{Op: "rel-1", Stop: hostclaim.ProbeStop{Box: "dst", Session: 12, Receipt: "fence-1", ReceiptSHA256: hex.EncodeToString(sum[:])}, |
| + Stamp: c.Holder.Stamp, Phase: hostclaim.PhaseRelocatingProbing, Generation: c.Holder.Generation}) |
| + c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "dst", Participants: []string{"dst"}, Journal: map[string]string{hostclaim.JournalCompleted: string(done)}} |
| + return nil |
| + }) |
| +} |
| + |
| +func (b *participationBox) servingCheck(o hostclaim.Observed) error { |
| + h := o.Claim.Holder |
| + return b.p.Check(b.t.Context(), b.actor, o.Claim, hostbox.Selection{Host: participationHost, Box: "dst", Region: "ie", Captured: hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp}}) |
| +} |
| + |
| +// Rev 3 finding 5, rulings 56 and 66, C10: completion without an agent |
| +// restart. While the participation slot is open the serving Check denies |
| +// (the wake cannot admit); DischargeCompleted closes the slot through main's |
| +// Prepare, whose cleanup is C10's discharge by the slot's binding and the |
| +// probe's receipt (no stop asked of root), ClearStale removes the record, |
| +// the serving Check passes, and N6's handover finds the probe's receipt. |
| +func TestRelocationCompletionDischargesParticipationWithoutRestart(t *testing.T) { |
| + ctx := t.Context() |
| + b := newParticipationBox(t) |
| + if err := b.p.Prepare(ctx, b.read()); err != nil { // acknowledged at installing |
| + t.Fatal(err) |
| + } |
| + _, target, receipt := b.probeStopped() |
| + serving := b.serve(receipt) |
| + if err := b.servingCheck(serving); err == nil { |
| + t.Fatal("the serving wake was admissible with the participation open") |
| + } |
| + if err := b.p.DischargeCompleted(ctx, serving); err != nil { |
| + t.Fatalf("discharge: %v", err) |
| + } |
| + if b.stops != 0 { |
| + t.Fatalf("the discharge asked root for %d stops", b.stops) |
| + } |
| + if len(b.p.slots) != 0 || len(b.records()) != 0 { |
| + t.Fatalf("slots %d records %v", len(b.p.slots), b.records()) |
| + } |
| + if err := b.servingCheck(b.read()); err != nil { |
| + t.Fatalf("the serving Check after the discharge: %v", err) |
| + } |
| + h := relocationCompletedHandover{Guard: b.guard, Box: "dst"} |
| + if err := h.Before(ctx, b.read()); err != nil { |
| + t.Fatalf("N6 before admission: %v", err) |
| + } |
| + if r, err := h.receipt(ctx, b.read()); err != nil || r.Target != target { |
| + t.Fatalf("N6's prior: %+v %v", r.Target, err) |
| + } |
| +} |
| + |
| +// Rev 4 finding 3, ruling 66: the agent restarts after the serving CAS, |
| +// with the participation's .hostexec record left behind. The new process's |
| +// DischargeCompleted reaches clearUntracked, C10 discharges by the |
| +// record's binding (no stop asked, the probe's fence untouched), |
| +// ClearStale removes the record, and the first serving wake's handover |
| +// from the probe's receipt is accepted by the guard. |
| +func TestRelocationRestartAfterServingCASRecoversParticipation(t *testing.T) { |
| + ctx := t.Context() |
| + b := newParticipationBox(t) |
| + if err := b.p.Prepare(ctx, b.read()); err != nil { // acknowledged at installing |
| + t.Fatal(err) |
| + } |
| + slot := b.p.slots[participationHost] |
| + _, target, receipt := b.probeStopped() |
| + serving := b.serve(receipt) |
| + // The crash: the process ends and its controller with it; the record |
| + // stays. The restarted agent has no slot. |
| + _ = slot.Controller.Close(ctx) |
| + b.p.mu.Lock() |
| + b.p.slots = map[string]*claimParticipation{} |
| + b.p.mu.Unlock() |
| + if len(b.records()) != 1 { |
| + t.Fatalf("records after the crash: %v", b.records()) |
| + } |
| + if err := b.p.DischargeCompleted(ctx, serving); err != nil { |
| + t.Fatalf("discharge after the restart: %v", err) |
| + } |
| + if b.stops != 0 || len(b.records()) != 0 { |
| + t.Fatalf("stops %d records %v", b.stops, b.records()) |
| + } |
| + h := serving.Claim.Holder |
| + got, err := b.guard.CompletedWriterStops(ctx, participationHost, hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp}, "rel-1", "dst") |
| + if err != nil || len(got) != 1 || got[0].Target != target { |
| + t.Fatalf("the probe's receipt changed: %+v %v", got, err) |
| + } |
| + if err := b.servingCheck(b.read()); err != nil { |
| + t.Fatalf("the serving Check: %v", err) |
| + } |
| + // The first serving wake: a cold admission at session and stamp 40; |
| + // the guard's handover barrier over the probe's receipt is accepted. |
| + admitted := b.update(func(c *hostclaim.Claim) error { |
| + c.Holder.Stamp, c.Holder.Session, c.Holder.Generation = 40, 40, "g40" |
| + c.Holder.Admissions = append(c.Holder.Admissions, hostclaim.AdmissionRecord{Session: 40, Box: "dst", Generation: "g40", Predecessor: "P1", Mode: "cold", Region: "ie"}) |
| + return nil |
| + }) |
| + barrier := target |
| + barrier.FenceID, barrier.Stamp, barrier.Session, barrier.Op, barrier.Phase, barrier.ObligationID, barrier.ClaimETag = "handover-1", 40, 40, "", "", "", admitted.ETag |
| + if err := b.guard.BeginColdHandover(ctx, target, barrier, func(*hostclaim.Claim) error { return nil }); err != nil { |
| + t.Fatalf("the serving handover from the probe's receipt: %v", err) |
| + } |
| +} |
diff --git a/internal/host/relocationdischarge_restart_unix_test.go b/internal/host/relocationdischarge_restart_unix_test.go| new file mode 100644 |
| index 000000000..8d8349416 |
| --- /dev/null |
| +++ b/internal/host/relocationdischarge_restart_unix_test.go |
| @@ -0,0 +1,174 @@ |
| +//go:build unix |
| + |
| +package host |
| + |
| +import ( |
| + "context" |
| + "crypto/sha256" |
| + "encoding/hex" |
| + "encoding/json" |
| + "errors" |
| + "os" |
| + "path/filepath" |
| + "sync" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/hostbox" |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + execlease "github.com/carlosframework/platform/internal/hostexec" |
| + "github.com/carlosframework/platform/internal/hostguard" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// guardClaims is the guard's claim reader, set by the test. |
| +type guardClaims struct { |
| + mu sync.Mutex |
| + obs hostclaim.Observed |
| +} |
| + |
| +func (g *guardClaims) set(o hostclaim.Observed) { g.mu.Lock(); g.obs = o; g.mu.Unlock() } |
| +func (g *guardClaims) Read(context.Context, string) (hostclaim.Observed, error) { |
| + g.mu.Lock() |
| + defer g.mu.Unlock() |
| + return g.obs, nil |
| +} |
| + |
| +// failFirstClaimRead fails the first Get of the claim key only: C10's |
| +// re-read (Astra round 4). |
| +type failFirstClaimRead struct { |
| + release.Store |
| + key string |
| + failed bool |
| + mu sync.Mutex |
| +} |
| + |
| +var errClaimReadDown = errors.New("control store unreachable") |
| + |
| +func (s *failFirstClaimRead) Get(ctx context.Context, key string) ([]byte, string, error) { |
| + s.mu.Lock() |
| + first := key == s.key && !s.failed |
| + if first { |
| + s.failed = true |
| + } |
| + s.mu.Unlock() |
| + if first { |
| + return nil, "", errClaimReadDown |
| + } |
| + return s.Store.Get(ctx, key) |
| +} |
| + |
| +// Astra round 4 (rev 4.2, ruling 70): after a restart between the serving |
| +// CAS and the participation discharge, a failure of only C10's claim |
| +// re-read never falls through to ordinary cleanup. Cleanup fails with that |
| +// error, nothing is stopped, the completed probe's fence is not replaced; |
| +// on retry the participation is discharged by the binding and the probe's |
| +// receipt, and the first serving wake's cold handover (N6's prior, the |
| +// probe's receipt) is accepted by the guard. |
| +func TestRelocationRestartDischargeSurvivesAFailedClaimRead(t *testing.T) { |
| + ctx := t.Context() |
| + const host, box = "shop.example", "dst" |
| + created := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) |
| + inc, err := hostclaim.Incarnation("acct", "app", host, created) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + holder := func(session, stamp uint64, gen string) *hostclaim.Holder { |
| + return &hostclaim.Holder{Family: "instance", Key: "apps/acct/app/instances/" + host + ".json", Account: "acct", App: "app", Incarnation: inc, CreatedAt: created, |
| + Location: "instances/" + host, Stamp: stamp, Session: session, Generation: gen, WriterBox: box, |
| + Admissions: []hostclaim.AdmissionRecord{{Session: 12, Box: box, Generation: "P1", Mode: "cold", Region: "ie"}}} |
| + } |
| + probing := hostclaim.Observed{Host: host, ETag: "e1", Claim: &hostclaim.Claim{V: 2, Gen: 30, Host: host, Holder: holder(12, 9, "P1"), |
| + Op: hostclaim.Op{Kind: hostclaim.KindRelocating, ID: "rel-1", Phase: hostclaim.PhaseRelocatingProbing, Exec: box, Participants: []string{box}}}} |
| + reader := &guardClaims{obs: probing} |
| + dataDir := t.TempDir() |
| + if err := os.Mkdir(filepath.Join(dataDir, "hostguard"), 0o770); err != nil { |
| + t.Fatal(err) |
| + } |
| + guard := hostguard.Guard{DataDir: dataDir, LockDir: t.TempDir(), Claims: reader} |
| + if err := guard.Activate(ctx, host, box, nil); err != nil { |
| + t.Fatal(err) |
| + } |
| + // The probe's writer-session stop, completed with root's receipt |
| + // (ruling 49), while the op was still relocating/probing. |
| + prior := hostguard.FenceTarget{FenceID: "fence-1", Region: "ie", Host: host, Incarnation: inc, Stamp: 9, Session: 12, Op: "rel-1", Phase: hostclaim.PhaseRelocatingProbing, |
| + ObligationID: hostguard.WriterStopObligation(12), Box: box, Enrollment: "enr", BootID: "boot", ControllerID: "ctl", ClaimETag: "e1", DirectoryRevision: "1"} |
| + if _, err := guard.BeginFence(ctx, prior); err != nil { |
| + t.Fatal(err) |
| + } |
| + at := time.Date(2026, 10, 11, 9, 0, 0, 0, time.UTC) |
| + receipt := mustMarshalStopReceipt(StopReceipt{V: 1, ID: "r1", Kind: StopReceiptStopped, Region: "ie", ClaimETag: "e1", ObligationID: prior.ObligationID, FenceID: "fence-1", Host: host, Incarnation: inc, |
| + InstallationStamp: 9, WriterSession: 12, OperationID: "rel-1", Phase: hostclaim.PhaseRelocatingProbing, Box: box, Enrollment: "enr", BootID: "boot", ControllerID: "ctl", DirectoryRevision: "1", EvidenceAt: at}) |
| + if err := guard.CompleteFence(ctx, prior, receipt, at, func(context.Context, hostguard.FenceTarget, []byte, time.Time) error { return nil }); err != nil { |
| + t.Fatal(err) |
| + } |
| + sum := sha256.Sum256(receipt) |
| + done, _ := json.Marshal(hostclaim.CompletedRelocation{Op: "rel-1", Stop: hostclaim.ProbeStop{Box: box, Session: 12, Receipt: "fence-1", ReceiptSHA256: hex.EncodeToString(sum[:])}, Stamp: 9, Phase: hostclaim.PhaseRelocatingProbing, Generation: "P1"}) |
| + serving := &hostclaim.Claim{V: 2, Gen: 31, Host: host, Holder: holder(12, 9, "P1"), |
| + Op: hostclaim.Op{Kind: hostclaim.KindServing, Exec: box, Participants: []string{box}, Journal: map[string]string{hostclaim.JournalCompleted: string(done)}}} |
| + raw, err := json.Marshal(serving) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + key, _ := hostclaim.Key(host) |
| + dir := release.NewDirStore(t.TempDir()) |
| + if _, err := dir.Put(ctx, key, raw, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + if obs, err := hostclaim.Read(ctx, dir, host); err != nil || obs.Claim.Op.Kind != hostclaim.KindServing { |
| + t.Fatalf("the serving claim does not read back: %v", err) |
| + } |
| + reader.set(hostclaim.Observed{Host: host, ETag: "e2", Claim: serving}) |
| + store := &failFirstClaimRead{Store: dir, key: key} |
| + stops := 0 |
| + a := &ClaimAssembly{Claims: store, Guard: guard, Wake: &ClaimWakeAssembly{}, Launches: &hostbox.LaunchJournal{}, |
| + Reconcile: &hostbox.ReconcileAdapter{Census: &hostbox.EmptyInstallationCensus{}}, |
| + collectStop: func(context.Context, StopRequest, StopDrainer) (StopResult, error) { |
| + stops++ |
| + return StopResult{}, errors.New("a serving-shaped stop") |
| + }} |
| + p := &productionClaimLocal{assembly: a, slots: map[string]*claimParticipation{}} |
| + // The restart: clearUntracked hands cleanup the current serving |
| + // observation and the persisted .hostexec binding. |
| + binding := execlease.Binding{Box: box, Enrollment: "enr", Host: host, Incarnation: inc, Op: "rel-1", BootID: "boot-0"} |
| + current := hostclaim.Observed{Host: host, ETag: "e2", Claim: serving} |
| + |
| + if err := p.cleanup(ctx, current, binding); !errors.Is(err, errClaimReadDown) { |
| + t.Fatalf("a failed C10 read fell through: %v", err) |
| + } |
| + if stops != 0 { |
| + t.Fatal("ordinary cleanup asked a stop while the claim read was failing") |
| + } |
| + if got, err := guard.CompletedWriterStops(ctx, host, hostguard.Captured{Incarnation: inc, Stamp: 9}, "rel-1", box); err != nil || len(got) != 1 || got[0].Target != prior { |
| + t.Fatalf("the probe's receipt changed: %+v %v", got, err) |
| + } |
| + |
| + // The retry: storage recovered. |
| + if err := p.cleanup(ctx, current, binding); err != nil { |
| + t.Fatalf("the retry did not discharge: %v", err) |
| + } |
| + if stops != 0 { |
| + t.Fatal("the discharge asked a stop") |
| + } |
| + // The first serving wake: N6's Before verifies the receipt, stopping |
| + // nothing; after its cold admission (session and stamp 40), the |
| + // guard's handover barrier over the probe's receipt is accepted, |
| + // which a replacement fence would have refused (ErrFenceActive). |
| + h := relocationCompletedHandover{Guard: guard, Box: box} |
| + if err := h.Before(ctx, current); err != nil { |
| + t.Fatalf("N6 before admission: %v", err) |
| + } |
| + r, err := h.receipt(ctx, current) |
| + if err != nil || r.Target != prior { |
| + t.Fatalf("N6's prior is not the probe's receipt: %+v %v", r.Target, err) |
| + } |
| + admitted := *serving |
| + admitted.Holder = holder(40, 40, "g40") |
| + admitted.Holder.Admissions = append(admitted.Holder.Admissions, hostclaim.AdmissionRecord{Session: 40, Box: box, Generation: "g40", Mode: "cold", Region: "ie"}) |
| + reader.set(hostclaim.Observed{Host: host, ETag: "e3", Claim: &admitted}) |
| + barrier := prior |
| + barrier.FenceID, barrier.Stamp, barrier.Session, barrier.Op, barrier.Phase, barrier.ObligationID, barrier.ClaimETag = "handover-1", 40, 40, "", "", "", "e3" |
| + if err := guard.BeginColdHandover(ctx, prior, barrier, func(*hostclaim.Claim) error { return nil }); err != nil { |
| + t.Fatalf("the serving handover from the probe's receipt: %v", err) |
| + } |
| +} |
diff --git a/internal/host/relocationhandover.go b/internal/host/relocationhandover.go| new file mode 100644 |
| index 000000000..585ee46f3 |
| --- /dev/null |
| +++ b/internal/host/relocationhandover.go |
| @@ -0,0 +1,105 @@ |
| +package host |
| + |
| +import ( |
| + "context" |
| + "crypto/sha256" |
| + "encoding/hex" |
| + "fmt" |
| + |
| + "github.com/carlosframework/platform/internal/hostbox" |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + "github.com/carlosframework/platform/internal/hostguard" |
| +) |
| + |
| +// relocationCompletedHandover is the first serving cold wake after a |
| +// relocation completed on this box (rulings 49 and 56; Astra round 2 |
| +// finding 6). The predecessor is the probe session S that the target |
| +// stopped at relocating/probing with root's writer-session receipt, and |
| +// the serving op journals that stop (CompletedRelocation). Nothing is |
| +// stopped again and nothing is relabelled: before admission the receipt |
| +// is found and verified in this box's guard journal (in place of main's |
| +// Wake.Stop, which would ask a serving-shaped re-stop of S); after |
| +// admission it is handed to root's ColdHandover.Run as the exact prior, |
| +// which Run accepts beside the separately validated successor without |
| +// matching their operation ids. |
| +type relocationCompletedHandover struct { |
| + Guard hostguard.Guard |
| + Identity hostbox.IdentityProvider |
| + Socket string |
| + Box string |
| +} |
| + |
| +// completedPrior reports whether prior is a serving claim whose writer is |
| +// the stopped probe session this box completed a relocation with. A |
| +// serving claim on this box whose completion record is present but cannot |
| +// be read is an error, never "not the completed prior" (absence is never |
| +// evidence): routed to main's stop it would re-stop S with a serving-shaped |
| +// fence that replaces the probe's. |
| +func (h relocationCompletedHandover) completedPrior(prior hostclaim.Observed) (hostclaim.CompletedRelocation, bool, error) { |
| + c := prior.Claim |
| + if c == nil || c.Holder == nil || c.Op.Kind != hostclaim.KindServing || c.Op.Exec != h.Box || c.Holder.WriterBox != h.Box { |
| + return hostclaim.CompletedRelocation{}, false, nil |
| + } |
| + done, ok, err := hostclaim.ReadCompletedRelocation(c) |
| + if err != nil { |
| + return hostclaim.CompletedRelocation{}, false, err |
| + } |
| + if !ok || done.Stop.Box != h.Box || done.Stop.Session != c.Holder.Session || done.Stamp != c.Holder.Stamp || done.Generation != c.Holder.Generation { |
| + return hostclaim.CompletedRelocation{}, false, nil |
| + } |
| + return done, true, nil |
| +} |
| + |
| +// receipt is root's completed writer-session stop of S, exactly the one |
| +// the serving op journals: fence id, phase and the receipt bytes' digest. |
| +func (h relocationCompletedHandover) receipt(ctx context.Context, prior hostclaim.Observed) (hostguard.FenceReceipt, error) { |
| + done, ok, err := h.completedPrior(prior) |
| + if err != nil { |
| + return hostguard.FenceReceipt{}, err |
| + } |
| + if !ok { |
| + return hostguard.FenceReceipt{}, hostbox.ErrChanged |
| + } |
| + r, err := writerSessionReceipt(ctx, h.Guard, prior.Host, hostguard.Captured{Incarnation: prior.Claim.Holder.Incarnation, Stamp: done.Stamp}, done.Op, h.Box, done.Stop.Session) |
| + if err != nil { |
| + return hostguard.FenceReceipt{}, err |
| + } |
| + sum := sha256.Sum256(r.Bytes) |
| + if r.Target.FenceID != done.Stop.Receipt || r.Target.Phase != done.Phase || hex.EncodeToString(sum[:]) != done.Stop.ReceiptSHA256 { |
| + return hostguard.FenceReceipt{}, fmt.Errorf("%w: session %d's stop under %s is not the one the serving op records", ErrClaimStopReceipt, done.Stop.Session, done.Op) |
| + } |
| + return r, nil |
| +} |
| + |
| +// writerSessionReceipt is root's completed writer-session stop of session |
| +// (hostguard.WriterStopObligation) for op on box, from this box's guard |
| +// journal: the one reader of a probe's stop, for the probe stop and for |
| +// the serving handover after it. |
| +func writerSessionReceipt(ctx context.Context, guard hostguard.Guard, host string, captured hostguard.Captured, op, box string, session uint64) (hostguard.FenceReceipt, error) { |
| + receipts, err := guard.CompletedWriterStops(ctx, host, captured, op, box) |
| + if err != nil { |
| + return hostguard.FenceReceipt{}, err |
| + } |
| + want := hostguard.WriterStopObligation(session) |
| + for _, r := range receipts { |
| + if r.Target.Session == session && r.Target.ObligationID == want { |
| + return r, nil |
| + } |
| + } |
| + return hostguard.FenceReceipt{}, fmt.Errorf("%w: root has no completed stop of session %d under %s", ErrClaimStopReceipt, session, op) |
| +} |
| + |
| +// Before replaces the predecessor stop for exactly this prior: the stop |
| +// already happened, so its receipt is found and verified, nothing more. |
| +func (h relocationCompletedHandover) Before(ctx context.Context, prior hostclaim.Observed) error { |
| + r, err := h.receipt(ctx, prior) |
| + if err != nil { |
| + return err |
| + } |
| + return verifyRetirementReceipt(ctx, r) |
| +} |
| + |
| +// After is main's cold handover with that receipt as the prior. |
| +func (h relocationCompletedHandover) After() func(context.Context, hostclaim.Observed, hostclaim.Observed) error { |
| + return coldHandoverWith(h.Identity, h.Socket, h.receipt) |
| +} |
diff --git a/internal/host/relocationhandover_unix_test.go b/internal/host/relocationhandover_unix_test.go| new file mode 100644 |
| index 000000000..8ece78885 |
| --- /dev/null |
| +++ b/internal/host/relocationhandover_unix_test.go |
| @@ -0,0 +1,98 @@ |
| +//go:build unix |
| + |
| +package host |
| + |
| +import ( |
| + "context" |
| + "crypto/sha256" |
| + "encoding/hex" |
| + "encoding/json" |
| + "net" |
| + "net/http" |
| + "path/filepath" |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/hostbox" |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| +) |
| + |
| +// N6, ruling 56: wireClaimAdmission with the completed-relocation branch. |
| +// The first serving wake after a relocation completed on this box hands |
| +// over from the probe's existing receipt: before admission the receipt is |
| +// found and verified and nothing is stopped; after it, root's |
| +// /claim/handover is posted that receipt's target as the prior. Any other |
| +// serving prior keeps main's stop and main's local handover. |
| +func TestRelocationCompletedHandoverRoutesOnlyTheCompletedPrior(t *testing.T) { |
| + ctx := t.Context() |
| + f := newStopProbeFixture(t) |
| + if _, err := f.stop(); err != nil { |
| + t.Fatal(err) |
| + } |
| + sum := sha256.Sum256(f.receipt) |
| + done, _ := json.Marshal(hostclaim.CompletedRelocation{Op: "rel-1", Stop: hostclaim.ProbeStop{Box: "dst", Session: 12, Receipt: "fence-1", ReceiptSHA256: hex.EncodeToString(sum[:])}, |
| + Stamp: 9, Phase: hostclaim.PhaseRelocatingProbing, Generation: "P1"}) |
| + serving := func(journal map[string]string) hostclaim.Observed { |
| + c := *f.obs.Claim |
| + c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "dst", Participants: []string{"dst"}, Journal: journal} |
| + return hostclaim.Observed{Host: f.obs.Host, ETag: "e2", Claim: &c} |
| + } |
| + completedPrior := serving(map[string]string{hostclaim.JournalCompleted: string(done)}) |
| + plainPrior := serving(nil) |
| + admitted := func(prior hostclaim.Observed) hostclaim.Observed { |
| + c := *prior.Claim |
| + h := *c.Holder |
| + h.Stamp, h.Session, h.Generation = 40, 40, "g40" |
| + c.Holder = &h |
| + return hostclaim.Observed{Host: prior.Host, ETag: "e3", Claim: &c} |
| + } |
| + |
| + // Root's handover endpoint, recording what it is posted. |
| + socket := filepath.Join(t.TempDir(), "c.sock") |
| + l, err := net.Listen("unix", socket) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var posted []coldHandoverRequest |
| + srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + var req coldHandoverRequest |
| + if r.URL.Path != "/claim/handover" || json.NewDecoder(r.Body).Decode(&req) != nil { |
| + w.WriteHeader(http.StatusBadRequest) |
| + return |
| + } |
| + posted = append(posted, req) |
| + w.WriteHeader(http.StatusNoContent) |
| + })} |
| + go srv.Serve(l) |
| + t.Cleanup(func() { srv.Close() }) |
| + |
| + stops, locals := 0, 0 |
| + w := &ClaimWakeAssembly{Stop: func(context.Context, hostclaim.Observed) error { stops++; return nil }} |
| + local := func(context.Context, hostclaim.Observed, hostclaim.Observed) error { locals++; return nil } |
| + identity := handoverRecoveryIdentity(t, hostbox.Actor{Box: "dst", Enrollment: "enr", Region: "ie", Revision: 1}) |
| + completed := &relocationCompletedHandover{Guard: f.a.Guard, Identity: identity, Socket: socket, Box: "dst"} |
| + wireClaimAdmission(w, "dst", local, claimCrossBoxHandover{}, &ClaimRelocationOwners{}, completed) |
| + |
| + if err := w.BeforeAdmission(ctx, completedPrior); err != nil || stops != 0 { |
| + t.Fatalf("before the completed prior's admission: %v (stops %d)", err, stops) |
| + } |
| + if err := w.AfterAdmission(ctx, completedPrior, admitted(completedPrior)); err != nil { |
| + t.Fatalf("after the completed prior's admission: %v", err) |
| + } |
| + if len(posted) != 1 || posted[0].Prior != f.target || posted[0].Next.Session != 40 || posted[0].Next.Box != "dst" || locals != 0 { |
| + t.Fatalf("root was posted %+v (locals %d)", posted, locals) |
| + } |
| + |
| + if err := w.BeforeAdmission(ctx, plainPrior); err != nil || stops != 1 { |
| + t.Fatalf("a serving prior without the record: %v (stops %d)", err, stops) |
| + } |
| + if err := w.AfterAdmission(ctx, plainPrior, admitted(plainPrior)); err != nil || locals != 1 || len(posted) != 1 { |
| + t.Fatalf("a serving prior's handover: %v (locals %d, posted %d)", err, locals, len(posted)) |
| + } |
| + |
| + // A record whose stop is not root's receipt is refused before anything. |
| + forged := serving(map[string]string{hostclaim.JournalCompleted: string(mustJSON(t, hostclaim.CompletedRelocation{Op: "rel-1", |
| + Stop: hostclaim.ProbeStop{Box: "dst", Session: 12, Receipt: "fence-1", ReceiptSHA256: hex.EncodeToString(make([]byte, 32))}, Stamp: 9, Phase: hostclaim.PhaseRelocatingProbing, Generation: "P1"}))}) |
| + if err := w.BeforeAdmission(ctx, forged); err == nil || stops != 1 { |
| + t.Fatalf("a forged record: %v (stops %d)", err, stops) |
| + } |
| +} |
diff --git a/internal/host/relocationowners.go b/internal/host/relocationowners.go| index 45fd801f6..14fff02a4 100644 |
| --- a/internal/host/relocationowners.go |
| +++ b/internal/host/relocationowners.go |
| @@ -10,6 +10,7 @@ import ( |
| "fmt" |
| "log" |
| "os" |
| + "path/filepath" |
| "slices" |
| "sync" |
| "time" |
| @@ -18,6 +19,7 @@ import ( |
| "github.com/carlosframework/platform/internal/hostactors" |
| "github.com/carlosframework/platform/internal/hostbox" |
| "github.com/carlosframework/platform/internal/hostclaim" |
| + "github.com/carlosframework/platform/internal/hostguard" |
| "github.com/carlosframework/platform/internal/hostrelocate" |
| "github.com/carlosframework/platform/internal/hostwrite" |
| "github.com/carlosframework/platform/internal/lease" |
| @@ -251,3 +253,206 @@ func (a ClaimAssembly) relocationAdmit(cfg RelocationAssemblyConfig) func(contex |
| } |
| } |
| } |
| + |
| +// relocationStopProbe is ruling 49 steps 1 to 3, while the op is still |
| +// relocating/probing so the captured session still matches: the |
| +// activator's local fence at the session and its stop of the probe entry; |
| +// main's writer-session stop through root (deterministic, so a repeat |
| +// meets the same fence and receipt), which also releases the attempt's |
| +// lease; then root's receipt for exactly that session. |
| +// |
| +// The fence is an owned one (ruling 75, Astra round 6 findings 1 and 2). |
| +// Before anything is asked of the activator, the owner this probe stop |
| +// will install under is persisted with the session in the op's journal |
| +// directory (probe-fence.json); a failed write installs nothing, and the |
| +// next pass writes it again. FenceAndStopAs then installs only on an |
| +// unfenced host, or meets again exactly the fence it installed under |
| +// that owner at that session, deciding under the activator's own lock. |
| +// Any other fence (another operation's, of any generation, including one |
| +// installed after an agent restart emptied the activator's fences while |
| +// the marker survived) is ErrFenceHeld, with nothing stopped and nothing |
| +// asked of root, and is never replaced, so ReleaseHold's lift can never |
| +// reopen admission another operation closed. |
| +func (a ClaimAssembly) relocationStopProbe(cfg RelocationAssemblyConfig) func(context.Context, hostclaim.Observed) (hostclaim.ProbeStop, error) { |
| + return func(ctx context.Context, o hostclaim.Observed) (hostclaim.ProbeStop, error) { |
| + if cfg.Activator == nil || a.Wake == nil || a.Wake.Stop == nil || o.Claim == nil || o.Claim.Holder == nil { |
| + return hostclaim.ProbeStop{}, ErrClaimStopUnavailable |
| + } |
| + local, err := a.localStopIdentity(ctx) |
| + if err != nil { |
| + return hostclaim.ProbeStop{}, err |
| + } |
| + c, h, box := o.Claim, o.Claim.Holder, local.Actor.Box |
| + var p hostclaim.Probe |
| + if json.Unmarshal([]byte(c.Op.Journal["probe"]), &p) != nil || p.Box != box || p.Session != h.Session || c.Op.Exec != box || c.Op.Phase != hostclaim.PhaseRelocatingProbing || h.WriterBox != box { |
| + return hostclaim.ProbeStop{}, hostbox.ErrChanged |
| + } |
| + marker, err := relocationProbeFencePath(cfg.DataDir, c.Op.ID) |
| + if err != nil { |
| + return hostclaim.ProbeStop{}, err |
| + } |
| + owner, err := relocationProbeFenceOwner(marker, o.Host, h.Session) |
| + if err != nil { |
| + return hostclaim.ProbeStop{}, err // nothing installed: the next pass persists it again |
| + } |
| + fenced, cancel := context.WithTimeout(ctx, hostbox.RequestLimit) |
| + _, err = cfg.Activator.FenceAndStopAs(fenced, o.Host, h.Session, owner) |
| + cancel() |
| + if err != nil && !errors.Is(err, activator.ErrStopUnproven) { |
| + return hostclaim.ProbeStop{}, err // not this probe's fence: nothing stopped, nothing asked |
| + } |
| + if err := a.Wake.Stop(ctx, o); err != nil { |
| + return hostclaim.ProbeStop{}, err |
| + } |
| + r, err := writerSessionReceipt(ctx, a.Guard, o.Host, hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp}, c.Op.ID, box, h.Session) |
| + if err != nil { |
| + return hostclaim.ProbeStop{}, err |
| + } |
| + sum := sha256.Sum256(r.Bytes) |
| + return hostclaim.ProbeStop{Box: box, Session: h.Session, Receipt: r.Target.FenceID, ReceiptSHA256: hex.EncodeToString(sum[:])}, nil |
| + } |
| +} |
| + |
| +// relocationProbeFencePath is where this box's probe stop persists the |
| +// owner of the activator fence it installs, with the session and host: |
| +// <DataDir>/relocate/ops/<op>/probe-fence.json. The journal sweep keeps |
| +// the op's directory until the completed relocation is marked discharged |
| +// (ruling 76), so the lift always finds it. |
| +func relocationProbeFencePath(dataDir, op string) (string, error) { |
| + dir, err := relocationOpDir(dataDir, op) |
| + if err != nil { |
| + return "", err |
| + } |
| + return filepath.Join(dir, "probe-fence.json"), nil |
| +} |
| + |
| +// relocationProbeFence is the persisted owner of one probe stop's fence. |
| +type relocationProbeFence struct { |
| + Host string `json:"host"` |
| + Session uint64 `json:"session"` |
| + Owner string `json:"owner"` |
| +} |
| + |
| +// relocationProbeFenceOwner returns the owner persisted for host's probe |
| +// session, or persists a new, random one first. A marker for another |
| +// session, or one that does not read, is replaced: a fresh owner can only |
| +// ever match a fence installed under it, so replacing one takes nothing |
| +// over. |
| +func relocationProbeFenceOwner(path, host string, session uint64) (string, error) { |
| + if rec, ok, err := readRelocationProbeFence(path); err == nil && ok && rec.Host == host && rec.Session == session && rec.Owner != "" { |
| + return rec.Owner, nil |
| + } |
| + var b [16]byte |
| + if _, err := rand.Read(b[:]); err != nil { |
| + return "", err |
| + } |
| + rec := relocationProbeFence{Host: host, Session: session, Owner: hex.EncodeToString(b[:])} |
| + raw, err := json.Marshal(rec) |
| + if err != nil { |
| + return "", err |
| + } |
| + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { |
| + return "", err |
| + } |
| + if err := writeFileAtomic(path, raw, 0o600); err != nil { |
| + return "", err |
| + } |
| + return rec.Owner, nil |
| +} |
| + |
| +// readRelocationProbeFence reads the persisted owner; false when there is |
| +// none. |
| +func readRelocationProbeFence(path string) (relocationProbeFence, bool, error) { |
| + raw, err := os.ReadFile(path) |
| + if errors.Is(err, os.ErrNotExist) { |
| + return relocationProbeFence{}, false, nil |
| + } |
| + if err != nil { |
| + return relocationProbeFence{}, false, err |
| + } |
| + var rec relocationProbeFence |
| + if err := json.Unmarshal(raw, &rec); err != nil { |
| + return relocationProbeFence{}, false, err |
| + } |
| + return rec, true, nil |
| +} |
| + |
| +// relocationReleaseHold finishes a completed relocation on the box it |
| +// completed on (ruling 49 step 5, rulings 76 and 80), in this order: while |
| +// the row is held for the op, the holder must still be the stopped probe |
| +// session, main's participation is discharged (C10) and the hold is |
| +// released (ruling 66: nothing is released before the discharge); then |
| +// the activator fence the probe stop installed under its persisted owner |
| +// is lifted; last, the serving op's completed relocation is marked |
| +// discharged, which is what lets any successor start (the completion |
| +// barrier, ruling 77). The pass calls it for every serving claim whose |
| +// completion record names this box and is not yet marked, never from the |
| +// row (ruling 80), and every step is idempotent, so a failure or a crash |
| +// at any point leaves the record unmarked and the next pass resumes where |
| +// it stopped. Once the row is no longer held for the op, the discharge has |
| +// happened (nothing else releases a relocating hold for a completed op), |
| +// and a replay only lifts and marks. A missing row is refused: absence is |
| +// never evidence of the discharge. Anything else leaves everything: an |
| +// excluded box's row is its park-local's. |
| +func (a ClaimAssembly) relocationReleaseHold(cfg RelocationAssemblyConfig) func(context.Context, string, string) error { |
| + return func(ctx context.Context, host, op string) error { |
| + if cfg.Registry == nil || cfg.Activator == nil || cfg.Discharge == nil { |
| + return ErrClaimStopUnavailable |
| + } |
| + local, err := a.localStopIdentity(ctx) |
| + if err != nil { |
| + return err |
| + } |
| + obs, err := hostclaim.Read(ctx, a.Claims, host) |
| + if err != nil { |
| + return err |
| + } |
| + if obs.Absent || obs.Claim == nil || obs.Claim.Holder == nil || obs.Claim.Op.Kind != hostclaim.KindServing || obs.Claim.Op.Exec != local.Actor.Box || obs.Claim.Holder.WriterBox != local.Actor.Box { |
| + return nil |
| + } |
| + done, ok, err := hostclaim.ReadCompletedRelocation(obs.Claim) |
| + if err != nil { |
| + return err // unreadable is not "no completion": nothing is released over it |
| + } |
| + if !ok || done.Op != op || done.Stop.Box != local.Actor.Box { |
| + return fmt.Errorf("%w: %s's serving op does not record %s as completed on this box", hostrelocate.ErrRefused, host, op) |
| + } |
| + if done.Discharged { |
| + return nil |
| + } |
| + row, err := cfg.Registry.Get(host) |
| + if err != nil { |
| + return err |
| + } |
| + if row == nil { |
| + return fmt.Errorf("%w: %s has no row on this box", hostrelocate.ErrRefused, host) |
| + } |
| + if row.HoldOp == op { |
| + if done.Stop.Session != obs.Claim.Holder.Session { |
| + return fmt.Errorf("%w: %s's serving holder is not %s's stopped probe session", hostrelocate.ErrRefused, host, op) |
| + } |
| + if err := cfg.Discharge(ctx, obs); err != nil { |
| + return err |
| + } |
| + if _, err := cfg.Registry.ReleaseHold(host, op); err != nil { |
| + return err |
| + } |
| + } |
| + marker, err := relocationProbeFencePath(cfg.DataDir, op) |
| + if err != nil { |
| + return err |
| + } |
| + fence, found, err := readRelocationProbeFence(marker) |
| + if err != nil { |
| + return err // the owner is unreadable: the fence stays, the claim is not marked |
| + } |
| + if found && fence.Host == host && fence.Session == done.Stop.Session { |
| + cfg.Activator.LiftFenceAs(host, fence.Session, fence.Owner) |
| + } |
| + _, err = hostclaim.Update(ctx, a.Claims, obs, hostclaim.MarkCompletedDischarged(op)) |
| + if errors.Is(err, hostclaim.ErrAlreadyRecorded) { |
| + return nil |
| + } |
| + return err |
| + } |
| +} |
diff --git a/internal/host/relocationreleasehold_test.go b/internal/host/relocationreleasehold_test.go| new file mode 100644 |
| index 000000000..086223773 |
| --- /dev/null |
| +++ b/internal/host/relocationreleasehold_test.go |
| @@ -0,0 +1,262 @@ |
| +//go:build unix |
| + |
| +package host |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "path/filepath" |
| + "slices" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/activator" |
| + "github.com/carlosframework/platform/internal/hostbox" |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + "github.com/carlosframework/platform/internal/hostrelocate" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// Ruling 66 (Astra round 3 finding 2) and ruling 76 (Astra round 6 |
| +// findings 1 and 3): after the serving CAS, the target discharges main's |
| +// participation, releases its row's relocating hold, lifts the fence its |
| +// probe stop installed under the persisted owner, and marks the completed |
| +// relocation discharged, in that order. A refused discharge leaves all of |
| +// it, so the edge keeps the row held, the activator wakes nothing and no |
| +// successor starts; a holder moved off the probe session while the row is |
| +// held is refused; another operation's fence is never lifted; a replay |
| +// after the hold went only lifts and marks; a claim that does not serve on |
| +// this box is left alone. |
| +func TestRelocationReleaseHoldWaitsForDischarge(t *testing.T) { |
| + const host, box = "shop.example", "dst" |
| + ctx := t.Context() |
| + created := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) |
| + inc, err := hostclaim.Incarnation("acct", "app", host, created) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + done, _ := json.Marshal(hostclaim.CompletedRelocation{Op: "rel-1", Stop: hostclaim.ProbeStop{Box: box, Session: 12, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)}, Stamp: 9, Phase: hostclaim.PhaseRelocatingProbing, Generation: "P1"}) |
| + claim := func(change func(*hostclaim.Claim)) *hostclaim.Claim { |
| + c := &hostclaim.Claim{V: 2, Gen: 31, Host: host, |
| + Holder: &hostclaim.Holder{Family: "instance", Key: "apps/acct/app/instances/" + host + ".json", Account: "acct", App: "app", Incarnation: inc, CreatedAt: created, |
| + Location: "instances/" + host, Stamp: 9, Session: 12, Generation: "P1", WriterBox: box, Admissions: []hostclaim.AdmissionRecord{{Session: 12, Box: box, Generation: "P1", Mode: "cold", Region: "ie"}}}, |
| + Op: hostclaim.Op{Kind: hostclaim.KindServing, Exec: box, Participants: []string{box}, Journal: map[string]string{hostclaim.JournalCompleted: string(done)}}} |
| + if change != nil { |
| + change(c) |
| + } |
| + return c |
| + } |
| + type world struct { |
| + order []string |
| + store release.Store |
| + reg *registry.Registry |
| + act *stopProbeActivator |
| + owner string |
| + discharge error |
| + hold func(context.Context, string, string) error |
| + } |
| + setup := func(t *testing.T, c *hostclaim.Claim) *world { |
| + t.Helper() |
| + w := &world{store: release.NewDirStore(t.TempDir())} |
| + raw, _ := json.Marshal(c) |
| + key, _ := hostclaim.Key(host) |
| + if _, err := w.store.Put(ctx, key, raw, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + if w.reg, err = registry.Open(filepath.Join(t.TempDir(), "registry.db")); err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { w.reg.Close() }) |
| + if err := w.reg.Upsert(registry.Route{Host: host, Status: "relocating", HoldOp: "rel-1", Socket: "/run/shop.sock"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + // The probe stop's fence, under the owner it persisted. |
| + dataDir := t.TempDir() |
| + marker, err := relocationProbeFencePath(dataDir, "rel-1") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if w.owner, err = relocationProbeFenceOwner(marker, host, 12); err != nil { |
| + t.Fatal(err) |
| + } |
| + w.act = newStopProbeActivator(&w.order) |
| + fctx, cancel := context.WithTimeout(ctx, time.Second) |
| + defer cancel() |
| + if _, err := w.act.svc.FenceAndStopAs(fctx, host, 12, w.owner); err != nil && !errors.Is(err, activator.ErrStopUnproven) { |
| + t.Fatal(err) |
| + } |
| + a := ClaimAssembly{Claims: w.store, stopLocal: func(context.Context) (claimStopLocal, error) { |
| + return claimStopLocal{Actor: hostbox.Actor{Box: box}}, nil |
| + }} |
| + cfg := RelocationAssemblyConfig{Registry: w.reg, Activator: w.act, DataDir: dataDir, Discharge: func(_ context.Context, o hostclaim.Observed) error { |
| + if row, _ := w.reg.Get(host); row == nil || row.HoldOp != "rel-1" { |
| + t.Error("the hold was released before the discharge") |
| + } |
| + w.order = append(w.order, "discharge") |
| + return w.discharge |
| + }} |
| + w.hold = a.relocationReleaseHold(cfg) |
| + return w |
| + } |
| + held := func(w *world) bool { |
| + row, err := w.reg.Get(host) |
| + return err == nil && row != nil && row.Status == "relocating" && row.HoldOp == "rel-1" |
| + } |
| + marked := func(w *world) bool { |
| + obs, err := hostclaim.Read(ctx, w.store, host) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + r, ok, err := hostclaim.ReadCompletedRelocation(obs.Claim) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + return ok && r.Discharged |
| + } |
| + t.Run("the discharge first, then the hold, the fence and the mark", func(t *testing.T) { |
| + w := setup(t, claim(nil)) |
| + if err := w.hold(ctx, host, "rel-1"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if want := []string{"discharge", "lift"}; !slices.Equal(w.order, want) || held(w) || w.act.svc.Fenced(host) || !marked(w) { |
| + t.Fatalf("order %v held=%v fenced=%v marked=%v", w.order, held(w), w.act.svc.Fenced(host), marked(w)) |
| + } |
| + // A replay of a marked relocation does nothing at all. |
| + w.order = nil |
| + if err := w.hold(ctx, host, "rel-1"); err != nil || len(w.order) != 0 { |
| + t.Fatalf("replay: %v %v", err, w.order) |
| + } |
| + }) |
| + t.Run("a refused discharge keeps everything", func(t *testing.T) { |
| + w := setup(t, claim(nil)) |
| + w.discharge = errors.New("the participation slot is still open") |
| + if err := w.hold(ctx, host, "rel-1"); !errors.Is(err, w.discharge) { |
| + t.Fatalf("got %v", err) |
| + } |
| + if want := []string{"discharge"}; !slices.Equal(w.order, want) || !held(w) || w.act.svc.FenceGeneration(host) != 12 || marked(w) { |
| + t.Fatalf("order %v held=%v fence=%d marked=%v", w.order, held(w), w.act.svc.FenceGeneration(host), marked(w)) |
| + } |
| + }) |
| + t.Run("another operation's fence after a restart stays", func(t *testing.T) { |
| + w := setup(t, claim(nil)) |
| + w.act.svc = &activator.Service{} // the restart: this box's fence is gone |
| + w.act.foreignFence(t, host, 12) |
| + if err := w.hold(ctx, host, "rel-1"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if held(w) || w.act.svc.FenceGeneration(host) != 12 || !marked(w) { |
| + t.Fatalf("held=%v fence=%d marked=%v", held(w), w.act.svc.FenceGeneration(host), marked(w)) |
| + } |
| + }) |
| + t.Run("a replay after the hold went only lifts and marks", func(t *testing.T) { |
| + // The hold was released, then the agent stopped before the lift; |
| + // a serving wake has since moved the holder on. |
| + w := setup(t, claim(func(c *hostclaim.Claim) { c.Holder.Session = 13 })) |
| + if _, err := w.reg.ReleaseHold(host, "rel-1"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := w.hold(ctx, host, "rel-1"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if want := []string{"lift"}; !slices.Equal(w.order, want) || w.act.svc.Fenced(host) || !marked(w) { |
| + t.Fatalf("order %v fenced=%v marked=%v", w.order, w.act.svc.Fenced(host), marked(w)) |
| + } |
| + }) |
| + t.Run("a holder moved off the probe session while held", func(t *testing.T) { |
| + w := setup(t, claim(func(c *hostclaim.Claim) { c.Holder.Session, c.Holder.Stamp = 40, 40 })) |
| + if err := w.hold(ctx, host, "rel-1"); !errors.Is(err, hostrelocate.ErrRefused) || len(w.order) != 0 || !held(w) || marked(w) { |
| + t.Fatalf("%v order %v held=%v", err, w.order, held(w)) |
| + } |
| + }) |
| + t.Run("no row is refused", func(t *testing.T) { |
| + w := setup(t, claim(nil)) |
| + if err := w.reg.Delete(host); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := w.hold(ctx, host, "rel-1"); !errors.Is(err, hostrelocate.ErrRefused) || len(w.order) != 0 || marked(w) { |
| + t.Fatalf("%v order %v", err, w.order) |
| + } |
| + }) |
| + for name, change := range map[string]func(*hostclaim.Claim){ |
| + "still relocating": func(c *hostclaim.Claim) { |
| + c.Op = hostclaim.Op{Kind: hostclaim.KindRelocating, ID: "rel-1", Phase: hostclaim.PhaseRelocatingProbing, Exec: box, Participants: []string{"src", box}, |
| + Intent: map[string]string{"region": "ie", "source_box": "src", "target_box": box}} |
| + }, |
| + "serving on another box": func(c *hostclaim.Claim) { |
| + c.Op.Exec, c.Op.Participants, c.Holder.WriterBox = "src", []string{"src"}, "src" |
| + }, |
| + } { |
| + t.Run(name, func(t *testing.T) { |
| + w := setup(t, claim(change)) |
| + if err := w.hold(ctx, host, "rel-1"); err != nil || len(w.order) != 0 || !held(w) || w.act.svc.FenceGeneration(host) != 12 { |
| + t.Fatalf("%v order %v held=%v", err, w.order, held(w)) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// Astra round 6, finding 3, through the target's real hold release and |
| +// the real start CAS: a successor relocation from the box a relocation |
| +// completed on starts only once that box has discharged its probe's |
| +// participation and marked it. |
| +func TestRelocationSuccessorStartsOnlyAfterTheTargetsDischarge(t *testing.T) { |
| + const host, box = "shop.example", "dst" |
| + ctx := t.Context() |
| + created := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) |
| + inc, err := hostclaim.Incarnation("acct", "app", host, created) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + done, _ := json.Marshal(hostclaim.CompletedRelocation{Op: "rel-1", Stop: hostclaim.ProbeStop{Box: box, Session: 12, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)}, Stamp: 9, Phase: hostclaim.PhaseRelocatingProbing, Generation: "P1"}) |
| + c := &hostclaim.Claim{V: 2, Gen: 31, Host: host, |
| + Holder: &hostclaim.Holder{Family: "instance", Key: "apps/acct/app/instances/" + host + ".json", Account: "acct", App: "app", Incarnation: inc, CreatedAt: created, |
| + Location: "instances/" + host, Stamp: 9, Session: 12, Generation: "P1", WriterBox: box, Admissions: []hostclaim.AdmissionRecord{{Session: 12, Box: box, Generation: "P1", Mode: "cold", Region: "ie"}}}, |
| + Op: hostclaim.Op{Kind: hostclaim.KindServing, Exec: box, Participants: []string{box}, Journal: map[string]string{hostclaim.JournalCompleted: string(done)}}} |
| + store := release.NewDirStore(t.TempDir()) |
| + raw, _ := json.Marshal(c) |
| + key, _ := hostclaim.Key(host) |
| + if _, err := store.Put(ctx, key, raw, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + defer reg.Close() |
| + if err := reg.Upsert(registry.Route{Host: host, Status: "relocating", HoldOp: "rel-1", Socket: "/run/shop.sock"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + var order []string |
| + refused := errors.New("the participation slot is still open") |
| + discharge := refused |
| + a := ClaimAssembly{Claims: store, stopLocal: func(context.Context) (claimStopLocal, error) { |
| + return claimStopLocal{Actor: hostbox.Actor{Box: box}}, nil |
| + }} |
| + hold := a.relocationReleaseHold(RelocationAssemblyConfig{Registry: reg, Activator: newStopProbeActivator(&order), DataDir: t.TempDir(), |
| + Discharge: func(context.Context, hostclaim.Observed) error { return discharge }}) |
| + start := func() error { |
| + obs, err := hostclaim.Read(ctx, store, host) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + _, err = hostclaim.StartRelocation(ctx, store, obs, hostclaim.RelocationStart{OpID: "rel-2", By: "box:wrk", Now: created.Add(3 * time.Hour), Region: "ie", Reason: hostclaim.ReasonDrain, |
| + SourceBox: box, SourceEnrollment: strings.Repeat("c", 32), TargetBox: "spa", TargetEnrollment: strings.Repeat("d", 32), DirectoryRevision: 2, AdmissionID: "adm-2"}, nil) |
| + return err |
| + } |
| + if err := hold(ctx, host, "rel-1"); !errors.Is(err, refused) { |
| + t.Fatalf("hold: %v", err) |
| + } |
| + if err := start(); !errors.Is(err, hostclaim.ErrPendingChore) { |
| + t.Fatalf("the successor started before the target's discharge: %v", err) |
| + } |
| + discharge = nil |
| + if err := hold(ctx, host, "rel-1"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := start(); err != nil { |
| + t.Fatalf("the successor after the discharge: %v", err) |
| + } |
| +} |
diff --git a/internal/host/relocationstopprobe_unix_test.go b/internal/host/relocationstopprobe_unix_test.go| new file mode 100644 |
| index 000000000..bb00976fc |
| --- /dev/null |
| +++ b/internal/host/relocationstopprobe_unix_test.go |
| @@ -0,0 +1,293 @@ |
| +//go:build unix |
| + |
| +package host |
| + |
| +import ( |
| + "context" |
| + "crypto/sha256" |
| + "encoding/hex" |
| + "encoding/json" |
| + "errors" |
| + "fmt" |
| + "os" |
| + "path/filepath" |
| + "slices" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/activator" |
| + "github.com/carlosframework/platform/internal/hostbox" |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + "github.com/carlosframework/platform/internal/hostguard" |
| +) |
| + |
| +// stopProbeActivator is a real activator.Service's fences, the calls |
| +// recorded in order. Replacing svc is an agent restart: the activator's |
| +// fences live only in its process. failNext, when set, is the next |
| +// FenceAndStopAs's answer instead of the service's. |
| +type stopProbeActivator struct { |
| + order *[]string |
| + svc *activator.Service |
| + failNext error |
| +} |
| + |
| +func newStopProbeActivator(order *[]string) *stopProbeActivator { |
| + return &stopProbeActivator{order: order, svc: &activator.Service{}} |
| +} |
| + |
| +func (a *stopProbeActivator) ProbeWake(context.Context, activator.ProbeConfig) (activator.ProbeResult, error) { |
| + return activator.ProbeResult{}, errors.New("not a probe") |
| +} |
| + |
| +func (a *stopProbeActivator) FenceAndStopAs(ctx context.Context, host string, generation uint64, owner string) (*activator.StopEvidence, error) { |
| + *a.order = append(*a.order, "fence-as") |
| + if err := a.failNext; err != nil { |
| + a.failNext = nil |
| + return nil, err |
| + } |
| + return a.svc.FenceAndStopAs(ctx, host, generation, owner) |
| +} |
| + |
| +func (a *stopProbeActivator) LiftFenceAs(host string, generation uint64, owner string) bool { |
| + *a.order = append(*a.order, "lift") |
| + return a.svc.LiftFenceAs(host, generation, owner) |
| +} |
| + |
| +// foreignFence installs another operation's plain fence on a.svc. |
| +func (a *stopProbeActivator) foreignFence(t *testing.T, host string, generation uint64) { |
| + t.Helper() |
| + ctx, cancel := context.WithTimeout(t.Context(), time.Second) |
| + defer cancel() |
| + if _, err := a.svc.FenceAndStop(ctx, host, generation); err != nil && !errors.Is(err, activator.ErrStopUnproven) { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +// stopProbeFixture is a target at relocating/probing whose probe passed |
| +// with session 12: the guard activated at the install stamp, main's |
| +// Wake.Stop standing for root's writer-session stop (it fences and |
| +// completes the stop in the guard journal with a receipt), and the probe |
| +// stop's provider bound to them. |
| +type stopProbeFixture struct { |
| + t *testing.T |
| + order []string |
| + act *stopProbeActivator |
| + a ClaimAssembly |
| + cfg RelocationAssemblyConfig |
| + obs hostclaim.Observed |
| + receipt []byte |
| + target hostguard.FenceTarget |
| +} |
| + |
| +func newStopProbeFixture(t *testing.T) *stopProbeFixture { |
| + t.Helper() |
| + ctx := t.Context() |
| + const host, box = "shop.example", "dst" |
| + created := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) |
| + inc, err := hostclaim.Incarnation("acct", "app", host, created) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + probe, _ := json.Marshal(hostclaim.Probe{Box: box, Session: 12, At: created.Add(time.Hour)}) |
| + f := &stopProbeFixture{t: t} |
| + f.obs = hostclaim.Observed{Host: host, ETag: "e1", Claim: &hostclaim.Claim{V: 2, Gen: 30, Host: host, |
| + Holder: &hostclaim.Holder{Family: "instance", Key: "apps/acct/app/instances/" + host + ".json", Account: "acct", App: "app", Incarnation: inc, CreatedAt: created, |
| + Location: "instances/" + host, Stamp: 9, Session: 12, Generation: "P1", WriterBox: box, Admissions: []hostclaim.AdmissionRecord{{Session: 12, Box: box, Generation: "P1", Mode: "cold", Region: "ie"}}}, |
| + Op: hostclaim.Op{Kind: hostclaim.KindRelocating, ID: "rel-1", Phase: hostclaim.PhaseRelocatingProbing, Exec: box, Participants: []string{"src", box}, Journal: map[string]string{"probe": string(probe)}}}} |
| + dataDir := t.TempDir() |
| + if err := os.Mkdir(filepath.Join(dataDir, "hostguard"), 0o770); err != nil { |
| + t.Fatal(err) |
| + } |
| + guard := hostguard.Guard{DataDir: dataDir, LockDir: t.TempDir(), Claims: &guardClaims{obs: f.obs}} |
| + if err := guard.Activate(ctx, host, box, nil); err != nil { |
| + t.Fatal(err) |
| + } |
| + f.target = hostguard.FenceTarget{FenceID: "fence-1", Region: "ie", Host: host, Incarnation: inc, Stamp: 9, Session: 12, Op: "rel-1", Phase: hostclaim.PhaseRelocatingProbing, |
| + ObligationID: hostguard.WriterStopObligation(12), Box: box, Enrollment: "enr", BootID: "boot", ControllerID: "ctl", ClaimETag: "e1", DirectoryRevision: "1"} |
| + at := created.Add(2 * time.Hour) |
| + f.receipt = mustMarshalStopReceipt(StopReceipt{V: 1, ID: "r1", Kind: StopReceiptStopped, Region: "ie", ClaimETag: "e1", ObligationID: f.target.ObligationID, FenceID: "fence-1", Host: host, Incarnation: inc, |
| + InstallationStamp: 9, WriterSession: 12, OperationID: "rel-1", Phase: hostclaim.PhaseRelocatingProbing, Box: box, Enrollment: "enr", BootID: "boot", ControllerID: "ctl", DirectoryRevision: "1", EvidenceAt: at}) |
| + wake := &ClaimWakeAssembly{Stop: func(ctx context.Context, o hostclaim.Observed) error { |
| + f.order = append(f.order, "wake-stop") |
| + // Root's writer-session stop: deterministic, so a repeat meets the |
| + // same, already completed, fence. |
| + if got, err := guard.CompletedWriterStops(ctx, host, hostguard.Captured{Incarnation: inc, Stamp: 9}, "rel-1", box); err == nil && len(got) == 1 { |
| + return nil |
| + } |
| + if _, err := guard.BeginFence(ctx, f.target); err != nil { |
| + return err |
| + } |
| + return guard.CompleteFence(ctx, f.target, f.receipt, at, func(context.Context, hostguard.FenceTarget, []byte, time.Time) error { return nil }) |
| + }} |
| + f.act = newStopProbeActivator(&f.order) |
| + f.a = ClaimAssembly{Guard: guard, Wake: wake, stopLocal: func(context.Context) (claimStopLocal, error) { |
| + return claimStopLocal{Actor: hostbox.Actor{Box: box, Enrollment: "enr", Region: "ie", Revision: 1}, BootID: "boot", ControllerID: "ctl", PID: 100, ProcessStart: "5000"}, nil |
| + }} |
| + f.cfg = RelocationAssemblyConfig{Activator: f.act, DataDir: dataDir} |
| + return f |
| +} |
| + |
| +func (f *stopProbeFixture) stop() (hostclaim.ProbeStop, error) { |
| + return f.a.relocationStopProbe(f.cfg)(f.t.Context(), f.obs) |
| +} |
| + |
| +// marker is the probe stop's persisted owner for rel-1. |
| +func (f *stopProbeFixture) marker() relocationProbeFence { |
| + f.t.Helper() |
| + path, err := relocationProbeFencePath(f.cfg.DataDir, "rel-1") |
| + if err != nil { |
| + f.t.Fatal(err) |
| + } |
| + rec, ok, err := readRelocationProbeFence(path) |
| + if err != nil || !ok { |
| + f.t.Fatalf("no probe-fence marker: %v", err) |
| + } |
| + return rec |
| +} |
| + |
| +// Ruling 49 steps 1 to 3 with ruling 75's owned fence, against the real |
| +// activator: the owner is persisted, then the fence goes in before root is |
| +// asked; an unproven stop (the probe's entry is gone) still asks root; |
| +// root's receipt is the guard journal's writer-session stop of the probe's |
| +// session; a repeat meets its own fence and returns the same ProbeStop; |
| +// another operation's fence and an uninstalled fence stop everything with |
| +// nothing asked of root and the other fence as it was. |
| +func TestRelocationStopProbeFencesStopsAndReadsRootsReceipt(t *testing.T) { |
| + sum := func(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) } |
| + const host = "shop.example" |
| + t.Run("the probe's session", func(t *testing.T) { |
| + f := newStopProbeFixture(t) |
| + s, err := f.stop() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if want := (hostclaim.ProbeStop{Box: "dst", Session: 12, Receipt: "fence-1", ReceiptSHA256: sum(f.receipt)}); s != want { |
| + t.Fatalf("stop %+v, want %+v", s, want) |
| + } |
| + if want := []string{"fence-as", "wake-stop"}; !slices.Equal(f.order, want) { |
| + t.Fatalf("order %v, want %v", f.order, want) |
| + } |
| + m := f.marker() |
| + if m.Host != host || m.Session != 12 || len(m.Owner) != 32 || f.act.svc.FenceGeneration(host) != 12 { |
| + t.Fatalf("marker %+v, fence %d", m, f.act.svc.FenceGeneration(host)) |
| + } |
| + // A repeat (the claim CAS lost, or a later pass) meets its own |
| + // fence under the same owner. |
| + f.order = nil |
| + again, err := f.stop() |
| + if err != nil || again != s { |
| + t.Fatalf("repeat: %+v %v", again, err) |
| + } |
| + if want := []string{"fence-as", "wake-stop"}; !slices.Equal(f.order, want) || f.marker() != m { |
| + t.Fatalf("repeat order %v, marker %+v", f.order, f.marker()) |
| + } |
| + }) |
| + for _, gen := range []uint64{5, 12, 20} { |
| + t.Run(fmt.Sprintf("another operation's fence at %d", gen), func(t *testing.T) { |
| + f := newStopProbeFixture(t) |
| + f.act.foreignFence(t, host, gen) |
| + if _, err := f.stop(); !errors.Is(err, activator.ErrFenceHeld) { |
| + t.Fatalf("got %v, want ErrFenceHeld", err) |
| + } |
| + if want := []string{"fence-as"}; !slices.Equal(f.order, want) || f.act.svc.FenceGeneration(host) != gen { |
| + t.Fatalf("root was asked or the fence changed: %v, fence %d", f.order, f.act.svc.FenceGeneration(host)) |
| + } |
| + }) |
| + } |
| + t.Run("no fence installed", func(t *testing.T) { |
| + f := newStopProbeFixture(t) |
| + f.act.failNext = activator.ErrFenceNotInstalled |
| + if _, err := f.stop(); !errors.Is(err, activator.ErrFenceNotInstalled) { |
| + t.Fatalf("got %v", err) |
| + } |
| + if want := []string{"fence-as"}; !slices.Equal(f.order, want) { |
| + t.Fatalf("root was asked: %v", f.order) |
| + } |
| + }) |
| + t.Run("not the probe's session", func(t *testing.T) { |
| + f := newStopProbeFixture(t) |
| + f.obs.Claim.Holder.Session = 13 |
| + if _, err := f.stop(); !errors.Is(err, hostbox.ErrChanged) || len(f.order) != 0 { |
| + t.Fatalf("%v %v", err, f.order) |
| + } |
| + }) |
| +} |
| + |
| +// Astra round 6, finding 1: the agent restarts after the probe stop |
| +// installed its fence and persisted its owner. The restart empties the |
| +// activator's fences; the marker survives. Another operation then fences |
| +// the host at a lower or equal generation. The surviving marker must not |
| +// let the probe stop take that fence over: ErrFenceHeld, the other fence |
| +// exactly as it was, root not asked. Once the other operation lifts its |
| +// fence, the next pass installs this probe's own fence and completes. |
| +func TestRelocationStopProbeAfterARestartNeverTakesAForeignFence(t *testing.T) { |
| + const host = "shop.example" |
| + for _, gen := range []uint64{5, 12} { |
| + t.Run(fmt.Sprint(gen), func(t *testing.T) { |
| + f := newStopProbeFixture(t) |
| + first, err := f.stop() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + before := f.marker() |
| + f.act.svc = &activator.Service{} // the restart |
| + f.act.foreignFence(t, host, gen) |
| + f.order = nil |
| + if _, err := f.stop(); !errors.Is(err, activator.ErrFenceHeld) { |
| + t.Fatalf("got %v, want ErrFenceHeld", err) |
| + } |
| + if want := []string{"fence-as"}; !slices.Equal(f.order, want) || f.act.svc.FenceGeneration(host) != gen || f.marker() != before { |
| + t.Fatalf("order %v, fence %d, marker %+v", f.order, f.act.svc.FenceGeneration(host), f.marker()) |
| + } |
| + if f.act.svc.LiftFenceAs(host, gen, before.Owner) { |
| + t.Fatal("the surviving owner lifted the other operation's fence") |
| + } |
| + if !f.act.svc.LiftFence(host, gen) { |
| + t.Fatal("the other operation's fence was not as it installed it") |
| + } |
| + f.order = nil |
| + again, err := f.stop() |
| + if err != nil || again != first || f.act.svc.FenceGeneration(host) != 12 { |
| + t.Fatalf("after the other fence went: %+v %v, fence %d", again, err, f.act.svc.FenceGeneration(host)) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// Astra round 6, finding 2: the owner is persisted before the activator is |
| +// asked, so a failed write installs nothing and blocks nothing. The |
| +// retry persists it and proceeds, under the owner it persisted. |
| +func TestRelocationStopProbeRetriesAFailedMarkerWrite(t *testing.T) { |
| + const host = "shop.example" |
| + f := newStopProbeFixture(t) |
| + path, err := relocationProbeFencePath(f.cfg.DataDir, "rel-1") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + // The op's journal directory cannot be made: a file stands in its way. |
| + if err := os.MkdirAll(filepath.Dir(filepath.Dir(path)), 0o700); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.WriteFile(filepath.Dir(path), []byte("in the way"), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := f.stop(); err == nil { |
| + t.Fatal("the probe stop went on without its owner persisted") |
| + } |
| + if len(f.order) != 0 || f.act.svc.Fenced(host) { |
| + t.Fatalf("the activator was asked before the owner was persisted: %v", f.order) |
| + } |
| + if err := os.Remove(filepath.Dir(path)); err != nil { |
| + t.Fatal(err) |
| + } |
| + s, err := f.stop() |
| + if err != nil || s.Session != 12 { |
| + t.Fatalf("retry: %+v %v", s, err) |
| + } |
| + if want := []string{"fence-as", "wake-stop"}; !slices.Equal(f.order, want) { |
| + t.Fatalf("retry order %v", f.order) |
| + } |
| + if m := f.marker(); !f.act.svc.LiftFenceAs(host, 12, m.Owner) { |
| + t.Fatal("the fence installed is not the persisted owner's") |
| + } |
| +} |
diff --git a/internal/hostbox/reconcile_relocation_target_test.go b/internal/hostbox/reconcile_relocation_target_test.go| new file mode 100644 |
| index 000000000..c8c9e4e57 |
| --- /dev/null |
| +++ b/internal/hostbox/reconcile_relocation_target_test.go |
| @@ -0,0 +1,200 @@ |
| +//go:build unix |
| + |
| +package hostbox |
| + |
| +import ( |
| + "context" |
| + "os" |
| + "path/filepath" |
| + "strconv" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + "github.com/carlosframework/platform/internal/hostguard" |
| + "github.com/carlosframework/platform/internal/hostwrite" |
| + "github.com/carlosframework/platform/internal/park" |
| + "github.com/carlosframework/platform/internal/provision" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// relocationReconcile is the box "box" as a relocation target: the |
| +// production adapter with the real census and park.Local, region HA's |
| +// relocation-aware reconciler (FenceClosed and HoldRow set), and the |
| +// target's row minted held by MintHeld at the install stamp. |
| +type relocationReconcile struct { |
| + f admissionFixture |
| + reg *registry.Registry |
| + adapter ReconcileAdapter |
| + cfg provision.ReconcileConfig |
| + captured hostguard.Captured |
| + holds []string |
| +} |
| + |
| +func newRelocationReconcile(t *testing.T) *relocationReconcile { |
| + t.Helper() |
| + ctx := t.Context() |
| + f := newAdmissionFixture(t, false) |
| + r := &relocationReconcile{f: f} |
| + var err error |
| + if r.reg, err = registry.Open(filepath.Join(t.TempDir(), "registry.db")); err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { r.reg.Close() }) |
| + inv, err := OpenStopInventoryDB(filepath.Join(t.TempDir(), "inventory.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { inv.Close() }) |
| + launches, err := OpenLaunchJournal(filepath.Join(t.TempDir(), "launches.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { launches.Close() }) |
| + census := &EmptyInstallationCensus{Inventory: inv, Registry: r.reg, Launches: launches, Box: "box", Enrollment: "en", DirectoryRevision: "1", |
| + Roots: map[string]string{"data": f.data, "opt": t.TempDir(), "units": t.TempDir()}, SharedPaths: map[string][]string{"data": {"hostguard"}}} |
| + guard := hostguard.Guard{DataDir: f.data, LockDir: t.TempDir(), Claims: hostguard.StoreReader{Store: f.store}} |
| + source := WorkSource{Guard: guard, Box: "box", Region: "ie", Now: time.Now, |
| + Policy: func(context.Context, *hostclaim.Claim, Selection, hostwrite.Identity) error { return nil }, |
| + Local: func(context.Context, *hostclaim.Claim, Selection) error { return nil }} |
| + r.adapter = ReconcileAdapter{Registry: r.reg, Inventory: inv, Census: census, Box: "box", Enrollment: "en", DirectoryRevision: "1", |
| + Reconciler: Reconciler{Claims: f.store, Box: "box", Region: "ie", Guard: guard, Writes: hostwrite.Config{Enabled: true, Selector: source}, |
| + Local: park.Local{Guard: guard, Claims: f.store, DataDir: f.data, Box: "box"}, |
| + Now: func() time.Time { return time.Unix(100, 0) }, CheckRecord: func(context.Context, RecordSelection) error { return nil }, CheckLocal: census.CheckLocal, |
| + FenceClosed: func(context.Context, hostclaim.Observed) (bool, error) { return false, nil }, |
| + HoldRow: func(_ context.Context, host, op string) error { |
| + r.holds = append(r.holds, op) |
| + row, err := r.reg.Get(host) |
| + if err != nil || row == nil { |
| + return ErrChanged |
| + } |
| + if row.HoldOp == op { |
| + return nil |
| + } |
| + return r.reg.SetHold(host, "relocating", op) |
| + }}} |
| + r.cfg = provision.ReconcileConfig{DataDir: f.data, Region: "ie"} |
| + r.cfg.Layout = func(context.Context, release.Store) (string, error) { return release.LayoutModeNested, nil } |
| + r.adapter.Configure(&r.cfg) |
| + // The installing CAS: this box executes rel-1 at the install stamp. |
| + o, err := hostclaim.Read(ctx, f.store, f.host) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if o, err = hostclaim.Update(ctx, f.store, o, func(c *hostclaim.Claim) error { |
| + c.Holder.Stamp = c.Gen + 1 |
| + c.Op = hostclaim.Op{Kind: hostclaim.KindRelocating, ID: "rel-1", Phase: hostclaim.PhaseInstalling, Exec: "box", Participants: []string{"src", "box"}, |
| + Intent: map[string]string{"region": "ie", "source_box": "src", "target_box": "box"}, Journal: map[string]string{"install_stamp": strconv.FormatUint(c.Holder.Stamp, 10)}} |
| + return nil |
| + }); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := guard.Activate(ctx, f.host, "box", nil); err != nil { |
| + t.Fatal(err) |
| + } |
| + r.captured = hostguard.Captured{Incarnation: o.Claim.Holder.Incarnation, Stamp: o.Claim.Holder.Stamp} |
| + record := provision.Record{Schema: 1, Host: f.host, Slug: "shop", App: "app", Account: f.account, Region: "ie", Channel: "stable", CreatedAt: time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC)} |
| + rec := provision.ClaimRecord{Host: f.host, Key: f.key, Family: "instance", Account: f.account, App: "app", CreatedAt: record.CreatedAt, Value: record, Route: provision.RouteFor(record, f.data, "", true)} |
| + if err := r.adapter.MintHeld(ctx, rec, r.captured, "relocating", "rel-1"); err != nil { |
| + t.Fatal(err) |
| + } |
| + return r |
| +} |
| + |
| +func (r *relocationReconcile) phase(t *testing.T, tr func(*hostclaim.Claim)) hostclaim.Observed { |
| + t.Helper() |
| + o, err := hostclaim.Read(t.Context(), r.f.store, r.f.host) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if o, err = hostclaim.Update(t.Context(), r.f.store, o, func(c *hostclaim.Claim) error { tr(c); return nil }); err != nil { |
| + t.Fatal(err) |
| + } |
| + return o |
| +} |
| + |
| +func targetGuardFiles(t *testing.T, data string) map[string]string { |
| + t.Helper() |
| + out := map[string]string{} |
| + entries, err := os.ReadDir(filepath.Join(data, "hostguard")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, e := range entries { |
| + raw, err := os.ReadFile(filepath.Join(data, "hostguard", e.Name())) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + out[e.Name()] = string(raw) |
| + } |
| + return out |
| +} |
| + |
| +// Ruling 31: under a relocating claim reconcile never activates the |
| +// target. The target's held row (minted by its install) is left to the |
| +// install at installing, installed and probing: stage A's check refuses |
| +// the executor's reconcile before any write (claim Plan 11 admits no |
| +// reconcile action under relocating), so the row, the guard record and |
| +// the claim are unchanged and no hold is written. |
| +func TestReconcileLeavesARelocatingTargetsRowToItsInstall(t *testing.T) { |
| + ctx := t.Context() |
| + r := newRelocationReconcile(t) |
| + before, err := r.reg.Get(r.f.host) |
| + if err != nil || before == nil { |
| + t.Fatal(err) |
| + } |
| + for _, phase := range []string{hostclaim.PhaseInstalling, hostclaim.PhaseInstalled, hostclaim.PhaseRelocatingProbing} { |
| + o := r.phase(t, func(c *hostclaim.Claim) { c.Op.Phase = phase }) |
| + guard := targetGuardFiles(t, r.f.data) |
| + r.holds = nil |
| + if err := r.f.pass(ctx, r.f.store, r.reg, r.cfg); err != nil { |
| + t.Fatalf("%s: %v", phase, err) |
| + } |
| + row, err := r.reg.Get(r.f.host) |
| + if err != nil || row == nil || *row != *before { |
| + t.Fatalf("%s: the row changed: %+v %v", phase, row, err) |
| + } |
| + if len(r.holds) != 0 { |
| + t.Fatalf("%s: a hold was written: %v", phase, r.holds) |
| + } |
| + if got := targetGuardFiles(t, r.f.data); len(got) != len(guard) || got[r.f.host+".json"] != guard[r.f.host+".json"] { |
| + t.Fatalf("%s: the guard record changed", phase) |
| + } |
| + after, err := hostclaim.Read(ctx, r.f.store, r.f.host) |
| + if err != nil || after.ETag != o.ETag { |
| + t.Fatalf("%s: the claim was written: %v", phase, err) |
| + } |
| + } |
| +} |
| + |
| +// Rev 1 finding 5: after completion the target serves at the install |
| +// stamp; relocationReleaseHold releases the row's hold (registry |
| +// ReleaseHold, under the op); reconcile's real adapter then finds the |
| +// installation's inventory coherent and keeps the row, ready and owned by |
| +// the same installation. |
| +func TestReconcileAdapterClearsTheHoldAfterCompletion(t *testing.T) { |
| + ctx := t.Context() |
| + r := newRelocationReconcile(t) |
| + minted, err := r.reg.Get(r.f.host) |
| + if err != nil || minted == nil { |
| + t.Fatal(err) |
| + } |
| + r.phase(t, func(c *hostclaim.Claim) { |
| + c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box", Participants: []string{"box"}} |
| + c.Holder.WriterBox = "box" |
| + }) |
| + if changed, err := r.reg.ReleaseHold(r.f.host, "rel-1"); err != nil || !changed { |
| + t.Fatalf("release hold: %v %v", changed, err) |
| + } |
| + if err := r.f.pass(ctx, r.f.store, r.reg, r.cfg); err != nil { |
| + t.Fatalf("the serving reconcile: %v", err) |
| + } |
| + row, err := r.reg.Get(r.f.host) |
| + if err != nil || row == nil || row.Status == "relocating" || row.HoldOp != "" || row.ClaimOwnership != minted.ClaimOwnership || !provision.NestedPaths(r.f.data, *row) { |
| + t.Fatalf("after completion: %+v %v (minted %+v)", row, err, minted.ClaimOwnership) |
| + } |
| + if len(r.holds) != 0 { |
| + t.Fatalf("a serving claim took the hold-only path: %v", r.holds) |
| + } |
| +} |