amadan
host: relocation reads main's participation owner and releases in Plan 11's order; hostclaim: a relocation completes only after its passed probe is stopped with root's receipt, and no operation starts from it before its target discharges it; activator: owned fences
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
28 files changed,
+2302
−2201
internal/activator/activator.go+4 −1internal/activator/fenceowner_test.go+112 −0internal/activator/maintenance.go+56 −4internal/host/claimcensus.go+5 −1internal/host/claimparticipation.go+27 −2internal/host/claimparticipation_relocation.go+43 −0internal/host/relocationassembly.go+190 −0internal/host/relocationcensus_test.go+88 −0internal/host/relocationcontrollers_linux_test.go+0 −357internal/host/relocationeffects.go+123 −677internal/host/relocationeffects_test.go+97 −326internal/host/relocationflush.go+36 −8internal/host/relocationflush_adoption_test.go+72 −0internal/host/relocationlifecycle_linux_test.go+0 −810internal/host/relocationparticipation_linux_test.go+255 −0internal/host/relocationsweep.go+136 −0internal/host/relocationsweep_test.go+155 −0internal/hostclaim/ops.go+5 −0internal/hostclaim/relocation.go+31 −1internal/hostclaim/relocation_probestop.go+139 −0internal/hostclaim/relocation_probestop_test.go+267 −0internal/hostclaim/relocation_test.go+9 −1internal/hostrelocate/participant.go+82 −6internal/hostrelocate/participant_gate_test.go+163 −0internal/hostrelocate/participant_test.go+45 −4internal/hostrelocate/worker.go+31 −3internal/hostrelocate/worker_bounds_test.go+129 −0internal/hostrelocate/worker_test.go+2 −0
diff --git a/internal/activator/activator.go b/internal/activator/activator.go| index b8725520c..027794350 100644 |
| --- a/internal/activator/activator.go |
| +++ b/internal/activator/activator.go |
| @@ -956,7 +956,10 @@ type Service struct { |
| // converges (recycleWorkerLocked), so the two share one pace. Under a.mu. |
| convergeBudget int |
| - fences map[string]uint64 |
| + fences map[string]uint64 |
| + // fenceOwners names the owner of an owned fence (FenceAndStopAs), by |
| + // host. A fence installed any other way has no entry. Under a.mu. |
| + fenceOwners map[string]string |
| tokenPreparations map[string]chan struct{} |
| stopHistory map[string]*stopObservation |
| // straySidecars are legacy sidecars whose termination is unproven, per |
diff --git a/internal/activator/fenceowner_test.go b/internal/activator/fenceowner_test.go| new file mode 100644 |
| index 000000000..540b8f3ea |
| --- /dev/null |
| +++ b/internal/activator/fenceowner_test.go |
| @@ -0,0 +1,112 @@ |
| +package activator |
| + |
| +import ( |
| + "errors" |
| + "fmt" |
| + "testing" |
| + "time" |
| +) |
| + |
| +// TestFenceAndStopAsMeetsOnlyItsOwnFence: an owned fence is the caller's |
| +// again only when it holds at the same generation with the same owner. |
| +// Any other fence (a plain one at a lower, equal or higher generation, or |
| +// another owner's at the same generation) refuses with ErrFenceHeld and |
| +// stays exactly as it was, with nothing stopped. |
| +func TestFenceAndStopAsMeetsOnlyItsOwnFence(t *testing.T) { |
| + t.Run("own fence, re-collected", func(t *testing.T) { |
| + a, e, _, _, _ := maintenanceAwake(t) |
| + proof, err := a.FenceAndStopAs(maintenanceContext(t, time.Second), e.host, 12, "owner-1") |
| + if err != nil || !proof.Matches(e.host, 12) { |
| + t.Fatalf("first install: %v", err) |
| + } |
| + again, err := a.FenceAndStopAs(maintenanceContext(t, time.Second), e.host, 12, "owner-1") |
| + if err != nil || !again.Matches(e.host, 12) { |
| + t.Fatalf("the owner's repeat: %v", err) |
| + } |
| + if g := a.FenceGeneration(e.host); g != 12 { |
| + t.Fatalf("generation %d, want 12", g) |
| + } |
| + }) |
| + type held struct { |
| + gen uint64 |
| + owner string |
| + } |
| + for _, h := range []held{{5, ""}, {12, ""}, {20, ""}, {12, "owner-2"}, {5, "owner-1"}} { |
| + t.Run(fmt.Sprintf("held %d by %q", h.gen, h.owner), func(t *testing.T) { |
| + a, e, main, _, _ := maintenanceAwake(t) |
| + a.mu.Lock() |
| + a.fences = map[string]uint64{e.host: h.gen} |
| + if h.owner != "" { |
| + a.fenceOwners = map[string]string{e.host: h.owner} |
| + } |
| + a.mu.Unlock() |
| + if _, err := a.FenceAndStopAs(maintenanceContext(t, time.Second), e.host, 12, "owner-1"); !errors.Is(err, ErrFenceHeld) { |
| + t.Fatalf("err = %v, want ErrFenceHeld", err) |
| + } |
| + a.mu.Lock() |
| + gen, owner := a.fences[e.host], a.fenceOwners[e.host] |
| + a.mu.Unlock() |
| + if gen != h.gen || owner != h.owner { |
| + t.Fatalf("held fence is %d by %q, want %d by %q", gen, owner, h.gen, h.owner) |
| + } |
| + if main.calls.Load() != 0 || !a.Owns(e.host) { |
| + t.Fatal("a refused owned fence stopped the host") |
| + } |
| + }) |
| + } |
| + t.Run("an empty owner", func(t *testing.T) { |
| + a, e, _, _, _ := maintenanceAwake(t) |
| + if _, err := a.FenceAndStopAs(maintenanceContext(t, time.Second), e.host, 12, ""); !errors.Is(err, ErrFenceGeneration) || a.Fenced(e.host) { |
| + t.Fatalf("err = %v, fenced %v", err, a.Fenced(e.host)) |
| + } |
| + }) |
| +} |
| + |
| +// TestAPlainFenceTakesOwnershipAway: FenceAndStop at the same generation |
| +// over an owned fence leaves a fence that is no longer the owner's, so |
| +// the owner can neither re-collect under it nor lift it. |
| +func TestAPlainFenceTakesOwnershipAway(t *testing.T) { |
| + a, e, _, _, _ := maintenanceAwake(t) |
| + if _, err := a.FenceAndStopAs(maintenanceContext(t, time.Second), e.host, 12, "owner-1"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := a.FenceAndStop(maintenanceContext(t, time.Second), e.host, 12); !errors.Is(err, ErrStopUnproven) && err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := a.FenceAndStopAs(maintenanceContext(t, time.Second), e.host, 12, "owner-1"); !errors.Is(err, ErrFenceHeld) { |
| + t.Fatalf("err = %v, want ErrFenceHeld", err) |
| + } |
| + if a.LiftFenceAs(e.host, 12, "owner-1") { |
| + t.Fatal("the owner lifted a fence it no longer owns") |
| + } |
| + if !a.Fenced(e.host) { |
| + t.Fatal("the plain fence was lifted") |
| + } |
| +} |
| + |
| +// TestLiftFenceAsLiftsOnlyTheOwnersFence: the lift names the generation |
| +// and the owner, and a plain LiftFence also clears the owner. |
| +func TestLiftFenceAsLiftsOnlyTheOwnersFence(t *testing.T) { |
| + a, e, _, _, _ := maintenanceAwake(t) |
| + if _, err := a.FenceAndStopAs(maintenanceContext(t, time.Second), e.host, 12, "owner-1"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if a.LiftFenceAs(e.host, 12, "owner-2") || a.LiftFenceAs(e.host, 11, "owner-1") || a.LiftFenceAs(e.host, 12, "") { |
| + t.Fatal("lifted with the wrong owner or generation") |
| + } |
| + if !a.LiftFenceAs(e.host, 12, "owner-1") || a.Fenced(e.host) { |
| + t.Fatal("the owner's lift did not lift its fence") |
| + } |
| + if _, err := a.FenceAndStopAs(maintenanceContext(t, time.Second), e.host, 13, "owner-1"); err != nil && !errors.Is(err, ErrStopUnproven) { |
| + t.Fatal(err) |
| + } |
| + if !a.LiftFence(e.host, 13) { |
| + t.Fatal("plain lift") |
| + } |
| + a.mu.Lock() |
| + _, left := a.fenceOwners[e.host] |
| + a.mu.Unlock() |
| + if left { |
| + t.Fatal("a plain lift left the owner behind") |
| + } |
| +} |
diff --git a/internal/activator/maintenance.go b/internal/activator/maintenance.go| index 3b6eb2e4a..b885f9e8b 100644 |
| --- a/internal/activator/maintenance.go |
| +++ b/internal/activator/maintenance.go |
| @@ -51,7 +51,7 @@ func (o *stopObservation) complete() bool { |
| // ErrStopUnproven guarantees this call installed a fence, but proves no stop. |
| // A missing entry is not evidence that a prior process or replica stopped. |
| func (a *Service) FenceAndStop(ctx context.Context, host string, generation uint64) (*StopEvidence, error) { |
| - return a.fenceAndStop(ctx, host, generation, false) |
| + return a.fenceAndStop(ctx, host, generation, false, "") |
| } |
| // FenceAndStopIfUnfenced is FenceAndStop for a caller that must never take |
| @@ -62,10 +62,28 @@ func (a *Service) FenceAndStop(ctx context.Context, host string, generation uint |
| // generation, it answers ErrFenceHeld, installs nothing and stops nothing. |
| // Otherwise it is FenceAndStop, answers included. |
| func (a *Service) FenceAndStopIfUnfenced(ctx context.Context, host string, generation uint64) (*StopEvidence, error) { |
| - return a.fenceAndStop(ctx, host, generation, true) |
| + return a.fenceAndStop(ctx, host, generation, true, "") |
| } |
| -func (a *Service) fenceAndStop(ctx context.Context, host string, generation uint64, exclusive bool) (*StopEvidence, error) { |
| +// FenceAndStopAs is FenceAndStopIfUnfenced for a caller that must be able |
| +// to meet its own fence again: a retry after a lost answer, or after the |
| +// caller's own later step failed. owner is the caller's persisted name for |
| +// this one fence. Under the lock that installs, a held fence is the |
| +// caller's only when it was installed by FenceAndStopAs with exactly this |
| +// owner at exactly this generation; then the call is FenceAndStop under it |
| +// (it re-collects the stop evidence). Any other held fence, of any |
| +// generation and however installed, answers ErrFenceHeld, installs nothing |
| +// and stops nothing. Fences live only in this process, so after a restart |
| +// no fence is the caller's, whatever it persisted. An empty owner is |
| +// ErrFenceGeneration. |
| +func (a *Service) FenceAndStopAs(ctx context.Context, host string, generation uint64, owner string) (*StopEvidence, error) { |
| + if owner == "" { |
| + return nil, ErrFenceGeneration |
| + } |
| + return a.fenceAndStop(ctx, host, generation, true, owner) |
| +} |
| + |
| +func (a *Service) fenceAndStop(ctx context.Context, host string, generation uint64, exclusive bool, owner string) (*StopEvidence, error) { |
| if a == nil || !controlname.ValidHostShape(host) || generation == 0 { |
| return nil, ErrFenceGeneration |
| } |
| @@ -100,17 +118,32 @@ func (a *Service) fenceAndStop(ctx context.Context, host string, generation uint |
| a.fences = make(map[string]uint64) |
| } |
| prior := a.fences[host] |
| + own := owner != "" && prior == generation && a.fenceOwners[host] == owner |
| // Only before this call installs: on a later round of the loop the |
| // fence held is this call's own. |
| - if exclusive && !installed && prior != 0 { |
| + if exclusive && !installed && prior != 0 && !own { |
| a.mu.Unlock() |
| return nil, ErrFenceHeld |
| } |
| + if owner != "" && installed && !own { |
| + // An owned fence replaced while this call waited is no longer |
| + // this call's: it installed one, and proves nothing more. |
| + a.mu.Unlock() |
| + return nil, ErrStopUnproven |
| + } |
| if prior > generation { |
| a.mu.Unlock() |
| return nil, ErrFenceGeneration |
| } |
| a.fences[host] = generation |
| + if owner != "" { |
| + if a.fenceOwners == nil { |
| + a.fenceOwners = make(map[string]string) |
| + } |
| + a.fenceOwners[host] = owner |
| + } else { |
| + delete(a.fenceOwners, host) |
| + } |
| installed = true |
| e := a.entries[host] |
| if e == nil { |
| @@ -165,6 +198,25 @@ func (a *Service) LiftFence(host string, generation uint64) bool { |
| return false |
| } |
| delete(a.fences, host) |
| + delete(a.fenceOwners, host) |
| + return true |
| +} |
| + |
| +// LiftFenceAs is LiftFence for an owned fence: it lifts host's fence only |
| +// when it is exactly the one FenceAndStopAs installed with owner at |
| +// generation, and reports whether it lifted anything. A fence another |
| +// operation installed at the same generation stays. |
| +func (a *Service) LiftFenceAs(host string, generation uint64, owner string) bool { |
| + if a == nil || generation == 0 || owner == "" { |
| + return false |
| + } |
| + a.mu.Lock() |
| + defer a.mu.Unlock() |
| + if a.fences[host] != generation || a.fenceOwners[host] != owner { |
| + return false |
| + } |
| + delete(a.fences, host) |
| + delete(a.fenceOwners, host) |
| return true |
| } |
diff --git a/internal/host/claimcensus.go b/internal/host/claimcensus.go| index 79a00c204..4d6bb8aa0 100644 |
| --- a/internal/host/claimcensus.go |
| +++ b/internal/host/claimcensus.go |
| @@ -45,7 +45,11 @@ func claimDataCensus(dataDir string) []string { |
| // census reads neither. The registry is bootstrap's file, shared by |
| // every host, plus SQLite's sidecars once something has opened it. |
| reg := claimCensusRel(dataDir, filepath.Join(dataDir, "registry.db")) |
| - return []string{".hostclaims", ".hostexec", "hostguard", hostbox.CensusFile(reg), hostbox.CensusFile(reg + "-wal"), hostbox.CensusFile(reg + "-shm")} |
| + // Region HA's relocation journals (N9) verify their own contents, like |
| + // the claim's: install and attempt journals, release positions and the |
| + // build's scratch, every one keyed by op id under relocate/ and swept |
| + // once its operation is over. No installation artifact lives there. |
| + return []string{".hostclaims", ".hostexec", "hostguard", "relocate", hostbox.CensusFile(reg), hostbox.CensusFile(reg + "-wal"), hostbox.CensusFile(reg + "-shm")} |
| } |
| func claimOptCensus(optDir string) []string { |
diff --git a/internal/host/claimparticipation.go b/internal/host/claimparticipation.go| index 963d5ed99..285cd3087 100644 |
| --- a/internal/host/claimparticipation.go |
| +++ b/internal/host/claimparticipation.go |
| @@ -42,6 +42,30 @@ type productionClaimLocal struct { |
| loopCancel context.CancelFunc |
| loopDone chan struct{} |
| stopIdentity *claimStopLocal |
| + // lockRoot is a test seam (region HA N12). Set, the participation's |
| + // hostguard and hostexec locks live under it and its controllers skip |
| + // the update-floor check (their UpdateGuardDir is empty; main's floor |
| + // tests cover it). Empty is production: /run/carlos's locks and the |
| + // floor under OptDir. |
| + lockRoot string |
| +} |
| + |
| +// lockDir is the named lock directory under lockRoot, or "" (the |
| +// package's default) when lockRoot is unset. |
| +func (p *productionClaimLocal) lockDir(name string) string { |
| + if p.lockRoot == "" { |
| + return "" |
| + } |
| + return filepath.Join(p.lockRoot, name) |
| +} |
| + |
| +// updateFloor is the controller's update-floor guard directory and |
| +// journal: production's under OptDir, none under the N12 test seam. |
| +func (p *productionClaimLocal) updateFloor() (string, string) { |
| + if p.lockRoot != "" { |
| + return "", "" |
| + } |
| + return filepath.Join(p.Resources.OptDir, "system", "guard"), filepath.Join(p.Resources.OptDir, "system", "router", "journal.json") |
| } |
| func (p *productionClaimLocal) bind(ctx context.Context, a *ClaimAssembly) error { |
| @@ -106,7 +130,7 @@ func (p *productionClaimLocal) prepareWith(ctx context.Context, actor hostbox.Ac |
| if err != nil { |
| return err |
| } |
| - return p.clearUntracked(ctx, actor, o, state, execlease.Config{Binding: state.Binding, Guard: hostguard.Guard{DataDir: p.Resources.DataDir, Claims: hostguard.StoreReader{Store: p.Store}}}) |
| + return p.clearUntracked(ctx, actor, o, state, execlease.Config{Binding: state.Binding, Guard: hostguard.Guard{DataDir: p.Resources.DataDir, LockDir: p.lockDir("hostguard"), Claims: hostguard.StoreReader{Store: p.Store}}, LockDir: p.lockDir("hostexec")}) |
| } |
| select { |
| case <-old.Ready: |
| @@ -179,7 +203,8 @@ func (p *productionClaimLocal) prepareWith(ctx context.Context, actor hostbox.Ac |
| p.slots[o.Host] = slot |
| p.mu.Unlock() |
| binding := execlease.Binding{Box: actor.Box, Enrollment: actor.Enrollment, Host: o.Host, Incarnation: c.Holder.Incarnation, Op: c.Op.ID, BootID: strings.TrimSpace(string(boot))} |
| - slot.Config = execlease.Config{Binding: binding, Store: p.Store, Guard: hostguard.Guard{DataDir: p.Resources.DataDir, Claims: hostguard.StoreReader{Store: p.Store}}, Directory: p.Directory, Clock: execlease.SystemClock{}, UpdateGuardDir: filepath.Join(p.Resources.OptDir, "system", "guard"), UpdateJournal: filepath.Join(p.Resources.OptDir, "system", "router", "journal.json"), |
| + floor, journal := p.updateFloor() |
| + slot.Config = execlease.Config{Binding: binding, Store: p.Store, Guard: hostguard.Guard{DataDir: p.Resources.DataDir, LockDir: p.lockDir("hostguard"), Claims: hostguard.StoreReader{Store: p.Store}}, Directory: p.Directory, Clock: execlease.SystemClock{}, UpdateGuardDir: floor, UpdateJournal: journal, LockDir: p.lockDir("hostexec"), |
| CloseAdmission: func() { p.cancelWork(o.Host) }, CancelWork: func() { p.cancelWork(o.Host) }, Cleanup: func(ctx context.Context) error { |
| err := p.cleanup(ctx, o) |
| slot.CleanupSucceeded = err == nil |
diff --git a/internal/host/claimparticipation_relocation.go b/internal/host/claimparticipation_relocation.go| new file mode 100644 |
| index 000000000..602c45b90 |
| --- /dev/null |
| +++ b/internal/host/claimparticipation_relocation.go |
| @@ -0,0 +1,43 @@ |
| +package host |
| + |
| +import ( |
| + "context" |
| + |
| + "github.com/carlosframework/platform/internal/hostbox" |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + execlease "github.com/carlosframework/platform/internal/hostexec" |
| +) |
| + |
| +// Acknowledged is region HA's read of main's participation for a |
| +// relocating operation (rev 2, ruling 42). It adds no authority: it is |
| +// Prepare, exactly as WorkSource.Prepare and the participation loop call |
| +// it, then the slot's own Ready. Whichever caller comes first opens the |
| +// box's one controller for {host, op}; every later caller gets that same |
| +// controller's state. A lost or failed controller is main's retained |
| +// denial (CC9), returned as its error and never replaced here. |
| +func (p *productionClaimLocal) Acknowledged(ctx context.Context, o hostclaim.Observed) (execlease.State, error) { |
| + if o.Claim == nil || o.Claim.Holder == nil || o.Claim.Op.Kind != hostclaim.KindRelocating { |
| + return execlease.State{}, hostbox.ErrChanged |
| + } |
| + if err := p.Prepare(ctx, o); err != nil { |
| + return execlease.State{}, err |
| + } |
| + p.mu.Lock() |
| + slot := p.slots[o.Host] |
| + p.mu.Unlock() |
| + if slot == nil { |
| + return execlease.State{}, execlease.ErrInactive |
| + } |
| + select { |
| + case <-slot.Ready: |
| + case <-ctx.Done(): |
| + return execlease.State{}, ctx.Err() |
| + } |
| + if slot.Err != nil { |
| + return execlease.State{}, slot.Err |
| + } |
| + if slot.Controller == nil { |
| + return execlease.State{}, execlease.ErrInactive |
| + } |
| + return slot.Controller.Ready(ctx) |
| +} |
diff --git a/internal/host/relocationassembly.go b/internal/host/relocationassembly.go| new file mode 100644 |
| index 000000000..9fcbd0708 |
| --- /dev/null |
| +++ b/internal/host/relocationassembly.go |
| @@ -0,0 +1,190 @@ |
| +package host |
| + |
| +import ( |
| + "context" |
| + "crypto/ecdh" |
| + "errors" |
| + "fmt" |
| + "path/filepath" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/activator" |
| + "github.com/carlosframework/platform/internal/controlsock" |
| + "github.com/carlosframework/platform/internal/genselect" |
| + "github.com/carlosframework/platform/internal/hostactors" |
| + "github.com/carlosframework/platform/internal/hostbox" |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + execctl "github.com/carlosframework/platform/internal/hostexec" |
| + "github.com/carlosframework/platform/internal/hostrelocate" |
| + "github.com/carlosframework/platform/internal/lease" |
| + "github.com/carlosframework/platform/internal/provision" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// RelocationActivator is the part of activator.Service relocation uses: |
| +// the probe attempt (Task 8d) and the local fence that ends a passed |
| +// probe's session before the serving CAS (ruling 49). The fence is an |
| +// owned one (ruling 75): taken and met again only under the owner the |
| +// probe stop persisted, and lifted only under it. |
| +type RelocationActivator interface { |
| + ProbeWake(context.Context, activator.ProbeConfig) (activator.ProbeResult, error) |
| + FenceAndStopAs(context.Context, string, uint64, string) (*activator.StopEvidence, error) |
| + LiftFenceAs(string, uint64, string) bool |
| +} |
| + |
| +var _ RelocationActivator = (*activator.Service)(nil) |
| + |
| +// RelocationAssemblyConfig is what the relocation providers need beyond |
| +// the bound claim assembly. Task 10 fills it once per agent. |
| +type RelocationAssemblyConfig struct { |
| + Directory hostactors.Directory |
| + // Data is the region's data bucket as a release.Store (files |
| + // manifests), built with release.NewS3Store (ruling 50); Versions is |
| + // the same bucket's versioned client (LTX, install manifests). |
| + Data release.Store |
| + Versions lease.VersionedStore |
| + Bucket string |
| + // Region is the logical placement region (genselect requests, pins, |
| + // the worker). LeaseRegion is the AWS region lease.Acquire signs for: |
| + // the roster's Cloud.Region, as main's Wake.LeaseRegion (ruling 51). |
| + Region, LeaseRegion string |
| + Registry *registry.Registry |
| + DataDir string // hostguard.Guard.DataDir, the managed data root |
| + WorkDir string // the build's scratch: <DataDir>/relocate/build |
| + ReleaseDir string // <DataDir>/relocate/release |
| + SecretsKey *ecdh.PrivateKey |
| + Activator RelocationActivator |
| + // Source is the bound assembly's WorkSource (ClaimProviders.Selector): |
| + // the install authority's guard, identity, Local and Prepare (ruling 46). |
| + Source hostbox.WorkSource |
| + // LaterProbeAttempts is set only once claim ask C11 is answered on |
| + // main: root then authorizes a relocating launch at the admitted |
| + // writer's stamp, not only at install_stamp (ruling 58). |
| + LaterProbeAttempts bool |
| + Now func() time.Time |
| +} |
| + |
| +// RelocationEffects binds every provider to this bound assembly. ack is |
| +// main's participation owner (productionClaimLocal.Acknowledged). |
| +func (a ClaimAssembly) RelocationEffects(cfg RelocationAssemblyConfig, ack func(context.Context, hostclaim.Observed) (execctl.State, error)) *RelocationEffects { |
| + if cfg.Now == nil { |
| + cfg.Now = time.Now |
| + } |
| + receipts := RelocationReleaseReceipts{Store: a.Claims, Directory: cfg.Directory} |
| + paths := relocationHostPaths(cfg.DataDir, cfg.Registry) |
| + return &RelocationEffects{ |
| + Participation: ack, |
| + Identity: func(ctx context.Context) (RelocationIdentity, error) { return a.localStopIdentity(ctx) }, |
| + Producers: func(ctx context.Context, o hostclaim.Observed) (StopDrainer, error) { |
| + if a.Wake == nil || a.Wake.Producers == nil { |
| + return nil, ErrClaimStopUnavailable |
| + } |
| + p, err := a.Wake.Producers(ctx, o) |
| + if err != nil { |
| + return nil, err |
| + } |
| + return p, nil |
| + }, |
| + Collect: func(ctx context.Context, r StopRequest, d StopDrainer) (StopResult, error) { |
| + if _, ok := d.(RecoveryReleaseDrain); ok { |
| + r.Recover = true // 6b's authenticated recovery path: root replays the durable receipt |
| + } |
| + if a.collectStop != nil { |
| + return a.collectStop(ctx, r, d) |
| + } |
| + return CollectClaimStop(ctx, controlsock.Path, r, d) |
| + }, |
| + WritersStopped: func(ctx context.Context, o hostclaim.Observed) (ClaimWritersDrained, WritersStoppedEvidence, error) { |
| + return a.WritersStopped(ctx, o.Host) |
| + }, |
| + Releases: ReleaseObligation{Receipts: receipts}, |
| + FlushDB: a.relocationFlushDB(cfg), |
| + FlushFiles: ReleaseFlushFiles{Data: cfg.Data, Registry: cfg.Registry, |
| + FilesDir: func(host string) string { return paths(host).FilesDir }, EnvPath: func(host string) string { return paths(host).EnvPath }}.Flush, |
| + Positions: ReleasePositionsFile{Dir: cfg.ReleaseDir}, |
| + PreflightKind: a.relocationPreflight(cfg), |
| + } |
| +} |
| + |
| +// relocationPreflight is ruling 48's gate before an attempt is counted: |
| +// root speaks the isolated launch protocol, and its capability for kind |
| +// relocating answers only with the attempt reader installed (Task 8d). |
| +func (a ClaimAssembly) relocationPreflight(cfg RelocationAssemblyConfig) func(context.Context, hostclaim.Observed) error { |
| + return func(ctx context.Context, o hostclaim.Observed) error { |
| + if a.Wake == nil || a.Wake.Roles == nil || o.Claim == nil { |
| + return activator.ErrClaimIsolationUnavailable |
| + } |
| + // Ruling 58: main's root pins every relocating launch to |
| + // install_stamp (claimLaunchAuthority), and a cold attempt after |
| + // P's opening runs at a new stamp. Until claim ask C11 relaxes the |
| + // pin to P's opening, such an attempt is not made. |
| + if hostclaim.PreparedOpened(o.Claim) && !cfg.LaterProbeAttempts { |
| + return hostrelocate.ErrLaterAttemptUnlaunchable |
| + } |
| + if err := a.Wake.Roles.Preflight(ctx, ClaimLaunchInputs{}); err != nil { |
| + return err |
| + } |
| + return a.Wake.Roles.PreflightKind(ctx, hostclaim.KindRelocating) |
| + } |
| +} |
| + |
| +// relocationPaths are one host's files directory and env file, derived |
| +// from its route by the layout the activator would classify it with |
| +// (design §3.1: a nested row's files live in its instance directory with |
| +// the same names). Every relocation target route is nested (ruling 45). |
| +type relocationPaths struct{ FilesDir, EnvPath string } |
| + |
| +func relocationPathsFor(dataDir string, rt registry.Route) relocationPaths { |
| + dir := dataDir |
| + if layout, _ := activator.Classify(rt, activator.LayoutInputs{DataDir: dataDir}); layout == activator.LayoutNested { |
| + dir = provision.InstanceDir(dataDir, rt.Host) |
| + } |
| + return relocationPaths{FilesDir: filepath.Join(dir, rt.Host), EnvPath: filepath.Join(dir, rt.Host+".env")} |
| +} |
| + |
| +// relocationHostPaths reads the host's own row: the source's flush |
| +// publishes the paths its writer used. No row is no paths, which the |
| +// flush refuses rather than guess. |
| +func relocationHostPaths(dataDir string, reg *registry.Registry) func(string) relocationPaths { |
| + return func(host string) relocationPaths { |
| + if reg == nil { |
| + return relocationPaths{} |
| + } |
| + rt, err := reg.Get(host) |
| + if err != nil || rt == nil { |
| + return relocationPaths{} |
| + } |
| + return relocationPathsFor(dataDir, *rt) |
| + } |
| +} |
| + |
| +// relocationFlushDB is release step 2 (ruling 8): SyncAndWait with every |
| +// writer stopped, then N is the newest level-0 MaxTXID, which must equal |
| +// the plan's MaxTXID. A listing with no LTX file is the stopped writer |
| +// vouching that it never committed: N is 0. |
| +func (a ClaimAssembly) relocationFlushDB(cfg RelocationAssemblyConfig) func(context.Context, hostclaim.Observed) (uint64, error) { |
| + return func(ctx context.Context, o hostclaim.Observed) (uint64, error) { |
| + if a.Wake == nil || o.Claim == nil || o.Claim.Holder == nil { |
| + return 0, ErrClaimStopUnavailable |
| + } |
| + att := a.Wake.get(o.Host) |
| + if att == nil || att.rep == nil || cfg.Versions == nil || cfg.Bucket == "" { |
| + return 0, ErrClaimStopUnavailable |
| + } |
| + if err := att.rep.SyncAndWait(ctx); err != nil { |
| + return 0, err |
| + } |
| + plan, err := genselect.Select(ctx, cfg.Versions, genselect.Request{Bucket: cfg.Bucket, Region: cfg.Region, Host: o.Host, Holder: *o.Claim.Holder}) |
| + if errors.Is(err, genselect.ErrEmpty) { |
| + return 0, nil |
| + } |
| + if err != nil { |
| + return 0, err |
| + } |
| + if plan.MaxLevel0TXID != plan.MaxTXID { |
| + return 0, fmt.Errorf("%w: level 0 at %d, the database at %d", ErrClaimStopUnavailable, plan.MaxLevel0TXID, plan.MaxTXID) |
| + } |
| + return plan.MaxTXID, nil |
| + } |
| +} |
diff --git a/internal/host/relocationcensus_test.go b/internal/host/relocationcensus_test.go| new file mode 100644 |
| index 000000000..d8ed9ea66 |
| --- /dev/null |
| +++ b/internal/host/relocationcensus_test.go |
| @@ -0,0 +1,88 @@ |
| +//go:build unix |
| + |
| +package host |
| + |
| +import ( |
| + "errors" |
| + "os" |
| + "path/filepath" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/hostbox" |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + "github.com/carlosframework/platform/internal/provision" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// Astra round 2 finding 8, ruling 61: an empty installation initialises with |
| +// a previous relocation's journals present under relocate/ (N9 classifies |
| +// the directory as the claim classifies its own journals); without the |
| +// classification the same tree refuses, which is the defect. |
| +func TestCensusInitializesWithOldRelocationJournals(t *testing.T) { |
| + ctx := t.Context() |
| + r := claimCensusBox(t) |
| + reg, err := registry.Open(filepath.Join(r.DataDir, "registry.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + defer reg.Close() |
| + inventory, err := hostbox.OpenStopInventoryDB(filepath.Join(r.DataDir, ".hostclaims", "inventory.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + defer inventory.Close() |
| + launches, err := hostbox.OpenLaunchJournal(filepath.Join(r.DataDir, ".hostclaims", "launches.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + defer launches.Close() |
| + for path, body := range map[string]string{ |
| + "relocate/ops/op-old/install.json": `{"v":1,"host":"old.example","op":"op-old"}`, |
| + "relocate/ops/op-old/attempts.json": `{"v":1,"host":"old.example","op":"op-old"}`, |
| + "relocate/release/op-old.json": `{"v":1,"request":{"Host":"old.example"}}`, |
| + "relocate/build/op-old/old.example.db": "scratch", |
| + } { |
| + full := filepath.Join(r.DataDir, filepath.FromSlash(path)) |
| + if err := os.MkdirAll(filepath.Dir(full), 0o700); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.WriteFile(full, []byte(body), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + store := release.NewDirStore(t.TempDir()) |
| + created := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) |
| + record := provision.Record{Schema: 1, Host: "shop.example", Slug: "shop", App: "app", Account: "acct", Region: "ie", Channel: "stable", CreatedAt: created} |
| + if err := provision.CreateRecord(ctx, store, record); err != nil { |
| + t.Fatal(err) |
| + } |
| + key := provision.RecordKey(record.Account, record.App, record.Host) |
| + obs, err := hostclaim.Read(ctx, store, record.Host) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + obs, err = hostclaim.Update(ctx, store, obs, hostclaim.Adopt(hostclaim.NewHolder{Family: "instance", Key: key, Account: record.Account, App: record.App, Host: record.Host, Region: record.Region, CreatedAt: created}, "box", nil, nil, "adopt", func() string { return "id" })) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + selected := hostbox.RecordSelection{Host: record.Host, Incarnation: obs.Claim.Holder.Incarnation} |
| + census := hostbox.EmptyInstallationCensus{Inventory: inventory, Registry: reg, Launches: launches, Box: "box", Enrollment: "enrollment", DirectoryRevision: "1", |
| + Roots: map[string]string{"data": r.DataDir, "opt": r.OptDir, "units": r.UnitsDir}, SharedPaths: claimGlobalSharedPaths(r)} |
| + unclassified := census |
| + unclassified.SharedPaths = map[string][]string{} |
| + for root, entries := range claimGlobalSharedPaths(r) { |
| + for _, e := range entries { |
| + if e != "relocate" { |
| + unclassified.SharedPaths[root] = append(unclassified.SharedPaths[root], e) |
| + } |
| + } |
| + } |
| + if err := unclassified.Initialize(ctx, obs, selected); !errors.Is(err, hostbox.ErrTypedInventory) { |
| + t.Fatalf("without N9 the journals were accepted: %v", err) |
| + } |
| + if err := census.Initialize(ctx, obs, selected); err != nil { |
| + t.Fatalf("initialisation with old relocation journals: %v", err) |
| + } |
| +} |
diff --git a/internal/host/relocationcontrollers_linux_test.go b/internal/host/relocationcontrollers_linux_test.go| deleted file mode 100644 |
| index 89028d43f..000000000 |
| --- a/internal/host/relocationcontrollers_linux_test.go |
| +++ /dev/null |
| @@ -1,357 +0,0 @@ |
| -//go:build linux |
| - |
| -package host |
| - |
| -import ( |
| - "context" |
| - "encoding/json" |
| - "errors" |
| - "os" |
| - "path/filepath" |
| - "strings" |
| - "testing" |
| - "time" |
| - |
| - "github.com/carlosframework/platform/internal/hostbox" |
| - "github.com/carlosframework/platform/internal/hostclaim" |
| - execctl "github.com/carlosframework/platform/internal/hostexec" |
| - "github.com/carlosframework/platform/internal/hostguard" |
| - "github.com/carlosframework/platform/internal/hostquarantine" |
| - "github.com/carlosframework/platform/internal/release" |
| -) |
| - |
| -type activeDirectory struct{} |
| - |
| -func (activeDirectory) Lookup(context.Context, string) (hostquarantine.Actor, error) { |
| - return hostquarantine.Actor{Enrollment: strings.Repeat("a", 32), State: "active"}, nil |
| -} |
| - |
| -// A real hostexec controller on disk: after the relocation completes, |
| -// Finish closes it and clears its durable record, so root's §5.10 check |
| -// (hostexecParticipation) lets an executable update through. While the |
| -// operation still runs, the clear refuses and the record stays. A cleanup |
| -// that failed keeps the record however often Close is called afterwards |
| -// (hostexec returns the failure once), and across a restart, until a |
| -// repeated cleanup succeeds. |
| -func TestFinishClearsTheDurableRecordSoUpdatesAreEligible(t *testing.T) { |
| - ctx := context.Background() |
| - dataDir := t.TempDir() |
| - for _, name := range []string{"hostguard", ".hostexec", "locks", "exec-locks"} { |
| - if err := os.Mkdir(filepath.Join(dataDir, name), 0o770); err != nil { |
| - t.Fatal(err) |
| - } |
| - } |
| - store := release.NewDirStore(t.TempDir()) |
| - const host = "shop.bab.oncarlos.com" |
| - created := time.Date(2026, 10, 4, 1, 0, 0, 0, time.UTC) |
| - nh := hostclaim.NewHolder{Family: "instance", Key: "apps/bab/jam/instances/" + host + ".json", Account: "bab", App: "jam", Host: host, Region: "ie", RecordRegion: "ie", CreatedAt: created} |
| - body := []byte(`{"host":"` + host + `","account":"bab","app":"jam","region":"ie","created_at":"2026-10-04T01:00:00Z"}`) |
| - obs, err := hostclaim.Read(ctx, store, host) |
| - if err != nil { |
| - t.Fatal(err) |
| - } |
| - step := func(o hostclaim.Observed, tr func(*hostclaim.Claim) error) hostclaim.Observed { |
| - t.Helper() |
| - next, err := hostclaim.Update(ctx, store, o, tr) |
| - if err != nil { |
| - t.Fatal(err) |
| - } |
| - return next |
| - } |
| - obs = step(obs, hostclaim.BeginCreateOn(nh, body, "c1", "console", created, hostclaim.ExecutorSelection{Box: "src", Enrollment: strings.Repeat("a", 32)})) |
| - obs = step(obs, hostclaim.CompleteCreate("c1", "src")) |
| - obs = step(obs, hostclaim.AdmitWriter(hostclaim.Admission{Box: "src", Mode: "cold", NewGeneration: "g1", Region: "ie", KeepStamp: true})) |
| - obs = step(obs, func(c *hostclaim.Claim) error { |
| - c.Op = hostclaim.Op{Kind: hostclaim.KindRelocating, ID: "rel-1", Phase: hostclaim.PhaseStarting, Exec: "src", Participants: []string{"src", "dst"}, |
| - Intent: map[string]string{"region": "ie", "source_box": "src", "target_box": "dst"}} |
| - return nil |
| - }) |
| - failing := false |
| - scoped := func(context.Context) error { |
| - if failing { |
| - return errors.New("cleanup: the partial install would not go") |
| - } |
| - return nil |
| - } |
| - cfg := execctl.Config{Store: store, Guard: hostguard.Guard{DataDir: dataDir, LockDir: filepath.Join(dataDir, "locks"), Claims: hostguard.StoreReader{Store: store}}, |
| - Directory: activeDirectory{}, Clock: execctl.SystemClock{}, LockDir: filepath.Join(dataDir, "exec-locks"), |
| - CloseAdmission: func() {}, CancelWork: func() {}, Cleanup: scoped} |
| - journal := filepath.Join(dataDir, "relocate", "participations") |
| - controllers := func() *RelocationControllers { |
| - return &RelocationControllers{Journal: journal, DataDir: dataDir, |
| - Cleanup: func(ctx context.Context, _ execctl.Binding) error { return scoped(ctx) }, |
| - Clear: func(ctx context.Context, b execctl.Binding) error { |
| - c := cfg |
| - c.Binding = b |
| - return execctl.ClearStale(ctx, c) |
| - }} |
| - } |
| - e, _, _, _ := effectsFixture(t) |
| - e.Controllers = controllers() |
| - e.Identity = func(context.Context) (RelocationIdentity, error) { |
| - return RelocationIdentity{Actor: hostbox.Actor{Box: "src", Enrollment: strings.Repeat("a", 32), Revision: 4}, BootID: "boot", ControllerID: "ctl"}, nil |
| - } |
| - e.Controller = func(ctx context.Context, b execctl.Binding) (RelocationController, error) { |
| - c := cfg |
| - c.Binding = b |
| - return OpenRelocationController(ctx, c) |
| - } |
| - if _, err := e.Acknowledge(ctx, obs); err != nil { |
| - t.Fatal(err) |
| - } |
| - if err := hostexecParticipation(dataDir); err == nil { |
| - t.Fatal("a participant's record did not block an update") |
| - } |
| - // The controller's cleanup fails: Close reports it, the record stays. |
| - failing = true |
| - if err := e.Finish(ctx, host, "rel-1"); err == nil { |
| - t.Fatal("a failed cleanup was not reported") |
| - } |
| - if err := hostexecParticipation(dataDir); err == nil || len(kept(e)) != 1 { |
| - t.Fatalf("a failed cleanup cleared the record: %v retained=%v", err, kept(e)) |
| - } |
| - // Still relocating: the clear refuses, the record and the entry stay. |
| - failing = false |
| - if err := e.Finish(ctx, host, "rel-1"); !errors.Is(err, execctl.ErrInactive) { |
| - t.Fatalf("finish during the operation: %v", err) |
| - } |
| - if err := hostexecParticipation(dataDir); err == nil || len(kept(e)) != 1 { |
| - t.Fatalf("the record was cleared while the operation ran: %v retained=%v", err, kept(e)) |
| - } |
| - // Served on the target. The cleanup fails again: hostexec's Close now |
| - // returns nil, and the next sweep must still not clear. |
| - step(obs, func(c *hostclaim.Claim) error { |
| - c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "dst", Participants: []string{"dst"}} |
| - return nil |
| - }) |
| - failing = true |
| - if err := e.Finish(ctx, host, "rel-1"); err == nil { |
| - t.Fatal("a sweep after a failed cleanup reported success") |
| - } |
| - if err := hostexecParticipation(dataDir); err == nil || len(kept(e)) != 1 { |
| - t.Fatalf("a nil Close cleared the record: %v retained=%v", err, kept(e)) |
| - } |
| - // The agent restarts with the cleanup still failing: nothing clears. |
| - restarted := controllers() |
| - if err := restarted.Finish(ctx, host, "rel-1"); err == nil { |
| - t.Fatal("a restart cleared without a successful cleanup") |
| - } |
| - if err := hostexecParticipation(dataDir); err == nil || len(kept(restarted)) != 1 { |
| - t.Fatalf("a restart cleared the record: %v retained=%v", err, kept(restarted)) |
| - } |
| - // The cleanup succeeds: the retry clears the record and the journal. |
| - failing = false |
| - if err := restarted.Finish(ctx, host, "rel-1"); err != nil { |
| - t.Fatal(err) |
| - } |
| - if err := hostexecParticipation(dataDir); err != nil { |
| - t.Fatalf("after the relocation an update is still refused: %v", err) |
| - } |
| - if len(kept(restarted)) != 0 { |
| - t.Fatalf("retained: %v", kept(restarted)) |
| - } |
| -} |
| - |
| -// Round 12 finding 1: relocation A completes and B starts before this box |
| -// sweeps A. A's real controller still holds the host's lifetime lock; |
| -// Acknowledge for B retires it first (hostexec refuses to clear A's record |
| -// while B lists this box, so B's Open replaces it), under the pass's |
| -// bounded context, and B's controller opens. A lock held by a controller |
| -// this box does not retain is waited for no longer than the pass. |
| -func TestASuccessorRelocationRetiresItsPredecessorsController(t *testing.T) { |
| - ctx := context.Background() |
| - dataDir := t.TempDir() |
| - for _, name := range []string{"hostguard", ".hostexec", "locks", "exec-locks"} { |
| - if err := os.Mkdir(filepath.Join(dataDir, name), 0o770); err != nil { |
| - t.Fatal(err) |
| - } |
| - } |
| - store := release.NewDirStore(t.TempDir()) |
| - const host = "shop.bab.oncarlos.com" |
| - created := time.Date(2026, 10, 4, 1, 0, 0, 0, time.UTC) |
| - nh := hostclaim.NewHolder{Family: "instance", Key: "apps/bab/jam/instances/" + host + ".json", Account: "bab", App: "jam", Host: host, Region: "ie", RecordRegion: "ie", CreatedAt: created} |
| - body := []byte(`{"host":"` + host + `","account":"bab","app":"jam","region":"ie","created_at":"2026-10-04T01:00:00Z"}`) |
| - obs, err := hostclaim.Read(ctx, store, host) |
| - if err != nil { |
| - t.Fatal(err) |
| - } |
| - step := func(o hostclaim.Observed, tr func(*hostclaim.Claim) error) hostclaim.Observed { |
| - t.Helper() |
| - next, err := hostclaim.Update(ctx, store, o, tr) |
| - if err != nil { |
| - t.Fatal(err) |
| - } |
| - return next |
| - } |
| - relocating := func(id, from, to string) func(*hostclaim.Claim) error { |
| - return func(c *hostclaim.Claim) error { |
| - c.Op = hostclaim.Op{Kind: hostclaim.KindRelocating, ID: id, Phase: hostclaim.PhaseStarting, Exec: from, Participants: []string{from, to}, |
| - Intent: map[string]string{"region": "ie", "source_box": from, "target_box": to}} |
| - return nil |
| - } |
| - } |
| - obs = step(obs, hostclaim.BeginCreateOn(nh, body, "c1", "console", created, hostclaim.ExecutorSelection{Box: "src", Enrollment: strings.Repeat("a", 32)})) |
| - obs = step(obs, hostclaim.CompleteCreate("c1", "src")) |
| - obs = step(obs, hostclaim.AdmitWriter(hostclaim.Admission{Box: "src", Mode: "cold", NewGeneration: "g1", Region: "ie", KeepStamp: true})) |
| - obs = step(obs, relocating("rel-1", "src", "dst")) |
| - cfg := execctl.Config{Store: store, Guard: hostguard.Guard{DataDir: dataDir, LockDir: filepath.Join(dataDir, "locks"), Claims: hostguard.StoreReader{Store: store}}, |
| - Directory: activeDirectory{}, Clock: execctl.SystemClock{}, LockDir: filepath.Join(dataDir, "exec-locks"), |
| - CloseAdmission: func() {}, CancelWork: func() {}, Cleanup: func(context.Context) error { return nil }} |
| - lifetime, stop := context.WithCancel(ctx) |
| - t.Cleanup(stop) |
| - e, _, _, _ := effectsFixture(t) |
| - e.Lifetime = lifetime |
| - e.Controllers = &RelocationControllers{Journal: filepath.Join(dataDir, "relocate", "participations"), DataDir: dataDir, |
| - Cleanup: func(context.Context, execctl.Binding) error { return nil }, |
| - Clear: func(ctx context.Context, b execctl.Binding) error { |
| - c := cfg |
| - c.Binding = b |
| - return execctl.ClearStale(ctx, c) |
| - }} |
| - e.Identity = func(context.Context) (RelocationIdentity, error) { |
| - return RelocationIdentity{Actor: hostbox.Actor{Box: "src", Enrollment: strings.Repeat("a", 32), Revision: 4}, BootID: "boot", ControllerID: "ctl"}, nil |
| - } |
| - e.Controller = func(ctx context.Context, b execctl.Binding) (RelocationController, error) { |
| - c := cfg |
| - c.Binding = b |
| - return OpenRelocationController(ctx, c) |
| - } |
| - if _, err := e.Acknowledge(ctx, obs); err != nil { |
| - t.Fatal(err) |
| - } |
| - // A completes on dst, and B (dst back to src) starts before this box's |
| - // sweep has finished A. |
| - obs = step(obs, func(c *hostclaim.Claim) error { |
| - c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "dst", Participants: []string{"dst"}} |
| - return nil |
| - }) |
| - obs = step(obs, relocating("rel-2", "dst", "src")) |
| - pass, cancel := context.WithTimeout(ctx, 10*time.Second) |
| - defer cancel() |
| - ack, err := e.Acknowledge(pass, obs) |
| - if err != nil { |
| - t.Fatalf("the successor's acknowledgement: %v", err) |
| - } |
| - if ack.Op != "rel-2" { |
| - t.Fatalf("ack: %+v", ack) |
| - } |
| - // A's journal entry stays until a sweep verifies the record is B's |
| - // (rev 14), then it goes; B's stays. |
| - if err := e.Finish(pass, host, "rel-1"); err != nil { |
| - t.Fatalf("sweeping A after the handover: %v", err) |
| - } |
| - got, err := e.Retained() |
| - if err != nil || len(got) != 1 || got[0].Op != "rel-2" { |
| - t.Fatalf("retained after the handover: %+v %v", got, err) |
| - } |
| - if err := hostexecParticipation(dataDir); err == nil { |
| - t.Fatal("B's record did not block an update") |
| - } |
| - |
| - // A lock held by a controller this box does not retain (here, another |
| - // operation's opened directly): the next open waits no longer than the |
| - // pass, and opens nothing. |
| - if err := e.Controllers.Close(ctx); err != nil { |
| - t.Fatal(err) |
| - } |
| - obs = step(obs, relocating("rel-3", "dst", "src")) |
| - foreign := cfg |
| - foreign.Binding = execctl.Binding{Box: "src", Enrollment: strings.Repeat("a", 32), Host: host, Incarnation: obs.Claim.Holder.Incarnation, Op: "rel-3", BootID: "boot"} |
| - held, err := execctl.Open(lifetime, foreign) |
| - if err != nil { |
| - t.Fatal(err) |
| - } |
| - defer held.Close(ctx) |
| - other := *e |
| - other.Controllers = &RelocationControllers{Journal: filepath.Join(dataDir, "relocate", "other"), Cleanup: func(context.Context, execctl.Binding) error { return nil }} |
| - short, cancelShort := context.WithTimeout(ctx, 300*time.Millisecond) |
| - defer cancelShort() |
| - began := time.Now() |
| - if _, err := other.Acknowledge(short, obs); err == nil { |
| - t.Fatal("opened while another controller held the host") |
| - } |
| - if waited := time.Since(began); waited > 5*time.Second { |
| - t.Fatalf("the open waited %v, past the pass's bound", waited) |
| - } |
| - if got, _ := other.Controllers.Retained(); len(got) != 1 || got[0].Op != "rel-3" { |
| - t.Fatalf("the bounded open retained a controller: %+v", got) // only the journal entry |
| - } |
| - other.Controllers.mu.Lock() |
| - n := len(other.Controllers.open) |
| - other.Controllers.mu.Unlock() |
| - if n != 0 { |
| - t.Fatal("a controller was kept from a cancelled open") |
| - } |
| -} |
| - |
| -// Round 14: a failed clear forgets a participation only on hostexec's own |
| -// validated reading of the record. A parseable record missing its |
| -// enrollment or incarnation is one hostexec refuses, so it is never |
| -// "another binding": the participation is kept, across a restart, until |
| -// the record is repaired and a cleanup and clear succeed. |
| -func TestARecordHostexecRefusesKeepsItsParticipation(t *testing.T) { |
| - ctx := context.Background() |
| - const host = "shop.bab.oncarlos.com" |
| - b := execctl.Binding{Box: "src", Enrollment: strings.Repeat("a", 32), Host: host, Incarnation: "inc", Op: "rel-1", BootID: "boot", ControllerID: "ctl"} |
| - valid := execctl.State{V: 1, Binding: b, Expires: time.Date(2026, 10, 7, 9, 0, 0, 0, time.UTC), SafeUntil: 1, State: "lost"} |
| - for _, missing := range []string{"enrollment", "incarnation"} { |
| - t.Run(missing, func(t *testing.T) { |
| - dataDir := t.TempDir() |
| - if err := os.Mkdir(filepath.Join(dataDir, ".hostexec"), 0o770); err != nil { |
| - t.Fatal(err) |
| - } |
| - path := filepath.Join(dataDir, ".hostexec", host+".json") |
| - write := func(st execctl.State, drop string) { |
| - t.Helper() |
| - raw, _ := json.Marshal(st) |
| - var m map[string]any |
| - _ = json.Unmarshal(raw, &m) |
| - delete(m, drop) |
| - raw, _ = json.Marshal(m) |
| - if err := os.WriteFile(path, raw, 0o660); err != nil { |
| - t.Fatal(err) |
| - } |
| - } |
| - write(valid, missing) |
| - cleared, cleanups := 0, 0 |
| - clearErr := execctl.ErrState // ClearStale refuses the invalid record |
| - controllers := func() *RelocationControllers { |
| - return &RelocationControllers{Journal: filepath.Join(dataDir, "relocate", "participations"), DataDir: dataDir, |
| - Cleanup: func(context.Context, execctl.Binding) error { cleanups++; return nil }, |
| - Clear: func(context.Context, execctl.Binding) error { |
| - if clearErr != nil { |
| - return clearErr |
| - } |
| - cleared++ |
| - return os.Remove(path) |
| - }} |
| - } |
| - r := controllers() |
| - if err := r.record(host, "rel-1", b); err != nil { |
| - t.Fatal(err) |
| - } |
| - if err := r.Finish(ctx, host, "rel-1"); err == nil { |
| - t.Fatal("a record hostexec refuses was taken as another binding") |
| - } |
| - if got := kept(&RelocationEffects{Controllers: r}); len(got) != 1 { |
| - t.Fatalf("the participation was forgotten: %+v", got) |
| - } |
| - // A restart: the journal still names it, and still nothing clears. |
| - r = controllers() |
| - if err := r.Finish(ctx, host, "rel-1"); err == nil || len(kept(&RelocationEffects{Controllers: r})) != 1 { |
| - t.Fatalf("after a restart: %v", err) |
| - } |
| - // Repaired: the record is valid and this binding's; the cleanup |
| - // and the clear run, and only then is the participation gone. |
| - write(valid, "") |
| - clearErr = nil |
| - before := cleanups |
| - if err := r.Finish(ctx, host, "rel-1"); err != nil || cleared != 1 || cleanups != before+1 { |
| - t.Fatalf("after the repair: %v cleared=%d cleanups=%d", err, cleared, cleanups-before) |
| - } |
| - if got := kept(&RelocationEffects{Controllers: r}); len(got) != 0 { |
| - t.Fatalf("retained after the clear: %+v", got) |
| - } |
| - }) |
| - } |
| -} |
diff --git a/internal/host/relocationeffects.go b/internal/host/relocationeffects.go| index 32901efc5..0be97f348 100644 |
| --- a/internal/host/relocationeffects.go |
| +++ b/internal/host/relocationeffects.go |
| @@ -3,13 +3,9 @@ package host |
| import ( |
| "context" |
| "encoding/json" |
| - "errors" |
| "fmt" |
| "os" |
| "path/filepath" |
| - "sort" |
| - "strings" |
| - "sync" |
| "github.com/carlosframework/platform/internal/hostclaim" |
| execctl "github.com/carlosframework/platform/internal/hostexec" |
| @@ -24,376 +20,6 @@ import ( |
| // claimLaunchOwner.local rule. |
| type RelocationIdentity = claimStopLocal |
| -// RelocationController is the execution controller as this package uses |
| -// it: hostexec.Controller's Ready and Close, and Clear, which removes the |
| -// controller's durable local record once its cleanup is done |
| -// (hostexec.ClearStale with the controller's config): Close leaves the |
| -// record in "lost", and under §5.10 any record blocks an executable |
| -// update until it is cleared. |
| -type RelocationController interface { |
| - Ready(context.Context) (execctl.State, error) |
| - Close(context.Context) error |
| - Clear(context.Context) error |
| - // Lost is closed once the controller has lost its lease for good |
| - // (hostexec latches the first failure: a failed renewal, a lost |
| - // authority, a clock fault). |
| - Lost() <-chan struct{} |
| -} |
| - |
| -// participation is one journaled participation whose controller may have |
| -// left a durable record: written before the controller is opened |
| -// (Acknowledge), removed only after the record is cleared (Finish). A |
| -// crash anywhere in between leaves the entry, so a restarted agent finds |
| -// every record it may have left, including one from a controller that |
| -// never reached its first Finish or whose Open failed after writing. |
| -type participation struct { |
| - V int `json:"v"` |
| - Host string `json:"host"` |
| - Op string `json:"op"` |
| - Binding execctl.Binding `json:"binding"` |
| -} |
| - |
| -// RelocationControllers retains the execution controllers this agent has |
| -// opened for its participations, one per {host, op}, for the agent's |
| -// lifetime. A controller is never bound to a pass's context (claim D6: |
| -// the lease lives as long as the participation). A restarted agent has |
| -// none and opens a fresh one; the claim keeps the first acknowledgement. |
| -type RelocationControllers struct { |
| - mu sync.Mutex |
| - open map[string]RelocationController |
| - // Journal is where participations are recorded before their |
| - // controller opens (<DataDir>/relocate/participations); Finish |
| - // retries a journaled one after a restart, when no controller object |
| - // exists, through the binding it recorded. Required: Acknowledge |
| - // opens nothing without it. |
| - Journal string |
| - // Cleanup is the participation's scoped cleanup for one binding |
| - // (Task 8a's relocationCleanup: it retires only that operation's own |
| - // partial installation of that host and incarnation, after checking |
| - // the binding's box and enrollment are this box's verified actor, and |
| - // does nothing once this box has admitted a writer under the |
| - // operation or serves the host). Finish repeats it, and needs it to |
| - // succeed, in the same call as every clear: hostexec.Controller.Close |
| - // returns a cleanup failure once and nil on every later call, and a |
| - // restarted agent has no controller at all, so neither a nil Close |
| - // nor a journal entry proves the cleanup ran. Nil clears nothing. |
| - Cleanup func(context.Context, execctl.Binding) error |
| - // Clear removes a journaled binding's durable record after a restart |
| - // (hostexec.ClearStale with the assembly's config for the binding). |
| - Clear func(context.Context, execctl.Binding) error |
| - // DataDir is hostexec's data directory (Guard.DataDir), whose |
| - // .hostexec/<host>.json is the durable record. When a clear fails, |
| - // the participation is forgotten only if that record is verifiably |
| - // not this binding's: absent, or naming another binding (rev 14). |
| - // Empty: never verifiable, so a failed clear always keeps it. |
| - DataDir string |
| - // bindings remembers each open controller's binding. |
| - bindings map[string]execctl.Binding |
| -} |
| - |
| -func controllerKey(host, op string) string { return host + "\x00" + op } |
| - |
| -func (r *RelocationControllers) get(host, op string) (RelocationController, bool) { |
| - r.mu.Lock() |
| - defer r.mu.Unlock() |
| - c, ok := r.open[controllerKey(host, op)] |
| - return c, ok |
| -} |
| - |
| -func (r *RelocationControllers) put(host, op string, b execctl.Binding, c RelocationController) (RelocationController, bool) { |
| - r.mu.Lock() |
| - defer r.mu.Unlock() |
| - if r.open == nil { |
| - r.open, r.bindings = map[string]RelocationController{}, map[string]execctl.Binding{} |
| - } |
| - if prior, ok := r.open[controllerKey(host, op)]; ok { |
| - return prior, false |
| - } |
| - r.open[controllerKey(host, op)], r.bindings[controllerKey(host, op)] = c, b |
| - return c, true |
| -} |
| - |
| -const journalSuffix = ".json" |
| - |
| -func (r *RelocationControllers) journalPath(host, op string) string { |
| - return filepath.Join(r.Journal, host+"~"+op+journalSuffix) |
| -} |
| - |
| -// record journals a participation before its controller opens. |
| -func (r *RelocationControllers) record(host, op string, b execctl.Binding) error { |
| - if r.Journal == "" { |
| - return ErrClaimStopUnavailable |
| - } |
| - raw, err := json.Marshal(participation{V: 1, Host: host, Op: op, Binding: b}) |
| - if err != nil { |
| - return err |
| - } |
| - if err := os.MkdirAll(r.Journal, 0o700); err != nil { |
| - return err |
| - } |
| - return writeFileSynced(r.journalPath(host, op), raw, 0o600) |
| -} |
| - |
| -// journaled lists the committed participations on disk. Only names |
| -// ending in .json are committed records: writeFileSynced's .json.tmp is |
| -// an uncommitted write a crash may have truncated, and is ignored (its |
| -// participation opened nothing, because the record is written before the |
| -// controller opens). A committed record that cannot be read is reported, |
| -// joined into the error, and every readable one is still returned, so |
| -// one damaged entry never hides the others' cleanup. |
| -func (r *RelocationControllers) journaled() ([]participation, error) { |
| - if r.Journal == "" { |
| - return nil, nil |
| - } |
| - entries, err := os.ReadDir(r.Journal) |
| - if errors.Is(err, os.ErrNotExist) { |
| - return nil, nil |
| - } |
| - if err != nil { |
| - return nil, err |
| - } |
| - var out []participation |
| - var errs []error |
| - for _, e := range entries { |
| - if e.IsDir() || !strings.HasSuffix(e.Name(), journalSuffix) { |
| - continue |
| - } |
| - raw, err := os.ReadFile(filepath.Join(r.Journal, e.Name())) |
| - if err != nil { |
| - errs = append(errs, err) |
| - continue |
| - } |
| - var f participation |
| - if err := json.Unmarshal(raw, &f); err != nil || f.V != 1 || f.Host == "" || f.Op == "" { |
| - errs = append(errs, fmt.Errorf("%w: participation journal %s", ErrClaimStopUnavailable, e.Name())) |
| - continue |
| - } |
| - out = append(out, f) |
| - } |
| - return out, errors.Join(errs...) |
| -} |
| - |
| -// Retained lists the participations whose controllers are open, and the |
| -// journaled ones whose durable record may still need clearing. A damaged |
| -// journal entry is returned as the error beside every readable one. |
| -func (r *RelocationControllers) Retained() ([]hostrelocate.Participation, error) { |
| - r.mu.Lock() |
| - seen := map[string]bool{} |
| - var out []hostrelocate.Participation |
| - for key := range r.open { |
| - host, op, _ := strings.Cut(key, "\x00") |
| - seen[key] = true |
| - out = append(out, hostrelocate.Participation{Host: host, Op: op}) |
| - } |
| - r.mu.Unlock() |
| - journaled, err := r.journaled() |
| - for _, f := range journaled { |
| - if !seen[controllerKey(f.Host, f.Op)] { |
| - seen[controllerKey(f.Host, f.Op)] = true |
| - out = append(out, hostrelocate.Participation{Host: f.Host, Op: f.Op}) |
| - } |
| - } |
| - sort.Slice(out, func(i, j int) bool { |
| - return out[i].Host < out[j].Host || (out[i].Host == out[j].Host && out[i].Op < out[j].Op) |
| - }) |
| - return out, err |
| -} |
| - |
| -// Finish closes and forgets one participation's controller, releasing the |
| -// host's lifetime lock for the next operation, and clears its durable |
| -// record. Closing runs hostexec's Cleanup hook: the relocation's |
| -// controller is opened with a cleanup scoped to the relocation's own |
| -// partial work (Task 8a), never the serving writer's session, so |
| -// finishing after a successful relocation stops nothing that serves. |
| -// |
| -// The entry stays until Close succeeds: hostexec retains the lifetime |
| -// lock on a timed-out join and needs another Close, so a controller whose |
| -// close the pass's context cut short is found by the next sweep. Then the |
| -// scoped cleanup is repeated (Cleanup) and the record is cleared only if |
| -// that cleanup succeeded in this call: a nil Close after an earlier |
| -// failed one, or a restart with no controller, proves nothing. The |
| -// journal entry goes only once the record is gone, so a restart in |
| -// between retries the cleanup and the clear through the recorded binding. |
| -func (r *RelocationControllers) Finish(ctx context.Context, host, op string) error { |
| - return r.finish(ctx, host, op, false) |
| -} |
| - |
| -// errSuperseded is finish's answer, for a successor's handover, when the |
| -// predecessor's controller is closed and cleaned up but hostexec refuses |
| -// to clear its record because the claim's running operation lists this |
| -// box (hostexec.ClearStale, controller.go: "a fresh successor operation |
| -// may exclude this box"; here it includes it). The lifetime lock is free, |
| -// and the successor's Open replaces the record under it. |
| -var errSuperseded = errors.New("relocation: the predecessor's record passes to its successor") |
| - |
| -func (r *RelocationControllers) finish(ctx context.Context, host, op string, successor bool) error { |
| - key := controllerKey(host, op) |
| - r.mu.Lock() |
| - c, ok := r.open[key] |
| - b := r.bindings[key] |
| - r.mu.Unlock() |
| - if !ok { |
| - // No controller object: a restart, or a crash before the first |
| - // Finish. Retry through the journaled binding. |
| - journaled, err := r.journaled() |
| - found := false |
| - for _, f := range journaled { |
| - if f.Host == host && f.Op == op { |
| - found, b = true, f.Binding |
| - } |
| - } |
| - if !found { |
| - return err // nil: nothing journaled; else the damaged entry may be this one |
| - } |
| - } |
| - if r.Cleanup == nil || (!ok && r.Clear == nil) { |
| - return ErrClaimStopUnavailable |
| - } |
| - if ok { |
| - if err := c.Close(ctx); err != nil { |
| - return err // cleanup failed or was cut short: the record stays, and so does the entry |
| - } |
| - } |
| - if err := r.Cleanup(ctx, b); err != nil { |
| - return err // the record stays: no clear without a cleanup that succeeded now |
| - } |
| - clear := func() error { return r.Clear(ctx, b) } |
| - if ok { |
| - clear = func() error { return c.Clear(ctx) } |
| - } |
| - if !ok { |
| - // No controller object: when the record is verifiably not this |
| - // binding's (gone, or a successor's that holds the host's lock), |
| - // there is nothing to clear, and ClearStale would only wait out |
| - // the pass on the successor's lock. |
| - if mine, verr := r.recordIsMine(host, b); verr == nil && !mine { |
| - return r.forget(host, op) |
| - } |
| - } |
| - superseded := false |
| - if err := clear(); err != nil { |
| - // Whatever the error, the participation goes only if the record |
| - // itself shows it is not this binding's (round 13 finding 2: |
| - // ClearStale's ErrState also means an absent claim or an invalid |
| - // local record, and those leave this binding's record in place). |
| - mine, verr := r.recordIsMine(host, b) |
| - switch { |
| - case verr != nil: |
| - return errors.Join(err, verr) |
| - case !mine: |
| - // Verified: no record, or another binding's (a successor |
| - // opened over it). |
| - case successor && errors.Is(err, execctl.ErrInactive): |
| - superseded = true // the lock is free; the journal stays until the successor opens |
| - default: |
| - return err |
| - } |
| - } |
| - r.mu.Lock() |
| - if ok && r.open[key] == c { |
| - delete(r.open, key) |
| - delete(r.bindings, key) |
| - } |
| - r.mu.Unlock() |
| - if superseded { |
| - return errSuperseded |
| - } |
| - return r.forget(host, op) |
| -} |
| - |
| -// recordIsMine reads hostexec's durable record for host through |
| -// hostexec's own validated reader (main's InspectLocal) and reports whether it is |
| -// b's binding (box, enrollment, host, incarnation, operation: what |
| -// ClearStale matches). Only an absent record, or a valid record of another |
| -// binding, is "not mine"; a record hostexec would refuse (one missing its |
| -// enrollment or incarnation, say) is an error, and the participation is |
| -// kept (round 14). |
| -func (r *RelocationControllers) recordIsMine(host string, b execctl.Binding) (bool, error) { |
| - if r.DataDir == "" { |
| - return false, fmt.Errorf("%w: no hostexec data directory to verify the record", ErrClaimStopUnavailable) |
| - } |
| - st, err := execctl.InspectLocal(r.DataDir, host) |
| - if errors.Is(err, os.ErrNotExist) { |
| - return false, nil |
| - } |
| - if err != nil { |
| - return false, fmt.Errorf("hostexec record for %s: %w", host, err) |
| - } |
| - return st.Box == b.Box && st.Enrollment == b.Enrollment && st.Incarnation == b.Incarnation && st.Op == b.Op, nil |
| -} |
| - |
| -// live returns the open controller for {host, op}, retiring it first if it |
| -// has lost its lease (round 13 finding 1): hostexec latches the first |
| -// failure, so a lost controller never becomes ready again. Retiring joins |
| -// it (Close, bounded by ctx), repeats the scoped cleanup, and forgets the |
| -// object but keeps the journal: the record on disk stays "lost" (durable |
| -// denial) until a fresh controller, opened with fresh authority and its |
| -// own lease, replaces it. ok false: none open (or just retired). |
| -func (r *RelocationControllers) live(ctx context.Context, host, op string) (RelocationController, bool, error) { |
| - key := controllerKey(host, op) |
| - r.mu.Lock() |
| - c, ok := r.open[key] |
| - b := r.bindings[key] |
| - r.mu.Unlock() |
| - if !ok { |
| - return nil, false, nil |
| - } |
| - select { |
| - case <-c.Lost(): |
| - default: |
| - return c, true, nil |
| - } |
| - if r.Cleanup == nil { |
| - return nil, false, ErrClaimStopUnavailable |
| - } |
| - if err := c.Close(ctx); err != nil { |
| - return nil, false, err // joined later; the entry stays |
| - } |
| - if err := r.Cleanup(ctx, b); err != nil { |
| - return nil, false, err // the entry stays: the next pass repeats both |
| - } |
| - r.mu.Lock() |
| - if r.open[key] == c { |
| - delete(r.open, key) |
| - delete(r.bindings, key) |
| - } |
| - r.mu.Unlock() |
| - return nil, false, nil |
| -} |
| - |
| -// forget removes one participation's journal entry. |
| -func (r *RelocationControllers) forget(host, op string) error { |
| - if r.Journal != "" { |
| - if err := os.Remove(r.journalPath(host, op)); err != nil && !errors.Is(err, os.ErrNotExist) { |
| - return err |
| - } |
| - } |
| - return nil |
| -} |
| - |
| -// handover retires every other participation of host this box still holds |
| -// before op's controller opens (round 12 finding 1): hostexec keeps one |
| -// controller per host, and a predecessor whose sweep has not yet run still |
| -// holds the host's lifetime lock, so the successor's Open would wait on it |
| -// for the whole pass. Each predecessor is closed and its scoped cleanup |
| -// repeated under the pass's context (bounded). One whose record hostexec |
| -// will not clear because the successor lists this box (errSuperseded) |
| -// keeps its journal entry: the successor's Open replaces the record under |
| -// the freed lock, and a later sweep forgets the entry once the record is |
| -// verifiably the successor's (rev 14). |
| -func (r *RelocationControllers) handover(ctx context.Context, host, op string) error { |
| - retained, _ := r.Retained() // a damaged entry is Sweep's to report |
| - for _, p := range retained { |
| - if p.Host != host || p.Op == op { |
| - continue |
| - } |
| - if err := r.finish(ctx, host, p.Op, true); err != nil && !errors.Is(err, errSuperseded) { |
| - return fmt.Errorf("relocation: retiring %s's controller for %s before %s: %w", p.Op, host, op, err) |
| - } |
| - } |
| - return nil |
| -} |
| - |
| // writeFileSynced writes path whole: a synced temporary file renamed into |
| // place, then the directory synced, so a committed entry survives a crash. |
| func writeFileSynced(path string, body []byte, mode os.FileMode) error { |
| @@ -421,21 +47,6 @@ func writeFileSynced(path string, body []byte, mode os.FileMode) error { |
| return dir.Sync() |
| } |
| -// Close closes every retained controller (agent shutdown). |
| -func (r *RelocationControllers) Close(ctx context.Context) error { |
| - r.mu.Lock() |
| - open := r.open |
| - r.open = nil |
| - r.mu.Unlock() |
| - var first error |
| - for _, c := range open { |
| - if err := c.Close(ctx); err != nil && first == nil { |
| - first = err |
| - } |
| - } |
| - return first |
| -} |
| - |
| // ReleasePosition is one recorded release: the exact stop request, the |
| // process that flushed it (PID and start time, beside the request's boot |
| // and controller), and the positions its flush reached. |
| @@ -472,174 +83,82 @@ type ReleasePositions interface { |
| Recorded(ctx context.Context, r StopRequest) (recorded ReleasePosition, ok bool, err error) |
| } |
| -// RelocationEffects is hostrelocate.Effects over providers Plan 6b's box |
| -// assembly supplies. It owns only the order of §5.8's release sequence and |
| -// the shape of each record; every provider acts under the host guard. |
| +// RelocationEffects is hostrelocate.Effects over providers the claim |
| +// assembly supplies (rev 2). It owns only the order of §5.8's release |
| +// sequence and the shape of each record; every provider acts under the |
| +// host guard. The execution controller is main's: one participation owner |
| +// per box (ruling 42), opened by productionClaimLocal for every non-serving |
| +// operation that lists this box, whichever of the claim's participation |
| +// loop, WorkSource.Prepare or this pass asks first. |
| type RelocationEffects struct { |
| - // Lifetime is the agent's context: controllers are opened under it, |
| - // never under a pass's. |
| - Lifetime context.Context |
| - Controllers *RelocationControllers |
| - Identity func(context.Context) (RelocationIdentity, error) |
| - // Controller is Plan 8's execution controller for one binding |
| - // (hostexec.Open with the assembly's CloseAdmission, CancelWork and |
| - // Cleanup). It writes pending state, checks root's update-begun marker, |
| - // and takes the execution lease (§5.10, claim D6). |
| - Controller func(context.Context, execctl.Binding) (RelocationController, error) |
| + // Participation is main's participation owner for the observed |
| + // operation (productionClaimLocal.Acknowledged): it opens or reuses |
| + // the box's one hostexec controller for {host, op} and returns its |
| + // ready state. A lost lease stays main's denial (CC9). |
| + Participation func(context.Context, hostclaim.Observed) (execctl.State, error) |
| + Identity func(context.Context) (RelocationIdentity, error) |
| // Producers is the admitted session's drain (ClaimWakeAssembly.Producers). |
| Producers func(context.Context, hostclaim.Observed) (StopDrainer, error) |
| - // Collect asks root for full stop evidence (Plan 6b's CollectClaimStop |
| - // over the instance-control socket). |
| + // Collect asks root for full stop evidence (CollectClaimStop over the |
| + // instance-control socket); a RecoveryReleaseDrain is submitted as a |
| + // recovery request. |
| Collect func(context.Context, StopRequest, StopDrainer) (StopResult, error) |
| - // WritersStopped is release step 1, action and observation (§5.8, |
| - // A4-1), as claim Plan 11 built it: main's ClaimAssembly.WritersStopped |
| - // for the claim's host, which runs the admitted attempt's StopWriters |
| - // (close admission, join launches and deliveries, empty the |
| - // containment, settle spawns, keeping the replicator and the files |
| - // publisher) and then asks root for its own writers-stopped |
| - // observation (R3: root's complete descriptor scan, the agent's |
| - // replicator descriptors only a narrowing hint). The witness and the |
| - // evidence must both name the session being released. |
| + // WritersStopped is release step 1, action and observation: main's |
| + // ClaimAssembly.WritersStopped for the claim's host. |
| WritersStopped func(context.Context, hostclaim.Observed) (ClaimWritersDrained, WritersStoppedEvidence, error) |
| - // Releases is main's relocation-release obligation (claim Plan 11): |
| - // queued in a claim CAS before root's stop is requested, since root |
| - // refuses a relocation-release stop the claim does not owe, and |
| - // discharged only by root's exact receipt (RelocationReleaseReceipts) |
| - // before the release is returned. ReleaseObligation is production's. |
| + // Releases is main's relocation-release obligation (claim Plan 11). |
| Releases RelocationReleaseDischarge |
| - // FlushDB is release step 2: final checkpoint and SyncAndWait, then the |
| - // stopped database's final TXID N, returned only once the newest |
| - // level-0 file's MaxTXID is N. A sync error is returned, never swallowed. |
| - FlushDB func(context.Context, hostclaim.Observed) (uint64, error) |
| - // FlushFiles is release step 3: publish every pending file, the wake |
| - // state and the env file into the session's generation, whatever |
| - // files_durability says (ReleaseFlushFiles), and return the manifest's |
| - // final sequence. |
| + // FlushDB and FlushFiles are release steps 2 and 3. |
| + FlushDB func(context.Context, hostclaim.Observed) (uint64, error) |
| FlushFiles func(context.Context, hostclaim.Observed) (int64, error) |
| // Positions is the durable record of the flushed positions. |
| Positions ReleasePositions |
| - // BuildP, Install and Admit are the target's (§5.8): moves' shared |
| - // install routine for P, the local install from P's files manifest, |
| - // and one cold probe admission followed by the probe. |
| - BuildP func(context.Context, hostclaim.Observed) error |
| - Install func(context.Context, hostclaim.Observed) (hostclaim.LocalInstall, error) |
| - // VerifyLocal reports whether the local install the receipt names is |
| - // complete on this box now: the files directory as P's prepared |
| - // manifest lists it, the env file and the wake state, under the |
| - // receipt's install_stamp (Task 8c's production body). |
| + // BuildP, Install, VerifyLocal, Receipt, Admit: Tasks 8b to 8d. |
| + BuildP func(context.Context, hostclaim.Observed) error |
| + Install func(context.Context, hostclaim.Observed) (hostclaim.LocalInstall, error) |
| VerifyLocal func(context.Context, hostclaim.Observed, hostclaim.LocalInstall) (bool, error) |
| - // Receipt reads this box's durable local install receipt for the |
| - // claim's host (Task 8c: the install journal written under its |
| - // install_stamp); false when there is none. |
| - Receipt func(context.Context, hostclaim.Observed) (hostclaim.LocalInstall, bool, error) |
| - Admit func(context.Context, hostclaim.Observed, bool) (uint64, bool, error) |
| + Receipt func(context.Context, hostclaim.Observed) (hostclaim.LocalInstall, bool, error) |
| + Admit func(context.Context, hostclaim.Observed, bool) (uint64, bool, error) |
| + // PreflightKind is root's isolated-launch capability for kind |
| + // relocating (ClaimRoleAssembly.Preflight then PreflightKind; ruling 48). |
| + PreflightKind func(context.Context, hostclaim.Observed) error |
| + // StopProbe is ruling 49's stop of the passed probe session (Task 9). |
| + StopProbe func(context.Context, hostclaim.Observed) (hostclaim.ProbeStop, error) |
| + // ReleaseHold releases this box's row hold for a completed operation it |
| + // serves, and lifts the activator fence StopProbe installed (Task 9). |
| + ReleaseHold func(ctx context.Context, host, op string) error |
| } |
| var _ hostrelocate.Effects = (*RelocationEffects)(nil) |
| -// Acknowledge opens this box's execution controller for the operation |
| -// under the agent's lifetime, retains it, and returns the lease it took. |
| -// A controller already open for the participation is reused. |
| +// Acknowledge returns the lease of main's participation controller for the |
| +// operation (ruling 42). The claim keeps the first acknowledgement; a |
| +// restarted agent's later controller is main's to open. |
| func (e *RelocationEffects) Acknowledge(ctx context.Context, obs hostclaim.Observed) (hostclaim.ExecutionAcknowledgement, error) { |
| - if e.Identity == nil || e.Controller == nil || e.Controllers == nil || e.Lifetime == nil || obs.Claim == nil || obs.Claim.Holder == nil { |
| + if e.Participation == nil || obs.Claim == nil || obs.Claim.Holder == nil { |
| return hostclaim.ExecutionAcknowledgement{}, ErrClaimStopUnavailable |
| } |
| - c := obs.Claim |
| - ctl, ok, err := e.Controllers.live(ctx, c.Host, c.Op.ID) |
| + st, err := e.Participation(ctx, obs) |
| if err != nil { |
| return hostclaim.ExecutionAcknowledgement{}, err |
| } |
| - if !ok { |
| - id, err := e.Identity(ctx) |
| - if err != nil { |
| - return hostclaim.ExecutionAcknowledgement{}, err |
| - } |
| - b := execctl.Binding{Box: id.Actor.Box, Enrollment: id.Actor.Enrollment, Host: c.Host, Incarnation: c.Holder.Incarnation, Op: c.Op.ID, BootID: id.BootID} |
| - // Journaled before the open: hostexec writes its durable record |
| - // inside Open, so a crash or a failed Open leaves a record only |
| - // the journal lets a restarted agent find and clear. |
| - if err := e.Controllers.record(c.Host, c.Op.ID, b); err != nil { |
| - return hostclaim.ExecutionAcknowledgement{}, err |
| - } |
| - // A predecessor this box has not swept yet still holds the host's |
| - // lifetime lock: retire it first, under this pass's context. |
| - if err := e.Controllers.handover(ctx, c.Host, c.Op.ID); err != nil { |
| - return hostclaim.ExecutionAcknowledgement{}, err |
| - } |
| - opened, err := openBounded(ctx, e.Lifetime, func(octx context.Context) (RelocationController, error) { return e.Controller(octx, b) }) |
| - if err != nil { |
| - return hostclaim.ExecutionAcknowledgement{}, err |
| - } |
| - if ctl, ok = e.Controllers.put(c.Host, c.Op.ID, b, opened); !ok { |
| - _ = opened.Close(ctx) // another pass opened one first |
| - } |
| - } |
| - st, err := ctl.Ready(ctx) |
| - if err != nil { |
| - return hostclaim.ExecutionAcknowledgement{}, err |
| - } |
| - return hostclaim.ExecutionAcknowledgement{Op: c.Op.ID, Enrollment: st.Enrollment, BootID: st.BootID, ControllerID: st.ControllerID, LeaseETag: st.LeaseETag, Expires: st.Expires}, nil |
| -} |
| - |
| -// openBounded opens a controller that lives under lifetime (claim D6) but |
| -// waits for the host's lifetime lock no longer than the pass's ctx: |
| -// hostexec.Open takes one context for both. If ctx ends first, the open is |
| -// cancelled (Open returns, releasing what it took); a controller that |
| -// opened in the same instant is closed, never kept with a cancelled life. |
| -func openBounded(ctx, lifetime context.Context, open func(context.Context) (RelocationController, error)) (RelocationController, error) { |
| - octx, cancel := context.WithCancel(lifetime) |
| - type result struct { |
| - c RelocationController |
| - err error |
| - } |
| - ch := make(chan result, 1) |
| - // hostexec.Open waits on its Cleanup hook without a context, so the |
| - // open runs apart and the pass waits for it no longer than ctx. |
| - go func() { |
| - c, err := open(octx) |
| - ch <- result{c, err} |
| - }() |
| - select { |
| - case res := <-ch: |
| - if res.err != nil { |
| - cancel() |
| - return nil, res.err |
| - } |
| - context.AfterFunc(lifetime, cancel) // octx ends only with lifetime |
| - return res.c, nil |
| - case <-ctx.Done(): |
| - cancel() |
| - go func() { |
| - if res := <-ch; res.c != nil { |
| - _ = res.c.Close(context.Background()) // opened too late: never kept |
| - } |
| - }() |
| - return nil, ctx.Err() |
| + if st.Op != obs.Claim.Op.ID || st.Incarnation != obs.Claim.Holder.Incarnation || st.Host != obs.Host { |
| + return hostclaim.ExecutionAcknowledgement{}, execctl.ErrLost |
| } |
| + return hostclaim.ExecutionAcknowledgement{Op: obs.Claim.Op.ID, Enrollment: st.Enrollment, BootID: st.BootID, ControllerID: st.ControllerID, LeaseETag: st.LeaseETag, Expires: st.Expires}, nil |
| } |
| -// Ready checks the retained controller for the operation is active now. |
| +// Ready checks main's participation controller is active now. A lost one |
| +// is main's retained denial, never reopened here. |
| func (e *RelocationEffects) Ready(ctx context.Context, obs hostclaim.Observed) error { |
| - if e.Controllers == nil || obs.Claim == nil { |
| - return ErrClaimStopUnavailable |
| - } |
| - ctl, ok, err := e.Controllers.live(ctx, obs.Claim.Host, obs.Claim.Op.ID) |
| - if err != nil { |
| - return err |
| - } |
| - if !ok { |
| - return hostrelocate.ErrNoController // the participant reopens one (Acknowledge) |
| - } |
| - _, err = ctl.Ready(ctx) |
| + _, err := e.Acknowledge(ctx, obs) |
| return err |
| } |
| -// Release runs the release sequence inside root's stop collection: root |
| -// records its durable fence and asks the agent to drain; the drain does |
| -// steps 1 to 3 before it closes the producers and acknowledges (§5.8, |
| -// claim P6-D1). The receipt is step 4. The flushed positions are durable |
| -// before the producers close, so a receipt root replays later still |
| -// carries them. |
| +// Release runs §5.8's release in claim Plan 11 item 4's order (ruling 28): |
| +// writers stopped and observed, the database and files flushed and the |
| +// positions recorded at the claim as observed; then the obligation queued; |
| +// then root's full stop with the producers' own drain; then Complete. |
| func (e *RelocationEffects) Release(ctx context.Context, obs hostclaim.Observed) (hostclaim.Release, error) { |
| if e.Identity == nil || e.Producers == nil || e.Collect == nil || e.WritersStopped == nil || e.Releases == nil || e.FlushDB == nil || e.FlushFiles == nil || e.Positions == nil || obs.Claim == nil || obs.Claim.Holder == nil { |
| return hostclaim.Release{}, ErrClaimStopUnavailable |
| @@ -649,89 +168,68 @@ func (e *RelocationEffects) Release(ctx context.Context, obs hostclaim.Observed) |
| return hostclaim.Release{}, err |
| } |
| if id.PID <= 0 || id.ProcessStart == "" { |
| - // Without this process's identity no flush could be told apart |
| - // from a restarted process's (localStopIdentity always has it). |
| return hostclaim.Release{}, ErrClaimStopUnavailable |
| } |
| if key, digest := obs.Claim.Op.Journal["release_stop_receipt"], obs.Claim.Op.Journal["release_stop_sha256"]; key != "" || digest != "" { |
| - // Root's receipt already discharged the obligation, and the release |
| - // was not journaled (the journal CAS lost a race, or the agent |
| - // restarted after Complete). Root would refuse a second stop: the |
| - // claim owes it nothing and has moved on. The release is the |
| - // journaled receipt, verified again, with the recorded positions. |
| return e.replayDischarged(ctx, obs, id, key, digest) |
| } |
| - // Root fences a relocation-release stop only while the claim owes it |
| - // (hostguard.StopAuthorityMatches): the obligation is queued first, and |
| - // the stop is asked at the claim that owes it. |
| - if obs, err = e.Releases.Queue(ctx, obs); err != nil { |
| - return hostclaim.Release{}, err |
| - } |
| h := obs.Claim.Holder |
| req := releaseStopRequest(obs, id) |
| - // A flush this box already made for this stop, by any controller of |
| - // this installation: after a restart the request is made under the |
| - // identity that flushed, so root finds that stop's durable receipt. |
| recorded, flushed, err := e.Positions.Recorded(ctx, req) |
| if err != nil { |
| return hostclaim.Release{}, err |
| } |
| - txid, seq := recorded.TXID, recorded.Seq |
| - if flushed { |
| - // The stop is the recorded one: root's fence and its inventory |
| - // check name the directory revision the flush was made under, so |
| - // a retry keeps it even when the roster has moved on since |
| - // (round 8 finding 2), whether or not this is a recovery. |
| - req.ExpectedDirectoryRevision = recorded.Request.ExpectedDirectoryRevision |
| + if !flushed { |
| + drained, seen, err := e.WritersStopped(ctx, obs) |
| + if err != nil { |
| + return hostclaim.Release{}, err |
| + } |
| + if err := drained.For(obs); err != nil { |
| + return hostclaim.Release{}, err |
| + } |
| + if seen.Host != obs.Host || seen.Incarnation != h.Incarnation || seen.InstallationStamp != h.Stamp || seen.WriterSession != h.Session || |
| + seen.Box != id.Actor.Box || seen.Enrollment != id.Actor.Enrollment || seen.CheckedAt.IsZero() { |
| + return hostclaim.Release{}, fmt.Errorf("%w: writers-stopped evidence names another session", ErrClaimStopIdentity) |
| + } |
| + txid, err := e.FlushDB(ctx, obs) |
| + if err != nil { |
| + return hostclaim.Release{}, err |
| + } |
| + seq, err := e.FlushFiles(ctx, obs) |
| + if err != nil { |
| + return hostclaim.Release{}, err |
| + } |
| + recorded = ReleasePosition{Request: req, PID: id.PID, ProcessStart: id.ProcessStart, TXID: txid, Seq: seq} |
| + if err := e.Positions.Record(ctx, recorded); err != nil { |
| + return hostclaim.Release{}, err |
| + } |
| + } |
| + // Root fences a relocation-release stop only while the claim owes it |
| + // (hostguard.StopAuthorityMatches): queued now, at the flushed claim. |
| + if obs, err = e.Releases.Queue(ctx, obs); err != nil { |
| + return hostclaim.Release{}, err |
| } |
| + req = releaseStopRequest(obs, id) |
| + req.ExpectedDirectoryRevision = recorded.Request.ExpectedDirectoryRevision |
| var d StopDrainer = RecoveryReleaseDrain{} |
| - drain := &releaseDrain{e: e, obs: obs, id: id} |
| - if flushed && !recorded.owner().local(id) { |
| - // Another process flushed (main's rule: another boot or |
| - // controller, or the same ids in a restarted process, whose |
| - // producers ended with it): only root's |
| - // durable receipt for the recorded identity can finish this |
| - // release. The request names the recorded owner (boot, controller, |
| - // directory revision) and the drain refuses; the Collect provider |
| - // marks it a recovery request (6b's authenticated Recover path, |
| - // Task 8a), under which root accepts another tuple for this box's |
| - // session and replays the receipt. |
| - req.BootID, req.ControllerID = recorded.Request.BootID, recorded.Request.ControllerID |
| - } else { |
| - if drain.inner, err = e.Producers(ctx, obs); err != nil { |
| + if recorded.owner().local(id) { |
| + inner, err := e.Producers(ctx, obs) |
| + if err != nil { |
| return hostclaim.Release{}, err |
| } |
| - if flushed { |
| - // This process already flushed and recorded this stop, and |
| - // root has no durable receipt yet (it failed after the drain |
| - // acknowledged, before CompleteFence: claimstop.go). Steps 1 |
| - // to 3 are done and the replicator may be closed: resume at |
| - // the producers' idempotent drain with the recorded positions, |
| - // and let root verify and persist the receipt. |
| - drain.resumed, drain.txid, drain.seq = true, txid, seq |
| - } |
| - d = drain |
| + d = &releaseDrain{inner: inner} |
| + } else { |
| + req.BootID, req.ControllerID = recorded.Request.BootID, recorded.Request.ControllerID |
| } |
| result, err := e.Collect(ctx, req, d) |
| if err != nil { |
| return hostclaim.Release{}, err |
| } |
| - if drain.done { |
| - txid, seq = drain.txid, drain.seq |
| - } else if !flushed { |
| - // Root replayed a durable receipt and nothing recorded a flush for |
| - // this stop: never guess. |
| - return hostclaim.Release{}, fmt.Errorf("%w: a replayed receipt has no recorded flushed positions for this stop", ErrClaimStopReceipt) |
| - } |
| - // Root's exact receipt discharges the obligation (main's |
| - // RelocationReleaseReceipts.Complete verifies it under the releasing |
| - // box's census key and journals its key and digest); a release is |
| - // returned only once that is durable. A replay finds it done. |
| if _, err := e.Releases.Complete(ctx, obs, result.Key, result.Digest); err != nil { |
| return hostclaim.Release{}, err |
| } |
| return hostclaim.Release{ReleasedBy: id.Actor.Box, Enrollment: id.Actor.Enrollment, Generation: h.Generation, Session: h.Session, |
| - FinalTXID: txid, ManifestSeq: seq, Receipt: result.Key, ReceiptSHA256: result.Digest}, nil |
| + FinalTXID: recorded.TXID, ManifestSeq: recorded.Seq, Receipt: result.Key, ReceiptSHA256: result.Digest}, nil |
| } |
| // releaseStopRequest is the release's stop request for obs's session, as |
| @@ -768,82 +266,35 @@ func (e *RelocationEffects) replayDischarged(ctx context.Context, obs hostclaim. |
| FinalTXID: recorded.TXID, ManifestSeq: recorded.Seq, Receipt: key, ReceiptSHA256: digest}, nil |
| } |
| -// releaseDrain is the agent's drain for a release: step 1's action, its |
| -// observation, steps 2 and 3, the durable positions, and only then the |
| -// producers' own drain and acknowledgement. |
| -type releaseDrain struct { |
| - e *RelocationEffects |
| - obs hostclaim.Observed |
| - id RelocationIdentity // the process recorded with the positions |
| - inner StopDrainer |
| - txid uint64 |
| - seq int64 |
| - done bool |
| - // resumed: this stop's positions are already durable (an earlier |
| - // attempt by this controller flushed and recorded them), so the |
| - // drain goes straight to the producers' own drain. |
| - resumed bool |
| -} |
| - |
| // RecoveryReleaseDrain refuses: a restarted agent's producers are not the |
| -// stop's, and root must replay the durable receipt or nothing. The Collect |
| -// provider recognises it and submits the request as a recovery (6b). It is |
| -// main's recoveryStopDrain (claimwakestop.go), the one drain a recovered |
| -// stop is offered, under the name Task 8a's Collect matches. |
| +// stop's, and root must replay the durable receipt or nothing. It is |
| +// main's recoveryStopDrain, under the name the Collect provider matches. |
| type RecoveryReleaseDrain = recoveryStopDrain |
| +// releaseDrain is the producers' own drain: steps 1 to 3 ran before the |
| +// obligation was queued (ruling 28), so root's collection closes the |
| +// producers and acknowledges, nothing more. |
| +type releaseDrain struct{ inner StopDrainer } |
| + |
| func (d *releaseDrain) DrainClaim(ctx context.Context, r StopRequest, fence string) (DrainAcknowledgement, error) { |
| - if d.resumed { |
| - ack, err := d.inner.DrainClaim(ctx, r, fence) |
| - if err != nil { |
| - return DrainAcknowledgement{}, err |
| - } |
| - d.done = true |
| - return ack, nil |
| - } |
| - drained, seen, err := d.e.WritersStopped(ctx, d.obs) |
| - if err != nil { |
| - return DrainAcknowledgement{}, err |
| - } |
| - // The witness and root's observation are of the session this stop |
| - // releases, by this box: anything else stopped another session. |
| - h := d.obs.Claim.Holder |
| - if err := drained.For(d.obs); err != nil { |
| - return DrainAcknowledgement{}, err |
| - } |
| - if seen.Host != d.obs.Host || seen.Incarnation != h.Incarnation || seen.InstallationStamp != h.Stamp || seen.WriterSession != h.Session || |
| - seen.Box != d.id.Actor.Box || seen.Enrollment != d.id.Actor.Enrollment || seen.CheckedAt.IsZero() { |
| - return DrainAcknowledgement{}, fmt.Errorf("%w: writers-stopped evidence names another session", ErrClaimStopIdentity) |
| - } |
| - if d.txid, err = d.e.FlushDB(ctx, d.obs); err != nil { |
| - return DrainAcknowledgement{}, err |
| - } |
| - if d.seq, err = d.e.FlushFiles(ctx, d.obs); err != nil { |
| - return DrainAcknowledgement{}, err |
| - } |
| - if err := d.e.Positions.Record(ctx, ReleasePosition{Request: r, PID: d.id.PID, ProcessStart: d.id.ProcessStart, TXID: d.txid, Seq: d.seq}); err != nil { |
| - return DrainAcknowledgement{}, err |
| + if d.inner == nil { |
| + return DrainAcknowledgement{}, ErrClaimStopUnavailable |
| } |
| - ack, err := d.inner.DrainClaim(ctx, r, fence) |
| - if err != nil { |
| - return DrainAcknowledgement{}, err |
| - } |
| - d.done = true |
| - return ack, nil |
| + return d.inner.DrainClaim(ctx, r, fence) |
| } |
| -func (e *RelocationEffects) Retained() ([]hostrelocate.Participation, error) { |
| - if e.Controllers == nil { |
| - return nil, nil |
| - } |
| - return e.Controllers.Retained() |
| -} |
| +// Retained is empty: main's participation owner keeps and finishes every |
| +// controller (ruling 42), so the participant's Sweep has nothing to do. |
| +func (e *RelocationEffects) Retained() ([]hostrelocate.Participation, error) { return nil, nil } |
| +// Finish releases the completed operation's row hold on the serving box |
| +// and lifts the probe's activator fence (ruling 49 step 5). The controller |
| +// is main's to close. |
| func (e *RelocationEffects) Finish(ctx context.Context, host, op string) error { |
| - if e.Controllers == nil { |
| + if e.ReleaseHold == nil { |
| return ErrClaimStopUnavailable |
| } |
| - return e.Controllers.Finish(ctx, host, op) |
| + return e.ReleaseHold(ctx, host, op) |
| } |
| func (e *RelocationEffects) Build(ctx context.Context, obs hostclaim.Observed) error { |
| @@ -860,10 +311,12 @@ func (e *RelocationEffects) InstallLocal(ctx context.Context, obs hostclaim.Obse |
| return e.Install(ctx, obs) |
| } |
| -// EnsureLocal is hostrelocate.Effects' check before P's first admission: |
| -// the journaled receipt's install is verified and, if a lost lease's scoped |
| -// cleanup retired it, restored by the same Install provider under the same |
| -// install_stamp, then verified again. An opened P is never reinstalled. |
| +// EnsureLocal is hostrelocate.Effects' check before P's first admission |
| +// (ruling 52): the journaled receipt's install is verified and, when it is |
| +// damaged while its install stamp is still the active one, made whole by |
| +// the same Install provider under that stamp. A retired stamp is never |
| +// reactivated: Install refuses it, and the probe deadline replaces the |
| +// target. An opened P is never reinstalled. |
| func (e *RelocationEffects) EnsureLocal(ctx context.Context, obs hostclaim.Observed) error { |
| if e.VerifyLocal == nil || e.Install == nil || obs.Claim == nil { |
| return ErrClaimStopUnavailable |
| @@ -919,26 +372,19 @@ func (e *RelocationEffects) Probe(ctx context.Context, obs hostclaim.Observed, c |
| return e.Admit(ctx, obs, certified) |
| } |
| -// execController is hostexec.Controller as a RelocationController: Clear |
| -// is hostexec.ClearStale with the controller's own config, the durable |
| -// record's removal once the claim no longer runs the operation with this |
| -// box (CC9). It is the Controller provider's production value (Task 8a |
| -// fills the assembly's hooks into cfg). |
| -type execController struct { |
| - *execctl.Controller |
| - cfg execctl.Config |
| +// Preflight refuses before an attempt is counted while root cannot launch |
| +// a relocating probe (ruling 48). |
| +func (e *RelocationEffects) Preflight(ctx context.Context, obs hostclaim.Observed) error { |
| + if e.PreflightKind == nil || obs.Claim == nil || obs.Claim.Op.Kind != hostclaim.KindRelocating { |
| + return ErrClaimStopUnavailable |
| + } |
| + return e.PreflightKind(ctx, obs) |
| } |
| -func (c *execController) Clear(ctx context.Context) error { return execctl.ClearStale(ctx, c.cfg) } |
| - |
| -func (c *execController) Lost() <-chan struct{} { return c.Controller.Lost() } |
| - |
| -// OpenRelocationController opens hostexec's controller for cfg and wraps |
| -// it with its durable clear. |
| -func OpenRelocationController(ctx context.Context, cfg execctl.Config) (RelocationController, error) { |
| - c, err := execctl.Open(ctx, cfg) |
| - if err != nil { |
| - return nil, err |
| +// EndProbe is ruling 49's stop of the passed probe session. |
| +func (e *RelocationEffects) EndProbe(ctx context.Context, obs hostclaim.Observed) (hostclaim.ProbeStop, error) { |
| + if e.StopProbe == nil { |
| + return hostclaim.ProbeStop{}, ErrClaimStopUnavailable |
| } |
| - return &execController{Controller: c, cfg: cfg}, nil // a pointer: the retained set compares controllers by identity |
| + return e.StopProbe(ctx, obs) |
| } |
diff --git a/internal/host/relocationeffects_test.go b/internal/host/relocationeffects_test.go| index f7a0b75cf..0562af2e4 100644 |
| --- a/internal/host/relocationeffects_test.go |
| +++ b/internal/host/relocationeffects_test.go |
| @@ -16,7 +16,6 @@ import ( |
| execctl "github.com/carlosframework/platform/internal/hostexec" |
| "github.com/carlosframework/platform/internal/hostfiles" |
| "github.com/carlosframework/platform/internal/hostguard" |
| - "github.com/carlosframework/platform/internal/hostrelocate" |
| "github.com/carlosframework/platform/internal/registry" |
| "github.com/carlosframework/platform/internal/release" |
| ) |
| @@ -28,65 +27,6 @@ func (d recordingDrain) DrainClaim(context.Context, StopRequest, string) (DrainA |
| return DrainAcknowledgement{FenceID: "fence"}, nil |
| } |
| -// readyController is one opened controller: it reports the context it was |
| -// opened under, so a test can see it was the agent's, not a pass's. |
| -type readyController struct { |
| - st execctl.State |
| - opened context.Context |
| - closed bool |
| - cleared bool |
| - lost error |
| - clearErr error |
| - closeErr error |
| -} |
| - |
| -func (c *readyController) Clear(context.Context) error { |
| - if c.clearErr != nil { |
| - return c.clearErr |
| - } |
| - c.cleared = true |
| - return nil |
| -} |
| - |
| -func (c *readyController) Lost() <-chan struct{} { |
| - ch := make(chan struct{}) |
| - if c.lost != nil { |
| - close(ch) |
| - } |
| - return ch |
| -} |
| - |
| -func (c *readyController) Ready(context.Context) (execctl.State, error) { |
| - if c.lost != nil { |
| - return execctl.State{}, c.lost |
| - } |
| - return c.st, nil |
| -} |
| -func (c *readyController) Close(ctx context.Context) error { |
| - if err := ctx.Err(); err != nil { |
| - return err // hostexec retains the lifetime lock on a cut-short join |
| - } |
| - if c.closed { |
| - return nil // hostexec: every Close after the first returns nil |
| - } |
| - c.closed = true |
| - return c.closeErr // hostexec: a cleanup failure is returned once |
| -} |
| - |
| -// testControllers is a RelocationControllers with a journal on disk and a |
| -// scoped cleanup that succeeds. |
| -func testControllers(t *testing.T) *RelocationControllers { |
| - return &RelocationControllers{Journal: filepath.Join(t.TempDir(), "participations"), Cleanup: func(context.Context, execctl.Binding) error { return nil }} |
| -} |
| - |
| -// kept is Retained's list, for tests whose journal holds nothing damaged. |
| -func kept(r interface { |
| - Retained() ([]hostrelocate.Participation, error) |
| -}) []hostrelocate.Participation { |
| - got, _ := r.Retained() |
| - return got |
| -} |
| - |
| type memPositions struct { |
| calls *[]string |
| p ReleasePosition |
| @@ -149,14 +89,12 @@ func effectsFixture(t *testing.T) (*RelocationEffects, hostclaim.Observed, *[]st |
| seen := &StopRequest{} |
| exp := time.Date(2026, 10, 7, 10, 0, 0, 0, time.UTC) |
| e := &RelocationEffects{ |
| - Lifetime: context.Background(), |
| - Controllers: testControllers(t), |
| + Participation: func(context.Context, hostclaim.Observed) (execctl.State, error) { |
| + return execctl.State{Binding: execctl.Binding{Host: "shop.example", Incarnation: "inc", Op: "rel-1", Enrollment: "enr", BootID: "boot", ControllerID: "c9"}, LeaseETag: "lease-1", Expires: exp}, nil |
| + }, |
| Identity: func(context.Context) (RelocationIdentity, error) { |
| return RelocationIdentity{Actor: hostbox.Actor{Box: "src", Enrollment: "enr", Revision: 4}, BootID: "boot", ControllerID: "ctl", PID: 100, ProcessStart: "5000"}, nil |
| }, |
| - Controller: func(ctx context.Context, b execctl.Binding) (RelocationController, error) { |
| - return &readyController{opened: ctx, st: execctl.State{Binding: execctl.Binding{Enrollment: "enr", BootID: "boot", ControllerID: "c9"}, LeaseETag: "lease-1", Expires: exp}}, nil |
| - }, |
| Producers: func(context.Context, hostclaim.Observed) (StopDrainer, error) { return recordingDrain{calls}, nil }, |
| WritersStopped: func(_ context.Context, o hostclaim.Observed) (ClaimWritersDrained, WritersStoppedEvidence, error) { |
| *calls = append(*calls, "writers-stopped") |
| @@ -187,13 +125,13 @@ func effectsFixture(t *testing.T) (*RelocationEffects, hostclaim.Observed, *[]st |
| return e, obs, calls, seen |
| } |
| -func TestRelocationReleaseRunsStepsOneToThreeInsideTheDrain(t *testing.T) { |
| +func TestReleaseStopsWritersAndFlushesBeforeItQueues(t *testing.T) { |
| e, obs, calls, seen := effectsFixture(t) |
| r, err := e.Release(context.Background(), obs) |
| if err != nil { |
| t.Fatal(err) |
| } |
| - if want := []string{"queue", "writers-stopped", "db", "files", "record", "producers", "receipt", "complete"}; !slices.Equal(*calls, want) { |
| + if want := []string{"writers-stopped", "db", "files", "record", "queue", "producers", "receipt", "complete"}; !slices.Equal(*calls, want) { |
| t.Fatalf("order %v, want %v", *calls, want) |
| } |
| if r.FinalTXID != 41 || r.ManifestSeq != 8 || r.Session != 6 || r.Generation != "g6" || r.ReleasedBy != "src" || r.Receipt == "" { |
| @@ -213,12 +151,12 @@ func TestRelocationReleaseStopsAtTheFirstFailedStep(t *testing.T) { |
| }{ |
| {"the obligation cannot be queued", func(e *RelocationEffects) { |
| e.Releases.(*fakeReleaseObligation).queueErr = failure |
| - }, []string{"queue"}}, |
| + }, []string{"writers-stopped", "db", "files", "record", "queue"}}, |
| {"the writers cannot be stopped", func(e *RelocationEffects) { |
| e.WritersStopped = func(context.Context, hostclaim.Observed) (ClaimWritersDrained, WritersStoppedEvidence, error) { |
| return ClaimWritersDrained{}, WritersStoppedEvidence{}, failure |
| } |
| - }, []string{"queue"}}, |
| + }, nil}, |
| {"writers still running", func(e *RelocationEffects) { |
| e.WritersStopped = func(_ context.Context, o hostclaim.Observed) (ClaimWritersDrained, WritersStoppedEvidence, error) { |
| *e.Positions.(*memPositions).calls = append(*e.Positions.(*memPositions).calls, "writers-stopped") |
| @@ -226,7 +164,7 @@ func TestRelocationReleaseStopsAtTheFirstFailedStep(t *testing.T) { |
| d.WorkDrained = false |
| return d, ev, err |
| } |
| - }, []string{"queue", "writers-stopped"}}, |
| + }, []string{"writers-stopped"}}, |
| {"root observed another session", func(e *RelocationEffects) { |
| e.WritersStopped = func(_ context.Context, o hostclaim.Observed) (ClaimWritersDrained, WritersStoppedEvidence, error) { |
| *e.Positions.(*memPositions).calls = append(*e.Positions.(*memPositions).calls, "writers-stopped") |
| @@ -234,16 +172,16 @@ func TestRelocationReleaseStopsAtTheFirstFailedStep(t *testing.T) { |
| ev.WriterSession++ |
| return d, ev, err |
| } |
| - }, []string{"queue", "writers-stopped"}}, |
| + }, []string{"writers-stopped"}}, |
| {"the database flush", func(e *RelocationEffects) { |
| e.FlushDB = func(context.Context, hostclaim.Observed) (uint64, error) { return 0, failure } |
| - }, []string{"queue", "writers-stopped"}}, |
| + }, []string{"writers-stopped"}}, |
| {"the files flush", func(e *RelocationEffects) { |
| e.FlushFiles = func(context.Context, hostclaim.Observed) (int64, error) { return 0, failure } |
| - }, []string{"queue", "writers-stopped", "db"}}, |
| + }, []string{"writers-stopped", "db"}}, |
| {"the receipt does not discharge the obligation", func(e *RelocationEffects) { |
| e.Releases.(*fakeReleaseObligation).completeErr = failure |
| - }, []string{"queue", "writers-stopped", "db", "files", "record", "producers", "receipt", "complete"}}, |
| + }, []string{"writers-stopped", "db", "files", "record", "queue", "producers", "receipt", "complete"}}, |
| } { |
| t.Run(tc.name, func(t *testing.T) { |
| e, obs, calls, _ := effectsFixture(t) |
| @@ -263,10 +201,16 @@ func TestRelocationReleaseReplaysOnlyRecordedPositions(t *testing.T) { |
| return StopResult{Key: "k", Digest: strings.Repeat("a", 64)}, nil // root replayed; no drain ran here |
| } |
| t.Run("no record", func(t *testing.T) { |
| - e, obs, _, _ := effectsFixture(t) |
| + // Ruling 28: without a recorded flush the release flushes first. |
| + // A process with no admitted attempt cannot stop or flush the |
| + // writers, so it never reaches the obligation or root. |
| + e, obs, calls, _ := effectsFixture(t) |
| e.Collect = replayed |
| - if _, err := e.Release(context.Background(), obs); !errors.Is(err, ErrClaimStopReceipt) { |
| - t.Fatalf("got %v", err) |
| + e.WritersStopped = func(context.Context, hostclaim.Observed) (ClaimWritersDrained, WritersStoppedEvidence, error) { |
| + return ClaimWritersDrained{}, WritersStoppedEvidence{}, ErrClaimStopUnavailable |
| + } |
| + if _, err := e.Release(context.Background(), obs); !errors.Is(err, ErrClaimStopUnavailable) || len(*calls) != 0 { |
| + t.Fatalf("got %v, ran %v", err, *calls) |
| } |
| }) |
| t.Run("the lost response", func(t *testing.T) { |
| @@ -288,42 +232,84 @@ func TestRelocationReleaseReplaysOnlyRecordedPositions(t *testing.T) { |
| t.Fatalf("replay: %+v %v", r, err) |
| } |
| }) |
| - t.Run("root fails after the producers closed", func(t *testing.T) { |
| - // Round 7: the drain flushed, recorded and closed the producers; |
| - // root then failed before its receipt was durable (kernel |
| - // verification or CompleteFence). The retry, same controller, no |
| - // restart, resumes at the producers' drain with the recorded |
| - // positions: no step of the release runs again. |
| - e, obs, calls, seen := effectsFixture(t) |
| - collect := e.Collect |
| - e.Collect = func(ctx context.Context, r StopRequest, d StopDrainer) (StopResult, error) { |
| - if _, err := d.DrainClaim(ctx, r, "fence"); err != nil { |
| - return StopResult{}, err |
| - } |
| - return StopResult{}, errors.New("verify stopped: cgroup not yet empty") // no receipt persisted |
| - } |
| - if _, err := e.Release(context.Background(), obs); err == nil { |
| - t.Fatal("a release without a receipt succeeded") |
| - } |
| - *calls = nil |
| - e.Collect = collect |
| - // Round 8 finding 2: the roster moved on between the attempts |
| - // (same boot, same controller); the retry still names the |
| - // revision the stop was flushed under. |
| + t.Run("an identity without its process", func(t *testing.T) { |
| + e, obs, calls, _ := effectsFixture(t) |
| e.Identity = func(context.Context) (RelocationIdentity, error) { |
| - return RelocationIdentity{Actor: hostbox.Actor{Box: "src", Enrollment: "enr", Revision: 5}, BootID: "boot", ControllerID: "ctl", PID: 100, ProcessStart: "5000"}, nil |
| + return RelocationIdentity{Actor: hostbox.Actor{Box: "src", Enrollment: "enr", Revision: 4}, BootID: "boot", ControllerID: "ctl"}, nil |
| } |
| - r, err := e.Release(context.Background(), obs) |
| - if err != nil || r.FinalTXID != 41 || r.ManifestSeq != 8 { |
| - t.Fatalf("retry: %+v %v", r, err) |
| + if _, err := e.Release(context.Background(), obs); !errors.Is(err, ErrClaimStopUnavailable) || len(*calls) != 0 { |
| + t.Fatalf("released without a process identity: %v ran %v", err, *calls) |
| } |
| - if want := []string{"queue", "producers", "receipt", "complete"}; !slices.Equal(*calls, want) { |
| - t.Fatalf("the retry ran %v, want %v (no flush, no record, no writers stop again)", *calls, want) |
| + }) |
| + t.Run("a record of another session", func(t *testing.T) { |
| + e, obs, _, _ := effectsFixture(t) |
| + if _, err := e.Release(context.Background(), obs); err != nil { |
| + t.Fatal(err) |
| } |
| - if seen.ExpectedDirectoryRevision != "4" || seen.BootID != "boot" || seen.ControllerID != "ctl" { |
| - t.Fatalf("the retry asked root under another stop: %+v", *seen) |
| + e.Collect = replayed |
| + obs.Claim.Holder.Session = 7 |
| + *e.Positions.(*memPositions).calls = nil |
| + if _, err := e.Release(context.Background(), obs); err != nil || !slices.Contains(*e.Positions.(*memPositions).calls, "writers-stopped") { |
| + t.Fatalf("another session's positions were used: %v ran %v", err, *e.Positions.(*memPositions).calls) |
| } |
| }) |
| +} |
| + |
| +// Review Focus 3: a release never returns before root's receipt. Root |
| +// fails after the producers closed (kernel verification or CompleteFence), |
| +// so no receipt is durable: the release returns an error with its |
| +// positions recorded, and the retry, same controller, no restart, resumes |
| +// at the producers' drain with the recorded positions. No step of the |
| +// release runs twice. |
| +func TestReleaseNeverReturnsBeforeRootsReceipt(t *testing.T) { |
| + // Round 7: the drain flushed, recorded and closed the producers; |
| + // root then failed before its receipt was durable (kernel |
| + // verification or CompleteFence). The retry, same controller, no |
| + // restart, resumes at the producers' drain with the recorded |
| + // positions: no step of the release runs again. |
| + e, obs, calls, seen := effectsFixture(t) |
| + collect := e.Collect |
| + e.Collect = func(ctx context.Context, r StopRequest, d StopDrainer) (StopResult, error) { |
| + if _, err := d.DrainClaim(ctx, r, "fence"); err != nil { |
| + return StopResult{}, err |
| + } |
| + return StopResult{}, errors.New("verify stopped: cgroup not yet empty") // no receipt persisted |
| + } |
| + if _, err := e.Release(context.Background(), obs); err == nil { |
| + t.Fatal("a release without a receipt succeeded") |
| + } |
| + if want := []string{"writers-stopped", "db", "files", "record", "queue", "producers"}; !slices.Equal(*calls, want) { |
| + t.Fatalf("the first call ran %v, want %v", *calls, want) |
| + } |
| + *calls = nil |
| + e.Collect = collect |
| + // Round 8 finding 2: the roster moved on between the attempts |
| + // (same boot, same controller); the retry still names the |
| + // revision the stop was flushed under. |
| + e.Identity = func(context.Context) (RelocationIdentity, error) { |
| + return RelocationIdentity{Actor: hostbox.Actor{Box: "src", Enrollment: "enr", Revision: 5}, BootID: "boot", ControllerID: "ctl", PID: 100, ProcessStart: "5000"}, nil |
| + } |
| + r, err := e.Release(context.Background(), obs) |
| + if err != nil || r.FinalTXID != 41 || r.ManifestSeq != 8 { |
| + t.Fatalf("retry: %+v %v", r, err) |
| + } |
| + if want := []string{"queue", "producers", "receipt", "complete"}; !slices.Equal(*calls, want) { |
| + t.Fatalf("the retry ran %v, want %v (no flush, no record, no writers stop again)", *calls, want) |
| + } |
| + if seen.ExpectedDirectoryRevision != "4" || seen.BootID != "boot" || seen.ControllerID != "ctl" { |
| + t.Fatalf("the retry asked root under another stop: %+v", *seen) |
| + } |
| + if p := e.Positions.(*memPositions); !p.ok || p.p.TXID != 41 || p.p.Seq != 8 { |
| + t.Fatalf("the positions were not kept for the retry: %+v", p) |
| + } |
| +} |
| + |
| +// A restarted agent asks root as the recorded owner's recovery: the |
| +// request names the boot and controller the flush was made under, offers |
| +// no drain, and pairs root's replayed receipt with the recorded positions. |
| +// Both restarts count: new ids, and the same ids in a new process (main's |
| +// claimLaunchOwner.local). |
| +func TestReleaseAfterARestartRecoversTheRecordedOwner(t *testing.T) { |
| t.Run("after a restart the recorded identity asks root", func(t *testing.T) { |
| // The agent flushed and recorded under controller "ctl", root |
| // completed its receipt, the agent restarted. The new process asks |
| @@ -386,26 +372,6 @@ func TestRelocationReleaseReplaysOnlyRecordedPositions(t *testing.T) { |
| t.Fatalf("not a recovery: recovery=%v asked=%+v ran %v", recovery, asked, *calls) |
| } |
| }) |
| - t.Run("an identity without its process", func(t *testing.T) { |
| - e, obs, calls, _ := effectsFixture(t) |
| - e.Identity = func(context.Context) (RelocationIdentity, error) { |
| - return RelocationIdentity{Actor: hostbox.Actor{Box: "src", Enrollment: "enr", Revision: 4}, BootID: "boot", ControllerID: "ctl"}, nil |
| - } |
| - if _, err := e.Release(context.Background(), obs); !errors.Is(err, ErrClaimStopUnavailable) || len(*calls) != 0 { |
| - t.Fatalf("released without a process identity: %v ran %v", err, *calls) |
| - } |
| - }) |
| - t.Run("a record of another session", func(t *testing.T) { |
| - e, obs, _, _ := effectsFixture(t) |
| - if _, err := e.Release(context.Background(), obs); err != nil { |
| - t.Fatal(err) |
| - } |
| - e.Collect = replayed |
| - obs.Claim.Holder.Session = 7 |
| - if _, err := e.Release(context.Background(), obs); !errors.Is(err, ErrClaimStopReceipt) { |
| - t.Fatalf("another session's positions were used: %v", err) |
| - } |
| - }) |
| } |
| func TestReleasePositionsFileBindsTheExactStop(t *testing.T) { |
| @@ -447,7 +413,7 @@ func TestReleasePositionsFileBindsTheExactStop(t *testing.T) { |
| t.Fatalf("%s: another stop's positions were returned", name) |
| } |
| } |
| - if err := os.WriteFile(filepath.Join(f.Dir, "shop.example.json"), []byte("{"), 0o600); err != nil { |
| + if err := os.WriteFile(filepath.Join(f.Dir, "rel-1.json"), []byte("{"), 0o600); err != nil { |
| t.Fatal(err) |
| } |
| if _, _, err := f.Recorded(ctx, r); !errors.Is(err, ErrClaimStopReceipt) { |
| @@ -455,7 +421,7 @@ func TestReleasePositionsFileBindsTheExactStop(t *testing.T) { |
| } |
| // A record that names no process would read as this process's own |
| // under claimLaunchOwner.local: it is damaged, never trusted. |
| - if err := os.WriteFile(filepath.Join(f.Dir, "shop.example.json"), []byte(`{"v":1,"request":{"Host":"shop.example"},"txid":41,"seq":8}`), 0o600); err != nil { |
| + if err := os.WriteFile(filepath.Join(f.Dir, "rel-1.json"), []byte(`{"v":1,"request":{"Host":"shop.example"},"txid":41,"seq":8}`), 0o600); err != nil { |
| t.Fatal(err) |
| } |
| if _, _, err := f.Recorded(ctx, r); !errors.Is(err, ErrClaimStopReceipt) { |
| @@ -463,93 +429,6 @@ func TestReleasePositionsFileBindsTheExactStop(t *testing.T) { |
| } |
| } |
| -func TestRelocationAcknowledgeRetainsTheControllerUnderTheAgentLifetime(t *testing.T) { |
| - e, obs, _, _ := effectsFixture(t) |
| - var opened []*readyController |
| - exp := time.Date(2026, 10, 7, 10, 0, 0, 0, time.UTC) |
| - var bound execctl.Binding |
| - e.Controller = func(ctx context.Context, b execctl.Binding) (RelocationController, error) { |
| - bound = b |
| - c := &readyController{opened: ctx, st: execctl.State{Binding: execctl.Binding{Enrollment: "enr", BootID: "boot", ControllerID: "c9"}, LeaseETag: "lease-1", Expires: exp}} |
| - opened = append(opened, c) |
| - return c, nil |
| - } |
| - pass, cancel := context.WithCancel(context.Background()) |
| - ack, err := e.Acknowledge(pass, obs) |
| - if err != nil { |
| - t.Fatal(err) |
| - } |
| - cancel() // the pass ends; the controller must not |
| - if bound.Op != "rel-1" || bound.Host != "shop.example" || bound.Incarnation != "inc" || ack.LeaseETag != "lease-1" || ack.ControllerID != "c9" || !ack.Expires.Equal(exp) { |
| - t.Fatalf("binding %+v ack %+v", bound, ack) |
| - } |
| - if opened[0].opened.Err() != nil { |
| - t.Fatal("the controller was opened under the pass's context") |
| - } |
| - // A second pass reuses the retained controller; Ready sees it. |
| - if _, err := e.Acknowledge(context.Background(), obs); err != nil || len(opened) != 1 { |
| - t.Fatalf("second pass: %v opened=%d", err, len(opened)) |
| - } |
| - if err := e.Ready(context.Background(), obs); err != nil { |
| - t.Fatal(err) |
| - } |
| - // A lost lease is recovered without a restart (round 13 finding 1): |
| - // Ready joins the lost controller and repeats the scoped cleanup, |
| - // reports no controller, and the participant's Acknowledge opens a |
| - // fresh one; a cleanup that fails keeps the lost one for the next pass. |
| - opened[0].lost = execctl.ErrLost |
| - cleanupErr := errors.New("cleanup: a process would not stop") |
| - e.Controllers.Cleanup = func(context.Context, execctl.Binding) error { return cleanupErr } |
| - if err := e.Ready(context.Background(), obs); !errors.Is(err, cleanupErr) || !opened[0].closed { |
| - t.Fatalf("lost controller, cleanup failing: %v closed=%v", err, opened[0].closed) |
| - } |
| - e.Controllers.Cleanup = func(context.Context, execctl.Binding) error { return nil } |
| - if err := e.Ready(context.Background(), obs); !errors.Is(err, hostrelocate.ErrNoController) { |
| - t.Fatalf("lost controller: %v", err) |
| - } |
| - if _, err := e.Acknowledge(context.Background(), obs); err != nil || len(opened) != 2 { |
| - t.Fatalf("recovery: %v opened=%d", err, len(opened)) |
| - } |
| - if err := e.Ready(context.Background(), obs); err != nil { |
| - t.Fatalf("after recovery: %v", err) |
| - } |
| - // An agent restart has no retained controller: Ready says so, and |
| - // Acknowledge opens a fresh one. |
| - restarted := *e |
| - restarted.Controllers = testControllers(t) |
| - if err := restarted.Ready(context.Background(), obs); !errors.Is(err, hostrelocate.ErrNoController) { |
| - t.Fatalf("after a restart: %v", err) |
| - } |
| - if _, err := restarted.Acknowledge(context.Background(), obs); err != nil || len(opened) != 3 { |
| - t.Fatalf("reopen: %v opened=%d", err, len(opened)) |
| - } |
| - // Finishing a participation closes its controller and forgets it; a |
| - // later Acknowledge for the same host opens a new one. |
| - if got := kept(&restarted); len(got) != 1 || got[0] != (hostrelocate.Participation{Host: "shop.example", Op: "rel-1"}) { |
| - t.Fatalf("retained: %+v", got) |
| - } |
| - // A close cut short by the pass's context keeps the entry, so a later |
| - // sweep can finish it; then it is forgotten. |
| - cut, cancel := context.WithCancel(context.Background()) |
| - cancel() |
| - if err := restarted.Finish(cut, "shop.example", "rel-1"); err == nil || opened[2].closed || len(kept(&restarted)) != 1 { |
| - t.Fatalf("a cut-short close forgot the controller: %v closed=%v retained=%v", err, opened[2].closed, kept(&restarted)) |
| - } |
| - if err := restarted.Finish(context.Background(), "shop.example", "rel-1"); err != nil || !opened[2].closed || !opened[2].cleared || len(kept(&restarted)) != 0 { |
| - t.Fatalf("finish: %v closed=%v cleared=%v retained=%v", err, opened[2].closed, opened[2].cleared, kept(&restarted)) |
| - } |
| - if err := restarted.Ready(context.Background(), obs); !errors.Is(err, hostrelocate.ErrNoController) { |
| - t.Fatalf("after finish: %v", err) |
| - } |
| - obs.Claim.Op.ID = "rel-2" |
| - if _, err := restarted.Acknowledge(context.Background(), obs); err != nil || len(opened) != 4 { |
| - t.Fatalf("the next relocation: %v opened=%d", err, len(opened)) |
| - } |
| - if err := restarted.Controllers.Close(context.Background()); err != nil || !opened[3].closed { |
| - t.Fatalf("close: %v closed=%v", err, opened[3].closed) |
| - } |
| -} |
| - |
| func TestReleaseFlushFilesCarriesTheLastWakeStateAndEnv(t *testing.T) { |
| ctx := context.Background() |
| data := release.NewDirStore(t.TempDir()) |
| @@ -637,118 +516,10 @@ func TestReleaseFlushFilesCarriesTheLastWakeStateAndEnv(t *testing.T) { |
| } |
| } |
| -func TestFinishClearsTheDurableRecordAndRetriesAcrossARestart(t *testing.T) { |
| - journal := filepath.Join(t.TempDir(), "participations") |
| - e, obs, _, _ := effectsFixture(t) |
| - cleanups := 0 |
| - var cleanupErr error |
| - cleanup := func(_ context.Context, b execctl.Binding) error { |
| - if b.Op != "rel-1" || b.Box != "src" || b.Host != "shop.example" || b.Incarnation != "inc" { |
| - t.Fatalf("cleanup for another binding: %+v", b) |
| - } |
| - cleanups++ |
| - return cleanupErr |
| - } |
| - e.Controllers = &RelocationControllers{Journal: journal, Cleanup: cleanup} |
| - var ctl *readyController |
| - exp := time.Date(2026, 10, 7, 10, 0, 0, 0, time.UTC) |
| - e.Controller = func(ctx context.Context, b execctl.Binding) (RelocationController, error) { |
| - ctl = &readyController{opened: ctx, st: execctl.State{Binding: b, LeaseETag: "lease-1", Expires: exp}} |
| - return ctl, nil |
| - } |
| - // Without a journal nothing opens: a record no restart could find. |
| - unjournaled := *e |
| - unjournaled.Controllers = &RelocationControllers{Cleanup: cleanup} |
| - if _, err := unjournaled.Acknowledge(context.Background(), obs); !errors.Is(err, ErrClaimStopUnavailable) || ctl != nil { |
| - t.Fatalf("a controller opened without a journal: %v", err) |
| - } |
| - if _, err := e.Acknowledge(context.Background(), obs); err != nil { |
| - t.Fatal(err) |
| - } |
| - // The controller's cleanup fails. hostexec's Close reports it once. |
| - ctl.closeErr = errors.New("cleanup: a process would not stop") |
| - cleanupErr = ctl.closeErr |
| - if err := e.Finish(context.Background(), "shop.example", "rel-1"); err == nil || !ctl.closed || ctl.cleared || len(kept(e)) != 1 { |
| - t.Fatalf("a failed cleanup: %v closed=%v cleared=%v retained=%v", err, ctl.closed, ctl.cleared, kept(e)) |
| - } |
| - // The next sweep: Close now returns nil, which proves nothing. The |
| - // repeated cleanup still fails, so the record stays. |
| - if err := e.Finish(context.Background(), "shop.example", "rel-1"); err == nil || ctl.cleared || len(kept(e)) != 1 { |
| - t.Fatalf("a nil Close after a failed cleanup cleared the record: %v cleared=%v retained=%v", err, ctl.cleared, kept(e)) |
| - } |
| - // The clear itself fails: the entry and the journal stay. |
| - cleanupErr = nil |
| - ctl.clearErr = errors.New("claim read failed") |
| - if err := e.Finish(context.Background(), "shop.example", "rel-1"); err == nil || ctl.cleared || len(kept(e)) != 1 { |
| - t.Fatalf("a failed clear: %v cleared=%v retained=%v", err, ctl.cleared, kept(e)) |
| - } |
| - // The agent restarts: no controller object, the journal remains. The |
| - // cleanup is repeated first; while it fails nothing is cleared. |
| - cleared := []execctl.Binding{} |
| - clear := func(_ context.Context, b execctl.Binding) error { cleared = append(cleared, b); return nil } |
| - again := &RelocationControllers{Journal: journal, Cleanup: cleanup, Clear: clear} |
| - if got := kept(again); len(got) != 1 || got[0].Op != "rel-1" { |
| - t.Fatalf("after a restart the journaled participation is forgotten: %+v", got) |
| - } |
| - cleanupErr = errors.New("cleanup: still failing") |
| - if err := again.Finish(context.Background(), "shop.example", "rel-1"); err == nil || len(cleared) != 0 || len(kept(again)) != 1 { |
| - t.Fatalf("a restart cleared without a successful cleanup: %v cleared=%+v", err, cleared) |
| - } |
| - cleanupErr = nil |
| - before := cleanups |
| - if err := again.Finish(context.Background(), "shop.example", "rel-1"); err != nil || cleanups != before+1 || len(cleared) != 1 || cleared[0].Op != "rel-1" || cleared[0].Box != "src" { |
| - t.Fatalf("retry after restart: %v cleanups=%d cleared=%+v", err, cleanups-before, cleared) |
| - } |
| - if got := kept(again); len(got) != 0 { |
| - t.Fatalf("the journal entry outlived the clear: %+v", got) |
| - } |
| -} |
| - |
| -// A crash before the first Finish (or a failed Open after hostexec wrote |
| -// its record) leaves no controller and no finishing state, only the |
| -// participation journaled before the open: a restart finds it, and an |
| -// uncommitted, truncated journal write beside it hides nothing. |
| -func TestARestartFindsAParticipationThatNeverReachedFinish(t *testing.T) { |
| - journal := filepath.Join(t.TempDir(), "participations") |
| - e, obs, _, _ := effectsFixture(t) |
| - e.Controllers = &RelocationControllers{Journal: journal, Cleanup: func(context.Context, execctl.Binding) error { return nil }} |
| - if _, err := e.Acknowledge(context.Background(), obs); err != nil { |
| - t.Fatal(err) |
| - } |
| - // The crash: the controller object is gone. Another participation's |
| - // journal write was cut short, leaving a truncated temporary file. |
| - if err := os.WriteFile(filepath.Join(journal, "spa.example~rel-9.json.tmp"), []byte(`{"v":1,"ho`), 0o600); err != nil { |
| - t.Fatal(err) |
| - } |
| - cleaned, cleared := []execctl.Binding{}, []execctl.Binding{} |
| - again := &RelocationControllers{Journal: journal, |
| - Cleanup: func(_ context.Context, b execctl.Binding) error { cleaned = append(cleaned, b); return nil }, |
| - Clear: func(_ context.Context, b execctl.Binding) error { cleared = append(cleared, b); return nil }} |
| - got, err := again.Retained() |
| - if err != nil || len(got) != 1 || got[0] != (hostrelocate.Participation{Host: "shop.example", Op: "rel-1"}) { |
| - t.Fatalf("restart with a truncated temporary file: %+v %v", got, err) |
| - } |
| - // A committed entry that cannot be read is reported, never hiding the |
| - // readable one. |
| - if err := os.WriteFile(filepath.Join(journal, "bad.example~rel-3.json"), []byte("{"), 0o600); err != nil { |
| - t.Fatal(err) |
| - } |
| - got, err = again.Retained() |
| - if !errors.Is(err, ErrClaimStopUnavailable) || len(got) != 1 || got[0].Host != "shop.example" { |
| - t.Fatalf("a damaged committed entry: %+v %v", got, err) |
| - } |
| - if err := again.Finish(context.Background(), "shop.example", "rel-1"); err != nil || len(cleaned) != 1 || len(cleared) != 1 || cleared[0].Host != "shop.example" || cleared[0].Incarnation != "inc" { |
| - t.Fatalf("finish after the crash: %v cleaned=%+v cleared=%+v", err, cleaned, cleared) |
| - } |
| - if got, _ := again.Retained(); len(got) != 0 { |
| - t.Fatalf("the participation outlived its clear: %+v", got) |
| - } |
| -} |
| - |
| // Round 14 finding 2: EnsureLocal verifies the receipt's install, restores |
| // a retired one under the same stamp and verifies it again, refuses a |
| // restore that is not the receipt's, and never touches an opened P. |
| -func TestEnsureLocalRestoresARetiredInstallOnlyBeforePOpens(t *testing.T) { |
| +func TestEnsureLocalRepairsOnlyUnderTheActiveStampBeforePOpens(t *testing.T) { |
| ctx := context.Background() |
| e, obs, _, _ := effectsFixture(t) |
| receipt := hostclaim.LocalInstall{Box: "dst", Enrollment: "enr-d", InstallStamp: 9, FilesManifestVersion: 1, JournalSHA256: strings.Repeat("e", 64)} |
| @@ -768,7 +539,7 @@ func TestEnsureLocalRestoresARetiredInstallOnlyBeforePOpens(t *testing.T) { |
| } |
| present = false |
| if err := e.EnsureLocal(ctx, obs); err != nil || installs != 1 || !present { |
| - t.Fatalf("a retired install: %v installs=%d", err, installs) |
| + t.Fatalf("a damaged install under its active stamp: %v installs=%d", err, installs) |
| } |
| present, restored.InstallStamp = false, 10 |
| if err := e.EnsureLocal(ctx, obs); !errors.Is(err, ErrClaimStopReceipt) { |
diff --git a/internal/host/relocationflush.go b/internal/host/relocationflush.go| index 6eade7bb3..ef88ff0a1 100644 |
| --- a/internal/host/relocationflush.go |
| +++ b/internal/host/relocationflush.go |
| @@ -7,10 +7,12 @@ import ( |
| "fmt" |
| "os" |
| "path/filepath" |
| + "strings" |
| "github.com/carlosframework/platform/internal/controlname" |
| "github.com/carlosframework/platform/internal/hostclaim" |
| "github.com/carlosframework/platform/internal/hostfiles" |
| + "github.com/carlosframework/platform/internal/instancekey" |
| "github.com/carlosframework/platform/internal/registry" |
| "github.com/carlosframework/platform/internal/release" |
| ) |
| @@ -73,6 +75,22 @@ func (f ReleaseFlushFiles) Flush(ctx context.Context, obs hostclaim.Observed) (i |
| default: |
| return 0, err |
| } |
| + // O5 (instance-key plan rev 3): the adoption record beside the env |
| + // file is staged in the same commit when it parses and binds exactly |
| + // the releasing writer; otherwise its deletion is staged, so a stale |
| + // record, or none, is never published as this writer's. |
| + writer := instancekey.Binding{Host: obs.Host, Incarnation: h.Incarnation, Session: h.Session, Generation: h.Generation} |
| + rec, err := os.ReadFile(filepath.Join(filepath.Dir(f.EnvPath(obs.Host)), obs.Host+instancekey.AdoptionSuffix)) |
| + if err != nil && !errors.Is(err, os.ErrNotExist) { |
| + return 0, err |
| + } |
| + if a, perr := instancekey.ParseAdoption(rec); err == nil && perr == nil && a.Binds(writer) { |
| + if err := p.StagePlatform(ctx, instancekey.AdoptionItem, rec); err != nil { |
| + return 0, err |
| + } |
| + } else if err := p.StagePlatformDelete(instancekey.AdoptionItem); err != nil { |
| + return 0, err |
| + } |
| st, err := hostfiles.SyncDir(ctx, p, f.FilesDir(obs.Host), f.Skip) |
| if err != nil { |
| return 0, err |
| @@ -93,24 +111,26 @@ type releasePosition struct { |
| ReleasePosition |
| } |
| -// ReleasePositionsFile is ReleasePositions on disk: one file per host under |
| -// Dir, written whole and renamed into place, read back against the exact |
| -// stop request. The claim's ETag and the directory revision are not part |
| +// ReleasePositionsFile is ReleasePositions on disk: one file per operation |
| +// under Dir, named by the op id so no file under a managed root names a |
| +// host (region HA ruling 44), written whole and renamed into place, read |
| +// back against the exact stop request (whose host it names). The claim's ETag and the directory revision are not part |
| // of the match: a target replacement moves both while the source's stop |
| // stays the same stop. |
| type ReleasePositionsFile struct { |
| Dir string |
| } |
| -func (f ReleasePositionsFile) path(host string) (string, error) { |
| - if f.Dir == "" || !controlname.ValidHostShape(host) { |
| +func (f ReleasePositionsFile) path(r StopRequest) (string, error) { |
| + op := r.OperationID |
| + if f.Dir == "" || !controlname.ValidHostShape(r.Host) || !validOpID(op) { |
| return "", ErrClaimStopUnavailable |
| } |
| - return filepath.Join(f.Dir, host+".json"), nil |
| + return filepath.Join(f.Dir, op+".json"), nil |
| } |
| func (f ReleasePositionsFile) Record(_ context.Context, p ReleasePosition) error { |
| - path, err := f.path(p.Request.Host) |
| + path, err := f.path(p.Request) |
| if err != nil { |
| return err |
| } |
| @@ -158,7 +178,7 @@ func sameStop(a, b StopRequest) bool { |
| } |
| func (f ReleasePositionsFile) Recorded(_ context.Context, r StopRequest) (ReleasePosition, bool, error) { |
| - path, err := f.path(r.Host) |
| + path, err := f.path(r) |
| if err != nil { |
| return ReleasePosition{}, false, err |
| } |
| @@ -180,3 +200,11 @@ func (f ReleasePositionsFile) Recorded(_ context.Context, r StopRequest) (Releas |
| } |
| return p.ReleasePosition, true, nil |
| } |
| + |
| +// validOpID is the one check that an op id can name a file or directory |
| +// under the data root (region HA ruling 44): non-empty, not "." or "..", |
| +// no separator or NUL. Release positions and every relocation journal use |
| +// it. |
| +func validOpID(op string) bool { |
| + return op != "" && op != "." && op != ".." && !strings.ContainsAny(op, "/\\\x00") |
| +} |
diff --git a/internal/host/relocationflush_adoption_test.go b/internal/host/relocationflush_adoption_test.go| new file mode 100644 |
| index 000000000..eda94903d |
| --- /dev/null |
| +++ b/internal/host/relocationflush_adoption_test.go |
| @@ -0,0 +1,72 @@ |
| +package host |
| + |
| +import ( |
| + "context" |
| + "os" |
| + "path/filepath" |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + "github.com/carlosframework/platform/internal/hostfiles" |
| + "github.com/carlosframework/platform/internal/instancekey" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// O5 (instance-key plan rev 3): the release flush publishes the adoption |
| +// record in the same commit as the env item only when it binds exactly the |
| +// releasing writer; a stale record, or none, publishes none. |
| +func TestReleaseFlushPublishesOnlyTheReleasingWritersAdoption(t *testing.T) { |
| + ctx := context.Background() |
| + const host = "shop.example" |
| + writer := instancekey.Binding{Host: host, Incarnation: "inc-1", Session: 6, Generation: "g6"} |
| + for _, tc := range []struct { |
| + name string |
| + record []byte // nil is no file |
| + want bool |
| + }{ |
| + {"the releasing writer's record", instancekey.AdoptionRecord(writer, instancekey.SourcePlatform, instancekey.Digest("k1")), true}, |
| + {"a stale record from session 5", instancekey.AdoptionRecord(instancekey.Binding{Host: host, Incarnation: "inc-1", Session: 5, Generation: "g5"}, instancekey.SourcePlatform, instancekey.Digest("k1")), false}, |
| + {"a malformed record", []byte(`{"v":1}`), false}, |
| + {"no record", nil, false}, |
| + } { |
| + t.Run(tc.name, func(t *testing.T) { |
| + data := release.NewDirStore(t.TempDir()) |
| + reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { reg.Close() }) |
| + if err := reg.Upsert(registry.Route{Host: host, Kind: "service", Addr: "10.0.0.1:3000", Status: "ready", Hibernate: true}); err != nil { |
| + t.Fatal(err) |
| + } |
| + dir, envDir := t.TempDir(), t.TempDir() |
| + if err := os.WriteFile(filepath.Join(envDir, host+".env"), []byte("CARLOS_INSTANCE_KEY=k1\n"), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + if tc.record != nil { |
| + if err := os.WriteFile(filepath.Join(envDir, host+instancekey.AdoptionSuffix), tc.record, 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + f := ReleaseFlushFiles{Data: data, Registry: reg, FilesDir: func(string) string { return dir }, EnvPath: func(h string) string { return filepath.Join(envDir, h+".env") }} |
| + obs := hostclaim.Observed{Host: host, Claim: &hostclaim.Claim{Host: host, Holder: &hostclaim.Holder{Location: "instances/" + host + "/inc", Incarnation: "inc-1", Session: 6, Generation: "g6"}}} |
| + seq, err := f.Flush(ctx, obs) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + prefix := "instances/" + host + "/inc/files/g6/" |
| + m, ok, err := hostfiles.Load(ctx, data, prefix) |
| + if err != nil || !ok || m.Seq != seq { |
| + t.Fatalf("manifest: %v %v", ok, err) |
| + } |
| + got, err := hostfiles.ReadPlatform(ctx, data, prefix, m, instancekey.AdoptionItem, nil) |
| + if tc.want && (err != nil || string(got) != string(tc.record)) { |
| + t.Fatalf("the writer's record was not published: %q %v", got, err) |
| + } |
| + if !tc.want && err == nil { |
| + t.Fatalf("a record not the releasing writer's was published: %q", got) |
| + } |
| + }) |
| + } |
| +} |
diff --git a/internal/host/relocationlifecycle_linux_test.go b/internal/host/relocationlifecycle_linux_test.go| deleted file mode 100644 |
| index 5d81ac341..000000000 |
| --- a/internal/host/relocationlifecycle_linux_test.go |
| +++ /dev/null |
| @@ -1,810 +0,0 @@ |
| -//go:build linux |
| - |
| -package host |
| - |
| -import ( |
| - "context" |
| - "encoding/json" |
| - "errors" |
| - "fmt" |
| - "math/rand" |
| - "os" |
| - "path/filepath" |
| - "slices" |
| - "strconv" |
| - "strings" |
| - "sync" |
| - "testing" |
| - "time" |
| - |
| - "github.com/carlosframework/platform/internal/hostbox" |
| - "github.com/carlosframework/platform/internal/hostclaim" |
| - execctl "github.com/carlosframework/platform/internal/hostexec" |
| - "github.com/carlosframework/platform/internal/hostguard" |
| - "github.com/carlosframework/platform/internal/hostrelocate" |
| - "github.com/carlosframework/platform/internal/release" |
| -) |
| - |
| -// lifeSeeds is the property test's seed count (rev 14: 200 by default). |
| -// Run it niced; it is not a sweep. |
| -const lifeSeeds = 200 |
| - |
| -// lifeStore is the bucket with faults: lease puts that fail (a renewal |
| -// that cannot reach the store) and claim reads that fail or find nothing. |
| -type lifeStore struct { |
| - release.Store |
| - mu sync.Mutex |
| - failLease bool |
| - claimErr error |
| - claimKey string |
| -} |
| - |
| -func (s *lifeStore) Put(ctx context.Context, key string, b []byte, etag string) (string, error) { |
| - s.mu.Lock() |
| - fail := s.failLease && strings.Contains(key, "/exec/") |
| - s.mu.Unlock() |
| - if fail { |
| - return "", errors.New("lease put: store unreachable") |
| - } |
| - return s.Store.Put(ctx, key, b, etag) |
| -} |
| - |
| -func (s *lifeStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| - s.mu.Lock() |
| - err := s.claimErr |
| - s.mu.Unlock() |
| - if err != nil && key == s.claimKey { |
| - return nil, "", err |
| - } |
| - return s.Store.Get(ctx, key) |
| -} |
| - |
| -func (s *lifeStore) DeleteIfMatch(ctx context.Context, key, etag string) error { |
| - return release.DeleteIfMatch(ctx, s.Store, key, etag) |
| -} |
| - |
| -// lifeClock is hostexec's clock under the test's hand: every After fires |
| -// on tick, so a renewal happens exactly when a step says so. |
| -type lifeClock struct { |
| - mu sync.Mutex |
| - wall time.Time |
| - boot int64 |
| - waiters []chan time.Time |
| -} |
| - |
| -func (c *lifeClock) Now() (time.Time, int64, error) { |
| - c.mu.Lock() |
| - defer c.mu.Unlock() |
| - return c.wall, c.boot, nil |
| -} |
| - |
| -func (c *lifeClock) After(time.Duration) <-chan time.Time { |
| - ch := make(chan time.Time, 1) |
| - c.mu.Lock() |
| - c.waiters = append(c.waiters, ch) |
| - c.mu.Unlock() |
| - return ch |
| -} |
| - |
| -func (c *lifeClock) tick() { |
| - c.mu.Lock() |
| - c.wall, c.boot = c.wall.Add(time.Second), c.boot+int64(time.Second) |
| - ws := c.waiters |
| - c.waiters = nil |
| - now := c.wall |
| - c.mu.Unlock() |
| - for _, w := range ws { |
| - w <- now |
| - } |
| -} |
| - |
| -// lifeWorld is one box ("src") taking part in a host's relocations over |
| -// real hostexec controllers, with the faults a seed injects. |
| -type lifeWorld struct { |
| - t *testing.T |
| - ctx context.Context |
| - inner release.Store |
| - store *lifeStore |
| - clock *lifeClock |
| - dataDir string |
| - host string |
| - cfg execctl.Config |
| - e *RelocationEffects |
| - p hostrelocate.Participant |
| - ops int |
| - mu sync.Mutex |
| - cleanErr bool // both cleanups fail |
| - block chan struct{} // non-nil: both cleanups wait for it (a close that times out) |
| - cleanOK map[string]bool |
| - // installed is this box's local install per operation (rev 15): the |
| - // Install provider makes it, a scoped cleanup of an operation whose P |
| - // is unopened retires it, VerifyLocal reads it. |
| - installed map[string]bool |
| - // gen is the agent process's generation: a crash starts the next one. |
| - // A dead generation's hooks never complete (rev 16). |
| - gen int |
| - // clears and opens count durable records cleared and controllers |
| - // opened in the current step, for invariant (h). |
| - clears, opens int |
| - // the record as the previous step left it. |
| - prevExists, prevReadable bool |
| - prevOp string |
| - saved []byte // a record the seed corrupted, to restore on recovery |
| - violated []string |
| -} |
| - |
| -const lifeDeadline = 400 * time.Millisecond |
| - |
| -func newLifeWorld(t *testing.T) *lifeWorld { |
| - t.Helper() |
| - ctx := context.Background() |
| - w := &lifeWorld{t: t, ctx: ctx, host: "shop.bab.oncarlos.com", clock: &lifeClock{wall: time.Date(2026, 10, 7, 9, 0, 0, 0, time.UTC), boot: int64(time.Hour)}, cleanOK: map[string]bool{}, installed: map[string]bool{}} |
| - w.dataDir = t.TempDir() |
| - for _, name := range []string{"hostguard", ".hostexec", "locks", "exec-locks"} { |
| - if err := os.Mkdir(filepath.Join(w.dataDir, name), 0o770); err != nil { |
| - t.Fatal(err) |
| - } |
| - } |
| - w.inner = release.NewDirStore(t.TempDir()) |
| - key, _ := hostclaim.Key(w.host) |
| - w.store = &lifeStore{Store: w.inner, claimKey: key} |
| - created := time.Date(2026, 10, 4, 1, 0, 0, 0, time.UTC) |
| - nh := hostclaim.NewHolder{Family: "instance", Key: "apps/bab/jam/instances/" + w.host + ".json", Account: "bab", App: "jam", Host: w.host, Region: "ie", RecordRegion: "ie", CreatedAt: created} |
| - body := []byte(`{"host":"` + w.host + `","account":"bab","app":"jam","region":"ie","created_at":"2026-10-04T01:00:00Z"}`) |
| - w.claim(hostclaim.BeginCreateOn(nh, body, "c1", "console", created, hostclaim.ExecutorSelection{Box: "src", Enrollment: strings.Repeat("a", 32)})) |
| - w.claim(hostclaim.CompleteCreate("c1", "src")) |
| - w.claim(hostclaim.AdmitWriter(hostclaim.Admission{Box: "src", Mode: "cold", NewGeneration: "g1", Region: "ie", KeepStamp: true})) |
| - w.cfg = execctl.Config{Store: w.store, Guard: hostguard.Guard{DataDir: w.dataLink(0), LockDir: filepath.Join(w.dataDir, "locks"), Claims: hostguard.StoreReader{Store: w.store}}, |
| - Directory: activeDirectory{}, Clock: w.clock, LockDir: filepath.Join(w.dataDir, "exec-locks"), |
| - CloseAdmission: func() {}, CancelWork: func() {}} |
| - lifetime, stop := context.WithCancel(ctx) |
| - t.Cleanup(stop) |
| - w.e = &RelocationEffects{Lifetime: lifetime, |
| - Identity: func(context.Context) (RelocationIdentity, error) { |
| - return RelocationIdentity{Actor: hostbox.Actor{Box: "src", Enrollment: strings.Repeat("a", 32), Revision: 4}, BootID: "boot", ControllerID: "ctl"}, nil |
| - }, |
| - Controller: func(ctx context.Context, b execctl.Binding) (RelocationController, error) { |
| - c := w.cfg |
| - c.Binding = b |
| - // hostexec's own Cleanup hook is the same scoped cleanup, |
| - // bound to the binding (Task 8a); it does not count for (a). |
| - if cur := w.current(); cur.Op.ID == b.Op && !slices.Contains(cur.Op.Participants, "src") { |
| - w.violate(fmt.Sprintf("(i) %s opened again after this box was excluded from it", b.Op)) |
| - } |
| - gen := w.generation() |
| - c.Cleanup = func(context.Context) error { return w.cleanupScoped(context.Background(), b.Op, false, gen) } |
| - opened, err := OpenRelocationController(ctx, c) |
| - if err != nil { |
| - return nil, err |
| - } |
| - w.mu.Lock() |
| - w.opens++ |
| - w.mu.Unlock() |
| - return &lifeController{RelocationController: opened, w: w, op: b.Op}, nil |
| - }, |
| - Install: func(_ context.Context, o hostclaim.Observed) (hostclaim.LocalInstall, error) { |
| - c := w.current() |
| - if hostclaim.PreparedOpened(c) { |
| - w.violate("(g) the local install was remade after P was opened") |
| - } |
| - w.mu.Lock() |
| - w.installed[c.Op.ID] = true |
| - w.mu.Unlock() |
| - n, _ := strconv.ParseUint(c.Op.Journal["install_stamp"], 10, 64) |
| - return hostclaim.LocalInstall{Box: "src", Enrollment: strings.Repeat("a", 32), InstallStamp: n, FilesManifestVersion: 1, JournalSHA256: strings.Repeat("e", 64)}, nil |
| - }, |
| - Receipt: func(_ context.Context, o hostclaim.Observed) (hostclaim.LocalInstall, bool, error) { |
| - var l hostclaim.LocalInstall |
| - _ = json.Unmarshal([]byte(o.Claim.Op.Journal["local_install"]), &l) |
| - w.mu.Lock() |
| - defer w.mu.Unlock() |
| - return l, w.installed[o.Claim.Op.ID], nil |
| - }, |
| - VerifyLocal: func(_ context.Context, o hostclaim.Observed, _ hostclaim.LocalInstall) (bool, error) { |
| - w.mu.Lock() |
| - defer w.mu.Unlock() |
| - return w.installed[o.Claim.Op.ID], nil |
| - }, |
| - Admit: func(_ context.Context, o hostclaim.Observed, _ bool) (uint64, bool, error) { |
| - // Production admits under hostguard.With, whose CheckLocal |
| - // refuses once the controller has lost its lease, and the |
| - // scoped cleanup retires the install under the same host |
| - // lock. Here the loss check, the install's presence and the |
| - // admission are one step under the lock cleanupScoped takes, |
| - // so a loss mid-pass refuses the admission as it would in |
| - // production; a missing install is still a violation. |
| - ctl, ok := w.e.Controllers.get(w.host, o.Claim.Op.ID) |
| - if !ok { |
| - return 0, false, hostbox.ErrChanged |
| - } |
| - w.mu.Lock() |
| - defer w.mu.Unlock() |
| - select { |
| - case <-ctl.Lost(): |
| - return 0, false, hostbox.ErrChanged |
| - default: |
| - } |
| - cur, err := hostclaim.Read(context.Background(), w.inner, w.host) |
| - if err != nil { |
| - return 0, false, err |
| - } |
| - if !hostclaim.PreparedOpened(cur.Claim) && !w.installed[cur.Claim.Op.ID] { |
| - w.violated = append(w.violated, "(f) P's first admission over a missing local install") |
| - } |
| - // The admission opens P (the claim journals it). |
| - if _, err := hostclaim.Update(context.Background(), w.inner, o, func(c *hostclaim.Claim) error { |
| - c.Op.Journal["prepared_opened"] = c.Op.Journal["prepared"] |
| - return nil |
| - }); err != nil { |
| - return 0, false, err |
| - } |
| - return 0, false, nil // the probe is still running |
| - }} |
| - w.e.Controllers = w.controllers() |
| - w.p = hostrelocate.Participant{Claims: w.store, Box: "src", Enrollment: strings.Repeat("a", 32), Effects: w.e, Now: func() time.Time { return w.clock.wall }} |
| - return w |
| -} |
| - |
| -// lifeController checks invariant (a) at every clear of a live |
| -// controller's record. |
| -type lifeController struct { |
| - RelocationController |
| - w *lifeWorld |
| - op string |
| -} |
| - |
| -func (c *lifeController) Clear(ctx context.Context) error { |
| - c.w.clearing(c.op) |
| - err := c.RelocationController.Clear(ctx) |
| - c.w.cleared(err) |
| - return err |
| -} |
| - |
| -func (w *lifeWorld) cleared(err error) { |
| - if err == nil { |
| - w.mu.Lock() |
| - w.clears++ |
| - w.mu.Unlock() |
| - } |
| -} |
| - |
| -func (w *lifeWorld) generation() int { |
| - w.mu.Lock() |
| - defer w.mu.Unlock() |
| - return w.gen |
| -} |
| - |
| -func (w *lifeWorld) controllers() *RelocationControllers { |
| - gen := w.generation() |
| - return &RelocationControllers{Journal: filepath.Join(w.dataDir, "relocate", "participations"), DataDir: w.dataDir, |
| - Cleanup: func(ctx context.Context, b execctl.Binding) error { return w.cleanupScoped(ctx, b.Op, true, gen) }, |
| - Clear: func(ctx context.Context, b execctl.Binding) error { |
| - w.clearing(b.Op) |
| - c := w.cfg |
| - c.Binding = b |
| - err := execctl.ClearStale(ctx, c) |
| - w.cleared(err) |
| - return err |
| - }} |
| -} |
| - |
| -// cleanupScoped is the scoped cleanup (Task 8a's relocationCleanup): |
| -// it retires the operation's local install unless the claim has opened |
| -// that operation's P (it never stops what serves). counts says whether it |
| -// is RelocationControllers' own call, the one invariant (a) asks for. |
| -func (w *lifeWorld) cleanupScoped(ctx context.Context, op string, counts bool, gen int) error { |
| - w.mu.Lock() |
| - block, fail, dead := w.block, w.cleanErr, gen != w.gen |
| - w.mu.Unlock() |
| - if dead { |
| - select {} // a crashed process's hooks never complete |
| - } |
| - w.mu.Lock() |
| - if counts { |
| - w.cleanOK[op] = false |
| - } |
| - w.mu.Unlock() |
| - if block != nil { |
| - select { |
| - case <-block: |
| - case <-ctx.Done(): |
| - return ctx.Err() |
| - } |
| - } |
| - if gen != w.generation() { |
| - select {} // the process died while its cleanup waited |
| - } |
| - if fail { |
| - return errors.New("cleanup: a partial install would not go") |
| - } |
| - // The claim is read under the lock the fake admission holds, so a |
| - // cleanup and an admission are ordered (production: the host lock). |
| - w.mu.Lock() |
| - c, err := hostclaim.Read(context.Background(), w.inner, w.host) |
| - opened := err == nil && !c.Absent && c.Claim.Op.ID == op && hostclaim.PreparedOpened(c.Claim) |
| - if !opened { |
| - w.installed[op] = false |
| - } |
| - if counts { |
| - w.cleanOK[op] = true |
| - } |
| - w.mu.Unlock() |
| - return nil |
| -} |
| - |
| -// clearing is invariant (a): a record is cleared only straight after a |
| -// scoped cleanup of the same participation succeeded, in the same call. |
| -func (w *lifeWorld) clearing(op string) { |
| - w.mu.Lock() |
| - defer w.mu.Unlock() |
| - if !w.cleanOK[op] { |
| - w.violated = append(w.violated, "(a) clear of "+op+" without a successful scoped cleanup first") |
| - } |
| - w.cleanOK[op] = false |
| -} |
| - |
| -func (w *lifeWorld) claim(tr func(*hostclaim.Claim) error) { |
| - w.t.Helper() |
| - o, err := hostclaim.Read(w.ctx, w.inner, w.host) |
| - if err != nil { |
| - w.t.Fatal(err) |
| - } |
| - if _, err := hostclaim.Update(w.ctx, w.inner, o, tr); err != nil { |
| - w.t.Fatal(err) |
| - } |
| -} |
| - |
| -func (w *lifeWorld) current() *hostclaim.Claim { |
| - w.t.Helper() |
| - o, err := hostclaim.Read(w.ctx, w.inner, w.host) |
| - if err != nil { |
| - w.t.Fatal(err) |
| - } |
| - return o.Claim |
| -} |
| - |
| -// startRelocation starts the next relocation, src taking part (as source |
| -// or as target), whatever the claim was doing. |
| -func (w *lifeWorld) startRelocation() { |
| - w.ops++ |
| - id := fmt.Sprintf("rel-%d", w.ops) |
| - from, to := "src", "dst" |
| - if exec := w.current().Op.Exec; exec != "" && exec != "src" && exec != "none" { |
| - from, to = exec, "src" |
| - } |
| - w.claim(func(c *hostclaim.Claim) error { |
| - c.Op = hostclaim.Op{Kind: hostclaim.KindRelocating, ID: id, Phase: hostclaim.PhaseStarting, Exec: from, Participants: []string{from, to}, |
| - Intent: map[string]string{"region": "ie", "source_box": from, "target_box": to, "target_enrollment": map[string]string{"src": strings.Repeat("a", 32), "dst": strings.Repeat("d", 32), "sp2": strings.Repeat("e", 32)}[to]}} |
| - return nil |
| - }) |
| -} |
| - |
| -// replaced is the worker replacing this box, the target, within the same |
| -// operation (rev 19, round 18 finding 1): after its release in |
| -// probe-failed, or at any target phase, the relocation goes on with sp2 |
| -// and this box is no longer a participant. Its controller, record and |
| -// journal must go, and nothing for this operation may open again here. |
| -func (w *lifeWorld) replaced() bool { |
| - c := w.current() |
| - if c.Op.Kind != hostclaim.KindRelocating || c.Op.Intent["target_box"] != "src" { |
| - return false |
| - } |
| - w.claim(func(c *hostclaim.Claim) error { |
| - if c.Op.Journal == nil { |
| - c.Op.Journal = map[string]string{} |
| - } |
| - c.Op.Participants = append(slices.DeleteFunc(slices.Clone(c.Op.Participants), func(b string) bool { return b == "src" }), "sp2") |
| - c.Op.Intent["target_box"], c.Op.Intent["target_enrollment"] = "sp2", strings.Repeat("e", 32) |
| - if c.Op.Phase != hostclaim.PhaseStarting && c.Op.Phase != hostclaim.PhaseReleasing { |
| - c.Op.Exec, c.Op.Phase = "none", hostclaim.PhasePinning |
| - for _, k := range []string{"local_install", "probe", "prepared", "prepared_opened", "install_stamp", "release", "release.phase", "probing_since", "probe_failed_since"} { |
| - delete(c.Op.Journal, k) |
| - } |
| - } |
| - c.Op.Journal["excluded_targets"] = `["src"]` |
| - c.Op.Journal["exec_ack.sp2"] = "{}" |
| - return nil |
| - }) |
| - return true |
| -} |
| - |
| -// probeFailedAndReleased moves this box, the target, to probe-failed with |
| -// its release journaled (the worker's and its own half, in short). |
| -func (w *lifeWorld) probeFailedAndReleased() { |
| - w.claim(func(c *hostclaim.Claim) error { |
| - if c.Op.Journal == nil { |
| - c.Op.Journal = map[string]string{} |
| - } |
| - c.Op.Phase = hostclaim.PhaseProbeFailed |
| - c.Op.Journal["probe_failed_since"] = w.clock.wall.Format(time.RFC3339Nano) |
| - c.Op.Journal["release"] = `{"released_by":"src"}` |
| - c.Op.Journal["release.phase"] = hostclaim.PhaseProbeFailed |
| - return nil |
| - }) |
| -} |
| - |
| -// install is the worker's half while this box is the target: P built and |
| -// the operation handed to it (installed), its local install still to make. |
| -func (w *lifeWorld) install() { |
| - c := w.current() |
| - if c.Op.Kind != hostclaim.KindRelocating || c.Op.Intent["target_box"] != "src" || (c.Op.Phase != hostclaim.PhaseStarting && c.Op.Phase != hostclaim.PhaseReleasing) { |
| - return |
| - } |
| - w.claim(func(c *hostclaim.Claim) error { |
| - c.Op.Phase, c.Op.Exec = hostclaim.PhaseInstalled, "src" |
| - if c.Op.Journal == nil { |
| - c.Op.Journal = map[string]string{} |
| - } |
| - c.Op.Journal["install_stamp"] = strconv.FormatUint(c.Gen+1, 10) |
| - c.Op.Journal["prepared"] = "P-" + c.Op.ID |
| - if c.Op.Journal["exec_ack.dst"] == "" { |
| - c.Op.Journal["exec_ack.dst"] = "{}" |
| - } |
| - return nil |
| - }) |
| -} |
| - |
| -func (w *lifeWorld) complete() { |
| - if c := w.current(); c.Op.Kind == hostclaim.KindRelocating { |
| - box := c.Op.Intent["target_box"] |
| - w.claim(func(c *hostclaim.Claim) error { |
| - c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: box, Participants: []string{box}} |
| - return nil |
| - }) |
| - } |
| -} |
| - |
| -// pass is one participant pass, Step then Sweep as the agent runs them, |
| -// under a deadline; (d) is that it returns within it. |
| -func (w *lifeWorld) pass() { |
| - w.mu.Lock() |
| - w.cleanOK = map[string]bool{} // "the same call": nothing carries over from an earlier pass |
| - w.mu.Unlock() |
| - done := make(chan struct{}) |
| - start := time.Now() |
| - go func() { |
| - defer close(done) |
| - ctx, cancel := context.WithTimeout(w.ctx, lifeDeadline) |
| - defer cancel() |
| - _, _ = w.p.Step(ctx, w.host) |
| - _ = w.p.Sweep(ctx) |
| - }() |
| - select { |
| - case <-done: |
| - if took := time.Since(start); took > lifeDeadline+time.Second { |
| - w.violate(fmt.Sprintf("(d) a pass took %v past its %v deadline", took, lifeDeadline)) |
| - } |
| - case <-time.After(lifeDeadline + 3*time.Second): |
| - w.t.Fatalf("(d) a pass never returned within %v", lifeDeadline+3*time.Second) |
| - } |
| -} |
| - |
| -func (w *lifeWorld) violate(s string) { |
| - w.mu.Lock() |
| - w.violated = append(w.violated, s) |
| - w.mu.Unlock() |
| -} |
| - |
| -// renew fires hostexec's timers so every controller renews (or loses). A |
| -// renewal or a loss may still be in flight when the next event runs; the |
| -// fake admission refuses after a loss as production does. |
| -func (w *lifeWorld) renew() { |
| - w.clock.tick() |
| - time.Sleep(20 * time.Millisecond) |
| -} |
| - |
| -func (w *lifeWorld) record() (execctl.State, bool, bool) { |
| - raw, err := os.ReadFile(filepath.Join(w.dataDir, ".hostexec", w.host+".json")) |
| - if errors.Is(err, os.ErrNotExist) { |
| - return execctl.State{}, false, true |
| - } |
| - var st execctl.State |
| - if err != nil || json.Unmarshal(raw, &st) != nil { |
| - return st, true, false |
| - } |
| - return st, true, true |
| -} |
| - |
| -func (w *lifeWorld) open() []RelocationController { |
| - r := w.e.Controllers |
| - r.mu.Lock() |
| - defer r.mu.Unlock() |
| - var out []RelocationController |
| - for _, c := range r.open { |
| - out = append(out, c) |
| - } |
| - return out |
| -} |
| - |
| -// check is invariants (a) to (c) after a step. |
| -func (w *lifeWorld) check(step string) { |
| - st, exists, readable := w.record() |
| - if exists && readable { |
| - retained, _ := w.e.Controllers.Retained() |
| - if !slices.ContainsFunc(retained, func(p hostrelocate.Participation) bool { return p.Host == w.host && p.Op == st.Op }) { |
| - w.violate(fmt.Sprintf("(b) after %s: %s's record exists but its participation is forgotten (retained %+v)", step, st.Op, retained)) |
| - } |
| - } |
| - if n := len(w.open()); n > 1 { |
| - w.violate(fmt.Sprintf("(c) after %s: %d controllers for one host", step, n)) |
| - } |
| - // (h): a record goes only by a clear or by a successor's open over it, |
| - // never by a crash or any other step; while one exists, an executable |
| - // update is refused (root's §5.10 check). |
| - w.mu.Lock() |
| - clears, opens := w.clears, w.opens |
| - w.clears, w.opens = 0, 0 |
| - w.mu.Unlock() |
| - if w.prevExists && w.prevReadable && (!exists || (readable && st.Op != w.prevOp)) && clears == 0 && opens == 0 { |
| - w.violate(fmt.Sprintf("(h) after %s: %s's record went with no clear and no successor's open", step, w.prevOp)) |
| - } |
| - if exists && hostexecParticipation(w.dataDir) == nil { |
| - w.violate(fmt.Sprintf("(h) after %s: a record exists but an executable update is not refused", step)) |
| - } |
| - w.prevExists, w.prevReadable, w.prevOp = exists, readable, st.Op |
| -} |
| - |
| -// dataLink is generation gen's path to the data directory: a symlink, so |
| -// a crash can cut a dead generation off from it. A dead process writes |
| -// nothing; in-process, a dead controller's goroutines are still running |
| -// (hostexec's save loop retries a "lost" write until it lands), so the |
| -// link it opens through is removed. |
| -func (w *lifeWorld) dataLink(gen int) string { |
| - link := filepath.Join(filepath.Dir(w.dataDir), fmt.Sprintf("data-%d", gen)) |
| - if err := os.Symlink(w.dataDir, link); err != nil && !os.IsExist(err) { |
| - w.t.Fatal(err) |
| - } |
| - return link |
| -} |
| - |
| -// crash is the agent process dying abruptly (rev 16, round 15): no |
| -// controller is closed and none of its hooks run or complete; its durable |
| -// records stay, and every external fault (a blocked or failing cleanup, a |
| -// store that refuses) stays as it was. The kernel releases a dead |
| -// process's locks: the next generation uses fresh lock files, and the dead |
| -// generation's clock never fires again, so its controllers never renew, |
| -// lose or clean up. Recovery is a separate, later event. |
| -func (w *lifeWorld) crash() { |
| - w.mu.Lock() |
| - w.gen++ |
| - gen := w.gen |
| - w.mu.Unlock() |
| - now, boot, _ := w.clock.Now() |
| - w.clock = &lifeClock{wall: now, boot: boot} |
| - for _, d := range []string{fmt.Sprintf("exec-locks-%d", gen), fmt.Sprintf("locks-%d", gen)} { |
| - if err := os.MkdirAll(filepath.Join(w.dataDir, d), 0o770); err != nil { |
| - w.t.Fatal(err) |
| - } |
| - } |
| - w.cfg.Clock = w.clock |
| - w.cfg.LockDir = filepath.Join(w.dataDir, fmt.Sprintf("exec-locks-%d", gen)) |
| - w.cfg.Guard.LockDir = filepath.Join(w.dataDir, fmt.Sprintf("locks-%d", gen)) |
| - if err := os.Remove(w.cfg.Guard.DataDir); err != nil { |
| - w.t.Fatal(err) |
| - } |
| - w.cfg.Guard.DataDir = w.dataLink(gen) |
| - w.e.Controllers = w.controllers() |
| -} |
| - |
| -func (w *lifeWorld) recoverCleanup() { |
| - w.mu.Lock() |
| - if w.block != nil { |
| - close(w.block) |
| - w.block = nil |
| - } |
| - w.cleanErr = false |
| - w.mu.Unlock() |
| -} |
| - |
| -func (w *lifeWorld) recoverAll() { |
| - w.recoverCleanup() |
| - w.store.mu.Lock() |
| - w.store.failLease, w.store.claimErr = false, nil |
| - w.store.mu.Unlock() |
| - if w.saved != nil { |
| - path := filepath.Join(w.dataDir, ".hostexec", w.host+".json") |
| - if raw, err := os.ReadFile(path); err == nil && string(raw) == "{" { |
| - _ = os.WriteFile(path, w.saved, 0o660) |
| - // hostexec retries a "lost" write every 100ms through a |
| - // directory it already holds open, so a controller that met |
| - // the invalid record (a dead generation's included: in-process |
| - // its goroutine outlives the crash) lands its write now, as |
| - // part of the repair, never after a later clear. |
| - time.Sleep(300 * time.Millisecond) |
| - } |
| - w.saved = nil |
| - } |
| -} |
| - |
| -func (w *lifeWorld) event(r *rand.Rand) string { |
| - switch n := r.Intn(16); n { |
| - case 15: |
| - if w.replaced() { |
| - return "this box replaced in the same operation" |
| - } |
| - return "no replacement (not the target)" |
| - case 14: |
| - w.install() |
| - return "the target's installation (P built)" |
| - case 0, 1, 2, 3: |
| - w.pass() |
| - return "pass" |
| - case 4: |
| - w.renew() |
| - return "renew" |
| - case 5: |
| - w.store.mu.Lock() |
| - w.store.failLease = true |
| - w.store.mu.Unlock() |
| - w.renew() |
| - return "lease renewal fails" |
| - case 6: |
| - w.mu.Lock() |
| - if w.block == nil { |
| - w.block = make(chan struct{}) |
| - } |
| - w.mu.Unlock() |
| - return "cleanup blocks (close times out)" |
| - case 7: |
| - w.mu.Lock() |
| - w.cleanErr = true |
| - w.mu.Unlock() |
| - return "cleanup fails" |
| - case 8: |
| - w.store.mu.Lock() |
| - if r.Intn(2) == 0 { |
| - w.store.claimErr = errors.New("claim read: store unreachable") |
| - } else { |
| - w.store.claimErr = os.ErrNotExist // ClearStale: ErrState for an absent claim |
| - } |
| - w.store.mu.Unlock() |
| - return "claim reads fail" |
| - case 9: |
| - path := filepath.Join(w.dataDir, ".hostexec", w.host+".json") |
| - if raw, err := os.ReadFile(path); err == nil && w.saved == nil { |
| - w.saved = raw |
| - _ = os.WriteFile(path, []byte("{"), 0o660) // ClearStale: an invalid local record |
| - } |
| - return "local record invalid" |
| - case 10: |
| - w.crash() |
| - return "agent crash (no hooks run, faults persist)" |
| - case 11: |
| - w.complete() |
| - w.startRelocation() // the successor starts before this box's sweep |
| - return "completion, successor before the sweep" |
| - case 12: |
| - w.complete() |
| - return "completion" |
| - default: |
| - w.recoverAll() |
| - return "storage recovers" |
| - } |
| -} |
| - |
| -// TestRelocationControllerLifecycleProperties drives seeded random |
| -// sequences of lease-renewal failures, close timeouts, cleanup failures, |
| -// ClearStale failures (claim unreadable, claim absent, invalid local |
| -// record, a successor that lists this box), agent restarts, successors |
| -// before the sweep, completions and recovery, over real hostexec |
| -// controllers, and checks after every step: (a) no clear without a |
| -// successful scoped cleanup in the same call; (b) no participation |
| -// forgotten while its record exists; (c) at most one controller per host; |
| -// (d) no pass past its deadline; and (e) once faults stop, within eight |
| -// passes the current relocation is ready again, then after completion |
| -// every record is cleared and no controller remains, and a new relocation |
| -// opens. |
| -func TestRelocationControllerLifecycleProperties(t *testing.T) { |
| - seeds := lifeSeeds |
| - if testing.Short() { |
| - seeds = 20 |
| - } |
| - for seed := 1; seed <= seeds; seed++ { |
| - t.Run(fmt.Sprint(seed), func(t *testing.T) { |
| - r := rand.New(rand.NewSource(int64(seed))) |
| - w := newLifeWorld(t) |
| - defer w.recoverAll() |
| - w.startRelocation() |
| - var trace []string |
| - if seed%2 == 0 { |
| - // Every other seed starts as the target with its local |
| - // install journaled (rev 15), so the random events fall on |
| - // an installation that a lost lease's cleanup can retire |
| - // before P's first admission. |
| - w.complete() |
| - w.startRelocation() // dst back to src: this box is the target |
| - w.pass() |
| - w.install() |
| - w.pass() |
| - trace = append(trace, "target, installed, local install journaled") |
| - if w.current().Op.Journal["local_install"] == "" { |
| - t.Fatalf("seed %d: the local install was not journaled: %+v", seed, w.current().Op) |
| - } |
| - if seed%4 == 2 { |
| - // Round 18's trace: the target's probe fails, it |
| - // releases, it is replaced while the relocation goes |
| - // on elsewhere, its cleanup blocks, the agent crashes. |
| - w.pass() |
| - w.probeFailedAndReleased() |
| - if !w.replaced() { |
| - t.Fatalf("seed %d: this box was not the target to replace: %+v", seed, w.current().Op) |
| - } |
| - w.mu.Lock() |
| - w.block = make(chan struct{}) |
| - w.mu.Unlock() |
| - w.pass() |
| - w.check("replaced with its cleanup blocked") |
| - w.crash() |
| - w.check("agent crash after the replacement") |
| - trace = append(trace, "probe failed, released, replaced, cleanup blocked, crash") |
| - } |
| - } |
| - for i := 0; i < 30; i++ { |
| - ev := w.event(r) |
| - trace = append(trace, ev) |
| - w.check(ev) |
| - if len(w.violated) > 0 { |
| - t.Fatalf("seed %d: %v\ntrace: %v", seed, w.violated, trace) |
| - } |
| - } |
| - // (e) Faults stop. |
| - w.recoverAll() |
| - trace = append(trace, "faults stop") |
| - // A participating box is ready again; an excluded one (rev 19) |
| - // has finished its cleanup: no controller, no record, no |
| - // journaled participation, while the relocation goes on. |
| - ready := func() bool { |
| - c := w.current() |
| - if c.Op.Kind != hostclaim.KindRelocating { |
| - return true |
| - } |
| - if !slices.Contains(c.Op.Participants, "src") { |
| - _, exists, _ := w.record() |
| - retained, err := w.e.Controllers.Retained() |
| - return !exists && err == nil && len(retained) == 0 && len(w.open()) == 0 |
| - } |
| - if c.Op.Journal["exec_ack.src"] == "" { |
| - return false |
| - } |
| - obs, err := hostclaim.Read(w.ctx, w.inner, w.host) |
| - return err == nil && w.e.Ready(w.ctx, obs) == nil |
| - } |
| - ok := false |
| - for i := 0; i < 8 && !ok; i++ { |
| - w.renew() |
| - w.pass() |
| - w.check("recovery pass") |
| - ok = ready() |
| - } |
| - if !ok || len(w.violated) > 0 { |
| - t.Fatalf("seed %d: the running relocation did not recover within 8 passes: %v\ntrace: %v", seed, w.violated, trace) |
| - } |
| - w.complete() |
| - clean := false |
| - for i := 0; i < 8 && !clean; i++ { |
| - w.renew() |
| - w.pass() |
| - w.check("sweep pass") |
| - _, exists, _ := w.record() |
| - retained, err := w.e.Controllers.Retained() |
| - clean = !exists && len(retained) == 0 && err == nil && len(w.open()) == 0 |
| - } |
| - if !clean || len(w.violated) > 0 { |
| - st, exists, _ := w.record() |
| - retained, _ := w.e.Controllers.Retained() |
| - t.Fatalf("seed %d: not clean within 8 passes after completion: record %v %+v retained %+v open %d %v\ntrace: %v", seed, exists, st, retained, len(w.open()), w.violated, trace) |
| - } |
| - w.startRelocation() |
| - for i := 0; i < 3; i++ { |
| - w.pass() |
| - } |
| - w.check("new relocation") |
| - if !ready() || len(w.open()) != 1 || len(w.violated) > 0 { |
| - t.Fatalf("seed %d: a new relocation did not open: open %d %v\ntrace: %v", seed, len(w.open()), w.violated, trace) |
| - } |
| - for _, c := range w.open() { |
| - _ = c.Close(context.Background()) |
| - } |
| - }) |
| - } |
| -} |
diff --git a/internal/host/relocationparticipation_linux_test.go b/internal/host/relocationparticipation_linux_test.go| new file mode 100644 |
| index 000000000..6634e267d |
| --- /dev/null |
| +++ b/internal/host/relocationparticipation_linux_test.go |
| @@ -0,0 +1,255 @@ |
| +//go:build linux |
| + |
| +package host |
| + |
| +import ( |
| + "context" |
| + "crypto/ed25519" |
| + "crypto/rand" |
| + "errors" |
| + "os" |
| + "path/filepath" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/hostactors" |
| + "github.com/carlosframework/platform/internal/hostbox" |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + execlease "github.com/carlosframework/platform/internal/hostexec" |
| + "github.com/carlosframework/platform/internal/hostguard" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +const participationHost = "shop.example" |
| + |
| +// participationBox is box "dst" of a relocation of shop.example from |
| +// "src", with main's participation owner as production builds it: a real |
| +// productionClaimLocal in agent mode over a DirStore control bucket with a |
| +// signed roster, real hostexec controllers (their locks under the N12 test |
| +// seam), the real census over empty roots, the launch journal, and a |
| +// recording transport where root would be asked for a stop. |
| +type participationBox struct { |
| + t *testing.T |
| + store *release.DirStore |
| + data string |
| + actor hostbox.Actor |
| + guard hostguard.Guard |
| + p *productionClaimLocal |
| + stops int // stops asked of root |
| + census *hostbox.EmptyInstallationCensus |
| + launches *hostbox.LaunchJournal |
| +} |
| + |
| +func newParticipationBox(t *testing.T) *participationBox { |
| + t.Helper() |
| + ctx := t.Context() |
| + b := &participationBox{t: t, store: release.NewDirStore(t.TempDir()), data: t.TempDir()} |
| + for _, name := range []string{"hostguard", ".hostexec"} { |
| + if err := os.Mkdir(filepath.Join(b.data, name), 0o770); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + pub, key, err := ed25519.GenerateKey(rand.Reader) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + enrollment := func(box string) string { return strings.Repeat(map[string]string{"src": "b", "dst": "c"}[box], 32) } |
| + roster := hostactors.Roster{V: 1, DeploymentBucket: "bucket", Revision: 4, Regions: map[string]hostactors.Region{"ie": {Account: "123", AWSRegion: "eu-west-1"}}, |
| + Boxes: map[string]hostactors.Box{}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}} |
| + for _, box := range []string{"src", "dst"} { |
| + roster.Boxes[box] = hostactors.Box{Enrollment: enrollment(box), Namespace: "platform", BoxID: box, Label: box, Region: "ie", Pool: "default", |
| + Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-" + box}, EvidenceSHA256: strings.Repeat("c", 64), State: "active"} |
| + } |
| + raw, err := release.SignRootEnvelope(release.RootSigner(key), roster) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := b.store.Put(ctx, hostactors.Key, raw, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + b.actor = hostbox.Actor{Box: "dst", Enrollment: enrollment("dst"), Region: "ie", Revision: 4} |
| + |
| + created := time.Date(2026, 10, 7, 1, 0, 0, 0, time.UTC) |
| + nh := hostclaim.NewHolder{Family: "instance", Key: "apps/acct/app/instances/" + participationHost + ".json", Account: "acct", App: "app", Host: participationHost, Region: "ie", RecordRegion: "ie", CreatedAt: created} |
| + body := []byte(`{"host":"` + participationHost + `","account":"acct","app":"app","region":"ie","created_at":"2026-10-07T01:00:00Z"}`) |
| + obs, err := hostclaim.Read(ctx, b.store, participationHost) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, tr := range []func(*hostclaim.Claim) error{ |
| + hostclaim.BeginCreateOn(nh, body, "c1", "console", created, hostclaim.ExecutorSelection{Box: "src", Enrollment: enrollment("src")}), |
| + hostclaim.CompleteCreate("c1", "src"), |
| + hostclaim.AdmitWriter(hostclaim.Admission{Box: "src", Mode: "cold", NewGeneration: "g1", Region: "ie", KeepStamp: true}), |
| + func(c *hostclaim.Claim) error { |
| + c.Op = hostclaim.Op{Kind: hostclaim.KindRelocating, ID: "rel-1", Phase: hostclaim.PhaseInstalling, Exec: "dst", Participants: []string{"src", "dst"}, |
| + Intent: map[string]string{"region": "ie", "source_box": "src", "target_box": "dst", "target_enrollment": enrollment("dst")}} |
| + return nil |
| + }, |
| + } { |
| + if obs, err = hostclaim.Update(ctx, b.store, obs, tr); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + locks := t.TempDir() |
| + for _, name := range []string{"hostguard", "hostexec"} { |
| + if err := os.Mkdir(filepath.Join(locks, name), 0o700); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + b.guard = hostguard.Guard{DataDir: b.data, LockDir: filepath.Join(locks, "hostguard"), Claims: hostguard.StoreReader{Store: b.store}} |
| + inventory, err := hostbox.OpenStopInventoryDB(filepath.Join(t.TempDir(), "inventory.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { inventory.Close() }) |
| + reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { reg.Close() }) |
| + launches, err := hostbox.OpenLaunchJournal(filepath.Join(t.TempDir(), "launches.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { launches.Close() }) |
| + b.launches = launches |
| + b.census = &hostbox.EmptyInstallationCensus{Inventory: inventory, Registry: reg, Launches: launches, Box: "dst", Enrollment: b.actor.Enrollment, DirectoryRevision: "4", |
| + Roots: map[string]string{"data": b.data, "opt": t.TempDir(), "units": t.TempDir()}, SharedPaths: map[string][]string{"data": {".hostexec", "hostguard"}}} |
| + a := &ClaimAssembly{Claims: b.store, Guard: b.guard, Wake: &ClaimWakeAssembly{}, Launches: launches, Reconcile: &hostbox.ReconcileAdapter{Census: b.census}, |
| + collectStop: func(context.Context, StopRequest, StopDrainer) (StopResult, error) { |
| + b.stops++ |
| + return StopResult{}, errors.New("root: no stop in this fixture") |
| + }} |
| + b.p = &productionClaimLocal{Resources: ClaimRuntimeResources{Mode: "agent", DataDir: b.data, OptDir: t.TempDir()}, Identity: handoverRecoveryIdentity(t, b.actor), |
| + Directory: hostactors.Directory{Store: b.store, PublicKey: pub, Bucket: "bucket"}, Store: b.store, slots: map[string]*claimParticipation{}, |
| + assembly: a, lifetime: ctx, lockRoot: locks} |
| + t.Cleanup(b.closeAll) |
| + return b |
| +} |
| + |
| +func (b *participationBox) read() hostclaim.Observed { |
| + b.t.Helper() |
| + o, err := hostclaim.Read(context.Background(), b.store, participationHost) |
| + if err != nil { |
| + b.t.Fatal(err) |
| + } |
| + return o |
| +} |
| + |
| +func (b *participationBox) update(tr func(*hostclaim.Claim) error) hostclaim.Observed { |
| + b.t.Helper() |
| + o, err := hostclaim.Update(context.Background(), b.store, b.read(), tr) |
| + if err != nil { |
| + b.t.Fatal(err) |
| + } |
| + return o |
| +} |
| + |
| +// records is the box's .hostexec records. |
| +func (b *participationBox) records() []string { |
| + b.t.Helper() |
| + entries, err := os.ReadDir(filepath.Join(b.data, ".hostexec")) |
| + if err != nil { |
| + b.t.Fatal(err) |
| + } |
| + var out []string |
| + for _, e := range entries { |
| + out = append(out, e.Name()) |
| + } |
| + return out |
| +} |
| + |
| +// closeAll stands for the agent's exit: every open controller is closed. |
| +func (b *participationBox) closeAll() { |
| + b.p.mu.Lock() |
| + slots := b.p.slots |
| + b.p.mu.Unlock() |
| + for _, s := range slots { |
| + if s.Controller != nil { |
| + _ = s.Controller.Close(context.Background()) |
| + } |
| + } |
| +} |
| + |
| +// Finding 1, ruling 42: one controller per {host, op}, whichever asks |
| +// first. The relocation pass's acknowledgement (Acknowledged, through |
| +// RelocationEffects) and main's participation (Prepare, as the loop and |
| +// WorkSource.Prepare call it) open or reuse the box's one slot: one |
| +// .hostexec record, one lease, one controller id. When the op ends without |
| +// the box, main's next Prepare closes the controller, runs its cleanup, |
| +// clears the record and drops the slot, and the relocation's Ready refuses. |
| +func TestRelocationAcknowledgesThroughMainsParticipation(t *testing.T) { |
| + for _, first := range []string{"relocation pass", "participation loop"} { |
| + t.Run(first+" first", func(t *testing.T) { |
| + ctx := t.Context() |
| + b := newParticipationBox(t) |
| + obs := b.read() |
| + effects := &RelocationEffects{Participation: b.p.Acknowledged} |
| + var ack hostclaim.ExecutionAcknowledgement |
| + acknowledge := func() { |
| + t.Helper() |
| + var err error |
| + if ack, err = effects.Acknowledge(ctx, obs); err != nil { |
| + t.Fatalf("acknowledge: %v", err) |
| + } |
| + } |
| + participate := func() { |
| + t.Helper() |
| + if err := b.p.Prepare(ctx, obs); err != nil { |
| + t.Fatalf("main's Prepare: %v", err) |
| + } |
| + } |
| + if first == "relocation pass" { |
| + acknowledge() |
| + participate() |
| + } else { |
| + participate() |
| + acknowledge() |
| + } |
| + b.p.mu.Lock() |
| + slot := b.p.slots[participationHost] |
| + n := len(b.p.slots) |
| + b.p.mu.Unlock() |
| + if slot == nil || n != 1 || slot.Controller == nil { |
| + t.Fatalf("want one slot with a controller, have %d", n) |
| + } |
| + opened, err := slot.Controller.Ready(ctx) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if ack.Op != "rel-1" || ack.ControllerID != opened.ControllerID || ack.LeaseETag == "" || ack.LeaseETag != opened.LeaseETag || ack.Enrollment != b.actor.Enrollment { |
| + t.Fatalf("the acknowledgement is not main's controller: %+v, opened %+v", ack, opened.Binding) |
| + } |
| + again, err := effects.Acknowledge(ctx, obs) |
| + if err != nil || again.ControllerID != ack.ControllerID || again.LeaseETag != ack.LeaseETag { |
| + t.Fatalf("a second acknowledgement saw another controller: %+v %v", again, err) |
| + } |
| + if err := effects.Ready(ctx, obs); err != nil { |
| + t.Fatalf("ready: %v", err) |
| + } |
| + if got := b.records(); len(got) != 1 || got[0] != participationHost+".json" { |
| + t.Fatalf("records %v", got) |
| + } |
| + |
| + // The op ends without this box: the source serves again. |
| + dropped := b.update(func(c *hostclaim.Claim) error { |
| + c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "src", Participants: []string{"src"}} |
| + return nil |
| + }) |
| + if err := b.p.Prepare(ctx, dropped); err != nil { |
| + t.Fatalf("main's Prepare did not discharge the participation: %v", err) |
| + } |
| + if !slot.CleanupSucceeded || len(b.p.slots) != 0 || len(b.records()) != 0 { |
| + t.Fatalf("the participation outlived its op: cleanup=%v slots=%d records=%v", slot.CleanupSucceeded, len(b.p.slots), b.records()) |
| + } |
| + if err := effects.Ready(ctx, dropped); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("ready after the op ended: %v", err) |
| + } |
| + if _, err := execlease.InspectLocal(b.data, participationHost); !errors.Is(err, os.ErrNotExist) { |
| + t.Fatalf("record: %v", err) |
| + } |
| + }) |
| + } |
| +} |
diff --git a/internal/host/relocationsweep.go b/internal/host/relocationsweep.go| new file mode 100644 |
| index 000000000..ae550ac6a |
| --- /dev/null |
| +++ b/internal/host/relocationsweep.go |
| @@ -0,0 +1,136 @@ |
| +package host |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "os" |
| + "path/filepath" |
| + "strings" |
| + |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// sweepRelocationJournals is the lifecycle half of N9 (ruling 61): every |
| +// region HA journal under <DataDir>/relocate names its host inside and its |
| +// op in its path, and is removed once the host's claim no longer runs that |
| +// op (completed and marked discharged, replaced by a successor, or the |
| +// claim gone). Positions, |
| +// attempts and install journals are needed only while their op runs; |
| +// the build scratch only while its op is installing. A journal that |
| +// cannot be read is left in place and reported: it is census-opaque, so |
| +// it blocks nothing, and absence is never evidence. |
| +func sweepRelocationJournals(ctx context.Context, dataDir string, claims release.Store) error { |
| + root := filepath.Join(dataDir, "relocate") |
| + var errs []error |
| + gone := func(host, op string) (bool, error) { |
| + obs, err := hostclaim.Read(ctx, claims, host) |
| + if err != nil { |
| + return false, err |
| + } |
| + if obs.Absent || obs.Claim == nil { |
| + return true, nil |
| + } |
| + // Ruling 76: a completed relocation's journals, the probe stop's |
| + // fence owner among them, stay until the box it completed on has |
| + // discharged, released its hold, lifted its fence and marked it. |
| + if done, ok := hostclaim.ReadCompletedRelocation(obs.Claim); ok && done.Op == op && !done.Discharged { |
| + return false, nil |
| + } |
| + return obs.Claim.Op.ID != op, nil |
| + } |
| + hostOf := func(path string) (string, error) { |
| + raw, err := os.ReadFile(path) |
| + if err != nil { |
| + return "", err |
| + } |
| + var probe struct { |
| + Host string `json:"host"` |
| + Request struct{ Host string } |
| + } |
| + if err := json.Unmarshal(raw, &probe); err != nil { |
| + return "", err |
| + } |
| + if probe.Host != "" { |
| + return probe.Host, nil |
| + } |
| + return probe.Request.Host, nil |
| + } |
| + // ops/<op>/{install,attempts}.json |
| + ops, err := os.ReadDir(filepath.Join(root, "ops")) |
| + if err != nil && !errors.Is(err, os.ErrNotExist) { |
| + return err |
| + } |
| + for _, e := range ops { |
| + dir := filepath.Join(root, "ops", e.Name()) |
| + host, readErr := "", error(nil) |
| + for _, name := range []string{"install.json", "attempts.json", "probe-fence.json"} { |
| + h, err := hostOf(filepath.Join(dir, name)) |
| + if err == nil && h != "" { |
| + host = h |
| + break |
| + } |
| + if err != nil && !errors.Is(err, os.ErrNotExist) { |
| + readErr = errors.Join(readErr, err) |
| + } |
| + } |
| + if host == "" { |
| + errs = append(errs, readErr) |
| + continue |
| + } |
| + if done, err := gone(host, e.Name()); err != nil { |
| + errs = append(errs, err) |
| + } else if done { |
| + errs = append(errs, os.RemoveAll(dir)) |
| + } |
| + } |
| + // release/<op>.json |
| + positions, err := os.ReadDir(filepath.Join(root, "release")) |
| + if err != nil && !errors.Is(err, os.ErrNotExist) { |
| + return err |
| + } |
| + for _, e := range positions { |
| + op, ok := strings.CutSuffix(e.Name(), ".json") |
| + if !ok { |
| + continue |
| + } |
| + path := filepath.Join(root, "release", e.Name()) |
| + host, err := hostOf(path) |
| + if err != nil || host == "" { |
| + errs = append(errs, err) |
| + continue |
| + } |
| + if done, err := gone(host, op); err != nil { |
| + errs = append(errs, err) |
| + } else if done { |
| + errs = append(errs, os.Remove(path)) |
| + } |
| + } |
| + // build/<op>/<host>.db: the scratch of a build whose op is over. |
| + builds, err := os.ReadDir(filepath.Join(root, "build")) |
| + if err != nil && !errors.Is(err, os.ErrNotExist) { |
| + return errors.Join(append(errs, err)...) |
| + } |
| + for _, e := range builds { |
| + dir := filepath.Join(root, "build", e.Name()) |
| + files, err := os.ReadDir(dir) |
| + if err != nil { |
| + errs = append(errs, err) |
| + continue |
| + } |
| + for _, f := range files { |
| + host, ok := strings.CutSuffix(f.Name(), ".db") |
| + if !ok { |
| + continue |
| + } |
| + if done, err := gone(host, e.Name()); err != nil { |
| + errs = append(errs, err) |
| + } else if done { |
| + errs = append(errs, os.RemoveAll(dir)) |
| + } |
| + break |
| + } |
| + } |
| + return errors.Join(errs...) |
| +} |
diff --git a/internal/host/relocationsweep_test.go b/internal/host/relocationsweep_test.go| new file mode 100644 |
| index 000000000..4f36ce4cd |
| --- /dev/null |
| +++ b/internal/host/relocationsweep_test.go |
| @@ -0,0 +1,155 @@ |
| +package host |
| + |
| +import ( |
| + "encoding/json" |
| + "errors" |
| + "os" |
| + "path/filepath" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + "github.com/carlosframework/platform/internal/provision" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// Ruling 61: a relocation journal lives while its op runs. Once the host's |
| +// claim runs another op, or the claim is gone, the sweep removes the op's |
| +// install and attempt journals, its release positions and its build |
| +// scratch. The live op's files stay, and a journal whose host cannot be |
| +// read stays and is reported: absence is never evidence. |
| +func TestRelocationJournalsAreSweptWhenTheirOpEnds(t *testing.T) { |
| + ctx := t.Context() |
| + store := release.NewDirStore(t.TempDir()) |
| + created := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) |
| + record := provision.Record{Schema: 1, Host: "shop.example", Slug: "shop", App: "app", Account: "acct", Region: "ie", Channel: "stable", CreatedAt: created} |
| + if err := provision.CreateRecord(ctx, store, record); err != nil { |
| + t.Fatal(err) |
| + } |
| + obs, err := hostclaim.Read(ctx, store, record.Host) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + obs, err = hostclaim.Update(ctx, store, obs, hostclaim.Adopt(hostclaim.NewHolder{Family: "instance", Key: provision.RecordKey(record.Account, record.App, record.Host), |
| + Account: record.Account, App: record.App, Host: record.Host, Region: record.Region, CreatedAt: created}, "src", nil, nil, "adopt", func() string { return "id" })) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err = hostclaim.Update(ctx, store, obs, func(c *hostclaim.Claim) error { |
| + c.Op = hostclaim.Op{Kind: hostclaim.KindRelocating, ID: "rel-live", Phase: hostclaim.PhaseInstalling, Exec: "dst", Participants: []string{"src", "dst"}, |
| + Intent: map[string]string{"region": "ie", "source_box": "src", "target_box": "dst"}} |
| + return nil |
| + }); err != nil { |
| + t.Fatal(err) |
| + } |
| + data := t.TempDir() |
| + files := map[string]string{ |
| + "relocate/ops/rel-live/install.json": `{"v":1,"host":"shop.example","op":"rel-live"}`, |
| + "relocate/ops/rel-old/install.json": `{"v":1,"host":"shop.example","op":"rel-old"}`, |
| + "relocate/ops/rel-old/attempts.json": `{"v":1,"host":"shop.example","op":"rel-old"}`, |
| + "relocate/ops/rel-gone/attempts.json": `{"v":1,"host":"gone.example","op":"rel-gone"}`, |
| + "relocate/ops/rel-torn/install.json": `{"v":1,"host":`, |
| + "relocate/release/rel-live.json": `{"request":{"Host":"shop.example"}}`, |
| + "relocate/release/rel-old.json": `{"request":{"Host":"shop.example"}}`, |
| + "relocate/build/rel-live/shop.example.db": "scratch", |
| + "relocate/build/rel-old/shop.example.db": "scratch", |
| + } |
| + for path, body := range files { |
| + full := filepath.Join(data, filepath.FromSlash(path)) |
| + if err := os.MkdirAll(filepath.Dir(full), 0o700); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.WriteFile(full, []byte(body), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + if err := sweepRelocationJournals(ctx, data, store); err == nil { |
| + t.Fatal("a torn journal was not reported") |
| + } |
| + for path := range files { |
| + _, err := os.Stat(filepath.Join(data, filepath.FromSlash(path))) |
| + live := filepath.Base(filepath.Dir(path)) == "rel-live" || filepath.Base(path) == "rel-live.json" || filepath.Base(filepath.Dir(path)) == "rel-torn" |
| + switch { |
| + case live && err != nil: |
| + t.Errorf("%s was swept while its op runs or its host is unread: %v", path, err) |
| + case !live && !errors.Is(err, os.ErrNotExist): |
| + t.Errorf("%s survived the end of its op: %v", path, err) |
| + } |
| + } |
| + for _, dir := range []string{"relocate/ops/rel-old", "relocate/ops/rel-gone", "relocate/build/rel-old"} { |
| + if _, err := os.Stat(filepath.Join(data, filepath.FromSlash(dir))); !errors.Is(err, os.ErrNotExist) { |
| + t.Errorf("%s was left behind: %v", dir, err) |
| + } |
| + } |
| + // A second pass over what is left removes nothing more and still |
| + // reports the torn journal. |
| + if err := sweepRelocationJournals(ctx, data, store); err == nil { |
| + t.Fatal("the torn journal stopped being reported") |
| + } |
| + if _, err := os.Stat(filepath.Join(data, "relocate/ops/rel-live/install.json")); err != nil { |
| + t.Fatalf("the second pass swept the live op: %v", err) |
| + } |
| +} |
| + |
| +// Ruling 76: the op a serving claim records as completed keeps its |
| +// journals, the probe stop's fence owner among them, until the completed |
| +// relocation is marked discharged, so the hold release always finds the |
| +// owner it lifts under. Then the next sweep removes them. |
| +func TestRelocationJournalsOfACompletedOpStayUntilItsDischarge(t *testing.T) { |
| + ctx := t.Context() |
| + store := release.NewDirStore(t.TempDir()) |
| + created := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) |
| + record := provision.Record{Schema: 1, Host: "shop.example", Slug: "shop", App: "app", Account: "acct", Region: "ie", Channel: "stable", CreatedAt: created} |
| + if err := provision.CreateRecord(ctx, store, record); err != nil { |
| + t.Fatal(err) |
| + } |
| + obs, err := hostclaim.Read(ctx, store, record.Host) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + obs, err = hostclaim.Update(ctx, store, obs, hostclaim.Adopt(hostclaim.NewHolder{Family: "instance", Key: provision.RecordKey(record.Account, record.App, record.Host), |
| + Account: record.Account, App: record.App, Host: record.Host, Region: record.Region, CreatedAt: created}, "dst", nil, nil, "adopt", func() string { return "id" })) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + done, _ := json.Marshal(hostclaim.CompletedRelocation{Op: "rel-done", Stop: hostclaim.ProbeStop{Box: "dst", Session: 12, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)}, Phase: hostclaim.PhaseRelocatingProbing}) |
| + if obs, err = hostclaim.Update(ctx, store, obs, func(c *hostclaim.Claim) error { |
| + c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "dst", Participants: []string{"dst"}, Journal: map[string]string{hostclaim.JournalCompleted: string(done)}} |
| + return nil |
| + }); err != nil { |
| + t.Fatal(err) |
| + } |
| + data := t.TempDir() |
| + files := []string{"relocate/ops/rel-done/install.json", "relocate/ops/rel-done/probe-fence.json", "relocate/ops/rel-fence-only/probe-fence.json"} |
| + bodies := []string{`{"v":1,"host":"shop.example","op":"rel-done"}`, `{"host":"shop.example","session":12,"owner":"0123"}`, `{"host":"shop.example","session":3,"owner":"4567"}`} |
| + for i, path := range files { |
| + full := filepath.Join(data, filepath.FromSlash(path)) |
| + if err := os.MkdirAll(filepath.Dir(full), 0o700); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.WriteFile(full, []byte(bodies[i]), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + if err := sweepRelocationJournals(ctx, data, store); err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, path := range files[:2] { |
| + if _, err := os.Stat(filepath.Join(data, filepath.FromSlash(path))); err != nil { |
| + t.Fatalf("%s was swept before the discharge: %v", path, err) |
| + } |
| + } |
| + if _, err := os.Stat(filepath.Join(data, "relocate/ops/rel-fence-only")); !errors.Is(err, os.ErrNotExist) { |
| + t.Fatalf("an ended op known only by its fence owner survived: %v", err) |
| + } |
| + if _, err := hostclaim.Update(ctx, store, obs, hostclaim.MarkCompletedDischarged("rel-done")); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := sweepRelocationJournals(ctx, data, store); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := os.Stat(filepath.Join(data, "relocate/ops/rel-done")); !errors.Is(err, os.ErrNotExist) { |
| + t.Fatalf("the discharged op's journals survived: %v", err) |
| + } |
| +} |
diff --git a/internal/hostclaim/ops.go b/internal/hostclaim/ops.go| index 85b3e740c..b50bf0652 100644 |
| --- a/internal/hostclaim/ops.go |
| +++ b/internal/hostclaim/ops.go |
| @@ -22,10 +22,15 @@ func RequireOp(id string) func(*Claim) error { |
| // StartGuard is §4: one operation at a time, started from serving, and |
| // CC11: never over a chore in a region or box the operation will write. |
| +// Region HA N13 (ruling 77): nor while a completed relocation's box has |
| +// not discharged its participation (completionBarrier). |
| func StartGuard(c *Claim, regions, boxes []string) error { |
| if c.Holder == nil || c.Op.Kind != KindServing { |
| return fmt.Errorf("%w: kind %q", ErrWrongState, c.Op.Kind) |
| } |
| + if err := completionBarrier(c, ""); err != nil { |
| + return err |
| + } |
| inR, inB := set(regions), set(boxes) |
| for _, o := range c.Obligations { |
| if (o.Region != "" && inR[o.Region]) || (o.Box != "" && inB[o.Box]) { |
diff --git a/internal/hostclaim/relocation.go b/internal/hostclaim/relocation.go| index 5c756f29b..b6b9d3e03 100644 |
| --- a/internal/hostclaim/relocation.go |
| +++ b/internal/hostclaim/relocation.go |
| @@ -165,6 +165,10 @@ type LocalInstall struct { |
| type Probe struct { |
| Box string `json:"box"` |
| Session uint64 `json:"session"` |
| + // At is when the probe passed (rev 4, ruling 67): the start of the |
| + // bound on obtaining its stop receipt. Zero in a probe journaled before |
| + // rev 4; the worker then counts from probing_since. |
| + At time.Time `json:"at,omitzero"` |
| } |
| // Target names a box for the relocation's target slot. |
| @@ -258,6 +262,13 @@ func RelocationStartGuard(c *Claim, region, source, target string, sourceFenced |
| if c.Holder == nil || c.Op.Kind != KindServing || c.Op.ID != "" { |
| return fmt.Errorf("%w: kind %q", ErrWrongState, c.Op.Kind) |
| } |
| + fenced := "" |
| + if sourceFenced { |
| + fenced = source |
| + } |
| + if err := completionBarrier(c, fenced); err != nil { |
| + return err |
| + } |
| for _, o := range c.Obligations { |
| switch { |
| case o.Kind == "park-replica" && o.Region == region: |
| @@ -1009,6 +1020,11 @@ func CompleteRelocation(opID string, newID func() string) func(*Claim) error { |
| if !readJournal(c.Op, "probe", &p) || p.Box != c.Op.Exec || p.Session != c.Holder.Session || c.Holder.WriterBox != p.Box { |
| return fmt.Errorf("%w: no passing probe of the current session", ErrRelocation) |
| } |
| + if !probeStopped(c, p) { |
| + // Ruling 49: the kind change ends the probe's session, so the |
| + // session is stopped, with root's receipt, before it. |
| + return fmt.Errorf("%w: the probe's session is not stopped", ErrRelocation) |
| + } |
| target := c.Op.Exec |
| var owed []string |
| readJournal(c.Op, "installed_boxes", &owed) |
| @@ -1052,7 +1068,21 @@ func CompleteRelocation(opID string, newID func() string) func(*Claim) error { |
| c.Obligations = append(c.Obligations, Obligation{ID: newID(), Op: opID, Kind: "park-generation", Region: region, Generation: g, Retirement: newID(), Owner: "region:" + region}) |
| } |
| } |
| - c.Op = Op{Kind: KindServing, Exec: target, Participants: []string{target}} |
| + var stop ProbeStop |
| + readJournal(c.Op, "probe_stop", &stop) |
| + done, err := json.Marshal(CompletedRelocation{Op: opID, Stop: stop, Stamp: c.Holder.Stamp, Phase: c.Op.Phase, Generation: c.Holder.Generation}) |
| + if err != nil { |
| + return err |
| + } |
| + // Ruling 56: the serving op keeps the stopped probe session's |
| + // receipt reference for the next cold wake's handover. |
| + next := Op{Kind: KindServing, Exec: target, Participants: []string{target}, Journal: map[string]string{JournalCompleted: string(done)}} |
| + if c.Op.Journal["conversion"] == "committed-landing" { |
| + // M1 on amadan #236: the move this relocation finished is named |
| + // on the serving op, so moves' close can find it. |
| + next.Journal["released_move"] = opID |
| + } |
| + c.Op = next |
| return nil |
| } |
| } |
diff --git a/internal/hostclaim/relocation_probestop.go b/internal/hostclaim/relocation_probestop.go| new file mode 100644 |
| index 000000000..31ebf7051 |
| --- /dev/null |
| +++ b/internal/hostclaim/relocation_probestop.go |
| @@ -0,0 +1,139 @@ |
| +package hostclaim |
| + |
| +import ( |
| + "fmt" |
| + "slices" |
| + "time" |
| +) |
| + |
| +// ProbeStop is the target's journaled stop of its passed probe session |
| +// (region HA ruling 49): root's full-stop receipt for the writer-session |
| +// stop (hostguard.WriterStopObligation) of exactly that session, taken |
| +// while the op is still relocating/probing. The serving CAS requires it, |
| +// so the kind change never strands a running session. |
| +type ProbeStop struct { |
| + Box string `json:"box"` |
| + Session uint64 `json:"session"` |
| + Receipt string `json:"receipt"` |
| + ReceiptSHA256 string `json:"receipt_sha256"` |
| +} |
| + |
| +// RecordProbeStop journals the executor's stop of its passed probe session. |
| +// It is bound to op.exec, the journaled passing probe and the holder's |
| +// current session; a different stop already journaled is refused. |
| +func RecordProbeStop(opID string, s ProbeStop) func(*Claim) error { |
| + return func(c *Claim) error { |
| + if err := requireRelocation(c, opID, PhaseRelocatingProbing); err != nil { |
| + return err |
| + } |
| + var p Probe |
| + if !readJournal(c.Op, "probe", &p) || p.Box != c.Op.Exec || p.Session != c.Holder.Session { |
| + return fmt.Errorf("%w: no passing probe of the current session to stop", ErrRelocation) |
| + } |
| + if s.Box != p.Box || s.Session != p.Session || s.Receipt == "" || !isHex64(s.ReceiptSHA256) { |
| + return fmt.Errorf("%w: the stop is not of the probe's session", ErrRelocation) |
| + } |
| + var old ProbeStop |
| + if readJournal(c.Op, "probe_stop", &old) { |
| + if old == s { |
| + return ErrAlreadyRecorded |
| + } |
| + return fmt.Errorf("%w: a different probe stop is journaled", ErrRelocation) |
| + } |
| + return journalJSON(c, "probe_stop", s) |
| + } |
| +} |
| + |
| +// probeStopped reports whether the journaled probe stop names the passed |
| +// probe's session (CompleteRelocation's gate). |
| +func probeStopped(c *Claim, p Probe) bool { |
| + var s ProbeStop |
| + return readJournal(c.Op, "probe_stop", &s) && s.Box == p.Box && s.Session == p.Session && s.Receipt != "" |
| +} |
| + |
| +// CompletedRelocation is the serving op's record of the relocation it |
| +// ended (region HA ruling 56): the op, and the probe session's stop that |
| +// root receipted at relocating/probing. The next serving cold wake on Box |
| +// hands over from exactly that receipt (ColdHandover.Run takes the prior |
| +// receipt's target as it is), and the participation discharge (claim ask |
| +// C10) reads it. Journaled by CompleteRelocation under JournalCompleted. |
| +type CompletedRelocation struct { |
| + Op string `json:"op"` |
| + Stop ProbeStop `json:"stop"` |
| + Stamp uint64 `json:"stamp"` // the stopped session's installation stamp |
| + Phase string `json:"phase"` // the stop's phase: probing |
| + Generation string `json:"generation"` // the stopped session's generation |
| + // Discharged is set by MarkCompletedDischarged once the box the |
| + // relocation completed on has discharged its probe's participation |
| + // (claim ask C10). Until then no successor relocation starts from that |
| + // box (RelocationStartGuard): its participation slot is still the old |
| + // op's, and main's Prepare for a successor would answer ErrLost. |
| + Discharged bool `json:"discharged,omitempty"` |
| +} |
| + |
| +// JournalCompleted is the serving op's journal key for CompletedRelocation. |
| +const JournalCompleted = "completed_relocation" |
| + |
| +// ReadCompletedRelocation reads the serving op's record; false when none. |
| +func ReadCompletedRelocation(c *Claim) (CompletedRelocation, bool) { |
| + var r CompletedRelocation |
| + if c == nil || c.Op.Kind != KindServing || !readJournal(c.Op, JournalCompleted, &r) || r.Op == "" || r.Stop.Box == "" || r.Stop.Session == 0 { |
| + return CompletedRelocation{}, false |
| + } |
| + return r, true |
| +} |
| + |
| +// completionBarrier is region HA ruling 77, the one barrier every start |
| +// from serving passes (StartGuard for moves, deletes, purges and |
| +// placements; RelocationStartGuard for relocations): while the serving |
| +// op's completed relocation is not marked discharged, the box it completed |
| +// on still holds that op's participation slot, and main's Prepare for any |
| +// successor operation would answer ErrLost, so nothing starts. A |
| +// completion box on lost, or the one the start itself cites as fenced |
| +// (fencedSource, a failover's source), holds nothing a successor needs: a |
| +// dead box's slot binds nothing on another box. |
| +func completionBarrier(c *Claim, fencedSource string) error { |
| + done, ok := ReadCompletedRelocation(c) |
| + if !ok || done.Discharged || slices.Contains(c.Lost, done.Stop.Box) || (fencedSource != "" && fencedSource == done.Stop.Box) { |
| + return nil |
| + } |
| + return fmt.Errorf("%w: relocation %s is not yet discharged on %s", ErrPendingChore, done.Op, done.Stop.Box) |
| +} |
| + |
| +// MarkCompletedDischarged records, on the serving op, that the box the |
| +// relocation opID completed on has discharged its participation. It |
| +// changes nothing else; a repeat is ErrAlreadyRecorded. A serving op with |
| +// no completed relocation is ErrWrongState, and one that records another |
| +// op is ErrRelocation. |
| +func MarkCompletedDischarged(opID string) func(*Claim) error { |
| + return func(c *Claim) error { |
| + r, ok := ReadCompletedRelocation(c) |
| + if !ok { |
| + return fmt.Errorf("%w: no completed relocation on the serving op", ErrWrongState) |
| + } |
| + if r.Op != opID { |
| + return fmt.Errorf("%w: the serving op records %s, not %s", ErrRelocation, r.Op, opID) |
| + } |
| + if r.Discharged { |
| + return ErrAlreadyRecorded |
| + } |
| + r.Discharged = true |
| + return journalJSON(c, JournalCompleted, r) |
| + } |
| +} |
| + |
| +// ToInstalledAt is ToInstalled that also journals installed_at, the start |
| +// of the installed bound (ruling 60). |
| +func ToInstalledAt(opID string, a PreparedAuthority, now time.Time) func(*Claim) error { |
| + inner := ToInstalled(opID, a) |
| + return func(c *Claim) error { |
| + if now.IsZero() { |
| + return fmt.Errorf("%w: installed needs its start time", ErrRelocation) |
| + } |
| + if err := inner(c); err != nil { |
| + return err |
| + } |
| + c.Op.Journal["installed_at"] = now.UTC().Format(time.RFC3339Nano) |
| + return nil |
| + } |
| +} |
diff --git a/internal/hostclaim/relocation_probestop_test.go b/internal/hostclaim/relocation_probestop_test.go| new file mode 100644 |
| index 000000000..9ce573b59 |
| --- /dev/null |
| +++ b/internal/hostclaim/relocation_probestop_test.go |
| @@ -0,0 +1,267 @@ |
| +package hostclaim |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "slices" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// probedDrain drives a drain of the serving fixture to a passed probe on |
| +// relTarget: released, pinned, installed, probing, P admitted, the probe |
| +// journaled with its time. |
| +func probedDrain(t *testing.T) (func(func(*Claim) error) Observed, func() Observed) { |
| + t.Helper() |
| + ctx := context.Background() |
| + s, cur := probedDrainIn(t) |
| + apply := func(tr func(*Claim) error) Observed { |
| + t.Helper() |
| + next, err := Update(ctx, s, cur, tr) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + cur = next |
| + return next |
| + } |
| + return apply, func() Observed { return cur } |
| +} |
| + |
| +// probedDrainIn is probedDrain's claim and its store. |
| +func probedDrainIn(t *testing.T) (release.Store, Observed) { |
| + t.Helper() |
| + ctx := context.Background() |
| + s, o := relServing(t) |
| + o, err := StartRelocation(ctx, s, o, relStart(ReasonDrain, nil), relAuthority{}) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + o = step(t, s, o, ack(relSource)) |
| + o = step(t, s, o, ack(relTarget)) |
| + o = step(t, s, o, ToReleasing("rel-1", relObserved)) |
| + h := o.Claim.Holder |
| + o = step(t, s, o, JournalRelease("rel-1", Release{ReleasedBy: relSource, Enrollment: relEnrollment(relSource), Generation: h.Generation, Session: h.Session, FinalTXID: 40, ManifestSeq: 7, Receipt: "control/stop/r.json", ReceiptSHA256: strings.Repeat("d", 64)})) |
| + if o, err = PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, nil); err != nil { |
| + t.Fatal(err) |
| + } |
| + o = step(t, s, o, JournalPin("rel-1", relPin(o, 40, 7, relObserved))) |
| + o = toProbing(t, s, o, "P1", 40) |
| + o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseRelocatingProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, |
| + Box: relTarget, Mode: "cold", NewGeneration: "P1", KeepStamp: true, PredecessorCertified: true, Region: "ie", PredecessorRegion: "ie"})) |
| + o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relTarget, Session: o.Claim.Holder.Session, At: relObserved.Add(time.Minute)})) |
| + return s, o |
| +} |
| + |
| +// copyOf is a deep copy, for trying a transition without writing it. |
| +func copyOf(t *testing.T, c *Claim) *Claim { |
| + t.Helper() |
| + raw, err := json.Marshal(c) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var out Claim |
| + if err := json.Unmarshal(raw, &out); err != nil { |
| + t.Fatal(err) |
| + } |
| + return &out |
| +} |
| + |
| +func stopOf(o Observed) ProbeStop { |
| + return ProbeStop{Box: relTarget, Session: o.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("e", 64)} |
| +} |
| + |
| +// Ruling 49 step 4: the serving CAS requires the probe's stop, journaled |
| +// for exactly the passed probe's box and session; it then records the |
| +// completed relocation on the serving op (ruling 56). |
| +func TestCompleteRelocationRequiresTheProbeStop(t *testing.T) { |
| + apply, cur := probedDrain(t) |
| + o := cur() |
| + var p Probe |
| + if !readJournal(o.Claim.Op, "probe", &p) || p.At.IsZero() { |
| + t.Fatalf("the probe has no time: %+v", p) |
| + } |
| + if err := CompleteRelocation("rel-1", counter())(copyOf(t, o.Claim)); !errors.Is(err, ErrRelocation) { |
| + t.Fatalf("completed with the probe's session running: %v", err) |
| + } |
| + for name, s := range map[string]ProbeStop{ |
| + "another session": {Box: relTarget, Session: o.Claim.Holder.Session + 1, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("e", 64)}, |
| + "another box": {Box: relSource, Session: o.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("e", 64)}, |
| + "no receipt": {Box: relTarget, Session: o.Claim.Holder.Session, ReceiptSHA256: strings.Repeat("e", 64)}, |
| + "a bad digest": {Box: relTarget, Session: o.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: "short"}, |
| + } { |
| + if err := RecordProbeStop("rel-1", s)(copyOf(t, o.Claim)); !errors.Is(err, ErrRelocation) { |
| + t.Fatalf("%s: recorded: %v", name, err) |
| + } |
| + } |
| + stop := stopOf(o) |
| + o = apply(RecordProbeStop("rel-1", stop)) |
| + if err := RecordProbeStop("rel-1", stop)(copyOf(t, o.Claim)); !errors.Is(err, ErrAlreadyRecorded) { |
| + t.Fatalf("a replay: %v", err) |
| + } |
| + other := stop |
| + other.Receipt = "fence-2" |
| + if err := RecordProbeStop("rel-1", other)(copyOf(t, o.Claim)); !errors.Is(err, ErrRelocation) { |
| + t.Fatalf("a second, different stop: %v", err) |
| + } |
| + o = apply(CompleteRelocation("rel-1", counter())) |
| + done, ok := ReadCompletedRelocation(o.Claim) |
| + if !ok || done.Op != "rel-1" || done.Stop != stop || done.Stamp != o.Claim.Holder.Stamp || done.Phase != PhaseRelocatingProbing || done.Generation != "P1" { |
| + t.Fatalf("completed relocation: %+v %v", done, ok) |
| + } |
| + if o.Claim.Op.Journal["released_move"] != "" { |
| + t.Fatal("a plain relocation named a move") |
| + } |
| +} |
| + |
| +// M1: a converted landing's completion names the move on the serving op, |
| +// beside the completed relocation. |
| +func TestCompleteRelocationJournalsReleasedMoveForAConvertedLanding(t *testing.T) { |
| + apply, cur := probedDrain(t) |
| + apply(func(c *Claim) error { c.Op.Journal["conversion"] = "committed-landing"; return nil }) |
| + apply(RecordProbeStop("rel-1", stopOf(cur()))) |
| + o := apply(CompleteRelocation("rel-1", counter())) |
| + if o.Claim.Op.Journal["released_move"] != "rel-1" { |
| + t.Fatalf("serving op journal: %v", o.Claim.Op.Journal) |
| + } |
| + if _, ok := ReadCompletedRelocation(o.Claim); !ok { |
| + t.Fatal("no completed relocation beside the released move") |
| + } |
| +} |
| + |
| +// Ruling 60: installed starts its own bound. |
| +func TestToInstalledAtJournalsItsStart(t *testing.T) { |
| + ctx := context.Background() |
| + s, o := relServing(t) |
| + o, err := StartRelocation(ctx, s, o, relStart(ReasonDrain, nil), relAuthority{}) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + o = step(t, s, o, ack(relSource)) |
| + o = step(t, s, o, ack(relTarget)) |
| + o = step(t, s, o, ToReleasing("rel-1", relObserved)) |
| + h := o.Claim.Holder |
| + o = step(t, s, o, JournalRelease("rel-1", Release{ReleasedBy: relSource, Enrollment: relEnrollment(relSource), Generation: h.Generation, Session: h.Session, FinalTXID: 40, ManifestSeq: 7, Receipt: "control/stop/r.json", ReceiptSHA256: strings.Repeat("d", 64)})) |
| + if o, err = PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, nil); err != nil { |
| + t.Fatal(err) |
| + } |
| + o = step(t, s, o, JournalPin("rel-1", relPin(o, 40, 7, relObserved))) |
| + o = step(t, s, o, ToInstalling("rel-1", Target{relTarget, relEnrollment(relTarget)}, "P1")) |
| + if err := ToInstalledAt("rel-1", relPrepared("P1", 40, 7), time.Time{})(copyOf(t, o.Claim)); !errors.Is(err, ErrRelocation) { |
| + t.Fatalf("installed without a start: %v", err) |
| + } |
| + o = step(t, s, o, ToInstalledAt("rel-1", relPrepared("P1", 40, 7), relObserved)) |
| + if o.Claim.Op.Phase != PhaseInstalled || o.Claim.Op.Journal["installed_at"] != relObserved.Format(time.RFC3339Nano) { |
| + t.Fatalf("installed: %s %v", o.Claim.Op.Phase, o.Claim.Op.Journal) |
| + } |
| +} |
| + |
| +// Astra round 6, finding 3: A to B completes, then a successor B to C |
| +// asks to start before B has discharged its probe's participation (C10). |
| +// Through the real start CAS: a drain or rebalance from B is refused, with |
| +// the claim unchanged, until MarkCompletedDischarged; a failover that |
| +// cites B's termination fence is not, since a dead B's slot binds nothing |
| +// on C. |
| +func TestRelocationSuccessorWaitsForTheCompletedDischarge(t *testing.T) { |
| + ctx := context.Background() |
| + successor := func(reason string, fence *FenceRef) RelocationStart { |
| + return RelocationStart{OpID: "rel-2", By: "worker", Now: relObserved.Add(time.Hour), Region: "ie", Reason: reason, |
| + SourceBox: relTarget, SourceEnrollment: relEnrollment(relTarget), TargetBox: relSpare, TargetEnrollment: relEnrollment(relSpare), |
| + DirectoryRevision: 9, AdmissionID: "adm-2", Fence: fence} |
| + } |
| + completed := completedDrain |
| + t.Run("a drain from B waits for B's discharge", func(t *testing.T) { |
| + s, o := completed(t) |
| + for _, reason := range []string{ReasonDrain, ReasonRebalance} { |
| + if _, err := StartRelocation(ctx, s, o, successor(reason, nil), relAuthority{}); !errors.Is(err, ErrPendingChore) { |
| + t.Fatalf("%s started before the discharge: %v", reason, err) |
| + } |
| + } |
| + if now, err := Read(ctx, s, o.Claim.Host); err != nil || now.ETag != o.ETag { |
| + t.Fatalf("a refused start wrote the claim: %v", err) |
| + } |
| + if _, err := Update(ctx, s, o, MarkCompletedDischarged("rel-9")); !errors.Is(err, ErrRelocation) { |
| + t.Fatalf("another op's discharge: %v", err) |
| + } |
| + o = step(t, s, o, MarkCompletedDischarged("rel-1")) |
| + if _, err := Update(ctx, s, o, MarkCompletedDischarged("rel-1")); !errors.Is(err, ErrAlreadyRecorded) { |
| + t.Fatalf("a repeat: %v", err) |
| + } |
| + if done, ok := ReadCompletedRelocation(o.Claim); !ok || !done.Discharged || done.Stop != stopOf(o) { |
| + t.Fatalf("the mark changed the record: %+v", done) |
| + } |
| + next, err := StartRelocation(ctx, s, o, successor(ReasonDrain, nil), relAuthority{}) |
| + if err != nil { |
| + t.Fatalf("the successor after the discharge: %v", err) |
| + } |
| + if next.Claim.Op.ID != "rel-2" || next.Claim.Op.Intent["source_box"] != relTarget || next.Claim.Op.Intent["target_box"] != relSpare { |
| + t.Fatalf("successor op %+v", next.Claim.Op) |
| + } |
| + }) |
| + t.Run("a failover from a fenced B does not wait", func(t *testing.T) { |
| + s, o := completed(t) |
| + ref := putFence(t, s, relTarget) |
| + next, err := StartRelocation(ctx, s, o, successor(ReasonFailover, &ref), relAuthority{}) |
| + if err != nil { |
| + t.Fatalf("failover from the fenced source: %v", err) |
| + } |
| + if next.Claim.Op.Exec != "none" || !slices.Contains(next.Claim.Lost, relTarget) { |
| + t.Fatalf("failover op %+v lost %v", next.Claim.Op, next.Claim.Lost) |
| + } |
| + }) |
| + t.Run("no completed relocation, nothing to mark", func(t *testing.T) { |
| + _, o := relServing(t) |
| + if err := MarkCompletedDischarged("rel-1")(copyOf(t, o.Claim)); !errors.Is(err, ErrWrongState) { |
| + t.Fatalf("%v", err) |
| + } |
| + }) |
| +} |
| + |
| +// completedDrain is A to B through every real transition, completed and |
| +// not yet discharged on B. |
| +func completedDrain(t *testing.T) (release.Store, Observed) { |
| + t.Helper() |
| + s, o := probedDrainIn(t) |
| + o = step(t, s, o, RecordProbeStop("rel-1", stopOf(o))) |
| + o = step(t, s, o, CompleteRelocation("rel-1", counter())) |
| + if done, ok := ReadCompletedRelocation(o.Claim); !ok || done.Discharged || o.Claim.Op.Exec != relTarget { |
| + t.Fatalf("A to B did not complete undischarged: %+v %v", done, ok) |
| + } |
| + return s, o |
| +} |
| + |
| +// Astra round 7, finding 2: the discharge barrier is StartGuard's too, so |
| +// a move from B (and a delete) after A to B completes is refused, with the |
| +// claim unchanged, until B marks the completion discharged; then the real |
| +// move start CAS succeeds. A completion box on lost holds nothing. |
| +func TestMoveSuccessorWaitsForTheCompletedDischarge(t *testing.T) { |
| + ctx := context.Background() |
| + s, o := completedDrain(t) |
| + move := BeginAdmittedMove("mv-1", "console", relTarget, relSpare, "uk", relObserved.Add(time.Hour), |
| + MoveAdmission{AppID: "app-1", AdmissionID: "mv-1", SourceEnrollment: relEnrollment(relTarget), TargetEnrollment: relEnrollment(relSpare)}) |
| + if _, err := Update(ctx, s, o, move); !errors.Is(err, ErrPendingChore) { |
| + t.Fatalf("a move started before B's discharge: %v", err) |
| + } |
| + if _, err := Update(ctx, s, o, BeginDelete("del-1", "console", "etag-1", relObserved.Add(time.Hour), []string{"ie"}, []string{relTarget})); !errors.Is(err, ErrPendingChore) { |
| + t.Fatalf("a delete started before B's discharge: %v", err) |
| + } |
| + if now, err := Read(ctx, s, o.Claim.Host); err != nil || now.ETag != o.ETag { |
| + t.Fatalf("a refused start wrote the claim: %v", err) |
| + } |
| + lost := copyOf(t, o.Claim) |
| + lost.Lost = append(lost.Lost, relTarget) |
| + if err := StartGuard(lost, []string{"uk"}, []string{relSpare}); err != nil { |
| + t.Fatalf("a lost completion box still holds the barrier: %v", err) |
| + } |
| + o = step(t, s, o, MarkCompletedDischarged("rel-1")) |
| + next, err := Update(ctx, s, o, move) |
| + if err != nil { |
| + t.Fatalf("the move after the discharge: %v", err) |
| + } |
| + if next.Claim.Op.Kind != KindMoving || next.Claim.Op.ID != "mv-1" || next.Claim.Op.Exec != relTarget { |
| + t.Fatalf("move op %+v", next.Claim.Op) |
| + } |
| +} |
diff --git a/internal/hostclaim/relocation_test.go b/internal/hostclaim/relocation_test.go| index c4ea1849b..783ce9d3e 100644 |
| --- a/internal/hostclaim/relocation_test.go |
| +++ b/internal/hostclaim/relocation_test.go |
| @@ -206,6 +206,7 @@ func TestRelocationDrainEndToEnd(t *testing.T) { |
| t.Fatalf("certified admission: %+v stamp %d", last, o.Claim.Holder.Stamp) |
| } |
| o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relTarget, Session: o.Claim.Holder.Session})) |
| + o = step(t, s, o, RecordProbeStop("rel-1", ProbeStop{Box: relTarget, Session: o.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)})) |
| o = step(t, s, o, CompleteRelocation("rel-1", counter())) |
| c := o.Claim |
| if c.Op.Kind != KindServing || c.Op.ID != "" || c.Op.Exec != relTarget || !slices.Equal(c.Op.Participants, []string{relTarget}) { |
| @@ -261,6 +262,7 @@ func TestRelocationFailoverCitesTheFenceAndDeclaresTheLoss(t *testing.T) { |
| t.Fatalf("failover must declare the source's session: %+v", last) |
| } |
| o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relTarget, Session: o.Claim.Holder.Session})) |
| + o = step(t, s, o, RecordProbeStop("rel-1", ProbeStop{Box: relTarget, Session: o.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)})) |
| o = step(t, s, o, CompleteRelocation("rel-1", counter())) |
| owes := slices.ContainsFunc(o.Claim.Obligations, func(ob Obligation) bool { return ob.Kind == "park-local" && ob.Box == relSource }) |
| if owes == qualified { |
| @@ -277,7 +279,8 @@ func TestRelocationFailoverCitesTheFenceAndDeclaresTheLoss(t *testing.T) { |
| l.Session != sourceSession || l.Generation != "g1" || l.Discharge.Qualified != qualified || l.Timers != TimersUnknown || l.Reported { |
| t.Fatalf("retained loss: %+v", l) |
| } |
| - if c.Op.Journal != nil || !slices.Contains(c.Lost, relSource) { |
| + done, ok := ReadCompletedRelocation(c) |
| + if len(c.Op.Journal) != 1 || !ok || done.Op != "rel-1" || done.Stop.Session != c.Holder.Session || done.Phase != PhaseRelocatingProbing || !slices.Contains(c.Lost, relSource) { |
| t.Fatalf("serving kept the journal, or lost the box: %+v %v", c.Op, c.Lost) |
| } |
| } |
| @@ -619,6 +622,7 @@ func TestRelocationProbeFailureIsANestedRelocation(t *testing.T) { |
| o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseRelocatingProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, |
| Box: relSpare, Mode: "cold", NewGeneration: "P2", KeepStamp: true, PredecessorCertified: true, Region: "ie", PredecessorRegion: "ie"})) |
| o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relSpare, Session: o.Claim.Holder.Session})) |
| + o = step(t, s, o, RecordProbeStop("rel-1", ProbeStop{Box: relSpare, Session: o.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)})) |
| o = step(t, s, o, CompleteRelocation("rel-1", counter())) |
| var owed []string |
| for _, ob := range o.Claim.Obligations { |
| @@ -863,6 +867,7 @@ func TestRelocationRetainsEveryFencedBoxAcrossCompletion(t *testing.T) { |
| o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseRelocatingProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, |
| Box: relSpare, Mode: "cold", NewGeneration: "P2", KeepStamp: true, PredecessorCertified: true, Region: "ie", PredecessorRegion: "ie"})) |
| o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relSpare, Session: o.Claim.Holder.Session})) |
| + o = step(t, s, o, RecordProbeStop("rel-1", ProbeStop{Box: relSpare, Session: o.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)})) |
| o = step(t, s, o, CompleteRelocation("rel-1", counter())) |
| c := o.Claim |
| if len(c.EvidenceRefs) != 1 || c.EvidenceRefs[0].Box != relTarget || c.EvidenceRefs[0].Role != "target" || c.EvidenceRefs[0].Session != 0 || !slices.Contains(c.LossRefs, c.EvidenceRefs[0].ID) { |
| @@ -1096,6 +1101,7 @@ func TestRelocationParksEveryAbandonedPreparedGeneration(t *testing.T) { |
| o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseRelocatingProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, |
| Box: relSpare, Mode: "cold", NewGeneration: "P2", KeepStamp: true, PredecessorCertified: true, Region: "ie", PredecessorRegion: "ie"})) |
| o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relSpare, Session: o.Claim.Holder.Session})) |
| + o = step(t, s, o, RecordProbeStop("rel-1", ProbeStop{Box: relSpare, Session: o.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)})) |
| o = step(t, s, o, CompleteRelocation("rel-1", counter())) |
| var parks []string |
| for _, ob := range o.Claim.Obligations { |
| @@ -1139,6 +1145,7 @@ func TestRelocationParksAPDroppedFromProbeFailed(t *testing.T) { |
| o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseRelocatingProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, |
| Box: relSpare, Mode: "cold", NewGeneration: "P2", KeepStamp: true, Region: "ie", PredecessorRegion: "ie"})) |
| o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relSpare, Session: o.Claim.Holder.Session})) |
| + o = step(t, s, o, RecordProbeStop("rel-1", ProbeStop{Box: relSpare, Session: o.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)})) |
| o = step(t, s, o, CompleteRelocation("rel-1", counter())) |
| parked := false |
| for _, ob := range o.Claim.Obligations { |
| @@ -1298,6 +1305,7 @@ func TestRelocationInstallReleaseFollowsThePin(t *testing.T) { |
| t.Run("the serving CAS drops both", func(t *testing.T) { |
| s, o := relAtPhase(t, "probing+P") |
| o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relTarget, Session: o.Claim.Holder.Session})) |
| + o = step(t, s, o, RecordProbeStop("rel-1", ProbeStop{Box: relTarget, Session: o.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)})) |
| o = step(t, s, o, CompleteRelocation("rel-1", counter())) |
| pair(t, o.Claim) |
| if o.Claim.Op.Journal["install_stamp"] != "" { |
diff --git a/internal/hostrelocate/participant.go b/internal/hostrelocate/participant.go| index ca7d606bf..431b6d53b 100644 |
| --- a/internal/hostrelocate/participant.go |
| +++ b/internal/hostrelocate/participant.go |
| @@ -4,6 +4,7 @@ import ( |
| "context" |
| "encoding/json" |
| "errors" |
| + "fmt" |
| "slices" |
| "time" |
| @@ -70,6 +71,15 @@ type Effects interface { |
| // caller counts an attempt in the claim before each call, so a Probe |
| // that returned early would spend the attempts without probing. |
| Probe(ctx context.Context, obs hostclaim.Observed, certified bool) (session uint64, passed bool, err error) |
| + // Preflight is called in probing before an attempt is counted (region |
| + // HA ruling 48): root's isolated launch must speak the protocol and |
| + // install an attempt reader for relocating, or nothing is counted. |
| + Preflight(context.Context, hostclaim.Observed) error |
| + // EndProbe stops this box's passed probe session while the op is still |
| + // relocating/probing and returns root's receipt for it (ruling 49): |
| + // the activator's local fence and stop, then the claim's writer-session |
| + // stop through root. Idempotent for one session. |
| + EndProbe(context.Context, hostclaim.Observed) (hostclaim.ProbeStop, error) |
| // Retained lists the participations whose execution controllers this |
| // box still holds; Finish closes one and releases its lifetime lock. |
| // A controller is finished once the claim no longer runs that |
| @@ -152,8 +162,15 @@ func (p Participant) Step(ctx context.Context, host string) (hostclaim.Observed, |
| if err := p.Effects.EnsureLocal(ctx, obs); err != nil { |
| return obs, err |
| } |
| + // A repairing EnsureLocal may have written the claim (the |
| + // typed inventory): the transition is made on a fresh read of |
| + // the same installation (rev 3, Astra round 2 finding 2). |
| + fresh, err := p.sameInstallation(ctx, obs, hostclaim.PhaseInstalled) |
| + if err != nil { |
| + return obs, err |
| + } |
| // The target enters probing itself (§5.8 phase table). |
| - return p.update(ctx, obs, hostclaim.ToProbing(id, p.Now())) |
| + return p.update(ctx, fresh, hostclaim.ToProbing(id, p.Now())) |
| } |
| l, err := p.Effects.InstallLocal(ctx, obs) |
| if err != nil { |
| @@ -162,7 +179,14 @@ func (p Participant) Step(ctx context.Context, host string) (hostclaim.Observed, |
| if l.Box != p.Box || l.Enrollment != p.Enrollment { |
| return obs, ErrRefused |
| } |
| - return p.update(ctx, obs, hostclaim.RecordLocalInstall(id, l)) |
| + // The install recorded its typed inventory in the claim (park's |
| + // RecordInventory): record the receipt on a fresh read of the |
| + // same installation, never on the stale observation. |
| + fresh, err := p.sameInstallation(ctx, obs, hostclaim.PhaseInstalled) |
| + if err != nil { |
| + return obs, err |
| + } |
| + return p.update(ctx, fresh, hostclaim.RecordLocalInstall(id, l)) |
| case executor && c.Op.Phase == hostclaim.PhaseRelocatingProbing && c.Op.Journal["probe"] == "": |
| if !hostclaim.PreparedOpened(c) { |
| // Before P's first admission the local install must be whole. |
| @@ -180,6 +204,24 @@ func (p Participant) Step(ctx context.Context, host string) (hostclaim.Observed, |
| return p.update(ctx, obs, hostclaim.MarkStuck(id, "the local install receipt is missing after P was admitted (§5.8, B2-5)")) |
| } |
| } |
| + // EnsureLocal may have written the claim: count on a fresh read |
| + // of the same installation. |
| + if !hostclaim.PreparedOpened(c) { |
| + fresh, err := p.sameInstallation(ctx, obs, hostclaim.PhaseRelocatingProbing) |
| + if err != nil { |
| + return obs, err |
| + } |
| + obs, c = fresh, fresh.Claim |
| + } |
| + // Root must be able to authorize a relocating launch before an |
| + // attempt is spent (ruling 48): a refusal counts nothing. A later |
| + // attempt root cannot launch (ruling 58, until claim ask C11) is |
| + // probe-failed now, not three uncountable refusals. |
| + if err := p.Effects.Preflight(ctx, obs); errors.Is(err, ErrLaterAttemptUnlaunchable) { |
| + return p.update(ctx, obs, hostclaim.ToProbeFailed(id, p.Now())) |
| + } else if err != nil { |
| + return obs, errors.Join(ErrNotYet, err) |
| + } |
| // The attempt is counted in the claim before it is made (§5.8: |
| // at most three); the fourth is probe-failed, by this CAS. |
| counted, err := hostclaim.Update(ctx, p.Claims, obs, hostclaim.BeginProbeAttempt(id, p.Box)) |
| @@ -190,7 +232,7 @@ func (p Participant) Step(ctx context.Context, host string) (hostclaim.Observed, |
| return obs, err |
| } |
| obs, c = counted, counted.Claim |
| - session, passed, err := p.Effects.Probe(ctx, obs, certifiedFor(c)) |
| + session, passed, err := p.Effects.Probe(ctx, obs, CertifiedFor(c)) |
| if err != nil || !passed { |
| return obs, errors.Join(ErrNotYet, err) |
| } |
| @@ -200,11 +242,45 @@ func (p Participant) Step(ctx context.Context, host string) (hostclaim.Observed, |
| if err != nil || fresh.Absent { |
| return obs, errors.Join(ErrNotYet, err) |
| } |
| - return p.update(ctx, fresh, hostclaim.RecordProbe(id, hostclaim.Probe{Box: p.Box, Session: session})) |
| + return p.update(ctx, fresh, hostclaim.RecordProbe(id, hostclaim.Probe{Box: p.Box, Session: session, At: p.Now()})) |
| + case executor && c.Op.Phase == hostclaim.PhaseRelocatingProbing && c.Op.Journal["probe_stop"] == "": |
| + // Ruling 49: the passed probe's session is stopped, with root's |
| + // receipt, before the serving CAS ends it. |
| + s, err := p.Effects.EndProbe(ctx, obs) |
| + if err != nil { |
| + return obs, err |
| + } |
| + fresh, err := hostclaim.Read(ctx, p.Claims, host) |
| + if err != nil || fresh.Absent { |
| + return obs, errors.Join(ErrNotYet, err) |
| + } |
| + return p.update(ctx, fresh, hostclaim.RecordProbeStop(id, s)) |
| } |
| return obs, ErrNotYet |
| } |
| +// ErrLaterAttemptUnlaunchable is Preflight's answer for a cold attempt |
| +// after P's opening while root pins every relocating launch to |
| +// install_stamp (claimLaunchAuthority, claim ask C11; ruling 58). |
| +var ErrLaterAttemptUnlaunchable = errors.New("hostrelocate: root cannot launch a relocation attempt after P's opening") |
| + |
| +// sameInstallation re-reads the claim after an effect that may have written |
| +// it and requires the same operation, executor, phase, prepared P and |
| +// install stamp, unstuck: only the effect's own bookkeeping may differ. |
| +func (p Participant) sameInstallation(ctx context.Context, obs hostclaim.Observed, phase string) (hostclaim.Observed, error) { |
| + fresh, err := hostclaim.Read(ctx, p.Claims, obs.Host) |
| + if err != nil { |
| + return obs, err |
| + } |
| + a, b := obs.Claim, fresh.Claim |
| + if fresh.Absent || b == nil || b.Holder == nil || b.Op.Kind != hostclaim.KindRelocating || b.Op.ID != a.Op.ID || b.Op.Exec != p.Box || b.Op.Phase != phase || |
| + b.Op.Journal["prepared"] != a.Op.Journal["prepared"] || b.Op.Journal["install_stamp"] != a.Op.Journal["install_stamp"] || b.Op.Journal["stuck"] != "" || |
| + b.Holder.Incarnation != a.Holder.Incarnation || b.Holder.Session != a.Holder.Session || b.Holder.Stamp != a.Holder.Stamp { |
| + return obs, errors.Join(ErrNotYet, fmt.Errorf("%w: the installation changed under the effect", ErrRefused)) |
| + } |
| + return fresh, nil |
| +} |
| + |
| // Sweep finishes every retained controller whose participation is over: |
| // the claim no longer runs that operation, or this box is no longer a |
| // participant of it. It runs once per pass, for hosts the pass itself |
| @@ -231,7 +307,7 @@ func (p Participant) Sweep(ctx context.Context) error { |
| return errors.Join(errs...) |
| } |
| -// certifiedFor is whether the probe admission certifies its predecessor |
| +// CertifiedFor is whether the probe admission certifies its predecessor |
| // (§5.8, L9): only the admission that opens P, when this operation |
| // journaled a release whose session is the predecessor writer's. Claim |
| // Plan 11 gives a relocation no coverage value of its own, so the release |
| @@ -239,7 +315,7 @@ func (p Participant) Sweep(ctx context.Context) error { |
| // opening only with that release and a pin reaching it |
| // (relocationProbeOpening). Otherwise false: a failover, a later attempt, |
| // or a host that never had a writer is an uncertified cold admission. |
| -func certifiedFor(c *hostclaim.Claim) bool { |
| +func CertifiedFor(c *hostclaim.Claim) bool { |
| var r hostclaim.Release |
| if json.Unmarshal([]byte(c.Op.Journal["release"]), &r) != nil || r.ReleasedBy == "" { |
| return false |
diff --git a/internal/hostrelocate/participant_gate_test.go b/internal/hostrelocate/participant_gate_test.go| new file mode 100644 |
| index 000000000..273a9b467 |
| --- /dev/null |
| +++ b/internal/hostrelocate/participant_gate_test.go |
| @@ -0,0 +1,163 @@ |
| +package hostrelocate |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "slices" |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| +) |
| + |
| +// driveToProbing runs a drain until dst's participant is in probing with |
| +// its local install recorded and no attempt counted. |
| +func driveToProbing(t *testing.T, f *fixture, src, dst *fakeBox) (Participant, Participant) { |
| + t.Helper() |
| + ctx := context.Background() |
| + if _, err := f.w.Start(ctx, host, Request{Reason: hostclaim.ReasonDrain, Source: "src", Target: "dst"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + ps := Participant{Claims: f.s, Box: "src", Enrollment: enrollment("src"), Effects: src, Now: f.clock} |
| + pd := Participant{Claims: f.s, Box: "dst", Enrollment: enrollment("dst"), Effects: dst, Now: f.clock} |
| + for i := 0; i < 40; i++ { |
| + if o := f.claim(); o.Claim.Op.Phase == hostclaim.PhaseRelocatingProbing { |
| + return ps, pd |
| + } |
| + f.w.Step(ctx, host) |
| + ps.Step(ctx, host) |
| + if f.claim().Claim.Op.Phase == hostclaim.PhaseInstalled && f.claim().Claim.Op.Journal["local_install"] != "" { |
| + pd.Step(ctx, host) // ToProbing |
| + continue |
| + } |
| + pd.Step(ctx, host) |
| + } |
| + t.Fatalf("never probing: %+v", f.claim().Claim.Op) |
| + return ps, pd |
| +} |
| + |
| +// Ruling 48: while root refuses the relocating kind, the participant |
| +// counts no attempt; once it answers, the next pass counts one. |
| +func TestParticipantCountsNoAttemptWhileRootRefusesTheKind(t *testing.T) { |
| + f := newFixture(t) |
| + ctx := context.Background() |
| + src, dst := &fakeBox{f: f, box: "src"}, &fakeBox{f: f, box: "dst", preflightErr: errors.New("root: relocating unsupported")} |
| + _, pd := driveToProbing(t, f, src, dst) |
| + for i := 0; i < 3; i++ { |
| + if _, err := pd.Step(ctx, host); !errors.Is(err, ErrNotYet) { |
| + t.Fatalf("pass %d: %v", i, err) |
| + } |
| + if n := f.claim().Claim.Op.Journal["probe_attempts.dst"]; n != "" { |
| + t.Fatalf("an attempt was counted while root refused: %s", n) |
| + } |
| + } |
| + dst.preflightErr = nil |
| + if _, err := pd.Step(ctx, host); err != nil && !errors.Is(err, ErrNotYet) { |
| + t.Fatal(err) |
| + } |
| + if n := f.claim().Claim.Op.Journal["probe_attempts.dst"]; n != "1" { |
| + t.Fatalf("attempts after root answered: %q", n) |
| + } |
| +} |
| + |
| +// Ruling 58: once P's opening failed, an attempt root cannot launch is not |
| +// made; the operation goes probe-failed at once, with one attempt counted. |
| +func TestParticipantFailsAProbeRootCannotRelaunch(t *testing.T) { |
| + f := newFixture(t) |
| + ctx := context.Background() |
| + src, dst := &fakeBox{f: f, box: "src"}, &fakeBox{f: f, box: "dst", failProbes: true} |
| + _, pd := driveToProbing(t, f, src, dst) |
| + if _, err := pd.Step(ctx, host); !errors.Is(err, ErrNotYet) { |
| + t.Fatalf("the first attempt: %v", err) |
| + } |
| + if !hostclaim.PreparedOpened(f.claim().Claim) { |
| + t.Fatal("P was not opened by the first attempt") |
| + } |
| + dst.preflightErr = ErrLaterAttemptUnlaunchable |
| + o, err := pd.Step(ctx, host) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if o.Claim.Op.Phase != hostclaim.PhaseProbeFailed || o.Claim.Op.Journal["probe_attempts.dst"] != "1" || len(dst.certified) != 1 { |
| + t.Fatalf("phase %s attempts %q admissions %d", o.Claim.Op.Phase, o.Claim.Op.Journal["probe_attempts.dst"], len(dst.certified)) |
| + } |
| +} |
| + |
| +// Ruling 62: an install that writes the claim (the typed inventory) has its |
| +// receipt recorded on a fresh read, not a conflict loop on the stale one. |
| +func TestParticipantRecordsItsReceiptAfterAnEffectWroteTheClaim(t *testing.T) { |
| + f := newFixture(t) |
| + ctx := context.Background() |
| + src, dst := &fakeBox{f: f, box: "src"}, &fakeBox{f: f, box: "dst", writesOnInstall: true} |
| + if _, err := f.w.Start(ctx, host, Request{Reason: hostclaim.ReasonDrain, Source: "src", Target: "dst"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + ps := Participant{Claims: f.s, Box: "src", Enrollment: enrollment("src"), Effects: src, Now: f.clock} |
| + pd := Participant{Claims: f.s, Box: "dst", Enrollment: enrollment("dst"), Effects: dst, Now: f.clock} |
| + for i := 0; i < 40 && f.claim().Claim.Op.Phase != hostclaim.PhaseInstalled; i++ { |
| + f.w.Step(ctx, host) |
| + ps.Step(ctx, host) |
| + pd.Step(ctx, host) |
| + } |
| + if f.claim().Claim.Op.Phase != hostclaim.PhaseInstalled { |
| + t.Fatalf("never installed: %+v", f.claim().Claim.Op) |
| + } |
| + if _, err := pd.Step(ctx, host); err != nil { |
| + t.Fatalf("the receipt after an effect wrote the claim: %v", err) |
| + } |
| + c := f.claim().Claim |
| + if c.Op.Journal["local_install"] == "" || c.Op.Journal["test.inventory"] != "dst" { |
| + t.Fatalf("receipt %q inventory %q", c.Op.Journal["local_install"], c.Op.Journal["test.inventory"]) |
| + } |
| +} |
| + |
| +// passProbe runs dst's first attempt to a journaled passing probe. |
| +func passProbe(t *testing.T, f *fixture, pd Participant) { |
| + t.Helper() |
| + if _, err := pd.Step(context.Background(), host); err != nil { |
| + t.Fatalf("the probe: %v", err) |
| + } |
| + if f.claim().Claim.Op.Journal["probe"] == "" { |
| + t.Fatal("no probe journaled") |
| + } |
| +} |
| + |
| +// Ruling 67's first scenario: the execution lease is lost right after |
| +// RecordProbe. The participant reopens a controller and ends nothing, so |
| +// no probe_stop is ever journaled by it (the worker's bound decides). |
| +func TestParticipantEndsNoProbeWithoutItsLease(t *testing.T) { |
| + f := newFixture(t) |
| + ctx := context.Background() |
| + src, dst := &fakeBox{f: f, box: "src"}, &fakeBox{f: f, box: "dst"} |
| + _, pd := driveToProbing(t, f, src, dst) |
| + passProbe(t, f, pd) |
| + dst.restarted = true // the lease is gone; Ready reports no controller |
| + dst.effects = nil |
| + if _, err := pd.Step(ctx, host); !errors.Is(err, ErrNotYet) { |
| + t.Fatalf("a step without the lease: %v", err) |
| + } |
| + if slices.Contains(dst.effects, "end-probe") || f.claim().Claim.Op.Journal["probe_stop"] != "" { |
| + t.Fatalf("ended the probe without a ready controller: %v", dst.effects) |
| + } |
| +} |
| + |
| +// Ruling 67's second scenario: root keeps refusing the stop. The error is |
| +// returned and nothing is journaled; a later pass that gets the receipt |
| +// records it. |
| +func TestParticipantRecordsNoProbeStopWhileRootRefuses(t *testing.T) { |
| + f := newFixture(t) |
| + ctx := context.Background() |
| + src, dst := &fakeBox{f: f, box: "src"}, &fakeBox{f: f, box: "dst"} |
| + _, pd := driveToProbing(t, f, src, dst) |
| + passProbe(t, f, pd) |
| + refused := errors.New("root: stop refused") |
| + dst.endProbeErr = refused |
| + for i := 0; i < 2; i++ { |
| + if _, err := pd.Step(ctx, host); !errors.Is(err, refused) || f.claim().Claim.Op.Journal["probe_stop"] != "" { |
| + t.Fatalf("pass %d: %v", i, err) |
| + } |
| + } |
| + dst.endProbeErr = nil |
| + if _, err := pd.Step(ctx, host); err != nil || f.claim().Claim.Op.Journal["probe_stop"] == "" { |
| + t.Fatalf("the stop once root answered: %v", err) |
| + } |
| +} |
diff --git a/internal/hostrelocate/participant_test.go b/internal/hostrelocate/participant_test.go| index d3311094e..06d151f40 100644 |
| --- a/internal/hostrelocate/participant_test.go |
| +++ b/internal/hostrelocate/participant_test.go |
| @@ -35,6 +35,14 @@ type fakeBox struct { |
| restores, reinstalls, probedMissing int |
| // receiptLost: the box's durable local install receipt is gone. |
| receiptLost bool |
| + // preflightErr is root refusing the relocating kind, or |
| + // ErrLaterAttemptUnlaunchable (rulings 48, 58). |
| + preflightErr error |
| + // endProbeErr is a stop root keeps refusing (ruling 67). |
| + endProbeErr error |
| + // writesOnInstall: the install writes the claim (the typed inventory), |
| + // as the production install's RecordInventory does (ruling 62). |
| + writesOnInstall bool |
| } |
| func (b *fakeBox) Acknowledge(_ context.Context, o hostclaim.Observed) (hostclaim.ExecutionAcknowledgement, error) { |
| @@ -48,10 +56,19 @@ func (b *fakeBox) Retained() ([]Participation, error) { |
| } |
| return []Participation{{Host: host, Op: b.op}}, b.retainedErr |
| } |
| -func (b *fakeBox) Finish(_ context.Context, h, op string) error { |
| +func (b *fakeBox) Finish(ctx context.Context, h, op string) error { |
| b.finished = true |
| b.effects = append(b.effects, "finish "+op) |
| - return nil |
| + // Production's Finish is the target's hold release, which ends by |
| + // marking the completed relocation discharged (region HA ruling 76). |
| + obs, err := hostclaim.Read(ctx, b.f.s, h) |
| + if err != nil { |
| + return err |
| + } |
| + if done, ok := hostclaim.ReadCompletedRelocation(obs.Claim); ok && done.Op == op && done.Stop.Box == b.box && !done.Discharged { |
| + _, err = hostclaim.Update(ctx, b.f.s, obs, hostclaim.MarkCompletedDischarged(op)) |
| + } |
| + return err |
| } |
| func (b *fakeBox) Ready(context.Context, hostclaim.Observed) error { |
| b.effects = append(b.effects, "ready") |
| @@ -71,6 +88,9 @@ func (b *fakeBox) Build(context.Context, hostclaim.Observed) error { |
| } |
| func (b *fakeBox) InstallLocal(_ context.Context, o hostclaim.Observed) (hostclaim.LocalInstall, error) { |
| b.effects = append(b.effects, "install") |
| + if b.writesOnInstall { |
| + b.f.participant(func(c *hostclaim.Claim) error { c.Op.Journal["test.inventory"] = b.box; return nil }) |
| + } |
| n, _ := strconv.ParseUint(o.Claim.Op.Journal["install_stamp"], 10, 64) |
| return hostclaim.LocalInstall{Box: b.box, Enrollment: enrollment(b.box), InstallStamp: n, FilesManifestVersion: 1, JournalSHA256: strings.Repeat("e", 64)}, nil |
| } |
| @@ -273,11 +293,19 @@ func TestAParticipantFinishesItsControllerOnceTheOperationIsOver(t *testing.T) { |
| if dst.opened != before+1 { |
| t.Fatalf("the second relocation did not open its own controller: %d -> %d", before, dst.opened) |
| } |
| - // Mid-operation, Sweep keeps the controller. |
| + // No third relocation starts from spa until spa has finished the |
| + // second one (region HA ruling 77); then, mid-operation, Sweep keeps |
| + // the controller. |
| + pspa := Participant{Claims: f.s, Box: "spa", Enrollment: enrollment("spa"), Effects: spa, Now: f.clock} |
| + if _, err := f.w.Start(ctx, host, Request{Reason: hostclaim.ReasonDrain, Source: "spa", Target: "sp2"}); !errors.Is(err, hostclaim.ErrPendingChore) { |
| + t.Fatalf("a successor started before spa finished: %v", err) |
| + } |
| + if err := pspa.Sweep(ctx); err != nil || !spa.finished { |
| + t.Fatalf("spa kept the second relocation's controller: %v", err) |
| + } |
| if _, err := f.w.Start(ctx, host, Request{Reason: hostclaim.ReasonDrain, Source: "spa", Target: "sp2"}); err != nil { |
| t.Fatal(err) |
| } |
| - pspa := Participant{Claims: f.s, Box: "spa", Enrollment: enrollment("spa"), Effects: spa, Now: f.clock} |
| spa.finished = false |
| if _, err := pspa.Step(ctx, host); err != nil { |
| t.Fatal(err) |
| @@ -390,3 +418,16 @@ func TestAParticipantRestoresALocalInstallRetiredAfterItEntersProbing(t *testing |
| t.Fatalf("restores %d, probes over a missing install %d, reinstalls after P %d", dst.restores, dst.probedMissing, dst.reinstalls) |
| } |
| } |
| + |
| +func (b *fakeBox) Preflight(context.Context, hostclaim.Observed) error { |
| + b.effects = append(b.effects, "preflight") |
| + return b.preflightErr |
| +} |
| + |
| +func (b *fakeBox) EndProbe(_ context.Context, o hostclaim.Observed) (hostclaim.ProbeStop, error) { |
| + b.effects = append(b.effects, "end-probe") |
| + if b.endProbeErr != nil { |
| + return hostclaim.ProbeStop{}, b.endProbeErr |
| + } |
| + return hostclaim.ProbeStop{Box: b.box, Session: o.Claim.Holder.Session, Receipt: "fence-" + b.box, ReceiptSHA256: strings.Repeat("a", 64)}, nil |
| +} |
diff --git a/internal/hostrelocate/worker.go b/internal/hostrelocate/worker.go| index 4255f604c..cd447030e 100644 |
| --- a/internal/hostrelocate/worker.go |
| +++ b/internal/hostrelocate/worker.go |
| @@ -394,13 +394,31 @@ func (w Worker) Step(ctx context.Context, host string) (hostclaim.Observed, erro |
| if err != nil { |
| return obs, err |
| } |
| - if c.Op.Journal["probe"] != "" { |
| + if c.Op.Journal["probe"] != "" && c.Op.Journal["probe_stop"] != "" { |
| next, err := w.update(ctx, obs, hostclaim.CompleteRelocation(id, w.NewID)) |
| if err == nil { |
| w.Notify(ctx) |
| } |
| return next, err |
| } |
| + if c.Op.Journal["probe"] != "" { |
| + // The target stops its probe session first (ruling 49). Ruling |
| + // 67: bounded. If root's stop receipt cannot be had (the |
| + // execution lease lost after RecordProbe, so the participant |
| + // never passes Ready; a stop root keeps refusing), the op is |
| + // stuck ReleaseDeadline after the probe passed, for an operator. |
| + // Nothing is released: the participation's denial, the row's |
| + // hold and the session stay as they are. |
| + var p hostclaim.Probe |
| + passed := since.Add(ProbeDeadline) |
| + if json.Unmarshal([]byte(c.Op.Journal["probe"]), &p) == nil && !p.At.IsZero() { |
| + passed = p.At |
| + } |
| + if w.Now().Sub(passed) > ReleaseDeadline { |
| + return w.update(ctx, obs, hostclaim.MarkStuck(id, fmt.Sprintf("%s passed its probe but no root stop receipt for session %d within %s: an operator must obtain the stop", c.Op.Exec, p.Session, ReleaseDeadline))) |
| + } |
| + return obs, ErrNotYet |
| + } |
| if w.Now().Sub(since) > ProbeDeadline { |
| return w.update(ctx, obs, hostclaim.ToProbeFailed(id, w.Now())) |
| } |
| @@ -410,9 +428,19 @@ func (w Worker) Step(ctx context.Context, host string) (hostclaim.Observed, erro |
| if err != nil || !done || !acked(c) { |
| return obs, errors.Join(ErrNotYet, err) |
| } |
| - return w.update(ctx, obs, hostclaim.ToInstalled(id, a)) |
| + return w.update(ctx, obs, hostclaim.ToInstalledAt(id, a, w.Now())) |
| + case hostclaim.PhaseInstalled: |
| + // The target's local install and its entry into probing (Plan 6b). |
| + // Ruling 60: bounded. A target that cannot enter probing within |
| + // ProbeDeadline (a retired stamp, a participation main denies) is |
| + // stuck for an operator: no automatic replacement of a live target |
| + // is made here. |
| + at, err := time.Parse(time.RFC3339Nano, c.Op.Journal["installed_at"]) |
| + if err == nil && w.Now().Sub(at) > ProbeDeadline { |
| + return w.update(ctx, obs, hostclaim.MarkStuck(id, fmt.Sprintf("%s did not enter probing within %s of installed", c.Op.Exec, ProbeDeadline))) |
| + } |
| + return obs, ErrNotYet |
| } |
| - // installed is the target's: its local install and its probe (Plan 6b). |
| return obs, ErrNotYet |
| } |
diff --git a/internal/hostrelocate/worker_bounds_test.go b/internal/hostrelocate/worker_bounds_test.go| new file mode 100644 |
| index 000000000..71dd1034f |
| --- /dev/null |
| +++ b/internal/hostrelocate/worker_bounds_test.go |
| @@ -0,0 +1,129 @@ |
| +package hostrelocate |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| +) |
| + |
| +// releasedAndInstalling drives a drain to installing on dst, released. |
| +func releasedAndInstalling(t *testing.T, f *fixture) string { |
| + t.Helper() |
| + ctx := context.Background() |
| + if _, err := f.w.Start(ctx, host, Request{Reason: hostclaim.ReasonDrain, Source: "src", Target: "dst"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + f.ack("src") |
| + f.ack("dst") |
| + f.worker(nil) // releasing |
| + h := f.claim().Claim.Holder |
| + id := f.claim().Claim.Op.ID |
| + f.participant(hostclaim.JournalRelease(id, hostclaim.Release{ReleasedBy: "src", Enrollment: enrollment("src"), Generation: h.Generation, Session: h.Session, FinalTXID: 40, ManifestSeq: 7, Receipt: "r", ReceiptSHA256: strings.Repeat("d", 64)})) |
| + f.worker(nil) // pinning |
| + f.worker(nil) // pin |
| + f.worker(nil) // installing on dst |
| + return id |
| +} |
| + |
| +// Astra round 3 finding 4, ruling 67: a passed probe whose stop receipt |
| +// never comes is bounded. ReleaseDeadline after the probe passed, the op |
| +// is stuck for an operator; before it, nothing changes; nothing is |
| +// released. Why the receipt never comes is the participant's (its two |
| +// scenarios are TestParticipantEndsNoProbeWithoutItsLease and |
| +// TestParticipantRecordsNoProbeStopWhileRootRefuses); the worker sees |
| +// only a probe with no probe_stop. |
| +func TestWorkerBoundsAPassedProbeAwaitingItsStop(t *testing.T) { |
| + f := newFixture(t) |
| + id := releasedAndInstalling(t, f) |
| + f.installAndProbe("dst") |
| + o := f.claim() |
| + f.participant(hostclaim.RecordProbe(id, hostclaim.Probe{Box: "dst", Session: o.Claim.Holder.Session, At: f.now})) |
| + f.now = f.now.Add(ReleaseDeadline) |
| + if o := f.worker(ErrNotYet); o.Claim.Op.Journal["stuck"] != "" || o.Claim.Op.Kind != hostclaim.KindRelocating { |
| + t.Fatalf("stuck before the bound: %+v", o.Claim.Op.Journal) |
| + } |
| + f.now = f.now.Add(time.Second) |
| + o = f.worker(nil) |
| + if o.Claim.Op.Journal["stuck"] == "" || o.Claim.Op.Kind != hostclaim.KindRelocating || o.Claim.Op.Exec != "dst" || o.Claim.Op.Journal["probe_stop"] != "" { |
| + t.Fatalf("not stuck for an operator, or something released: %+v", o.Claim.Op) |
| + } |
| +} |
| + |
| +// A probe journaled before rev 4 has no At: the bound counts from |
| +// probing_since plus ProbeDeadline. |
| +func TestWorkerBoundsAnUntimedPassedProbeFromProbingSince(t *testing.T) { |
| + f := newFixture(t) |
| + id := releasedAndInstalling(t, f) |
| + f.installAndProbe("dst") |
| + o := f.claim() |
| + f.participant(hostclaim.RecordProbe(id, hostclaim.Probe{Box: "dst", Session: o.Claim.Holder.Session})) |
| + f.now = f.now.Add(ProbeDeadline + ReleaseDeadline) |
| + if o := f.worker(ErrNotYet); o.Claim.Op.Journal["stuck"] != "" { |
| + t.Fatal("stuck before probing_since + ProbeDeadline + ReleaseDeadline") |
| + } |
| + f.now = f.now.Add(time.Second) |
| + if o := f.worker(nil); o.Claim.Op.Journal["stuck"] == "" { |
| + t.Fatalf("not stuck: %+v", o.Claim.Op) |
| + } |
| +} |
| + |
| +// Ruling 60: an install that never enters probing (its stamp retired, its |
| +// participation denied) is stuck ProbeDeadline after installed_at. |
| +func TestWorkerBoundsAnInstallThatNeverProbes(t *testing.T) { |
| + f := newFixture(t) |
| + releasedAndInstalling(t, f) |
| + f.worker(ErrNotYet) // P is not built yet |
| + p := f.claim().Claim.Op.Journal["prepared"] |
| + f.participant(func(c *hostclaim.Claim) error { c.Op.Journal["test.built"] = p; return nil }) |
| + if o := f.worker(nil); o.Claim.Op.Phase != hostclaim.PhaseInstalled || o.Claim.Op.Journal["installed_at"] == "" { |
| + t.Fatalf("installed with its start: %s %+v", o.Claim.Op.Phase, o.Claim.Op.Journal) |
| + } |
| + f.now = f.now.Add(ProbeDeadline) |
| + if o := f.worker(ErrNotYet); o.Claim.Op.Journal["stuck"] != "" { |
| + t.Fatal("stuck before the bound") |
| + } |
| + f.now = f.now.Add(time.Second) |
| + if o := f.worker(nil); o.Claim.Op.Journal["stuck"] == "" || o.Claim.Op.Phase != hostclaim.PhaseInstalled { |
| + t.Fatalf("not stuck: %+v", o.Claim.Op) |
| + } |
| +} |
| + |
| +// Ruling 60: a target that enters probing but never admits (its stamp |
| +// retired, root refusing) is probe-failed at the probe deadline, then |
| +// stuck ReleaseDeadline later for an operator. A live target is never |
| +// replaced automatically while claim ask C1 is open: exec stays dst |
| +// throughout. |
| +func TestWorkerBoundsARefusedProbe(t *testing.T) { |
| + f := newFixture(t) |
| + id := releasedAndInstalling(t, f) |
| + p := f.claim().Claim.Op.Journal["prepared"] |
| + f.worker(ErrNotYet) // P is not built yet |
| + f.participant(func(c *hostclaim.Claim) error { c.Op.Journal["test.built"] = p; return nil }) |
| + f.worker(nil) // installed |
| + var stamp uint64 |
| + if err := json.Unmarshal([]byte(f.claim().Claim.Op.Journal["install_stamp"]), &stamp); err != nil { |
| + t.Fatal(err) |
| + } |
| + f.participant(hostclaim.RecordLocalInstall(id, hostclaim.LocalInstall{Box: "dst", Enrollment: enrollment("dst"), InstallStamp: stamp, FilesManifestVersion: 1, JournalSHA256: strings.Repeat("d", 64)})) |
| + f.participant(hostclaim.ToProbing(id, f.now)) |
| + f.now = f.now.Add(ProbeDeadline) |
| + if o := f.worker(ErrNotYet); o.Claim.Op.Phase != hostclaim.PhaseRelocatingProbing { |
| + t.Fatalf("probe-failed before the probe deadline: %s", o.Claim.Op.Phase) |
| + } |
| + f.now = f.now.Add(time.Second) |
| + if o := f.worker(nil); o.Claim.Op.Phase != hostclaim.PhaseProbeFailed || o.Claim.Op.Exec != "dst" { |
| + t.Fatalf("not probe-failed on dst: %s %s", o.Claim.Op.Phase, o.Claim.Op.Exec) |
| + } |
| + f.now = f.now.Add(ReleaseDeadline) |
| + if o := f.worker(ErrNotYet); o.Claim.Op.Journal["stuck"] != "" || o.Claim.Op.Exec != "dst" { |
| + t.Fatalf("stuck or replaced before the release deadline: %+v", o.Claim.Op) |
| + } |
| + f.now = f.now.Add(time.Second) |
| + if o := f.worker(nil); o.Claim.Op.Journal["stuck"] == "" || o.Claim.Op.Exec != "dst" || o.Claim.Op.Phase != hostclaim.PhaseProbeFailed { |
| + t.Fatalf("not stuck on dst for an operator: %+v", o.Claim.Op) |
| + } |
| +} |
diff --git a/internal/hostrelocate/worker_test.go b/internal/hostrelocate/worker_test.go| index 3095526ff..1c2908800 100644 |
| --- a/internal/hostrelocate/worker_test.go |
| +++ b/internal/hostrelocate/worker_test.go |
| @@ -284,6 +284,8 @@ func TestWorkerDrivesADrain(t *testing.T) { |
| f.worker(ErrNotYet) // probing, no probe result, within the deadline |
| o = f.claim() |
| f.participant(hostclaim.RecordProbe(o.Claim.Op.ID, hostclaim.Probe{Box: "dst", Session: o.Claim.Holder.Session})) |
| + f.worker(ErrNotYet) // the probe passed; its session is not stopped yet (ruling 49) |
| + f.participant(hostclaim.RecordProbeStop(o.Claim.Op.ID, hostclaim.ProbeStop{Box: "dst", Session: o.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)})) |
| if o = f.worker(nil); o.Claim.Op.Kind != hostclaim.KindServing || o.Claim.Op.Exec != "dst" { |
| t.Fatalf("serving: %+v", o.Claim.Op) |
| } |