amadan

carlos / platform Public

Merge branch 'moves-offline-t3' into moves-offline

claude 7a2f37df8696afb0fc9cd9e8c43ed3bc496f29d1 parent 0bf0fc8parent 570194e
13 files changed, +2799 −7
  • internal/cgroup/hostprocs.go +28 −0
  • internal/cgroup/hostprocs_linux.go +109 −0
  • internal/cgroup/hostprocs_linux_test.go +115 −0
  • internal/cgroup/hostprocs_other.go +7 −0
  • internal/host/agent.go +23 −0
  • internal/host/edgesfile.go +30 −2
  • internal/host/movesoffline.go +976 −0
  • internal/host/movesoffline_land.go +30 −0
  • internal/host/movesoffline_seal.go +28 −0
  • internal/host/movesoffline_test.go +1283 −0
  • internal/host/movespass.go +154 −4
  • internal/layout/extras.go +15 −0
  • internal/layout/extras_test.go +1 −1

CI cancelled — superseded by a newer push — run details

CI log 
=== gate ===
cd launchctx && go vet ./... && go test ./... -count=1
ok  	amadan.net/carlos/platform.git/launchctx	0.011s
?   	amadan.net/carlos/platform.git/launchctx/controlname	[no test files]
?   	amadan.net/carlos/platform.git/launchctx/wireid	[no test files]
go vet ./...
gofmt -l .: clean
go test ./... -count=1 -timeout 30m

amadan: job cancelled — a newer push superseded it
diff --git a/internal/cgroup/hostprocs.go b/internal/cgroup/hostprocs.go
new file mode 100644
index 000000000..d2f950abc
--- /dev/null
+++ b/internal/cgroup/hostprocs.go
@@ -0,0 +1,28 @@
+package cgroup
+
+import "errors"
+
+// ErrUnknown is HostProcs saying it cannot tell how many processes a host
+// has left: the platform has no cgroup v2 to read (darwin, windows), or the
+// agent runs without a delegated subtree, so its exec children share the
+// agent's own cgroup and cannot be told apart. A caller records it as "not
+// stopped", never as zero (the offline move's source report, Task 3 of
+// docs/superpowers/plans/2026-10-20-offline-operator-move.md).
+var ErrUnknown = errors.New("cgroup: the host's processes cannot be counted on this box")
+
+// HostProcs counts the processes left for host on this box:
+//
+// - in the host's exec cgroup, root/carlos-exec@<host> (ExecName), and
+// every cgroup below it (an exec child's sidecar and worker groups);
+// - when unit is set, in the cgroup directory unit names and every cgroup
+// below it. unit is an absolute path in the cgroup filesystem: for a
+// nested row the host's slice, which holds its run, generation, seed,
+// sidecar and cmd units alike.
+//
+// root is the delegated subtree (Manager.Root). A cgroup that does not
+// exist holds no process and counts 0. An empty root is ErrUnknown, since
+// without delegation the exec children run in the agent's own cgroup.
+// Reading never writes, moves or signals anything.
+func HostProcs(root, host, unit string) (int, error) {
+ return hostProcs(root, host, unit)
+}
diff --git a/internal/cgroup/hostprocs_linux.go b/internal/cgroup/hostprocs_linux.go
new file mode 100644
index 000000000..c4204423c
--- /dev/null
+++ b/internal/cgroup/hostprocs_linux.go
@@ -0,0 +1,109 @@
+package cgroup
+
+import (
+ "bufio"
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+)
+
+// hostProcsMaxDepth bounds the walk below a host's cgroup. The platform
+// nests at most a few levels (exec group, role child, generation); a tree
+// deeper than this is not one it made, and is reported rather than walked.
+const hostProcsMaxDepth = 16
+
+func hostProcs(root, host, unit string) (int, error) {
+ if root == "" {
+ return 0, ErrUnknown
+ }
+ if host == "" || strings.ContainsAny(host, "/\x00") || host == "." || host == ".." {
+ return 0, fmt.Errorf("cgroup: %q is not a host name", host)
+ }
+ n, err := countProcs(filepath.Join(root, ExecName(host)), 0)
+ if err != nil {
+ return 0, err
+ }
+ if unit != "" {
+ if !filepath.IsAbs(unit) {
+ return 0, fmt.Errorf("cgroup: the unit cgroup %q is not an absolute path", unit)
+ }
+ m, err := countProcs(filepath.Clean(unit), 0)
+ if err != nil {
+ return 0, err
+ }
+ n += m
+ }
+ return n, nil
+}
+
+// countProcs is the number of pids listed in dir's cgroup.procs and in every
+// cgroup below it. An absent dir is 0. Symlinks are never followed: cgroupfs
+// has none, so a link is not a cgroup.
+func countProcs(dir string, depth int) (int, error) {
+ if depth > hostProcsMaxDepth {
+ return 0, fmt.Errorf("cgroup: %s is nested deeper than %d levels", dir, hostProcsMaxDepth)
+ }
+ st, err := os.Lstat(dir)
+ if errors.Is(err, os.ErrNotExist) {
+ return 0, nil
+ }
+ if err != nil {
+ return 0, err
+ }
+ if !st.IsDir() {
+ return 0, fmt.Errorf("cgroup: %s is not a cgroup directory", dir)
+ }
+ n, err := readProcsFile(filepath.Join(dir, "cgroup.procs"))
+ if err != nil {
+ return 0, err
+ }
+ ents, err := os.ReadDir(dir)
+ if errors.Is(err, os.ErrNotExist) {
+ return n, nil // removed while read: its processes are gone with it
+ }
+ if err != nil {
+ return 0, err
+ }
+ for _, e := range ents {
+ if !e.IsDir() || e.Type()&os.ModeSymlink != 0 {
+ continue
+ }
+ m, err := countProcs(filepath.Join(dir, e.Name()), depth+1)
+ if err != nil {
+ return 0, err
+ }
+ n += m
+ }
+ return n, nil
+}
+
+// readProcsFile counts the non-empty lines of a cgroup.procs file. A file
+// that is gone (the cgroup was removed between the listing and the read)
+// lists nothing.
+func readProcsFile(path string) (int, error) {
+ f, err := os.Open(path)
+ if errors.Is(err, os.ErrNotExist) {
+ return 0, nil
+ }
+ if err != nil {
+ return 0, err
+ }
+ defer f.Close()
+ n := 0
+ sc := bufio.NewScanner(f)
+ for sc.Scan() {
+ if strings.TrimSpace(sc.Text()) != "" {
+ n++
+ }
+ }
+ if err := sc.Err(); err != nil {
+ // A cgroup removed while it was read answers ENODEV.
+ if _, statErr := os.Stat(filepath.Dir(path)); errors.Is(statErr, os.ErrNotExist) {
+ return 0, nil
+ }
+ return 0, err
+ }
+ return n, nil
+}
diff --git a/internal/cgroup/hostprocs_linux_test.go b/internal/cgroup/hostprocs_linux_test.go
new file mode 100644
index 000000000..e96d1528f
--- /dev/null
+++ b/internal/cgroup/hostprocs_linux_test.go
@@ -0,0 +1,115 @@
+package cgroup
+
+import (
+ "errors"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strconv"
+ "testing"
+)
+
+// writeProcs makes dir a fixture cgroup listing pids.
+func writeProcs(t *testing.T, dir string, pids ...int) {
+ t.Helper()
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ body := ""
+ for _, p := range pids {
+ body += strconv.Itoa(p) + "\n"
+ }
+ if err := os.WriteFile(filepath.Join(dir, "cgroup.procs"), []byte(body), 0o644); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// HostProcs over a fixture cgroupfs: the exec group and every group below
+// it count, and so does the unit's group; absent groups count 0; another
+// host's group never counts.
+func TestHostProcsCountsTheHostsGroupsOnly(t *testing.T) {
+ root := t.TempDir()
+ const host = "shop.example.com"
+ if n, err := HostProcs(root, host, ""); err != nil || n != 0 {
+ t.Fatalf("no group: (%d, %v), want (0, nil)", n, err)
+ }
+ writeProcs(t, filepath.Join(root, ExecName(host)))
+ if n, err := HostProcs(root, host, ""); err != nil || n != 0 {
+ t.Fatalf("an empty group: (%d, %v), want (0, nil)", n, err)
+ }
+ writeProcs(t, filepath.Join(root, ExecName(host)), 101)
+ writeProcs(t, filepath.Join(root, ExecName(host), "sidecar"), 102, 103)
+ writeProcs(t, filepath.Join(root, ExecName("other.example.com")), 201, 202, 203)
+ if n, err := HostProcs(root, host, ""); err != nil || n != 3 {
+ t.Fatalf("exec group and child: (%d, %v), want (3, nil)", n, err)
+ }
+ slice := filepath.Join(t.TempDir(), "carlos.slice", "carlos-instances.slice", "carlos-instances-shop.slice")
+ if n, err := HostProcs(root, host, slice); err != nil || n != 3 {
+ t.Fatalf("an absent unit group: (%d, %v), want (3, nil)", n, err)
+ }
+ writeProcs(t, filepath.Join(slice, "carlos-instance@shop.service"), 301)
+ if n, err := HostProcs(root, host, slice); err != nil || n != 4 {
+ t.Fatalf("with the unit's group: (%d, %v), want (4, nil)", n, err)
+ }
+ if _, err := HostProcs(root, host, "relative/slice"); err == nil {
+ t.Fatal("a relative unit path was read")
+ }
+ for _, bad := range []string{"", "..", "a/b"} {
+ if _, err := HostProcs(root, bad, ""); err == nil {
+ t.Fatalf("host %q was read", bad)
+ }
+ }
+}
+
+// Without a delegated subtree the exec children cannot be told apart from
+// the agent's own processes: unknown, never 0.
+func TestHostProcsWithoutDelegationIsUnknown(t *testing.T) {
+ if _, err := HostProcs("", "shop.example.com", ""); !errors.Is(err, ErrUnknown) {
+ t.Fatalf("no root: %v, want ErrUnknown", err)
+ }
+}
+
+// A real process in a real cgroup, where the kernel lets this test make one:
+// a child group of the test's own cgroup, with a sleep moved into it. Where
+// the cgroup is not writable (the usual case outside a delegated unit) the
+// fixture test above is the witness.
+func TestHostProcsSeesARealProcess(t *testing.T) {
+ b, err := os.ReadFile("/proc/self/cgroup")
+ if err != nil {
+ t.Skipf("no /proc/self/cgroup: %v", err)
+ }
+ rel, ok := ParseSelfCgroup(string(b))
+ if !ok {
+ t.Skip("no cgroup v2 path")
+ }
+ parent := filepath.Join("/sys/fs/cgroup", rel)
+ root := filepath.Join(parent, "hostprocs-test-"+strconv.Itoa(os.Getpid()))
+ const host = "shop.example.com"
+ group := filepath.Join(root, ExecName(host))
+ if err := os.MkdirAll(group, 0o755); err != nil {
+ t.Skipf("this cgroup is not writable: %v", err)
+ }
+ t.Cleanup(func() {
+ _ = os.Remove(group)
+ _ = os.Remove(root)
+ })
+ cmd := exec.Command("sleep", "30")
+ if err := cmd.Start(); err != nil {
+ t.Skipf("no sleep: %v", err)
+ }
+ t.Cleanup(func() {
+ _ = cmd.Process.Kill()
+ _ = cmd.Wait()
+ })
+ if err := os.WriteFile(filepath.Join(group, "cgroup.procs"), []byte(strconv.Itoa(cmd.Process.Pid)), 0o644); err != nil {
+ t.Skipf("the kernel refused the move: %v", err)
+ }
+ if n, err := HostProcs(root, host, ""); err != nil || n != 1 {
+ t.Fatalf("a sleep in the host's group: (%d, %v), want (1, nil)", n, err)
+ }
+ _ = cmd.Process.Kill()
+ _ = cmd.Wait()
+ if n, err := HostProcs(root, host, ""); err != nil || n != 0 {
+ t.Fatalf("after the sleep exited: (%d, %v), want (0, nil)", n, err)
+ }
+}
diff --git a/internal/cgroup/hostprocs_other.go b/internal/cgroup/hostprocs_other.go
new file mode 100644
index 000000000..ede2ef0df
--- /dev/null
+++ b/internal/cgroup/hostprocs_other.go
@@ -0,0 +1,7 @@
+//go:build !linux
+
+package cgroup
+
+// hostProcs has no cgroup v2 to read off Linux: every answer is unknown,
+// which a report records as not stopped.
+func hostProcs(string, string, string) (int, error) { return 0, ErrUnknown }
diff --git a/internal/host/agent.go b/internal/host/agent.go
index 5ccaa3955..06c3f9c5e 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -1452,6 +1452,14 @@ var reconcileMu sync.Mutex
// not hide an instance-side one, and neither must stop the other running —
// each pass is independently safe to run and independently valuable.
func reconcileSerialized(ctx context.Context, s release.Store, reg *registry.Registry, cfg provision.ReconcileConfig) error {
+ return reconcileSerializedWith(ctx, s, reg, cfg, armedMovesWorker.Load())
+}
+
+// reconcileSerializedWith is reconcileSerialized with the moves worker whose
+// offline hooks the pass takes (movesoffline.go): the armed worker in the
+// agent, a box's own worker in a test. A nil worker, or one on a box that
+// cannot take part in an offline move, adds nothing.
+func reconcileSerializedWith(ctx context.Context, s release.Store, reg *registry.Registry, cfg provision.ReconcileConfig, w *movesWorker) error {
reconcileMu.Lock()
defer reconcileMu.Unlock()
// A fresh row is minted at the layout this box's intent names
@@ -1472,6 +1480,17 @@ func reconcileSerialized(ctx context.Context, s release.Store, reg *registry.Reg
// Move holds (moves spec §9.6): participant rows minted held and kept.
// Unchanged with the Phase 1 switch off.
cfg = withMovesHolds(cfg)
+ // Offline move holds: the hold from the record alone, the keep from the
+ // view, which is read here first if no listing has been read yet or the
+ // last one failed, so keep never fails closed on a box whose listing
+ // reads. Nothing on a box that cannot take part, and nothing where
+ // Phase 1's hooks are set.
+ if w != nil && w.cfg.OfflineCapable && w.cfg.Offline != nil {
+ if !w.cfg.Offline.built() || w.cfg.Offline.failed() {
+ refreshOfflineView(ctx, s, w.cfg.Offline)
+ }
+ cfg = withOfflineHolds(cfg, offlineHold(w.cfg), w.cfg.Offline)
+ }
// Runs ride this pass's record reads (run spec §5.2): the worker notes
// every in-scope record with a run, and acts after the pass.
cfg = withRunsHook(cfg)
@@ -2594,6 +2613,10 @@ func fastConvergeLoop(reg *registry.Registry, svc *activator.Service, sd *schedu
// runs before the first moves pass, and an unbuilt view freezes every
// host (moves spec §9.4). Nothing with the Phase 1 switch off.
refreshMovesHolds(ctx, store)
+ // The offline keep view, likewise before any reconcile pass: once it is
+ // read, keep answers false for every host with no offline move, so the
+ // first pass removes what it always has (movesoffline.go).
+ refreshOfflineView(ctx, store, armedOfflineView())
// carlos run's boot pass (run spec §5.2, index item 2): reconcile's
// first pass is a minute away and the CLI expires a run at 60 s.
runRunsBootPass(ctx)
diff --git a/internal/host/edgesfile.go b/internal/host/edgesfile.go
index c02520656..150f394a0 100644
--- a/internal/host/edgesfile.go
+++ b/internal/host/edgesfile.go
@@ -9,6 +9,7 @@ import (
"log"
"math/rand/v2"
"os"
+ "slices"
"strings"
"sync"
"time"
@@ -162,7 +163,8 @@ type edgeHeadroom struct {
const capRegionHA1 = "region-ha-1"
// edgesFileCapabilities publishes region-ha-1 only when CARLOS_REGION_HA is
-// set to something other than "" or "0".
+// set to something other than "" or "0". offline-move-1 is added per pass
+// by withOfflineCapability.
func edgesFileCapabilities(getenv func(string) string) []string {
if !regionHAEnabled(getenv) {
return nil
@@ -170,6 +172,28 @@ func edgesFileCapabilities(getenv func(string) string) []string {
return []string{capRegionHA1}
}
+// withOfflineCapability is caps with offline-move-1 when capable. The loop
+// passes armedOfflineCapable(): the armed moves worker's own answer, the one
+// the reconcile hooks act on, so a box never publishes the capability
+// without the hold that makes it safe (a target that published it with no
+// armed worker would mint a landing row ready). caps is not modified.
+func withOfflineCapability(caps []string, capable bool) []string {
+ if !capable {
+ return caps
+ }
+ return append(slices.Clone(caps), capOfflineMove1)
+}
+
+// headroomCapabilities is what headroomFor is asked about: region HA's
+// capability alone. offline-move-1 publishes no headroom, so a box that
+// gains only it publishes the file it always has, plus the one string.
+func headroomCapabilities(caps []string) []string {
+ if slices.Contains(caps, capRegionHA1) {
+ return caps
+ }
+ return nil
+}
+
// awakeCount is how many routes this box has running now. An error reads
// as zero: headroom is advisory.
func awakeCount(reg *registry.Registry) int {
@@ -815,6 +839,9 @@ func edgesFileLoop(reg *registry.Registry) {
BoxID: box,
Capabilities: edgesFileCapabilities(os.Getenv),
}
+ // The flag-derived capabilities; offline-move-1 joins them on each pass
+ // once the moves worker is armed and capable.
+ baseCaps := ident.Capabilities
dataDir := provisionTickConfig(os.Getenv).DataDir
optDir := release.OptDir()
// The local floor files this file publishes are refreshed hourly by
@@ -859,7 +886,8 @@ func edgesFileLoop(reg *registry.Registry) {
if state.timed() {
eip = readPublicIP()
}
- ident.Headroom = headroomFor(ident.Capabilities, dataDir, reg)
+ ident.Capabilities = withOfflineCapability(baseCaps, armedOfflineCapable())
+ ident.Headroom = headroomFor(headroomCapabilities(ident.Capabilities), dataDir, reg)
ident = withMovesFields(ident, dataDir, optDir)
ident = withSeedsFields(ident, dataDir, optDir)
err = edgesFileWriteClaims(ctx, store, reg, ident, eip, &last, state.timed(), claims)
diff --git a/internal/host/movesoffline.go b/internal/host/movesoffline.go
new file mode 100644
index 000000000..9b1832ef6
--- /dev/null
+++ b/internal/host/movesoffline.go
@@ -0,0 +1,976 @@
+package host
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log"
+ "os"
+ "path"
+ "path/filepath"
+ "sort"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "syscall"
+ "time"
+
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/hostwrite"
+ "github.com/carlosframework/platform/internal/instancectl"
+ "github.com/carlosframework/platform/internal/layout"
+ "github.com/carlosframework/platform/internal/lease"
+ "github.com/carlosframework/platform/internal/moves"
+ "github.com/carlosframework/platform/internal/park"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// The offline operator move on a box (docs/superpowers/plans/2026-10-20-
+// offline-operator-move.md, Task 3): the hold reconcile asks, the keep view,
+// the capability, the moves pass's dispatch, and the source boxes' steps.
+// The sealer's step is movesoffline_seal.go (Task 4) and the target's is
+// movesoffline_land.go (Task 5).
+//
+// The stop has three layers. This file holds the first two on every
+// source-region box: reconcile holds the row from the record alone
+// (offlineHold), and each pass fences the activator and writes the box's
+// source report. The third, the tombstone lease, is the sealer's.
+//
+// Dormant, with no offline record anywhere, nothing here acts: the hold
+// answers false for every record, keep answers false once the startup
+// listing is read, the dispatch matches no record, and the edges file
+// carries one more capability string.
+
+// capOfflineMove1 is the edges-file capability a box publishes when it can
+// take part in an offline move (offlineCapable). The console's start door
+// refuses a region holding a box that does not publish it.
+const capOfflineMove1 = "offline-move-1"
+
+// offlineCapable is the box's own witness that it can take part in an
+// offline move: hibernation is configured and neither CARLOS_HOST_CLAIM nor
+// CARLOS_MOVES_PHASE1 is set to anything at all. The guard mode is not read
+// (platform-1e's ruling of 2026-10-20, the plan's Global Constraints): the
+// target's build stages through ordinary adoption, which root's own floor
+// check already covers on a guarded box. A box that is not capable publishes
+// no capability and runs no offline step.
+func offlineCapable(getenv func(string) string, hibernation bool) bool {
+ if !hibernation || hostwrite.Enabled(getenv) {
+ return false
+ }
+ return strings.TrimSpace(getenv(moves.SwitchEnv)) == "" && !moves.Enabled(getenv)
+}
+
+// hibernationConfigured is the edges file's HibernationConfigured, read the
+// same way, so the capability and the field beside it never disagree.
+func hibernationConfigured(getenv func(string) string) bool {
+ return strings.TrimSpace(getenv("CARLOS_HIBERNATE_BUCKET")) != ""
+}
+
+// offlineFenceOwner is the activator fence owner of one offline move.
+func offlineFenceOwner(id string) string { return "offline:" + id }
+
+// offlineHold is reconcile's Hold for offline moves: a pure function of the
+// record it is handed and this box's placement, never of a snapshot (Fable's
+// B1). A target box that restarts after the flip reconciles before its first
+// moves pass, and must mint the landing row held even then.
+//
+// - A source-region box (placement equals move.from) holds the host's row
+// "moving" under the move id from offline-stopping through
+// offline-sealed. After the flip the record names the target, the row
+// is out of this box's scope and offlineView.keep keeps it, held as it
+// is, until offline-finishing.
+// - A target-region box (placement equals move.to) holds it "landing"
+// while the record names the target and the move is offline-landing or
+// offline-landed. offline-open is the target's to release (Task 5).
+//
+// Placements are compared as movesNamesBox compares them, by
+// park.PlacementMatches under the verified home binding, so the home's ""
+// and its roster name are one.
+func offlineHold(box moveCfg) func(rec provision.Record) (status, op string, ok bool) {
+ home, here := box.ParkHomeRegion, box.parkRegion()
+ return func(rec provision.Record) (string, string, bool) {
+ m := rec.Move
+ if !m.Offline() || m.ID == "" || here == "" {
+ return "", "", false
+ }
+ switch m.State {
+ case provision.OfflineStopping, provision.OfflineSealing, provision.OfflineSealed:
+ if park.PlacementMatches(home, m.From, here) {
+ return "moving", m.ID, true
+ }
+ case provision.OfflineLanding, provision.OfflineLanded:
+ if rec.Region == m.To && park.PlacementMatches(home, m.To, here) {
+ return "landing", m.ID, true
+ }
+ }
+ return "", "", false
+ }
+}
+
+// offlineView is what reconcile's KeepParticipant asks about offline moves:
+// the offline moves in flight, as the last listing read them. Only keep
+// reads it; the hold never does (offlineHold).
+//
+// A source box's view has seen the move before the record flips: the seal
+// door needs this box's report for the move, and the pass that writes it
+// refreshes the view first; an agent restart reads the listing again at
+// start. From then until finishing every listing carries the move.
+//
+// It is refreshed from every moves pass's record list and once at agent
+// start, before the reconcile loops (refreshOfflineView). keep fails closed,
+// keeping every row, only while no listing has been read (reconcileSerialized-
+// With reads one first when the view is unbuilt or its last listing failed,
+// so a failed startup listing heals on the next reconcile), and for a host
+// whose record does not decode. Once a listing is read, a host with no
+// offline move is not kept, so removals happen on the first pass as they
+// always have.
+type offlineView struct {
+ home, region string
+ // written is reconcile's HoldWritten for the offline hooks: the
+ // stage-requested marker after a hold write (registry.SetHold's
+ // contract, G10). Nil writes nothing.
+ written func(host string)
+ snap atomic.Pointer[offlineSnapshot]
+ logged sync.Map // host -> time.Time: the last unreadable-record line
+}
+
+type offlineSnapshot struct {
+ moves map[string]provision.MoveState // host -> its offline move
+ failed bool // the listing failed: keep everything
+ // unreadable are the hosts whose record read but did not decode, or
+ // disagreed with its path (provision.ListAllRecordsStrict): their move
+ // cannot be known, so their rows are kept.
+ unreadable map[string]bool
+}
+
+// newOfflineView builds the view for the box cfg describes. Unbuilt until
+// its first refresh.
+func newOfflineView(cfg moveCfg, written func(host string)) *offlineView {
+ return &offlineView{home: cfg.ParkHomeRegion, region: cfg.parkRegion(), written: written}
+}
+
+// refresh rebuilds the view from a record list that was read, with the
+// hosts provision.ListAllRecordsStrict could not decode: those are kept
+// (fail closed for keep only; the hold never reads the view) and logged at
+// most once a minute each.
+func (v *offlineView) refresh(recs []provision.Record, unreadable map[string]bool) {
+ next := &offlineSnapshot{moves: map[string]provision.MoveState{}, unreadable: map[string]bool{}}
+ for _, rec := range recs {
+ if rec.Move.Offline() {
+ next.moves[rec.Host] = *rec.Move
+ }
+ }
+ now := time.Now()
+ for host := range unreadable {
+ next.unreadable[host] = true
+ if last, ok := v.logged.Load(host); !ok || now.Sub(last.(time.Time)) >= time.Minute {
+ v.logged.Store(host, now)
+ log.Printf("moves: offline view: %s: the record does not decode; its row is kept until it does", host)
+ }
+ }
+ v.snap.Store(next)
+}
+
+// fail records a listing that could not be read, before any was: keep
+// answers true for every host until a listing is read.
+func (v *offlineView) fail() { v.snap.Store(&offlineSnapshot{failed: true}) }
+
+// built reports whether any listing, read or failed, has been recorded.
+func (v *offlineView) built() bool { return v.snap.Load() != nil }
+
+// failed reports whether the last listing recorded failed: keep answers true
+// for every host until one reads.
+func (v *offlineView) failed() bool {
+ snap := v.snap.Load()
+ return snap != nil && snap.failed
+}
+
+// keep is reconcile's KeepParticipant: the row of a host whose offline move
+// names this box's region stays, whatever the record's region says. The
+// source keeps it until offline-finishing (when it is removed, as the plan's
+// finishing step relies on); the target keeps it until the move is cleared
+// (the target's revert deletes it itself). An unknown state keeps.
+func (v *offlineView) keep(host string) bool {
+ snap := v.snap.Load()
+ if snap == nil || snap.failed || snap.unreadable[host] {
+ return true
+ }
+ m, ok := snap.moves[host]
+ if !ok || v.region == "" {
+ return false
+ }
+ if park.PlacementMatches(v.home, m.To, v.region) {
+ return true
+ }
+ if park.PlacementMatches(v.home, m.From, v.region) {
+ return m.State != provision.OfflineFinishing
+ }
+ return false
+}
+
+// refreshOfflineView reads the record list once, outside a moves pass: at
+// agent start, before the reconcile loops, next to refreshMovesHolds. A
+// failed listing leaves keep failing closed until a listing is read.
+func refreshOfflineView(ctx context.Context, s release.Store, v *offlineView) {
+ if v == nil || s == nil {
+ return
+ }
+ recs, unreadable, err := provision.ListAllRecordsStrict(ctx, s)
+ if err != nil {
+ log.Printf("moves: offline view: list records: %v; every row is kept until a listing reads", err)
+ v.fail()
+ return
+ }
+ v.refresh(recs, unreadable)
+}
+
+// armedOfflineView is the armed worker's view, nil when no worker is armed
+// or the box cannot take part in an offline move.
+func armedOfflineView() *offlineView {
+ w := armedMovesWorker.Load()
+ if w == nil || !w.cfg.OfflineCapable {
+ return nil
+ }
+ return w.cfg.Offline
+}
+
+// armedOfflineCapable is whether this box publishes offline-move-1: the
+// armed worker is capable and has its keep view, exactly the condition
+// under which reconcileSerializedWith installs the offline hooks.
+func armedOfflineCapable() bool { return armedOfflineView() != nil }
+
+// withOfflineHolds gives a reconcile pass the offline hooks, only when the
+// caller and Phase 1's view set none (withMovesHolds runs first): the two
+// are never armed together, since a box with Phase 1 on is not capable.
+func withOfflineHolds(cfg provision.ReconcileConfig, hold func(provision.Record) (string, string, bool), v *offlineView) provision.ReconcileConfig {
+ if hold == nil || v == nil || cfg.Hold != nil || cfg.KeepParticipant != nil {
+ return cfg
+ }
+ cfg.Hold, cfg.KeepParticipant = hold, v.keep
+ if cfg.HoldWritten == nil && v.written != nil {
+ cfg.HoldWritten = v.written
+ }
+ return cfg
+}
+
+// newMovesWorker is the worker for cfg: armMovesPass's, and each test box's.
+func newMovesWorker(cfg moveCfg) *movesWorker {
+ return &movesWorker{cfg: cfg, mirrors: map[string]lease.MirrorStore{}}
+}
+
+// fenceAsWith is moveCfg.FenceAs over an activator: FenceAndStopAs under a
+// short deadline, so one slow stop does not hold up the pass.
+// ErrStopUnproven still means the fence is installed, which is all the step
+// asks of it; Quiesced answers the rest.
+func fenceAsWith(svc *activator.Service) func(ctx context.Context, host string, generation uint64, owner string) error {
+ return func(ctx context.Context, host string, generation uint64, owner string) error {
+ fctx, cancel := context.WithTimeout(ctx, 2*time.Second)
+ defer cancel()
+ if _, err := svc.FenceAndStopAs(fctx, host, generation, owner); err != nil && !errors.Is(err, activator.ErrStopUnproven) {
+ return err
+ }
+ return nil
+ }
+}
+
+// hostSliceCgroup is the absolute cgroup directory of a nested host's
+// systemd slice under cgroupRoot (the cgroup v2 mount): every role and
+// generation unit of the host runs in it.
+func hostSliceCgroup(cgroupRoot, host string) string {
+ return filepath.Join(cgroupRoot, filepath.FromSlash(path.Dir(instancectl.ClaimUnitControlGroup(host))))
+}
+
+// offlineRound is the moves pass's round for a record with an offline move,
+// reached before every other branch of the pass (the Phase 1 skip and the
+// lost-box gate included). A box that is not capable logs one notice per
+// host and does nothing. Otherwise it runs the source step when its
+// placement is move.from, the sealer's when it is move.sealer, and the
+// target's when its placement is move.to, in that order, so the sealer
+// reads the report this box has just written.
+func (w *movesWorker) offlineRound(ctx context.Context, s release.Store, rec provision.Record) error {
+ cfg := w.cfg
+ m := rec.Move
+ if !m.Offline() {
+ return nil
+ }
+ here := cfg.parkRegion()
+ source := here != "" && park.PlacementMatches(cfg.ParkHomeRegion, m.From, here)
+ target := here != "" && park.PlacementMatches(cfg.ParkHomeRegion, m.To, here)
+ sealer := cfg.Box != "" && cfg.Box == m.Sealer
+ if !cfg.OfflineCapable {
+ if source || target || sealer {
+ w.notice(rec.Host+"#offline", "moves: %s: offline move %s waits: this box cannot take part in an offline move (it needs hibernation configured, and %s and %s unset)", rec.Host, m.ID, "CARLOS_HOST_CLAIM", moves.SwitchEnv)
+ }
+ return nil
+ }
+ var errs []error
+ if source {
+ errs = append(errs, w.offlineSourceStep(ctx, s, rec))
+ }
+ if sealer {
+ errs = append(errs, w.offlineSealerStep(ctx, s, rec))
+ }
+ if target {
+ errs = append(errs, w.offlineTargetStep(ctx, s, rec))
+ }
+ return errors.Join(errs...)
+}
+
+// now is cfg.Now, or the wall clock when a test left it unset.
+func (c moveCfg) now() time.Time {
+ if c.Now != nil {
+ return c.Now()
+ }
+ return time.Now()
+}
+
+// writeOfflineReport writes r at its key (moves.OfflineReportKey), stamped
+// with the report version, after the shape check every reader makes. The
+// source, sealer and target steps all write through it.
+func writeOfflineReport(ctx context.Context, s release.Store, r moves.OfflineReport) error {
+ r.V = moves.OfflineReportVersion
+ key := moves.OfflineReportKey(r.Host, r.Op, r.Role, r.Box)
+ if key == "" {
+ return fmt.Errorf("offline report: host %q, op %q, role %q or box %q is not usable", r.Host, r.Op, r.Role, r.Box)
+ }
+ if err := moves.ValidateOfflineReport(r); err != nil {
+ return err
+ }
+ body, err := json.Marshal(r)
+ if err != nil {
+ return err
+ }
+ // A report is rewritten every pass: read its etag and replace it
+ // (release.Store.Put with no etag is create-only).
+ return moves.PutReplace(ctx, s, key, body)
+}
+
+// putOfflineCreateOnly writes body at key only if nothing is there yet:
+// the immutable seal record (control/offline-moves/<host>/<id>/seal.json,
+// moves.OfflineSealKey, written once by the sealer and never rewritten).
+// It is an ensure: when the key already exists it writes nothing and
+// returns what is stored, with created false, so a sealer re-running after
+// a crash compares rather than overwrites, and a reader (flip, the landing)
+// always sees the first write. key must be non-empty.
+func putOfflineCreateOnly(ctx context.Context, s release.Store, key string, body []byte) (stored []byte, created bool, err error) {
+ if key == "" {
+ return nil, false, errors.New("offline move: no key to write")
+ }
+ _, err = s.Put(ctx, key, body, "")
+ if err == nil {
+ return body, true, nil
+ }
+ if !errors.Is(err, release.ErrCASConflict) {
+ return nil, false, fmt.Errorf("write %s: %w", key, err)
+ }
+ stored, _, err = s.Get(ctx, key)
+ if err != nil {
+ return nil, false, fmt.Errorf("read %s after a create-only write found it: %w", key, err)
+ }
+ return stored, false, nil
+}
+
+// readOfflineReports reads every report of one role for one move, keyed by
+// box. A report that does not validate is an error, never skipped.
+func readOfflineReports(ctx context.Context, s release.Store, host, op, role string) (map[string]moves.OfflineReport, error) {
+ prefix := moves.OfflineReportPrefix(host, op)
+ if prefix == "" {
+ return nil, fmt.Errorf("offline report: host %q or op %q is not usable", host, op)
+ }
+ prefix += role + "/"
+ keys, err := s.List(ctx, prefix)
+ if err != nil {
+ return nil, err
+ }
+ out := map[string]moves.OfflineReport{}
+ for _, key := range keys {
+ if !strings.HasSuffix(key, ".json") {
+ continue
+ }
+ body, _, err := s.Get(ctx, key)
+ if err != nil {
+ return nil, err
+ }
+ var r moves.OfflineReport
+ if err := json.Unmarshal(body, &r); err != nil {
+ return nil, fmt.Errorf("%s: %w", key, err)
+ }
+ if err := moves.ValidateOfflineReport(r); err != nil {
+ return nil, fmt.Errorf("%s: %w", key, err)
+ }
+ if moves.OfflineReportKey(r.Host, r.Op, r.Role, r.Box) != key {
+ return nil, fmt.Errorf("%s: the report names %s/%s/%s/%s", key, r.Host, r.Op, r.Role, r.Box)
+ }
+ out[r.Box] = r
+ }
+ return out, nil
+}
+
+// offlineSourceStep is a source-region box's part of an offline move. Every
+// state's step is an ensure: it reads what is on disk, in the registry and
+// in the activator, does only what is missing, and reports what it found.
+func (w *movesWorker) offlineSourceStep(ctx context.Context, s release.Store, rec provision.Record) error {
+ if w.cfg.Box == "" {
+ return errors.New("offline move: this box has no label to report under")
+ }
+ switch rec.Move.State {
+ case provision.OfflineStopping, provision.OfflineSealing, provision.OfflineSealed, provision.OfflineLanding, provision.OfflineLanded:
+ return w.offlineSourceStop(ctx, s, rec)
+ case provision.OfflineOpen:
+ return w.offlineSourcePark(ctx, s, rec)
+ case provision.OfflineFinishing:
+ return w.offlineSourceGone(ctx, s, rec)
+ case provision.OfflineReverting:
+ return w.offlineSourceRevert(ctx, s, rec)
+ }
+ return nil
+}
+
+// offlineFence installs, or meets again, this move's activator fence for
+// host. Fences live in the activator's memory, so every pass installs it
+// again and an agent restart regains it on its next pass. True when the
+// fence is this move's now.
+func (w *movesWorker) offlineFence(ctx context.Context, host string, m *provision.MoveState) bool {
+ if w.cfg.FenceAs == nil {
+ return false
+ }
+ if err := w.cfg.FenceAs(ctx, host, moveFenceGeneration(m), offlineFenceOwner(m.ID)); err != nil {
+ w.notice(host+"#offline-fence", "moves: %s: offline move %s: the wake fence is not installed: %v", host, m.ID, err)
+ return false
+ }
+ return true
+}
+
+// offlineLiftFence lifts this move's fence, and only it.
+func (w *movesWorker) offlineLiftFence(host string, m *provision.MoveState) {
+ if w.cfg.LiftFenceAs != nil && w.cfg.LiftFenceAs(host, moveFenceGeneration(m), offlineFenceOwner(m.ID)) {
+ log.Printf("moves: %s: offline move %s: wake fence lifted", host, m.ID)
+ }
+}
+
+// offlineRow reads the host's row on this box; nil when it has none.
+func (w *movesWorker) offlineRow(host string) (*registry.Route, error) {
+ if w.cfg.Route == nil {
+ return nil, errors.New("offline move: no registry on this box")
+ }
+ return w.cfg.Route(host)
+}
+
+// heldBy reports whether rt is held by op. An absent row counts: nothing
+// can serve or wake a host this box has no row for.
+func heldBy(rt *registry.Route, op string) bool {
+ return rt == nil || (rt.HoldOp == op && registry.IsHold(rt.Status))
+}
+
+// offlineLocal is what a box keeps on disk for a host, judged by lstat and
+// directory listings only, never by following a link. A row's own layout is
+// read at the row's paths; a box with no row, or a leftover of the other
+// layout, is read at both.
+type offlineLocal struct {
+ LocalDB, Files, Env, Sidecar bool
+ LocalTXID uint64
+}
+
+// flatDBPath is where a flat row's database is: the row's own path when it
+// has one and is flat, else the default.
+func (c moveCfg) flatDBPath(host string, rt *registry.Route) string {
+ if rt != nil && rt.DBPath != "" && !provision.NestedPaths(c.DataDir, *rt) {
+ return rt.DBPath
+ }
+ return provision.DefaultDBPath(c.DataDir, host)
+}
+
+// activatorMetaDir is the activator's litestream meta directory beside a
+// database (localLTXPosition reads it).
+func activatorMetaDir(dbPath string) string {
+ return filepath.Join(filepath.Dir(dbPath), ".activator-"+filepath.Base(dbPath)+"-litestream")
+}
+
+func lexists(p string) bool {
+ _, err := os.Lstat(p)
+ return err == nil
+}
+
+func isDir(p string) bool {
+ st, err := os.Lstat(p)
+ return err == nil && st.IsDir()
+}
+
+// offlinePlatformNames are a nested instance directory's platform entries:
+// everything else in it is the app's files.
+func offlinePlatformNames(host string) map[string]bool {
+ db := host + ".db"
+ return map[string]bool{
+ db: true, db + "-wal": true, db + "-shm": true, db + "-litestream": true,
+ ".activator-" + db + "-litestream": true, db + ".corrupt": true, db + ".gen": true,
+ host + ".env": true, host + ".config.env": true, host + "-sidecar.env": true,
+ host + ".instance-key-adoption.json": true, ".files-gen": true,
+ }
+}
+
+// localState reads what this box keeps for host: the DB file or either
+// litestream meta dir (S8: a meta dir alone is local state, since
+// localLTXPosition reads it), the files directory, the env file and a
+// sidecar env file, under the row's layout and any leftover of the other.
+func (w *movesWorker) localState(host string, rt *registry.Route) (offlineLocal, error) {
+ cfg := w.cfg
+ var out offlineLocal
+ nestedRow := rt != nil && provision.NestedPaths(cfg.DataDir, *rt)
+ // Flat: a flat row's own paths, or a nested row's flat leftovers.
+ {
+ db := cfg.flatDBPath(host, rt)
+ out.LocalDB = lexists(db) || lexists(db+"-litestream") || lexists(activatorMetaDir(db))
+ out.Files = isDir(filepath.Join(cfg.DataDir, host))
+ out.Env = lexists(filepath.Join(cfg.DataDir, host+".env"))
+ out.Sidecar = lexists(filepath.Join(cfg.DataDir, host+"-sidecar.env"))
+ pos, err := localLTXPosition(db)
+ if err != nil {
+ return out, err
+ }
+ out.LocalTXID = pos
+ }
+ // Nested, read through the raw directory (layout.RawInstanceDir).
+ raw := layout.RawInstanceDir(cfg.DataDir, host)
+ if nestedRow || isDir(raw) {
+ db := filepath.Join(raw, host+".db")
+ out.LocalDB = out.LocalDB || lexists(db) || lexists(db+"-litestream") || lexists(activatorMetaDir(db))
+ out.Env = out.Env || lexists(filepath.Join(raw, host+".env"))
+ out.Sidecar = out.Sidecar || lexists(filepath.Join(raw, host+"-sidecar.env"))
+ ents, err := os.ReadDir(raw)
+ if err != nil && !errors.Is(err, os.ErrNotExist) {
+ return out, err
+ }
+ platform := offlinePlatformNames(host)
+ for _, e := range ents {
+ if !platform[e.Name()] {
+ out.Files = true
+ break
+ }
+ }
+ dbPath := db
+ if nestedRow && rt.DBPath != "" {
+ dbPath = rt.DBPath
+ }
+ pos, err := localLTXPosition(dbPath)
+ if err != nil {
+ return out, err
+ }
+ out.LocalTXID = max(out.LocalTXID, pos)
+ }
+ return out, nil
+}
+
+// procs is the process count for the report: -1 when it cannot be read,
+// which the flip door reads as not stopped.
+func (w *movesWorker) procs(host string) int {
+ if w.cfg.Procs == nil {
+ return -1
+ }
+ n, err := w.cfg.Procs(host)
+ if err != nil {
+ w.notice(host+"#offline-procs", "moves: %s: the host's processes cannot be counted on this box: %v", host, err)
+ return -1
+ }
+ return n
+}
+
+// sourceReport is the source report's evidence as it stands now.
+func (w *movesWorker) sourceReport(host string, m *provision.MoveState, rt *registry.Route, fenced bool, phase string) (moves.OfflineReport, error) {
+ cfg := w.cfg
+ local, err := w.localState(host, rt)
+ if err != nil {
+ return moves.OfflineReport{}, err
+ }
+ quiesced := cfg.Quiesced != nil && cfg.Quiesced(host)
+ return moves.OfflineReport{
+ Op: m.ID, Host: host, Box: cfg.Box, Role: moves.OfflineRoleSource, Phase: phase, At: cfg.now().UTC(),
+ Held: heldBy(rt, m.ID), Fenced: fenced, Quiesced: quiesced, RowPresent: rt != nil,
+ LocalDB: local.LocalDB, Files: local.Files, Env: local.Env, Sidecar: local.Sidecar, LocalTXID: local.LocalTXID,
+ Procs: w.procs(host), LayoutJournal: layout.JournalPresent(cfg.DataDir, host),
+ }, nil
+}
+
+// offlineSourceStop is the source step from offline-stopping through
+// offline-landed: install the fence, read the row and the local state, and
+// rewrite this box's source report with At on every pass (the open door
+// needs it current). The row's hold is reconcile's (offlineHold) and,
+// after the flip, its keep (offlineView.keep).
+func (w *movesWorker) offlineSourceStop(ctx context.Context, s release.Store, rec provision.Record) error {
+ m := rec.Move
+ fenced := w.offlineFence(ctx, rec.Host, m)
+ rt, err := w.offlineRow(rec.Host)
+ if err != nil {
+ return err
+ }
+ r, err := w.sourceReport(rec.Host, m, rt, fenced, moves.OfflinePhaseStopped)
+ if err != nil {
+ return err
+ }
+ return writeOfflineReport(ctx, s, r)
+}
+
+// retirePath is one path of the manifest's source-retire row on this box:
+// where it is, its name in the park directory, and, for a path the agent
+// never renames, why.
+type retirePath struct {
+ path, name, left string
+}
+
+// Why a path is left where it is.
+const (
+ leftRootOwned = "root-owned: listed for the census, not renamed"
+ leftView = "the mount view of the instance directory: it stays, empty"
+ leftInstance = "the instance directory stays, empty"
+)
+
+// nestedParkPrefix is where a nested instance directory's entries go inside
+// the park directory, apart from the flat names.
+const nestedParkPrefix = "instance"
+
+// retirePaths is the manifest's source-retire row as paths on this box, in
+// a fixed order: the row's layout in full, the other layout's paths only
+// where they exist, the app's own entries of a nested instance directory as
+// found there and in the park directory, then what is never renamed.
+func (w *movesWorker) retirePaths(host, parkDir string, rt *registry.Route) ([]retirePath, error) {
+ cfg := w.cfg
+ nestedRow := rt != nil && provision.NestedPaths(cfg.DataDir, *rt)
+ raw := layout.RawInstanceDir(cfg.DataDir, host)
+ var out []retirePath
+ add := func(p, name string, always bool) {
+ if always || lexists(p) {
+ out = append(out, retirePath{path: p, name: name})
+ }
+ }
+ flatDB := cfg.flatDBPath(host, rt)
+ flat := !nestedRow
+ add(flatDB, filepath.Base(flatDB), flat)
+ add(flatDB+"-wal", filepath.Base(flatDB)+"-wal", flat)
+ add(flatDB+"-shm", filepath.Base(flatDB)+"-shm", flat)
+ add(flatDB+"-litestream", filepath.Base(flatDB)+"-litestream", flat)
+ add(activatorMetaDir(flatDB), filepath.Base(activatorMetaDir(flatDB)), flat)
+ add(filepath.Join(cfg.DataDir, host), host, flat)
+ add(filepath.Join(cfg.DataDir, host+".env"), host+".env", flat)
+ add(filepath.Join(cfg.DataDir, host+".config.env"), host+".config.env", flat)
+
+ nested := nestedRow || isDir(raw)
+ if nested {
+ named := map[string]bool{}
+ for _, n := range []string{host + ".db", host + ".db-wal", host + ".db-shm", host + ".db-litestream", ".activator-" + host + ".db-litestream", host + ".env", host + ".config.env"} {
+ named[n] = true
+ add(filepath.Join(raw, n), nestedParkPrefix+"/"+n, true)
+ }
+ // The app's files, and every other entry the directory holds: what
+ // is still there, and what an earlier pass already parked.
+ // A numbered copy in the park directory (parkName) belongs to the
+ // entry it is a copy of, not to the app.
+ others := map[string]bool{}
+ listed := map[string][]string{}
+ for _, dir := range []string{raw, filepath.Join(parkDir, nestedParkPrefix)} {
+ ents, err := os.ReadDir(dir)
+ if err != nil && !errors.Is(err, os.ErrNotExist) {
+ return nil, err
+ }
+ for _, e := range ents {
+ listed[dir] = append(listed[dir], e.Name())
+ }
+ }
+ known := map[string]bool{}
+ for n := range named {
+ known[n] = true
+ }
+ for _, n := range listed[raw] {
+ known[n] = true
+ }
+ for _, n := range listed[filepath.Join(parkDir, nestedParkPrefix)] {
+ known[n] = true
+ }
+ for _, n := range listed[raw] {
+ if !named[n] {
+ others[n] = true
+ }
+ }
+ for _, n := range listed[filepath.Join(parkDir, nestedParkPrefix)] {
+ if !named[n] && !isNumberedCopy(n, known) {
+ others[n] = true
+ }
+ }
+ names := make([]string, 0, len(others))
+ for n := range others {
+ names = append(names, n)
+ }
+ sort.Strings(names)
+ for _, n := range names {
+ add(filepath.Join(raw, n), nestedParkPrefix+"/"+n, true)
+ }
+ }
+ add(filepath.Join(cfg.DataDir, "programs", host+".json"), host+".json", true)
+ if nested {
+ out = append(out, retirePath{path: provision.InstanceDir(cfg.DataDir, host), left: leftView})
+ out = append(out, retirePath{path: raw, left: leftInstance})
+ }
+ optDir := cfg.OptDir
+ if optDir == "" {
+ optDir = release.OptDir()
+ }
+ out = append(out, retirePath{path: filepath.Join(optDir, "live", host), left: leftRootOwned})
+ return out, nil
+}
+
+// parkCopySep separates a name from its copy number in the park directory
+// (parkName). Chosen to be unlikely in an app's own file names.
+const parkCopySep = ".parked-"
+
+// isNumberedCopy reports whether n is a known entry's numbered park copy
+// (parkName's "<name>.parked-<n>"), which belongs to that entry.
+func isNumberedCopy(n string, known map[string]bool) bool {
+ i := strings.LastIndex(n, parkCopySep)
+ if i <= 0 || !known[n[:i]] || i+len(parkCopySep) == len(n) {
+ return false
+ }
+ for _, c := range n[i+len(parkCopySep):] {
+ if c < '0' || c > '9' {
+ return false
+ }
+ }
+ return true
+}
+
+// parkName is where name goes in dir: name itself, or name.parked-<n> for
+// the first free n when something of that name was parked before (the
+// source path came back after an earlier pass parked it). The park
+// directory is the move's own, so the first park of every path lands at its
+// plain name and a later pass reports it from there.
+func parkName(dir, name string) string {
+ dst := filepath.Join(dir, name)
+ if !lexists(dst) {
+ return dst
+ }
+ for n := 1; ; n++ {
+ if c := dst + parkCopySep + fmt.Sprint(n); !lexists(c) {
+ return c
+ }
+ }
+}
+
+// Why a rename the park attempted did not happen. A path whose rename fails
+// with an error a later pass would meet again (a mount, EXDEV, EBUSY; a path
+// the agent may not rename, EPERM or EACCES; a read-only filesystem) is
+// left for good: reported once, with leftCannot and the error, and never
+// tried again for this move, so a pass does not re-fence, fail and lift
+// over it. Any other error is leftNotYet, and the next pass tries again.
+const (
+ leftCannot = "cannot be renamed: "
+ leftNotYet = "not renamed yet: "
+)
+
+// renameTerminal reports whether a rename error is one a later pass would
+// meet again.
+func renameTerminal(err error) bool {
+ return errors.Is(err, os.ErrPermission) || errors.Is(err, syscall.EXDEV) ||
+ errors.Is(err, syscall.EBUSY) || errors.Is(err, syscall.EROFS)
+}
+
+// retire reports one path's action from what is on disk, renaming it into
+// the park directory first when park is true and it is there to rename.
+// cannot is the reason an earlier pass recorded for a path it found it can
+// never rename ("" when none): such a path is reported left again, as it
+// is, and not tried. Nothing is ever deleted.
+func retire(p retirePath, parkDir string, park bool, cannot string) moves.RetiredPath {
+ present := lexists(p.path)
+ switch {
+ case p.left != "" && present:
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredLeft, Reason: p.left}
+ case p.left != "":
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredAbsent}
+ case present && cannot != "":
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredLeft, Reason: cannot}
+ case present && park:
+ dst := parkName(parkDir, p.name)
+ err := os.MkdirAll(filepath.Dir(dst), 0o700)
+ if err == nil {
+ err = os.Rename(p.path, dst)
+ }
+ switch {
+ case err == nil:
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredParked}
+ case errors.Is(err, os.ErrNotExist) && !lexists(p.path):
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredAbsent}
+ case renameTerminal(err):
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredLeft, Reason: leftCannot + err.Error()}
+ }
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredLeft, Reason: leftNotYet + err.Error()}
+ case present:
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredLeft, Reason: leftNotYet + "the stop could not be confirmed this pass"}
+ case lexists(filepath.Join(parkDir, p.name)):
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredParked}
+ }
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredAbsent}
+}
+
+// cannotRename reads this box's own last report for the move and returns
+// the paths it recorded as never renameable (leftCannot), with their
+// reasons. Reading them from the report, not from memory, keeps the answer
+// across an agent restart.
+func (w *movesWorker) cannotRename(ctx context.Context, s release.Store, host, op string) (map[string]string, error) {
+ out := map[string]string{}
+ body, _, err := s.Get(ctx, moves.OfflineReportKey(host, op, moves.OfflineRoleSource, w.cfg.Box))
+ if errors.Is(err, os.ErrNotExist) {
+ return out, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+ var r moves.OfflineReport
+ if err := json.Unmarshal(body, &r); err != nil {
+ return nil, err
+ }
+ for _, rp := range r.Retired {
+ if rp.Action == moves.RetiredLeft && strings.HasPrefix(rp.Reason, leftCannot) {
+ out[rp.Path] = rp.Reason
+ }
+ }
+ return out, nil
+}
+
+// offlineSourcePark is the source step at offline-open: with the fence held,
+// the row held (or absent), the host quiesced, no process left and no layout
+// journal, rename every path of the source-retire row that is still here
+// into the park directory; then lift the fence and report parked with the
+// inventory. A pass that finds nothing left to rename does not fence again:
+// it reports from what is parked. A path that can never be renamed
+// (leftCannot) is reported left with its reason and does not count as left
+// to rename, so it neither blocks the parked report nor brings the fence
+// back on every pass. A rename that may yet succeed (leftNotYet) reports
+// failed, and the next pass tries again. Nothing is deleted.
+func (w *movesWorker) offlineSourcePark(ctx context.Context, s release.Store, rec provision.Record) error {
+ cfg := w.cfg
+ m, host := rec.Move, rec.Host
+ parkDir := moves.OfflineParkDir(cfg.DataDir, host, m.ID, moves.OfflineRoleSource)
+ if parkDir == "" {
+ return fmt.Errorf("offline move %s: no park directory for %s under %q", m.ID, host, cfg.DataDir)
+ }
+ rt, err := w.offlineRow(host)
+ if err != nil {
+ return err
+ }
+ paths, err := w.retirePaths(host, parkDir, rt)
+ if err != nil {
+ return err
+ }
+ cannot, err := w.cannotRename(ctx, s, host, m.ID)
+ if err != nil {
+ return err
+ }
+ pending := false
+ for _, p := range paths {
+ if p.left == "" && cannot[p.path] == "" && lexists(p.path) {
+ pending = true
+ break
+ }
+ }
+ if pending {
+ fenced := w.offlineFence(ctx, host, m)
+ quiesced := cfg.Quiesced != nil && cfg.Quiesced(host)
+ procs := w.procs(host)
+ journal := layout.JournalPresent(cfg.DataDir, host)
+ if !fenced || !heldBy(rt, m.ID) || !quiesced || procs != 0 || journal {
+ return fmt.Errorf("offline move %s: not parking %s yet: fenced %v, held %v, quiesced %v, processes %d, layout journal %v",
+ m.ID, host, fenced, heldBy(rt, m.ID), quiesced, procs, journal)
+ }
+ if err := os.MkdirAll(parkDir, 0o700); err != nil {
+ return err
+ }
+ }
+ retired := make([]moves.RetiredPath, 0, len(paths))
+ retry := false
+ for _, p := range paths {
+ rp := retire(p, parkDir, pending, cannot[p.path])
+ if rp.Action == moves.RetiredLeft && strings.HasPrefix(rp.Reason, leftNotYet) {
+ retry = true
+ }
+ retired = append(retired, rp)
+ }
+ w.offlineLiftFence(host, m)
+ phase := moves.OfflinePhaseParked
+ if retry {
+ phase = moves.OfflinePhaseFailed
+ }
+ r, err := w.sourceReport(host, m, rt, false, phase)
+ if err != nil {
+ return err
+ }
+ r.Retired = retired
+ if err := writeOfflineReport(ctx, s, r); err != nil {
+ return err
+ }
+ if retry {
+ return fmt.Errorf("offline move %s: %s: some paths were not renamed this pass; the next pass tries again", m.ID, host)
+ }
+ return nil
+}
+
+// offlineSourceGone is the source step at offline-finishing: keep no longer
+// holds the row, so reconcile removes it; once it is gone, report gone. The
+// fence was lifted at open and is lifted again, should a restart have
+// re-fenced in between.
+func (w *movesWorker) offlineSourceGone(ctx context.Context, s release.Store, rec provision.Record) error {
+ m, host := rec.Move, rec.Host
+ rt, err := w.offlineRow(host)
+ if err != nil {
+ return err
+ }
+ w.offlineLiftFence(host, m)
+ if rt != nil {
+ w.notice(host+"#offline-gone", "moves: %s: offline move %s: waiting for reconcile to remove the row", host, m.ID)
+ return nil
+ }
+ r, err := w.sourceReport(host, m, nil, false, moves.OfflinePhaseGone)
+ if err != nil {
+ return err
+ }
+ return writeOfflineReport(ctx, s, r)
+}
+
+// offlineSourceRevert is every source box's part of offline-reverting: the
+// row leaves this move's hold (ReleaseHold under the host's hold lock), the
+// fence is lifted, and the box reports reverted. The sealer's extra part is
+// its own step's (Task 4).
+//
+// The release also holds reconcileMu: a reconcile pass that listed the
+// records before the abort would otherwise mint a missing row held again
+// from its stale listing after the release, and a row held by a move that
+// has finished is never released. A pass that starts after the release
+// lists the record at offline-reverting, which holds nothing.
+func (w *movesWorker) offlineSourceRevert(ctx context.Context, s release.Store, rec provision.Record) error {
+ cfg := w.cfg
+ m, host := rec.Move, rec.Host
+ if cfg.ReleaseHold == nil {
+ return errors.New("offline move: no registry to release the hold through")
+ }
+ var relErr error
+ reconcileMu.Lock()
+ provision.WithHostHoldLock(host, func() {
+ _, relErr = cfg.ReleaseHold(host, m.ID)
+ })
+ reconcileMu.Unlock()
+ if relErr != nil {
+ return relErr
+ }
+ w.offlineLiftFence(host, m)
+ rt, err := w.offlineRow(host)
+ if err != nil {
+ return err
+ }
+ if rt != nil && rt.HoldOp == m.ID {
+ return fmt.Errorf("offline move %s: %s is still held by the move", m.ID, host)
+ }
+ r, err := w.sourceReport(host, m, rt, false, moves.OfflinePhaseReverted)
+ if err != nil {
+ return err
+ }
+ r.Held = false
+ return writeOfflineReport(ctx, s, r)
+}
diff --git a/internal/host/movesoffline_land.go b/internal/host/movesoffline_land.go
new file mode 100644
index 000000000..d3ea217f5
--- /dev/null
+++ b/internal/host/movesoffline_land.go
@@ -0,0 +1,30 @@
+package host
+
+import (
+ "context"
+
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// offlineTargetStep is the target box's part of an offline move: the
+// landing, open and the target's revert
+// (docs/superpowers/plans/2026-10-20-offline-operator-move.md, Task 5, which
+// owns this file). offlineRound calls it on a box whose placement equals
+// rec.Move.To, in every offline state, after the source and sealer steps.
+// rec carries an offline move (rec.Move.Offline()) and the box is capable
+// (moveCfg.OfflineCapable). Reconcile mints the row held "landing" from the
+// record alone (offlineHold) and keeps it until the move is cleared
+// (offlineView.keep); ending that hold (open, revert) must take
+// provision.WithHostHoldLock around the release. Every step must be an
+// ensure; an error is logged by the pass, throttled per host, and the next
+// pass runs the step again.
+//
+// The landing compares against the immutable seal record
+// (moves.OfflineSealKey(host, id), added on Task 6's branch), read with
+// s.Get; never against the sealer's per-pass report, which keeps changing.
+//
+// Task 3 leaves it a no-op.
+func (w *movesWorker) offlineTargetStep(ctx context.Context, s release.Store, rec provision.Record) error {
+ return nil
+}
diff --git a/internal/host/movesoffline_seal.go b/internal/host/movesoffline_seal.go
new file mode 100644
index 000000000..e13489c57
--- /dev/null
+++ b/internal/host/movesoffline_seal.go
@@ -0,0 +1,28 @@
+package host
+
+import (
+ "context"
+
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// offlineSealerStep is the sealer's part of an offline move
+// (docs/superpowers/plans/2026-10-20-offline-operator-move.md, Task 4, which
+// owns this file). offlineRound calls it on the box whose moveCfg.Box equals
+// rec.Move.Sealer, in every offline state, after this box's source step has
+// written its report for the pass. rec carries an offline move
+// (rec.Move.Offline()) and the box is capable (moveCfg.OfflineCapable).
+// Every step must be an ensure; an error is logged by the pass, throttled
+// per host, and the next pass runs the step again.
+//
+// The immutable seal record (moves.OfflineSealKey(host, id), added on
+// Task 6's branch) is written once through putOfflineCreateOnly, never
+// rewritten: on a re-run it returns the stored body with created false, and
+// the step compares instead of overwriting. The per-pass sealer report goes
+// through writeOfflineReport, which replaces.
+//
+// Task 3 leaves it a no-op.
+func (w *movesWorker) offlineSealerStep(ctx context.Context, s release.Store, rec provision.Record) error {
+ return nil
+}
diff --git a/internal/host/movesoffline_test.go b/internal/host/movesoffline_test.go
new file mode 100644
index 000000000..b1b8da306
--- /dev/null
+++ b/internal/host/movesoffline_test.go
@@ -0,0 +1,1283 @@
+package host
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "os"
+ "path/filepath"
+ "reflect"
+ "slices"
+ "sort"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/cgroup"
+ "github.com/carlosframework/platform/internal/domains"
+ "github.com/carlosframework/platform/internal/hostwrite"
+ "github.com/carlosframework/platform/internal/instancectl"
+ "github.com/carlosframework/platform/internal/layout"
+ "github.com/carlosframework/platform/internal/moves"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// The offline operator move's box side (Task 3 of
+// docs/superpowers/plans/2026-10-20-offline-operator-move.md), tested
+// through the functions the agent calls: newMovesWorker, armOffline,
+// reconcileSerializedWith, the production edges-file writer and movesPass,
+// over one DirStore, real registries and real activator services (the
+// package's fake wake backend, which only the fence needs).
+
+const (
+ offHost = "shop.example.com"
+ offID = "mv_zyxwvutsrqponmlkjihgfedcba"
+ offID2 = "mv_abcdefghijklmnopqrstuvwxy2"
+)
+
+type offWorld struct {
+ t *testing.T
+ s release.Store
+ acct string
+ mu sync.Mutex
+ now time.Time
+}
+
+func (wd *offWorld) clock() time.Time {
+ wd.mu.Lock()
+ defer wd.mu.Unlock()
+ return wd.now
+}
+
+func (wd *offWorld) advance(d time.Duration) {
+ wd.mu.Lock()
+ wd.now = wd.now.Add(d)
+ wd.mu.Unlock()
+}
+
+// newOffWorld is a deployment with one account, one app and one instance
+// record for offHost at the home placement ("").
+func newOffWorld(t *testing.T) *offWorld {
+ t.Helper()
+ t.Setenv("CARLOS_HOST_CLAIM", "") // the appconfig and provision guards read the process env
+ ctx := context.Background()
+ wd := &offWorld{t: t, s: release.NewDirStore(t.TempDir()), acct: domains.Sqid(2), now: time.Date(2026, 10, 20, 9, 0, 0, 0, time.UTC)}
+ manifest, err := json.Marshal(map[string]any{"id": 2, "sqid": wd.acct, "name": "fixture", "created_at": wd.now})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := wd.s.Put(ctx, "accounts/"+wd.acct+"/account.json", manifest, ""); err != nil {
+ t.Fatal(err)
+ }
+ if err := provision.PutManifest(ctx, wd.s, wd.acct, "app", provision.Manifest{Schema: 1, App: "app", Domain: "example.com"}); err != nil {
+ t.Fatal(err)
+ }
+ wd.create(offHost)
+ return wd
+}
+
+func (wd *offWorld) create(host string) {
+ wd.t.Helper()
+ rec := provision.Record{Schema: 1, Host: host, Slug: strings.SplitN(host, ".", 2)[0], Account: wd.acct, App: "app", Region: "", Channel: "stable", CreatedAt: wd.now}
+ if err := provision.CreateRecord(context.Background(), wd.s, rec); err != nil {
+ wd.t.Fatal(err)
+ }
+}
+
+// move applies one offline transition to offHost's record.
+func (wd *offWorld) move(mut func(*provision.Record) error) provision.Record {
+ wd.t.Helper()
+ return wd.moveHost(offHost, mut)
+}
+
+func (wd *offWorld) moveHost(host string, mut func(*provision.Record) error) provision.Record {
+ wd.t.Helper()
+ rec, err := provision.SetRecordMove(context.Background(), wd.s, wd.acct, "app", host, mut)
+ if err != nil {
+ wd.t.Fatal(err)
+ }
+ return *rec
+}
+
+func (wd *offWorld) record() provision.Record {
+ wd.t.Helper()
+ rec, err := provision.GetRecord(context.Background(), wd.s, wd.acct, "app", offHost)
+ if err != nil {
+ wd.t.Fatal(err)
+ }
+ return *rec
+}
+
+// The legal path to a state, from the record at rest.
+func (wd *offWorld) start() {
+ wd.move(provision.OfflineStart(offID, "", "uk", "op@example.com", wd.clock()))
+}
+func (wd *offWorld) seal() { wd.move(provision.OfflineSeal(offID, "ie-a", []string{"ie-a", "ie-b"})) }
+func (wd *offWorld) sealed() {
+ wd.move(provision.OfflineSealedBy(offID, "ie-a", provision.MoveSource{}))
+}
+func (wd *offWorld) flip() { wd.move(provision.OfflineFlip(offID)) }
+func (wd *offWorld) landed() { wd.move(provision.OfflineLandedBy(offID, "uk")) }
+func (wd *offWorld) open() { wd.move(provision.OfflineOpenTransition(offID, wd.clock())) }
+func (wd *offWorld) finishing() {
+ wd.move(provision.OfflineFinishingTransition(offID))
+}
+func (wd *offWorld) abort() { wd.move(provision.OfflineAbort(offID)) }
+
+// report reads one box's report for offID, or fails.
+func (wd *offWorld) report(role, box string) moves.OfflineReport {
+ wd.t.Helper()
+ r, ok := wd.reportIf(role, box)
+ if !ok {
+ wd.t.Fatalf("no %s report from %s", role, box)
+ }
+ return r
+}
+
+func (wd *offWorld) reportIf(role, box string) (moves.OfflineReport, bool) {
+ wd.t.Helper()
+ body, _, err := wd.s.Get(context.Background(), moves.OfflineReportKey(offHost, offID, role, box))
+ if errors.Is(err, os.ErrNotExist) {
+ return moves.OfflineReport{}, false
+ }
+ if err != nil {
+ wd.t.Fatal(err)
+ }
+ var r moves.OfflineReport
+ if err := json.Unmarshal(body, &r); err != nil {
+ wd.t.Fatal(err)
+ }
+ if err := moves.ValidateOfflineReport(r); err != nil {
+ wd.t.Fatal(err)
+ }
+ return r, true
+}
+
+// offBox is one box: its own data directory, registry, activator and moves
+// worker, armed as armMovesPass arms it (armOffline).
+type offBox struct {
+ wd *offWorld
+ name, region string
+ dd, opt, cg string
+ env map[string]string
+ reg *registry.Registry
+ svc *activator.Service
+ w *movesWorker
+ nested bool
+ last []byte
+}
+
+func (wd *offWorld) box(name, region string, env map[string]string) *offBox {
+ wd.t.Helper()
+ b := &offBox{wd: wd, name: name, region: region, dd: wd.t.TempDir(), opt: wd.t.TempDir(), cg: wd.t.TempDir(), env: env}
+ reg, err := registry.Open(filepath.Join(b.dd, "registry.db"))
+ if err != nil {
+ wd.t.Fatal(err)
+ }
+ wd.t.Cleanup(func() { reg.Close() })
+ b.reg = reg
+ b.restart()
+ return b
+}
+
+func (b *offBox) getenv(k string) string {
+ if v, ok := b.env[k]; ok {
+ return v
+ }
+ if k == "CARLOS_HIBERNATE_BUCKET" {
+ return "data"
+ }
+ return ""
+}
+
+// restart is an agent restart: a fresh activator (no fences) and a fresh
+// worker over the same registry and data directory.
+func (b *offBox) restart() {
+ b.svc = activator.New(activator.Config{Bucket: "data", Idle: time.Hour}, &fakeWakeBackend{}, b.reg)
+ cfg := moveCfg{Region: b.region, Box: b.name, DataDir: b.dd, OptDir: b.opt, Quiesced: b.svc.Quiesced, Route: b.reg.Get, Now: b.wd.clock}
+ armOffline(&cfg, b.dd, b.reg, b.svc, nil, hostwrite.Config{}, b.getenv)
+ // The stage-requested marker is root's business, and its throttled
+ // timer would outlive the test's directories.
+ cfg.Offline = newOfflineView(cfg, nil)
+ cfg.ReleaseHold = releaseHeld(b.reg, nil)
+ cfg.Procs = func(host string) (int, error) { return cgroup.HostProcs(b.cg, host, "") }
+ b.w = newMovesWorker(cfg)
+}
+
+func (b *offBox) reconcile() {
+ b.wd.t.Helper()
+ cfg := provision.ReconcileConfig{Region: b.region, DataDir: b.dd}
+ if b.nested {
+ cfg.Layout = func(context.Context, release.Store) (string, error) { return release.LayoutModeNested, nil }
+ } else {
+ cfg.Layout = func(context.Context, release.Store) (string, error) { return release.LayoutModeFlat, nil }
+ }
+ if err := reconcileSerializedWith(context.Background(), b.wd.s, b.reg, cfg, b.w); err != nil {
+ b.wd.t.Fatal(err)
+ }
+}
+
+func (b *offBox) edges() {
+ b.wd.t.Helper()
+ if err := edgesFileWrite(context.Background(), b.wd.s, b.reg, edgesFile{Box: b.name, Placement: b.region}, "", &b.last, false); err != nil {
+ b.wd.t.Fatal(err)
+ }
+}
+
+func (b *offBox) pass() {
+ b.wd.t.Helper()
+ if err := movesPass(context.Background(), b.wd.s, b.w); err != nil {
+ b.wd.t.Fatal(err)
+ }
+}
+
+// tick is the agent's order on a serial bump: reconcile, the edges file,
+// the moves pass.
+func (b *offBox) tick() {
+ b.wd.t.Helper()
+ b.reconcile()
+ b.edges()
+ b.pass()
+}
+
+func (b *offBox) row(host string) *registry.Route {
+ b.wd.t.Helper()
+ rt, err := b.reg.Get(host)
+ if err != nil {
+ b.wd.t.Fatal(err)
+ }
+ return rt
+}
+
+// lists reports whether this box's published edges file lists host.
+func (b *offBox) lists(host string) bool {
+ b.wd.t.Helper()
+ body, _, err := b.wd.s.Get(context.Background(), edgesFileKey(b.name))
+ if err != nil {
+ b.wd.t.Fatal(err)
+ }
+ var f edgesFile
+ if err := json.Unmarshal(body, &f); err != nil {
+ b.wd.t.Fatal(err)
+ }
+ _, ok := f.Hosts[host]
+ return ok
+}
+
+func writeFile(t *testing.T, p, body string) {
+ t.Helper()
+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// --- the hold ---------------------------------------------------------------
+
+// offlineHold's table: a pure function of the record and the box's
+// placement. The home's "" and a named region; every state.
+func TestOfflineHoldIsAFunctionOfTheRecord(t *testing.T) {
+ home := moveCfg{Region: ""}
+ uk := moveCfg{Region: "uk"}
+ fr := moveCfg{Region: "fr"}
+ rec := func(region, state string) provision.Record {
+ return provision.Record{Host: offHost, Region: region, Move: &provision.MoveState{Kind: provision.MoveKindOffline, ID: offID, From: "", To: "uk", State: state}}
+ }
+ cases := []struct {
+ name string
+ box moveCfg
+ rec provision.Record
+ wantStatus, wantOp string
+ }{
+ {"source stopping", home, rec("", provision.OfflineStopping), "moving", offID},
+ {"source sealing", home, rec("", provision.OfflineSealing), "moving", offID},
+ {"source sealed", home, rec("", provision.OfflineSealed), "moving", offID},
+ {"source reverting", home, rec("", provision.OfflineReverting), "", ""},
+ {"source after the flip", home, rec("uk", provision.OfflineLanding), "", ""},
+ {"target before the flip", uk, rec("", provision.OfflineSealed), "", ""},
+ {"target landing", uk, rec("uk", provision.OfflineLanding), "landing", offID},
+ {"target landed", uk, rec("uk", provision.OfflineLanded), "landing", offID},
+ {"target landing, region not yet the target's", uk, rec("", provision.OfflineLanding), "", ""},
+ {"target open", uk, rec("uk", provision.OfflineOpen), "", ""},
+ {"target finishing", uk, rec("uk", provision.OfflineFinishing), "", ""},
+ {"target reverting", uk, rec("", provision.OfflineReverting), "", ""},
+ {"a third region", fr, rec("", provision.OfflineStopping), "", ""},
+ {"a third region after the flip", fr, rec("uk", provision.OfflineLanding), "", ""},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ status, op, ok := offlineHold(c.box)(c.rec)
+ if status != c.wantStatus || op != c.wantOp || ok != (c.wantStatus != "") {
+ t.Fatalf("hold = (%q, %q, %v), want (%q, %q)", status, op, ok, c.wantStatus, c.wantOp)
+ }
+ })
+ }
+ // A record at rest, a legacy move and a Phase 1 move are never held.
+ for _, m := range []*provision.MoveState{nil, {To: "uk", State: provision.MoveCutting}, {ID: offID, From: "", To: "uk", State: provision.OfflineStopping}} {
+ if _, _, ok := offlineHold(home)(provision.Record{Host: offHost, Move: m}); ok {
+ t.Fatalf("held: %+v", m)
+ }
+ }
+}
+
+// B1, the pinning test: a fresh target worker whose keep view has never
+// seen the move (built from a list with no offline record in it, as a
+// box's view is before the flip) mints the row held "landing" on its first
+// reconcile after the flip. And with the view never built at all, reconcile
+// through the offline hooks alone does the same.
+func TestOfflineFreshTargetMintsTheLandingRowHeld(t *testing.T) {
+ wd := newOffWorld(t)
+ wd.start()
+ wd.seal()
+ wd.sealed()
+ wd.flip()
+
+ uk := wd.box("uk-box", "uk", nil)
+ uk.w.cfg.Offline.refresh(nil, nil) // stale: no offline move in it
+ uk.reconcile()
+ rt := uk.row(offHost)
+ if rt == nil || rt.Status != "landing" || rt.HoldOp != offID {
+ t.Fatalf("after the flip the target minted %+v, want held landing by %s", rt, offID)
+ }
+ uk.edges()
+ if uk.lists(offHost) {
+ t.Fatal("the target's edges file lists a held landing row")
+ }
+
+ // The hooks with a view that was never built, straight into
+ // provision.Reconcile: the hold needs no view.
+ fresh := wd.box("uk-box-2", "uk", nil)
+ if fresh.w.cfg.Offline.built() {
+ t.Fatal("fixture: the view is built")
+ }
+ cfg := withOfflineHolds(provision.ReconcileConfig{Region: "uk", DataDir: fresh.dd}, offlineHold(fresh.w.cfg), fresh.w.cfg.Offline)
+ if err := provision.Reconcile(context.Background(), wd.s, fresh.reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ if rt := fresh.row(offHost); rt == nil || rt.Status != "landing" || rt.HoldOp != offID {
+ t.Fatalf("an unbuilt view: minted %+v", rt)
+ }
+}
+
+// --- the source boxes -------------------------------------------------------
+
+// start holds the row on both source boxes (one minting it held, one
+// holding the row it has, keeping its other columns), both report held,
+// fenced and quiesced, and neither edges file lists the host. An agent
+// restart re-fences and re-reports. After the flip the rows stay, kept.
+func TestOfflineStartHoldsEverySourceRow(t *testing.T) {
+ wd := newOffWorld(t)
+ a := wd.box("ie-a", "", nil)
+ b := wd.box("ie-b", "", nil)
+ uk := wd.box("uk-box", "uk", nil)
+ a.tick() // ie-a serves the host before the move; ie-b has no row yet
+ if rt := a.row(offHost); rt == nil || rt.Status != "ready" || !a.lists(offHost) {
+ t.Fatalf("fixture: ie-a does not serve the host: %+v", rt)
+ }
+ if err := a.reg.SetReplication(offHost, "daily"); err != nil {
+ t.Fatal(err)
+ }
+ if err := a.reg.SetNextDue(offHost, 1234); err != nil {
+ t.Fatal(err)
+ }
+ before := *a.row(offHost)
+
+ wd.start()
+ a.tick()
+ b.tick()
+ uk.tick()
+ for _, bx := range []*offBox{a, b} {
+ rt := bx.row(offHost)
+ if rt == nil || rt.Status != "moving" || rt.HoldOp != offID {
+ t.Fatalf("%s: row %+v, want held moving by the move", bx.name, rt)
+ }
+ if bx.lists(offHost) {
+ t.Fatalf("%s's edges file lists a held host", bx.name)
+ }
+ if !bx.svc.Fenced(offHost) {
+ t.Fatalf("%s: no wake fence", bx.name)
+ }
+ r := wd.report(moves.OfflineRoleSource, bx.name)
+ if !r.Held || !r.Fenced || !r.Quiesced || !r.RowPresent || r.Procs != 0 || r.LayoutJournal || r.Phase != moves.OfflinePhaseStopped || !r.At.Equal(wd.clock()) {
+ t.Fatalf("%s: report %+v", bx.name, r)
+ }
+ }
+ after := *a.row(offHost)
+ after.Status, after.HoldOp = before.Status, before.HoldOp
+ if !reflect.DeepEqual(after, before) {
+ t.Fatalf("holding the existing row changed its other columns:\n before %+v\n after %+v", before, after)
+ }
+ if uk.row(offHost) != nil {
+ t.Fatal("the target minted a row before the flip")
+ }
+ if _, ok := wd.reportIf(moves.OfflineRoleSource, "uk-box"); ok {
+ t.Fatal("the target wrote a source report")
+ }
+
+ // An agent restart: the fence is gone with the process, and the next
+ // pass installs it again and rewrites the report.
+ a.restart()
+ if a.svc.Fenced(offHost) {
+ t.Fatal("fixture: a fresh activator is fenced")
+ }
+ wd.advance(time.Minute)
+ a.tick()
+ if !a.svc.Fenced(offHost) {
+ t.Fatal("after a restart the pass did not fence again")
+ }
+ if r := wd.report(moves.OfflineRoleSource, "ie-a"); !r.Fenced || !r.At.Equal(wd.clock()) {
+ t.Fatalf("after a restart: %+v", r)
+ }
+
+ // The flip: the record names uk; both source rows stay, held, and keep
+ // reporting; the target mints its row held landing.
+ wd.seal()
+ wd.sealed()
+ wd.flip()
+ wd.advance(time.Minute)
+ for _, bx := range []*offBox{a, b, uk} {
+ bx.tick()
+ }
+ for _, bx := range []*offBox{a, b} {
+ if rt := bx.row(offHost); rt == nil || rt.Status != "moving" || rt.HoldOp != offID {
+ t.Fatalf("%s after the flip: %+v, want the row kept, held", bx.name, rt)
+ }
+ if r := wd.report(moves.OfflineRoleSource, bx.name); !r.Held || !r.Fenced || !r.At.Equal(wd.clock()) {
+ t.Fatalf("%s after the flip: %+v", bx.name, r)
+ }
+ }
+ if rt := uk.row(offHost); rt == nil || rt.Status != "landing" || rt.HoldOp != offID {
+ t.Fatalf("target after the flip: %+v", rt)
+ }
+ if uk.lists(offHost) {
+ t.Fatal("the target lists the landing row")
+ }
+}
+
+// What the report reads from disk: a box with only a meta dir has local
+// state; a process left in the host's cgroup and a layout journal show.
+func TestOfflineSourceReportReadsTheBox(t *testing.T) {
+ wd := newOffWorld(t)
+ a := wd.box("ie-a", "", nil)
+ b := wd.box("ie-b", "", nil)
+ a.tick()
+ wd.start()
+ // ie-b has no DB file, only the activator's meta dir with one LTX file.
+ meta := activatorMetaDir(provision.DefaultDBPath(b.dd, offHost))
+ writeFile(t, filepath.Join(meta, "ltx", "0", "0000000000000001-0000000000000007.ltx"), "x")
+ // ie-a: files, env, a process in the host's exec cgroup, a journal.
+ writeFile(t, filepath.Join(a.dd, offHost, "instance.key"), "k")
+ writeFile(t, filepath.Join(a.dd, offHost+".env"), "CARLOS_INSTANCE_KEY=k\n")
+ writeFile(t, filepath.Join(a.cg, cgroup.ExecName(offHost), "cgroup.procs"), "4242\n")
+ writeFile(t, provision.LayoutJournalPath(a.dd, offHost), "{}")
+ a.tick()
+ b.tick()
+ rb := wd.report(moves.OfflineRoleSource, "ie-b")
+ if !rb.LocalDB || rb.LocalTXID != 7 || rb.Files || rb.Env || rb.Sidecar {
+ t.Fatalf("a meta dir alone: %+v", rb)
+ }
+ ra := wd.report(moves.OfflineRoleSource, "ie-a")
+ if ra.LocalDB || !ra.Files || !ra.Env || ra.Procs != 1 || !ra.LayoutJournal {
+ t.Fatalf("ie-a: %+v", ra)
+ }
+ // A box that cannot count its processes reports -1, never 0.
+ a.w.cfg.Procs = func(string) (int, error) { return 0, cgroup.ErrUnknown }
+ a.pass()
+ if r := wd.report(moves.OfflineRoleSource, "ie-a"); r.Procs != -1 {
+ t.Fatalf("unknown processes reported as %d", r.Procs)
+ }
+ // A sidecar file shows.
+ writeFile(t, filepath.Join(b.dd, offHost+"-sidecar.env"), "X=1\n")
+ b.pass()
+ if r := wd.report(moves.OfflineRoleSource, "ie-b"); !r.Sidecar {
+ t.Fatalf("sidecar not seen: %+v", r)
+ }
+}
+
+// offline-reverting, before and after the flip: every source row is
+// released and served again, the fence is lifted, the boxes report
+// reverted, and the next reconcile does not hold the rows again.
+func TestOfflineRevertReleasesEverySourceRow(t *testing.T) {
+ for _, afterFlip := range []bool{false, true} {
+ t.Run(map[bool]string{false: "before the flip", true: "after the flip"}[afterFlip], func(t *testing.T) {
+ wd := newOffWorld(t)
+ a := wd.box("ie-a", "", nil)
+ b := wd.box("ie-b", "", nil)
+ uk := wd.box("uk-box", "uk", nil)
+ a.tick()
+ b.tick()
+ wd.start()
+ if afterFlip {
+ wd.seal()
+ wd.sealed()
+ wd.flip()
+ }
+ for _, bx := range []*offBox{a, b, uk} {
+ bx.tick()
+ }
+ wd.abort()
+ for i := 0; i < 2; i++ {
+ for _, bx := range []*offBox{a, b, uk} {
+ bx.tick()
+ }
+ }
+ for _, bx := range []*offBox{a, b} {
+ rt := bx.row(offHost)
+ if rt == nil || rt.Status != "ready" || rt.HoldOp != "" {
+ t.Fatalf("%s: %+v, want ready and unheld", bx.name, rt)
+ }
+ if !bx.lists(offHost) {
+ t.Fatalf("%s does not serve the host again", bx.name)
+ }
+ if bx.svc.Fenced(offHost) {
+ t.Fatalf("%s: the fence is still up", bx.name)
+ }
+ if r := wd.report(moves.OfflineRoleSource, bx.name); r.Phase != moves.OfflinePhaseReverted || r.Held {
+ t.Fatalf("%s: %+v", bx.name, r)
+ }
+ }
+ // The move finishes; nothing holds the rows again.
+ wd.move(provision.OfflineFinish(offID, wd.clock()))
+ for _, bx := range []*offBox{a, b} {
+ bx.tick()
+ if rt := bx.row(offHost); rt == nil || rt.Status != "ready" || rt.HoldOp != "" {
+ t.Fatalf("%s after finish: %+v", bx.name, rt)
+ }
+ }
+ })
+ }
+}
+
+// offline-open on a flat source: every path of the source-retire row that
+// is there is renamed into the park directory byte for byte, nothing is
+// deleted, the fence is lifted, and the parked report lists every path with
+// its action. A process left in the host's cgroup stops the park first.
+// offline-finishing then removes the row and the box reports gone.
+func TestOfflineOpenParksTheFlatSource(t *testing.T) {
+ wd := newOffWorld(t)
+ a := wd.box("ie-a", "", nil)
+ uk := wd.box("uk-box", "uk", nil)
+ a.tick()
+ db := a.row(offHost).DBPath
+ contents := map[string]string{
+ db: "sqlite",
+ db + "-wal": "wal",
+ filepath.Join(activatorMetaDir(db), "ltx", "0", "0000000000000001-0000000000000003.ltx"): "ltx",
+ filepath.Join(a.dd, offHost, "instance.key"): "key",
+ filepath.Join(a.dd, offHost+".env"): "CARLOS_INSTANCE_KEY=k\n",
+ filepath.Join(a.dd, offHost+".config.env"): "SECRET=s\n",
+ filepath.Join(a.dd, "programs", offHost+".json"): `{"programs":[]}`,
+ filepath.Join(a.opt, "live", offHost, "app"): "binary",
+ filepath.Join(a.dd, "neighbour.example.com.db"): "not ours",
+ }
+ for p, body := range contents {
+ writeFile(t, p, body)
+ }
+ wd.start()
+ a.tick() // the source reports before the seal, as the seal door requires
+ wd.seal()
+ wd.sealed()
+ wd.flip()
+ wd.landed()
+ a.tick()
+ uk.tick()
+ wd.open()
+
+ // A process still in the host's cgroup: nothing is renamed.
+ procs := filepath.Join(a.cg, cgroup.ExecName(offHost), "cgroup.procs")
+ writeFile(t, procs, "4242\n")
+ a.tick()
+ if _, err := os.Stat(db); err != nil {
+ t.Fatalf("parked with a process left: %v", err)
+ }
+ if _, ok := wd.reportIf(moves.OfflineRoleSource, "ie-a"); !ok {
+ t.Fatal("fixture: no report")
+ }
+ if r := wd.report(moves.OfflineRoleSource, "ie-a"); r.Phase == moves.OfflinePhaseParked {
+ t.Fatalf("reported parked with a process left: %+v", r)
+ }
+ writeFile(t, procs, "")
+
+ a.tick()
+ park := moves.OfflineParkDir(a.dd, offHost, offID, moves.OfflineRoleSource)
+ for p, body := range contents {
+ switch {
+ case strings.HasPrefix(p, a.opt), strings.Contains(p, "neighbour"):
+ if got, err := os.ReadFile(p); err != nil || string(got) != body {
+ t.Fatalf("%s was touched: %q %v", p, got, err)
+ }
+ continue
+ }
+ if _, err := os.Lstat(p); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("%s is still in place: %v", p, err)
+ }
+ }
+ for rel, body := range map[string]string{
+ filepath.Base(db): "sqlite",
+ filepath.Base(db) + "-wal": "wal",
+ filepath.Join(filepath.Base(activatorMetaDir(db)), "ltx", "0", "0000000000000001-0000000000000003.ltx"): "ltx",
+ filepath.Join(offHost, "instance.key"): "key",
+ offHost + ".env": "CARLOS_INSTANCE_KEY=k\n",
+ offHost + ".config.env": "SECRET=s\n",
+ offHost + ".json": `{"programs":[]}`,
+ } {
+ if got, err := os.ReadFile(filepath.Join(park, rel)); err != nil || string(got) != body {
+ t.Fatalf("park/%s = %q %v, want %q", rel, got, err, body)
+ }
+ }
+ r := wd.report(moves.OfflineRoleSource, "ie-a")
+ if r.Phase != moves.OfflinePhaseParked || r.LocalDB || r.Files || r.Env || r.Fenced {
+ t.Fatalf("parked report: %+v", r)
+ }
+ if a.svc.Fenced(offHost) {
+ t.Fatal("the fence is still up after the park")
+ }
+ want := map[string]string{
+ db: moves.RetiredParked, db + "-wal": moves.RetiredParked, db + "-shm": moves.RetiredAbsent,
+ db + "-litestream": moves.RetiredAbsent, activatorMetaDir(db): moves.RetiredParked,
+ filepath.Join(a.dd, offHost): moves.RetiredParked, filepath.Join(a.dd, offHost+".env"): moves.RetiredParked,
+ filepath.Join(a.dd, offHost+".config.env"): moves.RetiredParked, filepath.Join(a.dd, "programs", offHost+".json"): moves.RetiredParked,
+ filepath.Join(a.opt, "live", offHost): moves.RetiredLeft,
+ }
+ assertRetired(t, r.Retired, want)
+
+ // A second pass changes nothing and reports the same inventory.
+ before := r.Retired
+ a.tick()
+ if r := wd.report(moves.OfflineRoleSource, "ie-a"); !reflect.DeepEqual(r.Retired, before) || r.Phase != moves.OfflinePhaseParked {
+ t.Fatalf("second pass: %+v", r.Retired)
+ }
+ // The row stays held through open.
+ if rt := a.row(offHost); rt == nil || rt.HoldOp != offID {
+ t.Fatalf("row at open: %+v", rt)
+ }
+
+ // finishing: keep no longer holds the row; reconcile removes it and
+ // the box reports gone. The parked state stays.
+ // The view the reconcile reads is the last moves pass's, so the row
+ // goes on the pass after the one that first sees finishing.
+ wd.finishing()
+ a.tick()
+ a.tick()
+ if rt := a.row(offHost); rt != nil {
+ t.Fatalf("row at finishing: %+v", rt)
+ }
+ a.pass()
+ if r := wd.report(moves.OfflineRoleSource, "ie-a"); r.Phase != moves.OfflinePhaseGone || r.RowPresent || r.LocalDB {
+ t.Fatalf("gone report: %+v", r)
+ }
+ if got, err := os.ReadFile(filepath.Join(park, filepath.Base(db))); err != nil || string(got) != "sqlite" {
+ t.Fatalf("the parked DB: %q %v", got, err)
+ }
+}
+
+func assertRetired(t *testing.T, got []moves.RetiredPath, want map[string]string) {
+ t.Helper()
+ seen := map[string]string{}
+ for _, rp := range got {
+ if _, dup := seen[rp.Path]; dup {
+ t.Fatalf("%s listed twice", rp.Path)
+ }
+ seen[rp.Path] = rp.Action
+ if rp.Action == moves.RetiredLeft && rp.Reason == "" {
+ t.Fatalf("%s left with no reason", rp.Path)
+ }
+ }
+ for p, action := range want {
+ if seen[p] != action {
+ t.Fatalf("%s: %q, want %q (inventory %+v)", p, seen[p], action, got)
+ }
+ }
+ if len(seen) != len(want) {
+ var extra []string
+ for p := range seen {
+ if _, ok := want[p]; !ok {
+ extra = append(extra, p+"="+seen[p])
+ }
+ }
+ sort.Strings(extra)
+ t.Fatalf("unexpected entries: %v", extra)
+ }
+}
+
+// M3: a path the agent can never rename (here a file in a directory it may
+// not write, as a root-owned /data/programs is) is reported left once, with
+// its reason, and is not tried again: later passes, an agent restart
+// included, neither fence nor rename, and the report stays parked.
+func TestOfflineParkLeavesWhatCannotBeRenamedOnce(t *testing.T) {
+ if os.Geteuid() == 0 {
+ t.Skip("root can rename anything")
+ }
+ wd := newOffWorld(t)
+ a := wd.box("ie-a", "", nil)
+ a.tick()
+ db := a.row(offHost).DBPath
+ writeFile(t, db, "sqlite")
+ programs := filepath.Join(a.dd, "programs")
+ writeFile(t, filepath.Join(programs, offHost+".json"), "{}")
+ if err := os.Chmod(programs, 0o555); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { os.Chmod(programs, 0o755) })
+ fences := 0
+ count := func() {
+ inner := a.w.cfg.FenceAs
+ a.w.cfg.FenceAs = func(ctx context.Context, host string, gen uint64, owner string) error {
+ fences++
+ return inner(ctx, host, gen, owner)
+ }
+ }
+ wd.start()
+ a.tick()
+ wd.seal()
+ wd.sealed()
+ wd.flip()
+ wd.landed()
+ a.tick()
+ wd.open()
+ count()
+ before := fences
+ a.tick()
+ if fences != before+1 {
+ t.Fatalf("the first park fenced %d times", fences-before)
+ }
+ r := wd.report(moves.OfflineRoleSource, "ie-a")
+ if r.Phase != moves.OfflinePhaseParked {
+ t.Fatalf("phase %s", r.Phase)
+ }
+ var left moves.RetiredPath
+ for _, rp := range r.Retired {
+ if rp.Path == filepath.Join(programs, offHost+".json") {
+ left = rp
+ }
+ if rp.Path == db && rp.Action != moves.RetiredParked {
+ t.Fatalf("the DB: %+v", rp)
+ }
+ }
+ if left.Action != moves.RetiredLeft || !strings.HasPrefix(left.Reason, leftCannot) {
+ t.Fatalf("the unrenameable path: %+v", left)
+ }
+ for i := 0; i < 2; i++ {
+ before = fences
+ a.tick()
+ if fences != before {
+ t.Fatal("a later pass fenced again for a path it cannot rename")
+ }
+ if again := wd.report(moves.OfflineRoleSource, "ie-a"); again.Phase != moves.OfflinePhaseParked || !reflect.DeepEqual(again.Retired, r.Retired) {
+ t.Fatalf("pass %d: %+v", i, again)
+ }
+ }
+ a.restart()
+ count()
+ before = fences
+ a.tick()
+ if fences != before || a.svc.Fenced(offHost) {
+ t.Fatal("after a restart the pass fenced again for a path it cannot rename")
+ }
+ if again := wd.report(moves.OfflineRoleSource, "ie-a"); !reflect.DeepEqual(again.Retired, r.Retired) {
+ t.Fatalf("after a restart: %+v", again.Retired)
+ }
+ if _, err := os.Stat(filepath.Join(programs, offHost+".json")); err != nil {
+ t.Fatalf("the left path: %v", err)
+ }
+}
+
+// offline-open on a nested source: the contents of the raw instance
+// directory are renamed into the park directory, the directory and its
+// view stay, empty, and the inventory names each path.
+func TestOfflineOpenParksTheNestedSource(t *testing.T) {
+ wd := newOffWorld(t)
+ a := wd.box("ie-a", "", nil)
+ a.nested = true
+ raw := layout.RawInstanceDir(a.dd, offHost)
+ if err := os.MkdirAll(raw, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ // The bind mount at <dd>/instances, as a link in the fixture.
+ if err := os.Symlink(filepath.Base(filepath.Dir(raw)), filepath.Join(a.dd, "instances")); err != nil {
+ t.Fatal(err)
+ }
+ a.tick()
+ rt := a.row(offHost)
+ if rt == nil || !provision.NestedPaths(a.dd, *rt) {
+ t.Fatalf("fixture: not a nested row: %+v", rt)
+ }
+ contents := map[string]string{
+ offHost + ".db": "sqlite",
+ offHost + ".db-shm": "shm",
+ offHost + ".env": "CARLOS_INSTANCE_KEY=k\n",
+ "instance.key": "key",
+ "uploads/a.png": "png",
+ }
+ for rel, body := range contents {
+ writeFile(t, filepath.Join(raw, rel), body)
+ }
+ wd.start()
+ a.tick()
+ if r := wd.report(moves.OfflineRoleSource, "ie-a"); !r.LocalDB || !r.Files || !r.Env {
+ t.Fatalf("nested report: %+v", r)
+ }
+ wd.seal()
+ wd.sealed()
+ wd.flip()
+ wd.landed()
+ wd.open()
+ a.tick()
+ park := moves.OfflineParkDir(a.dd, offHost, offID, moves.OfflineRoleSource)
+ for rel, body := range contents {
+ if got, err := os.ReadFile(filepath.Join(park, nestedParkPrefix, rel)); err != nil || string(got) != body {
+ t.Fatalf("park/instance/%s = %q %v", rel, got, err)
+ }
+ }
+ ents, err := os.ReadDir(raw)
+ if err != nil || len(ents) != 0 {
+ t.Fatalf("the instance directory: %v %v, want it there and empty", ents, err)
+ }
+ r := wd.report(moves.OfflineRoleSource, "ie-a")
+ if r.Phase != moves.OfflinePhaseParked || r.LocalDB || r.Files || r.Env {
+ t.Fatalf("parked report: %+v", r)
+ }
+ want := map[string]string{
+ filepath.Join(raw, offHost+".db"): moves.RetiredParked,
+ filepath.Join(raw, offHost+".db-wal"): moves.RetiredAbsent,
+ filepath.Join(raw, offHost+".db-shm"): moves.RetiredParked,
+ filepath.Join(raw, offHost+".db-litestream"): moves.RetiredAbsent,
+ filepath.Join(raw, ".activator-"+offHost+".db-litestream"): moves.RetiredAbsent,
+ filepath.Join(raw, offHost+".env"): moves.RetiredParked,
+ filepath.Join(raw, offHost+".config.env"): moves.RetiredAbsent,
+ filepath.Join(raw, "instance.key"): moves.RetiredParked,
+ filepath.Join(raw, "uploads"): moves.RetiredParked,
+ filepath.Join(a.dd, "programs", offHost+".json"): moves.RetiredAbsent,
+ provision.InstanceDir(a.dd, offHost): moves.RetiredLeft,
+ raw: moves.RetiredLeft,
+ filepath.Join(a.opt, "live", offHost): moves.RetiredAbsent,
+ }
+ assertRetired(t, r.Retired, want)
+ // Re-run: the same inventory, from what is parked.
+ a.tick()
+ if again := wd.report(moves.OfflineRoleSource, "ie-a"); !reflect.DeepEqual(again.Retired, r.Retired) {
+ t.Fatalf("second pass:\n %+v\n %+v", again.Retired, r.Retired)
+ }
+}
+
+// --- the dispatcher and the capability --------------------------------------
+
+// An offline record reaches offlineRound with Phase 1 off, ahead of the
+// Phase 1 skip and the lost-box gate: no host claim is read. A
+// Phase-1-shaped record with Phase 1 off is still logged and skipped.
+func TestOfflineDispatchPrecedesThePhase1Skip(t *testing.T) {
+ wd := newOffWorld(t)
+ wd.create("other.example.com")
+ a := wd.box("ie-a", "", nil)
+ a.tick()
+ wd.start()
+ // A Phase 1 move on the other host (its id is what makes it Phase 1).
+ if _, err := provision.SetRecordMove(context.Background(), wd.s, wd.acct, "app", "other.example.com", func(rec *provision.Record) error {
+ rec.Move = &provision.MoveState{ID: offID2, From: "", To: "uk", State: provision.MovePreparing, RequestedAt: wd.clock()}
+ return nil
+ }); err != nil {
+ t.Fatal(err)
+ }
+ a.tick()
+ if _, ok := wd.reportIf(moves.OfflineRoleSource, "ie-a"); !ok {
+ t.Fatal("the offline record never reached offlineRound")
+ }
+ if a.w.lost != nil {
+ t.Fatalf("the lost-box gate ran: %v", a.w.lost)
+ }
+ if _, ok := a.w.notices.Load("other.example.com#switch"); !ok {
+ t.Fatal("the Phase 1 record was not logged as waiting on the switch")
+ }
+ if rt := a.row("other.example.com"); rt == nil || rt.HoldOp != "" || rt.Status != "ready" {
+ t.Fatalf("the Phase 1 record's row was touched: %+v", rt)
+ }
+ keys, err := wd.s.List(context.Background(), "control/offline-moves/other.example.com/")
+ if err != nil || len(keys) != 0 {
+ t.Fatalf("the Phase 1 record was reported as offline: %v %v", keys, err)
+ }
+}
+
+// The capability is the box's own witness: published with hibernation
+// configured and both switches unset, withheld otherwise. A box without it
+// runs no offline step and installs no offline hook: its row is never
+// held, nothing is reported, and one notice is logged. The guard mode is
+// not an input (platform-1e's ruling of 2026-10-20).
+func TestOfflineCapabilityAndAnIncapableBox(t *testing.T) {
+ env := func(kv ...string) func(string) string {
+ m := map[string]string{"CARLOS_HIBERNATE_BUCKET": "data"}
+ for i := 0; i+1 < len(kv); i += 2 {
+ m[kv[i]] = kv[i+1]
+ }
+ return func(k string) string { return m[k] }
+ }
+ // The published capability is the armed worker's own answer (M5).
+ prev := armedMovesWorker.Load()
+ t.Cleanup(func() { armedMovesWorker.Store(prev) })
+ armedMovesWorker.Store(nil)
+ if armedOfflineCapable() {
+ t.Fatal("capable with no worker armed")
+ }
+ if got := withOfflineCapability(edgesFileCapabilities(env()), armedOfflineCapable()); len(got) != 0 {
+ t.Fatalf("no worker armed: %v", got)
+ }
+ wdCap := newOffWorld(t)
+ armed := wdCap.box("armed", "", nil)
+ armedMovesWorker.Store(armed.w)
+ if !armedOfflineCapable() {
+ t.Fatal("a capable armed worker does not publish")
+ }
+ if got := withOfflineCapability(edgesFileCapabilities(env()), armedOfflineCapable()); !slices.Equal(got, []string{capOfflineMove1}) {
+ t.Fatalf("capable box: %v", got)
+ }
+ base := edgesFileCapabilities(env("CARLOS_REGION_HA", "1"))
+ if got := withOfflineCapability(base, true); !slices.Equal(got, []string{capRegionHA1, capOfflineMove1}) || !slices.Equal(base, []string{capRegionHA1}) {
+ t.Fatalf("region HA too: %v (base %v)", got, base)
+ }
+ for _, kv := range [][]string{
+ {"CARLOS_HIBERNATE_BUCKET", ""},
+ {"CARLOS_HOST_CLAIM", "1"},
+ {"CARLOS_HOST_CLAIM", "0"}, // set at all
+ {moves.SwitchEnv, "on"},
+ {moves.SwitchEnv, "off"}, // set at all
+ } {
+ if offlineCapable(env(kv...), hibernationConfigured(env(kv...))) {
+ t.Fatalf("%s=%q is capable", kv[0], kv[1])
+ }
+ incapable := wdCap.box("incapable", "", map[string]string{kv[0]: kv[1]})
+ armedMovesWorker.Store(incapable.w)
+ if got := withOfflineCapability(edgesFileCapabilities(env(kv...)), armedOfflineCapable()); slices.Contains(got, capOfflineMove1) {
+ t.Fatalf("%s=%q publishes: %v", kv[0], kv[1], got)
+ }
+ }
+ armedMovesWorker.Store(prev)
+
+ wd := newOffWorld(t)
+ a := wd.box("ie-a", "", map[string]string{"CARLOS_HOST_CLAIM": "1"})
+ if a.w.cfg.OfflineCapable {
+ t.Fatal("armOffline: a claims box is capable")
+ }
+ a.tick()
+ wd.start()
+ a.tick()
+ if rt := a.row(offHost); rt == nil || rt.Status != "ready" || rt.HoldOp != "" {
+ t.Fatalf("an incapable box held the row: %+v", rt)
+ }
+ if a.svc.Fenced(offHost) {
+ t.Fatal("an incapable box fenced")
+ }
+ if _, ok := wd.reportIf(moves.OfflineRoleSource, "ie-a"); ok {
+ t.Fatal("an incapable box reported")
+ }
+ if _, ok := a.w.notices.Load(offHost + "#offline"); !ok {
+ t.Fatal("no notice from an incapable box")
+ }
+}
+
+// --- dormant ---------------------------------------------------------------
+
+// flatLayout is a reconcile config's layout read, always flat.
+func flatLayout(context.Context, release.Store) (string, error) { return release.LayoutModeFlat, nil }
+
+// dormantWorld is a deployment with no offline record anywhere: offHost at
+// rest, gone.example.com (whose record the test deletes), a legacy move and
+// a Phase 1 move on records of their own.
+func dormantWorld(t *testing.T) *offWorld {
+ t.Helper()
+ wd := newOffWorld(t)
+ for _, h := range []string{"gone.example.com", "legacy.example.com", "phase1.example.com"} {
+ wd.create(h)
+ }
+ wd.moveHost("legacy.example.com", func(rec *provision.Record) error {
+ rec.Move = &provision.MoveState{To: "uk", State: provision.MovePreparing}
+ return nil
+ })
+ wd.moveHost("phase1.example.com", func(rec *provision.Record) error {
+ rec.Move = &provision.MoveState{ID: offID2, From: "", To: "uk", State: provision.MovePreparing, RequestedAt: wd.clock()}
+ return nil
+ })
+ return wd
+}
+
+// dormantBoxes are two home boxes minted from the same records with no
+// hooks at all, so their registries start equal: the plain one stands for
+// main, the hooked one has a capable worker whose view no listing has read.
+func dormantBoxes(t *testing.T, wd *offWorld) (plain, hooked *offBox) {
+ t.Helper()
+ plain, hooked = wd.box("main", "", nil), wd.box("hooked", "", nil)
+ for _, b := range []*offBox{plain, hooked} {
+ if err := reconcileSerializedWith(context.Background(), wd.s, b.reg, provision.ReconcileConfig{DataDir: b.dd, Layout: flatLayout}, nil); err != nil {
+ t.Fatal(err)
+ }
+ if b.row("gone.example.com") == nil {
+ t.Fatalf("fixture: %s has no row to retire", b.name)
+ }
+ }
+ if hooked.w.cfg.Offline.built() || !hooked.w.cfg.OfflineCapable {
+ t.Fatal("fixture: the hooked box's view is built, or the box is not capable")
+ }
+ return plain, hooked
+}
+
+// sameRegistries fails unless the two boxes' registries match, paths and
+// creation times aside (each box has its own data directory and clock).
+func sameRegistries(t *testing.T, a, b *offBox) {
+ t.Helper()
+ strip := func(rs []registry.Route) []registry.Route {
+ out := append([]registry.Route(nil), rs...)
+ for i := range out {
+ out[i].DBPath, out[i].Socket, out[i].CreatedAt = "", "", 0
+ }
+ return out
+ }
+ ar, err := a.reg.List()
+ if err != nil {
+ t.Fatal(err)
+ }
+ br, err := b.reg.List()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !reflect.DeepEqual(strip(ar), strip(br)) {
+ t.Fatalf("the registries differ:\n %s %+v\n %s %+v", a.name, ar, b.name, br)
+ }
+}
+
+// With no offline record anywhere (a legacy and a Phase 1 move included):
+// the very first reconcile on a box whose view no listing has read removes
+// a retired row exactly as main does; after it the hold answers false for
+// every record and keep false for every host; and the edges file differs
+// by the one capability string and nothing else.
+func TestOfflineDormantIsMain(t *testing.T) {
+ ctx := context.Background()
+ wd := dormantWorld(t)
+ plain, hooked := dormantBoxes(t, wd)
+ if err := wd.s.Delete(ctx, provision.RecordKey(wd.acct, "app", "gone.example.com")); err != nil {
+ t.Fatal(err)
+ }
+ for _, b := range []*offBox{plain, hooked} {
+ w := b.w
+ if b == plain {
+ w = nil
+ }
+ if err := reconcileSerializedWith(ctx, wd.s, b.reg, provision.ReconcileConfig{DataDir: b.dd, Layout: flatLayout}, w); err != nil {
+ t.Fatal(err)
+ }
+ if rt := b.row("gone.example.com"); rt != nil {
+ t.Fatalf("%s kept a retired row on its first pass: %+v", b.name, rt)
+ }
+ }
+ sameRegistries(t, plain, hooked)
+ v := hooked.w.cfg.Offline
+ if !v.built() || v.failed() {
+ t.Fatal("the first reconcile did not read the view")
+ }
+ for _, h := range []string{offHost, "gone.example.com", "legacy.example.com", "phase1.example.com", "never.example.com"} {
+ if v.keep(h) {
+ t.Fatalf("keep(%s)", h)
+ }
+ }
+ recs, err := provision.ListAllRecords(ctx, wd.s)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(recs) != 3 {
+ t.Fatalf("fixture: %d records", len(recs))
+ }
+ for _, rec := range recs {
+ if _, _, ok := offlineHold(hooked.w.cfg)(rec); ok {
+ t.Fatalf("held %s", rec.Host)
+ }
+ }
+ // A further pass changes nothing on either box.
+ for _, b := range []*offBox{plain, hooked} {
+ w := b.w
+ if b == plain {
+ w = nil
+ }
+ if err := reconcileSerializedWith(ctx, wd.s, b.reg, provision.ReconcileConfig{DataDir: b.dd, Layout: flatLayout}, w); err != nil {
+ t.Fatal(err)
+ }
+ }
+ sameRegistries(t, plain, hooked)
+
+ // The edges file: the one string, and no headroom with it.
+ var last1, last2 []byte
+ none := func(string) string { return "" }
+ plainFile := edgesFile{Box: "plain", Placement: "", Capabilities: withOfflineCapability(edgesFileCapabilities(none), false)}
+ capable := edgesFile{Box: "plain", Placement: "", Capabilities: withOfflineCapability(edgesFileCapabilities(none), true)}
+ if headroomCapabilities(capable.Capabilities) != nil {
+ t.Fatal("offline-move-1 asks for headroom")
+ }
+ s1, s2 := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir())
+ if err := edgesFileWrite(ctx, s1, plain.reg, plainFile, "1.2.3.4", &last1, false); err != nil {
+ t.Fatal(err)
+ }
+ if err := edgesFileWrite(ctx, s2, plain.reg, capable, "1.2.3.4", &last2, false); err != nil {
+ t.Fatal(err)
+ }
+ b1, _, _ := s1.Get(ctx, edgesFileKey("plain"))
+ b2, _, _ := s2.Get(ctx, edgesFileKey("plain"))
+ var m1, m2 map[string]any
+ if json.Unmarshal(b1, &m1) != nil || json.Unmarshal(b2, &m2) != nil {
+ t.Fatal("edges files do not parse")
+ }
+ if caps, _ := m2["capabilities"].([]any); len(caps) != 1 || caps[0] != capOfflineMove1 {
+ t.Fatalf("capabilities: %v", m2["capabilities"])
+ }
+ delete(m2, "capabilities")
+ delete(m1, "generation")
+ delete(m2, "generation")
+ if !reflect.DeepEqual(m1, m2) {
+ t.Fatalf("the edges file differs beyond the capability:\n %s\n %s", b1, b2)
+ }
+ if bytes.Contains(b1, []byte("capabilities")) {
+ t.Fatal("fixture: the plain file has capabilities")
+ }
+}
+
+// M1: a failed startup listing keeps every row only until the next
+// reconcile, which reads the listing again: the retired row then goes as on
+// main, without waiting for a moves pass.
+func TestOfflineFailedListingHealsOnTheNextReconcile(t *testing.T) {
+ ctx := context.Background()
+ wd := dormantWorld(t)
+ plain, hooked := dormantBoxes(t, wd)
+ v := hooked.w.cfg.Offline
+ refreshOfflineView(ctx, failingListStore{Store: wd.s, prefix: "apps/", err: errors.New("listing refused")}, v)
+ if !v.built() || !v.failed() || !v.keep(offHost) {
+ t.Fatal("a failed startup listing does not keep")
+ }
+ if err := wd.s.Delete(ctx, provision.RecordKey(wd.acct, "app", "gone.example.com")); err != nil {
+ t.Fatal(err)
+ }
+ for _, b := range []*offBox{plain, hooked} {
+ w := b.w
+ if b == plain {
+ w = nil
+ }
+ if err := reconcileSerializedWith(ctx, wd.s, b.reg, provision.ReconcileConfig{DataDir: b.dd, Layout: flatLayout}, w); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if v.failed() || v.keep("gone.example.com") {
+ t.Fatal("the reconcile did not read the listing again")
+ }
+ if rt := hooked.row("gone.example.com"); rt != nil {
+ t.Fatalf("the retired row stayed after the view healed: %+v", rt)
+ }
+ sameRegistries(t, plain, hooked)
+ // While the listing keeps failing the view stays closed.
+ refreshOfflineView(ctx, failingListStore{Store: wd.s, prefix: "apps/", err: errors.New("listing refused")}, v)
+ if !v.keep(offHost) {
+ t.Fatal("a failed listing does not keep")
+ }
+ unbuilt := newOfflineView(moveCfg{Region: ""}, nil)
+ if !unbuilt.keep(offHost) {
+ t.Fatal("an unbuilt view keeps nothing")
+ }
+}
+
+// M4: a record that does not decode is kept by keep alone (its move cannot
+// be known), logged, and every other host is judged as usual. The moves
+// pass's listing carries the same answer.
+func TestOfflineUndecodableRecordIsKept(t *testing.T) {
+ ctx := context.Background()
+ wd := newOffWorld(t)
+ wd.create("broken.example.com")
+ if _, err := wd.s.Put(ctx, provision.RecordKey(wd.acct, "app", "broken.example.com"), []byte("{not json"), mustETag(t, wd.s, provision.RecordKey(wd.acct, "app", "broken.example.com"))); err != nil {
+ t.Fatal(err)
+ }
+ a := wd.box("ie-a", "", nil)
+ refreshOfflineView(ctx, wd.s, a.w.cfg.Offline)
+ v := a.w.cfg.Offline
+ if v.failed() || !v.keep("broken.example.com") || v.keep(offHost) {
+ t.Fatalf("startup view: failed %v, keep(broken) %v, keep(%s) %v", v.failed(), v.keep("broken.example.com"), offHost, v.keep(offHost))
+ }
+ fresh := newOfflineView(a.w.cfg, nil)
+ a.w.cfg.Offline = fresh
+ a.pass()
+ if !fresh.keep("broken.example.com") || fresh.keep(offHost) {
+ t.Fatal("the moves pass's listing does not keep the undecodable host")
+ }
+ if _, _, ok := offlineHold(a.w.cfg)(provision.Record{Host: "broken.example.com"}); ok {
+ t.Fatal("held a record with no move")
+ }
+}
+
+func mustETag(t *testing.T, s release.Store, key string) string {
+ t.Helper()
+ _, etag, err := s.Get(context.Background(), key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return etag
+}
+
+// hostSliceCgroup is the slice instancectl's units run in.
+func TestOfflineHostSliceIsInstancectlsSlice(t *testing.T) {
+ got := hostSliceCgroup("/sys/fs/cgroup", offHost)
+ want := filepath.Join("/sys/fs/cgroup", "carlos.slice", "carlos-instances.slice", instancectl.SliceName(offHost))
+ if got != want {
+ t.Fatalf("hostSliceCgroup = %s, want %s", got, want)
+ }
+ if filepath.Dir(filepath.Join("/sys/fs/cgroup", filepath.FromSlash(instancectl.ClaimUnitControlGroup(offHost)))) != got {
+ t.Fatal("the run unit is not in the slice")
+ }
+}
+
+// The seal record's writer is create-only: the first body stays, a re-run
+// gets it back and writes nothing, and the report writer beside it replaces.
+func TestOfflineCreateOnlyKeepsTheFirstWrite(t *testing.T) {
+ ctx := context.Background()
+ s := release.NewDirStore(t.TempDir())
+ key := moves.OfflineReportPrefix(offHost, offID) + "seal.json"
+ got, created, err := putOfflineCreateOnly(ctx, s, key, []byte(`{"n":1}`))
+ if err != nil || !created || string(got) != `{"n":1}` {
+ t.Fatalf("first write: %q %v %v", got, created, err)
+ }
+ got, created, err = putOfflineCreateOnly(ctx, s, key, []byte(`{"n":2}`))
+ if err != nil || created || string(got) != `{"n":1}` {
+ t.Fatalf("second write: %q %v %v, want the first body back, nothing written", got, created, err)
+ }
+ if body, _, err := s.Get(ctx, key); err != nil || string(body) != `{"n":1}` {
+ t.Fatalf("stored: %q %v", body, err)
+ }
+ if _, _, err := putOfflineCreateOnly(ctx, s, "", nil); err == nil {
+ t.Fatal("an empty key was written")
+ }
+}
+
+// keep's table: the source keeps until finishing, the target until the
+// move is cleared, a third region never.
+func TestOfflineKeep(t *testing.T) {
+ rec := func(region, state string) provision.Record {
+ return provision.Record{Host: offHost, Region: region, Move: &provision.MoveState{Kind: provision.MoveKindOffline, ID: offID, From: "", To: "uk", State: state}}
+ }
+ states := []string{provision.OfflineStopping, provision.OfflineSealing, provision.OfflineSealed, provision.OfflineLanding,
+ provision.OfflineLanded, provision.OfflineOpen, provision.OfflineFinishing, provision.OfflineReverting}
+ for _, st := range states {
+ recs := []provision.Record{rec("uk", st)}
+ src := newOfflineView(moveCfg{Region: ""}, nil)
+ src.refresh(recs, nil)
+ if got, want := src.keep(offHost), st != provision.OfflineFinishing; got != want {
+ t.Fatalf("source at %s: keep %v", st, got)
+ }
+ tgt := newOfflineView(moveCfg{Region: "uk"}, nil)
+ tgt.refresh(recs, nil)
+ if !tgt.keep(offHost) {
+ t.Fatalf("target at %s: not kept", st)
+ }
+ other := newOfflineView(moveCfg{Region: "fr"}, nil)
+ other.refresh(recs, nil)
+ if other.keep(offHost) {
+ t.Fatalf("a third region keeps at %s", st)
+ }
+ }
+}
diff --git a/internal/host/movespass.go b/internal/host/movespass.go
index 5ca6fce15..16efd306e 100644
--- a/internal/host/movespass.go
+++ b/internal/host/movespass.go
@@ -3,6 +3,7 @@ package host
import (
"context"
"crypto/ecdh"
+ "crypto/ed25519"
"encoding/json"
"errors"
"fmt"
@@ -17,6 +18,7 @@ import (
"github.com/carlosframework/platform/internal/activator"
"github.com/carlosframework/platform/internal/appconfig"
+ "github.com/carlosframework/platform/internal/cgroup"
"github.com/carlosframework/platform/internal/controlsock"
"github.com/carlosframework/platform/internal/dbverify"
"github.com/carlosframework/platform/internal/hostbox"
@@ -174,7 +176,7 @@ func (w *movesWorker) setStrict(recs []provision.Record) {
// rule (park.PlacementMatches under the verified home binding,
// ParkHomeRegion), so the home's roster name and its "" code are one.
func movesNamesBox(cfg moveCfg, rec provision.Record) bool {
- if rec.Move == nil || rec.Move.Legacy() {
+ if rec.Move == nil || rec.Move.Legacy() || rec.Move.Offline() {
return false
}
here := cfg.parkRegion()
@@ -321,7 +323,12 @@ func armMovesPass(hib activator.Config, dataDir string, reg *registry.Registry,
Events: events,
LogEvent: moveLogWriter(events),
}
- w := &movesWorker{mirrors: map[string]lease.MirrorStore{}}
+ // The edges box label and the version store, whatever the switch says:
+ // the offline move reports under the label (movesoffline.go). armPhase1
+ // sets the same two values.
+ cfg.Box = metricsBoxLabel(os.Getenv, readMachineID())
+ cfg.OptDir = release.OptDir()
+ w := newMovesWorker(cfg)
if cfg.Enabled {
armPhase1(&cfg, w, hib, dataDir, reg, svc, eb, writes, os.Getenv)
if writes.Enabled && providers.guard.FenceView != nil {
@@ -338,10 +345,86 @@ func armMovesPass(hib activator.Config, dataDir string, reg *registry.Registry,
cancel()
}
}
+ armOffline(&cfg, dataDir, reg, svc, eb, writes, os.Getenv)
w.cfg = cfg
armedMovesWorker.Store(w)
}
+// armOffline sets every moveCfg field the offline operator move reads
+// (movesoffline.go and Tasks 4 and 5 of docs/superpowers/plans/2026-10-20-
+// offline-operator-move.md), whatever the Phase 1 switch says, after
+// armPhase1 so a hook it already set is kept. Nothing here acts: the fields
+// are read only by offline steps, which run only on a capable box and only
+// for a record with an offline move.
+func armOffline(cfg *moveCfg, dataDir string, reg *registry.Registry, svc *activator.Service, eb *activator.ExecBackend, writes hostwrite.Config, getenv func(string) string) {
+ cfg.OfflineCapable = offlineCapable(getenv, hibernationConfigured(getenv))
+ cfg.Offline = newOfflineView(*cfg, holdStageRequester(dataDir))
+ if svc != nil {
+ cfg.FenceAs = fenceAsWith(svc)
+ cfg.LiftFenceAs = svc.LiftFenceAs
+ }
+ if reg != nil {
+ cfg.Registry = reg
+ // The caller holds provision.WithHostHoldLock; a release that
+ // landed touches the stage-requested marker, so root re-judges the
+ // now-ready row (registry.SetHold's contract, G10).
+ cfg.ReleaseHold = releaseHeld(reg, holdStageRequester(dataDir))
+ // The row and the timers the move landed on it, in one
+ // transaction, only while op holds the row.
+ cfg.DeleteIfHold = reg.DeleteIfHoldWithTimers
+ }
+ if eb != nil {
+ cfg.LaunchLayers = eb.LaunchLayers
+ cgRoot := eb.CgroupRoot
+ if cgRoot == "" {
+ cgRoot = "/sys/fs/cgroup"
+ }
+ execRoot := eb.Cgroups.Root()
+ cfg.Procs = func(host string) (int, error) {
+ return cgroup.HostProcs(execRoot, host, hostSliceCgroup(cgRoot, host))
+ }
+ }
+ if raw := strings.TrimSpace(getenv("CARLOS_SECRETS_KEY")); raw != "" {
+ if priv, err := appconfig.ParsePrivateKey(raw, "CARLOS_SECRETS_KEY"); err == nil {
+ cfg.ConfigPriv, cfg.ConfigPub = priv, priv.PublicKey()
+ if cfg.SecretsKey == "" {
+ cfg.SecretsKey = raw
+ }
+ }
+ }
+ if cfg.RenderHeld == nil {
+ cfg.RenderHeld = heldRenderer(writes, cfg.ConfigPriv, dataDir, cfg.Box)
+ }
+ optDir := cfg.OptDir
+ cfg.LiveVersion = func(host string) (string, error) { return release.LiveVersion(optDir, host) }
+ pub, perr := release.ParsePublicKey(getenv("CARLOS_RELEASE_PUBKEY"), "CARLOS_RELEASE_PUBKEY")
+ cfg.ChannelVersion = func(ctx context.Context, s release.Store, rt registry.Route) (string, error) {
+ if perr != nil {
+ return "", perr
+ }
+ return channelVersion(ctx, s, pub, rt)
+ }
+ if cfg.Now == nil {
+ cfg.Now = time.Now
+ }
+}
+
+// channelVersion is the build the row's channel points at, as adoption
+// resolves it: the account-qualified pointer of the row's account, app and
+// channel ("stable" when the row names none), verified under the release
+// key.
+func channelVersion(ctx context.Context, s release.Store, pub ed25519.PublicKey, rt registry.Route) (string, error) {
+ channel := rt.Channel
+ if channel == "" {
+ channel = "stable"
+ }
+ ptr, err := release.Channel(ctx, s, pub, release.RouteAccount(rt), rt.App, channel)
+ if err != nil {
+ return "", err
+ }
+ return ptr.Version, nil
+}
+
// movesDataDir is the data directory the moves worker is built with,
// resolved once to an absolute, cleaned path (final re-review 1, I2): the
// outboxes under it sync every directory from an entry's own up to it, and
@@ -892,6 +975,55 @@ type moveCfg struct {
// laneParkTarget's, armed as the revert's (revertLaneArmed). abFault is
// its test seam, nil in production.
abFault func(point string) error
+
+ // The offline operator move (movesoffline.go; docs/superpowers/plans/
+ // 2026-10-20-offline-operator-move.md, Tasks 3 to 5). armOffline sets
+ // every field below whatever the Phase 1 switch says, and so Box,
+ // OptDir, SecretsKey, RenderHeld and Now above; tests set them by hand.
+ // The sealer (Task 4) and the target (Task 5) read only these, plus the
+ // hibernate coordinates, DataDir, Region, Mirror, Quiesced, Route,
+ // Checkpoint and CarryRequest above.
+ //
+ // OfflineCapable is offlineCapable for this box: false, and every
+ // offline step logs one notice per host and does nothing. Offline is the
+ // keep view reconcile reads (offlineView.keep); the hold needs none.
+ OfflineCapable bool
+ Offline *offlineView
+ // FenceAs installs, or meets again, an owned activator fence
+ // (activator.Service.FenceAndStopAs under a 2 s deadline,
+ // ErrStopUnproven counted as installed: fenceAsWith); LiftFenceAs lifts
+ // exactly that fence. Nil (no activator): never fenced.
+ FenceAs func(ctx context.Context, host string, generation uint64, owner string) error
+ LiftFenceAs func(host string, generation uint64, owner string) bool
+ // ReleaseHold opens a row held by op (registry.ReleaseHold, touching the
+ // stage-requested marker when it landed); DeleteIfHold deletes a row
+ // held by op, with its timers rows, in one transaction
+ // (registry.DeleteIfHoldWithTimers). Both answer false when op does not
+ // hold the row. The caller holds provision.WithHostHoldLock.
+ ReleaseHold func(host, op string) (bool, error)
+ DeleteIfHold func(host, op string) (bool, error)
+ // Registry is this box's registry: TimersFor, ReplaceTimers,
+ // SetNextDue, SetReplication and Get.
+ Registry *registry.Registry
+ // LaunchLayers is the exec backend's composed launch environment for a
+ // row (activator.ExecBackend.LaunchLayers).
+ LaunchLayers func(rt registry.Route) (env map[string]string, instance map[string]bool, shadowed []string, err error)
+ // ConfigPriv and ConfigPub are the deployment's config keypair, parsed
+ // from CARLOS_SECRETS_KEY: ConfigPub seals the files archive, ConfigPriv
+ // opens it and renders the held config. Nil without the key.
+ ConfigPriv *ecdh.PrivateKey
+ ConfigPub *ecdh.PublicKey
+ // LiveVersion is the build a row's live link names on this box
+ // (release.LiveVersion over OptDir). ChannelVersion is the build the
+ // row's channel points at, as adoption resolves it (channelVersion,
+ // verified under CARLOS_RELEASE_PUBKEY; an error when that key does
+ // not parse).
+ LiveVersion func(host string) (string, error)
+ ChannelVersion func(ctx context.Context, s release.Store, rt registry.Route) (string, error)
+ // Procs counts the processes left for a host in its exec cgroup and its
+ // nested slice (cgroup.HostProcs). Nil, or an error: unknown, which a
+ // source report records as -1, not stopped.
+ Procs func(host string) (int, error)
}
// defaultMoveReadyDelta is the readiness bar when
@@ -942,7 +1074,9 @@ func movesPass(ctx context.Context, s release.Store, w *movesWorker) error {
w.flushPageOutbox(ctx)
w.flushLogOutbox(ctx, s)
}
- recs, err := provision.ListAllRecords(ctx, s)
+ // Strict returns exactly ListAllRecords' records and errors, plus the
+ // hosts whose record did not decode, which the offline keep view keeps.
+ recs, unreadable, err := provision.ListAllRecordsStrict(ctx, s)
if err != nil {
return fmt.Errorf("moves: list records: %w", err)
}
@@ -966,7 +1100,23 @@ func movesPass(ctx context.Context, s release.Store, w *movesWorker) error {
w.holds.refresh(ctx, s, recs)
}
}
+ // The offline keep view, from the list this pass read (movesoffline.go).
+ if cfg.Offline != nil {
+ cfg.Offline.refresh(recs, unreadable)
+ }
for _, rec := range recs {
+ // An offline operator move is routed first, ahead of the Phase 1
+ // skip and the lost-box gate below: its id makes it look like a
+ // Phase 1 record, which with the switch off is skipped, and it
+ // never reads a host claim (movesoffline.go).
+ if rec.Move.Offline() {
+ if err := w.offlineRound(ctx, s, rec); err != nil {
+ w.logTrouble(rec.Host, err)
+ continue
+ }
+ w.trouble.Delete(rec.Host)
+ continue
+ }
// A move closed or replaced by another writer is forgotten by the
// move log's memory (movesevents.go). With the switch off the pass
// touches no outbox, index or memory of the move log at all (final
@@ -1134,7 +1284,7 @@ const legacyMoveSentence = "this move was started by an older console and has no
// hasPhase1Into reports whether any Phase 1 move targets region.
func hasPhase1Into(recs []provision.Record, region string) bool {
for _, rec := range recs {
- if rec.Move != nil && !rec.Move.Legacy() && rec.Move.To == region {
+ if rec.Move != nil && !rec.Move.Legacy() && !rec.Move.Offline() && rec.Move.To == region {
return true
}
}
diff --git a/internal/layout/extras.go b/internal/layout/extras.go
index 5fef52ff6..329fdf930 100644
--- a/internal/layout/extras.go
+++ b/internal/layout/extras.go
@@ -321,6 +321,17 @@ func FlatLeftovers(dataDir, host string, hosts []string) []string {
return ours
}
+// RawInstanceDir is a nested host's instance directory through the raw
+// directory behind the /data/instances bind mount, for a caller outside this
+// package that renames entries out of it. The offline operator move parks a
+// nested source's contents this way (docs/superpowers/plans/2026-10-20-
+// offline-operator-move.md, Task 3): a rename through the mount answers
+// EXDEV, as the pass's own renames would (Artifact). rename(2) and lstat(2)
+// act on names, never on what a planted link points at; a caller uses only
+// those and a listing of the directory itself, and never follows a link
+// beneath it.
+func RawInstanceDir(dataDir, host string) string { return rawInstanceDir(dataDir, host) }
+
// platformNames are the entries the platform itself keeps directly under
// the data directory. UnattributedEntries leaves them out of its report.
var platformNames = map[string]bool{
@@ -328,6 +339,10 @@ var platformNames = map[string]bool{
"instances": true, ".instances": true, "layout": true, "system": true,
"moves": true, ".moves": true, ".hostexec": true, "litestream.env": true,
"lost+found": true,
+ // The offline operator move's park directory: local state it renames
+ // aside, never deletes (docs/superpowers/plans/2026-10-20-offline-
+ // operator-move.md).
+ ".offline-moves": true,
}
// UnattributedEntries lists the entries directly under dataDir that no
diff --git a/internal/layout/extras_test.go b/internal/layout/extras_test.go
index 42b105af4..4737ca688 100644
--- a/internal/layout/extras_test.go
+++ b/internal/layout/extras_test.go
@@ -88,7 +88,7 @@ func TestValidateJudgesTheJournalsExtraNames(t *testing.T) {
// wrote, which no move can take.
func TestUnattributedEntriesSkipsHostsAndPlatformNames(t *testing.T) {
dd := t.TempDir()
- for _, name := range []string{"blobs", "registry.db", "layout", ".instances", "instances", host, host + "-blobs", "cache.tmp"} {
+ for _, name := range []string{"blobs", "registry.db", "layout", ".instances", "instances", ".offline-moves", host, host + "-blobs", "cache.tmp"} {
if err := os.MkdirAll(filepath.Join(dd, name), 0o700); err != nil {
t.Fatal(err)
}