amadan

carlos / platform Public

Merge branch 'moves-offline' into moves-offline-t7

claude b8ec65b30c89413d1d81b0b5ef3b9bc6fd2b95b8 parent 1680286parent 8487b8b
23 files changed, +4655 −114
  • internal/cgroup/hostprocs.go +28 −0
  • internal/cgroup/hostprocs_linux.go +109 −0
  • internal/cgroup/hostprocs_linux_test.go +115 −0
  • internal/cgroup/hostprocs_other.go +7 −0
  • internal/console/apiinstancemove_offline.go +246 −60
  • internal/console/apiinstancemove_offline_test.go +335 −36
  • internal/console/authcli.go +7 −1
  • internal/console/svcroutes_test.go +15 −5
  • internal/host/agent.go +23 −0
  • internal/host/edgesfile.go +30 −2
  • internal/host/movesoffline.go +976 −0
  • internal/host/movesoffline_land.go +30 −0
  • internal/host/movesoffline_seal.go +28 −0
  • internal/host/movesoffline_test.go +1283 −0
  • internal/host/movespass.go +154 −4
  • internal/layout/extras.go +15 −0
  • internal/layout/extras_test.go +1 −1
  • internal/lease/leasetest/s3server.go +11 −0
  • internal/lease/leasetest/s3server_test.go +32 −0
  • internal/lease/tombstone.go +308 −0
  • internal/lease/tombstone_test.go +784 −0
  • internal/moves/offline.go +66 −5
  • internal/moves/offline_test.go +52 −0
diff --git a/internal/cgroup/hostprocs.go b/internal/cgroup/hostprocs.go
new file mode 100644
index 000000000..d2f950abc
--- /dev/null
+++ b/internal/cgroup/hostprocs.go
@@ -0,0 +1,28 @@
+package cgroup
+
+import "errors"
+
+// ErrUnknown is HostProcs saying it cannot tell how many processes a host
+// has left: the platform has no cgroup v2 to read (darwin, windows), or the
+// agent runs without a delegated subtree, so its exec children share the
+// agent's own cgroup and cannot be told apart. A caller records it as "not
+// stopped", never as zero (the offline move's source report, Task 3 of
+// docs/superpowers/plans/2026-10-20-offline-operator-move.md).
+var ErrUnknown = errors.New("cgroup: the host's processes cannot be counted on this box")
+
+// HostProcs counts the processes left for host on this box:
+//
+// - in the host's exec cgroup, root/carlos-exec@<host> (ExecName), and
+// every cgroup below it (an exec child's sidecar and worker groups);
+// - when unit is set, in the cgroup directory unit names and every cgroup
+// below it. unit is an absolute path in the cgroup filesystem: for a
+// nested row the host's slice, which holds its run, generation, seed,
+// sidecar and cmd units alike.
+//
+// root is the delegated subtree (Manager.Root). A cgroup that does not
+// exist holds no process and counts 0. An empty root is ErrUnknown, since
+// without delegation the exec children run in the agent's own cgroup.
+// Reading never writes, moves or signals anything.
+func HostProcs(root, host, unit string) (int, error) {
+ return hostProcs(root, host, unit)
+}
diff --git a/internal/cgroup/hostprocs_linux.go b/internal/cgroup/hostprocs_linux.go
new file mode 100644
index 000000000..c4204423c
--- /dev/null
+++ b/internal/cgroup/hostprocs_linux.go
@@ -0,0 +1,109 @@
+package cgroup
+
+import (
+ "bufio"
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+)
+
+// hostProcsMaxDepth bounds the walk below a host's cgroup. The platform
+// nests at most a few levels (exec group, role child, generation); a tree
+// deeper than this is not one it made, and is reported rather than walked.
+const hostProcsMaxDepth = 16
+
+func hostProcs(root, host, unit string) (int, error) {
+ if root == "" {
+ return 0, ErrUnknown
+ }
+ if host == "" || strings.ContainsAny(host, "/\x00") || host == "." || host == ".." {
+ return 0, fmt.Errorf("cgroup: %q is not a host name", host)
+ }
+ n, err := countProcs(filepath.Join(root, ExecName(host)), 0)
+ if err != nil {
+ return 0, err
+ }
+ if unit != "" {
+ if !filepath.IsAbs(unit) {
+ return 0, fmt.Errorf("cgroup: the unit cgroup %q is not an absolute path", unit)
+ }
+ m, err := countProcs(filepath.Clean(unit), 0)
+ if err != nil {
+ return 0, err
+ }
+ n += m
+ }
+ return n, nil
+}
+
+// countProcs is the number of pids listed in dir's cgroup.procs and in every
+// cgroup below it. An absent dir is 0. Symlinks are never followed: cgroupfs
+// has none, so a link is not a cgroup.
+func countProcs(dir string, depth int) (int, error) {
+ if depth > hostProcsMaxDepth {
+ return 0, fmt.Errorf("cgroup: %s is nested deeper than %d levels", dir, hostProcsMaxDepth)
+ }
+ st, err := os.Lstat(dir)
+ if errors.Is(err, os.ErrNotExist) {
+ return 0, nil
+ }
+ if err != nil {
+ return 0, err
+ }
+ if !st.IsDir() {
+ return 0, fmt.Errorf("cgroup: %s is not a cgroup directory", dir)
+ }
+ n, err := readProcsFile(filepath.Join(dir, "cgroup.procs"))
+ if err != nil {
+ return 0, err
+ }
+ ents, err := os.ReadDir(dir)
+ if errors.Is(err, os.ErrNotExist) {
+ return n, nil // removed while read: its processes are gone with it
+ }
+ if err != nil {
+ return 0, err
+ }
+ for _, e := range ents {
+ if !e.IsDir() || e.Type()&os.ModeSymlink != 0 {
+ continue
+ }
+ m, err := countProcs(filepath.Join(dir, e.Name()), depth+1)
+ if err != nil {
+ return 0, err
+ }
+ n += m
+ }
+ return n, nil
+}
+
+// readProcsFile counts the non-empty lines of a cgroup.procs file. A file
+// that is gone (the cgroup was removed between the listing and the read)
+// lists nothing.
+func readProcsFile(path string) (int, error) {
+ f, err := os.Open(path)
+ if errors.Is(err, os.ErrNotExist) {
+ return 0, nil
+ }
+ if err != nil {
+ return 0, err
+ }
+ defer f.Close()
+ n := 0
+ sc := bufio.NewScanner(f)
+ for sc.Scan() {
+ if strings.TrimSpace(sc.Text()) != "" {
+ n++
+ }
+ }
+ if err := sc.Err(); err != nil {
+ // A cgroup removed while it was read answers ENODEV.
+ if _, statErr := os.Stat(filepath.Dir(path)); errors.Is(statErr, os.ErrNotExist) {
+ return 0, nil
+ }
+ return 0, err
+ }
+ return n, nil
+}
diff --git a/internal/cgroup/hostprocs_linux_test.go b/internal/cgroup/hostprocs_linux_test.go
new file mode 100644
index 000000000..e96d1528f
--- /dev/null
+++ b/internal/cgroup/hostprocs_linux_test.go
@@ -0,0 +1,115 @@
+package cgroup
+
+import (
+ "errors"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strconv"
+ "testing"
+)
+
+// writeProcs makes dir a fixture cgroup listing pids.
+func writeProcs(t *testing.T, dir string, pids ...int) {
+ t.Helper()
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ body := ""
+ for _, p := range pids {
+ body += strconv.Itoa(p) + "\n"
+ }
+ if err := os.WriteFile(filepath.Join(dir, "cgroup.procs"), []byte(body), 0o644); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// HostProcs over a fixture cgroupfs: the exec group and every group below
+// it count, and so does the unit's group; absent groups count 0; another
+// host's group never counts.
+func TestHostProcsCountsTheHostsGroupsOnly(t *testing.T) {
+ root := t.TempDir()
+ const host = "shop.example.com"
+ if n, err := HostProcs(root, host, ""); err != nil || n != 0 {
+ t.Fatalf("no group: (%d, %v), want (0, nil)", n, err)
+ }
+ writeProcs(t, filepath.Join(root, ExecName(host)))
+ if n, err := HostProcs(root, host, ""); err != nil || n != 0 {
+ t.Fatalf("an empty group: (%d, %v), want (0, nil)", n, err)
+ }
+ writeProcs(t, filepath.Join(root, ExecName(host)), 101)
+ writeProcs(t, filepath.Join(root, ExecName(host), "sidecar"), 102, 103)
+ writeProcs(t, filepath.Join(root, ExecName("other.example.com")), 201, 202, 203)
+ if n, err := HostProcs(root, host, ""); err != nil || n != 3 {
+ t.Fatalf("exec group and child: (%d, %v), want (3, nil)", n, err)
+ }
+ slice := filepath.Join(t.TempDir(), "carlos.slice", "carlos-instances.slice", "carlos-instances-shop.slice")
+ if n, err := HostProcs(root, host, slice); err != nil || n != 3 {
+ t.Fatalf("an absent unit group: (%d, %v), want (3, nil)", n, err)
+ }
+ writeProcs(t, filepath.Join(slice, "carlos-instance@shop.service"), 301)
+ if n, err := HostProcs(root, host, slice); err != nil || n != 4 {
+ t.Fatalf("with the unit's group: (%d, %v), want (4, nil)", n, err)
+ }
+ if _, err := HostProcs(root, host, "relative/slice"); err == nil {
+ t.Fatal("a relative unit path was read")
+ }
+ for _, bad := range []string{"", "..", "a/b"} {
+ if _, err := HostProcs(root, bad, ""); err == nil {
+ t.Fatalf("host %q was read", bad)
+ }
+ }
+}
+
+// Without a delegated subtree the exec children cannot be told apart from
+// the agent's own processes: unknown, never 0.
+func TestHostProcsWithoutDelegationIsUnknown(t *testing.T) {
+ if _, err := HostProcs("", "shop.example.com", ""); !errors.Is(err, ErrUnknown) {
+ t.Fatalf("no root: %v, want ErrUnknown", err)
+ }
+}
+
+// A real process in a real cgroup, where the kernel lets this test make one:
+// a child group of the test's own cgroup, with a sleep moved into it. Where
+// the cgroup is not writable (the usual case outside a delegated unit) the
+// fixture test above is the witness.
+func TestHostProcsSeesARealProcess(t *testing.T) {
+ b, err := os.ReadFile("/proc/self/cgroup")
+ if err != nil {
+ t.Skipf("no /proc/self/cgroup: %v", err)
+ }
+ rel, ok := ParseSelfCgroup(string(b))
+ if !ok {
+ t.Skip("no cgroup v2 path")
+ }
+ parent := filepath.Join("/sys/fs/cgroup", rel)
+ root := filepath.Join(parent, "hostprocs-test-"+strconv.Itoa(os.Getpid()))
+ const host = "shop.example.com"
+ group := filepath.Join(root, ExecName(host))
+ if err := os.MkdirAll(group, 0o755); err != nil {
+ t.Skipf("this cgroup is not writable: %v", err)
+ }
+ t.Cleanup(func() {
+ _ = os.Remove(group)
+ _ = os.Remove(root)
+ })
+ cmd := exec.Command("sleep", "30")
+ if err := cmd.Start(); err != nil {
+ t.Skipf("no sleep: %v", err)
+ }
+ t.Cleanup(func() {
+ _ = cmd.Process.Kill()
+ _ = cmd.Wait()
+ })
+ if err := os.WriteFile(filepath.Join(group, "cgroup.procs"), []byte(strconv.Itoa(cmd.Process.Pid)), 0o644); err != nil {
+ t.Skipf("the kernel refused the move: %v", err)
+ }
+ if n, err := HostProcs(root, host, ""); err != nil || n != 1 {
+ t.Fatalf("a sleep in the host's group: (%d, %v), want (1, nil)", n, err)
+ }
+ _ = cmd.Process.Kill()
+ _ = cmd.Wait()
+ if n, err := HostProcs(root, host, ""); err != nil || n != 0 {
+ t.Fatalf("after the sleep exited: (%d, %v), want (0, nil)", n, err)
+ }
+}
diff --git a/internal/cgroup/hostprocs_other.go b/internal/cgroup/hostprocs_other.go
new file mode 100644
index 000000000..ede2ef0df
--- /dev/null
+++ b/internal/cgroup/hostprocs_other.go
@@ -0,0 +1,7 @@
+//go:build !linux
+
+package cgroup
+
+// hostProcs has no cgroup v2 to read off Linux: every answer is unknown,
+// which a report records as not stopped.
+func hostProcs(string, string, string) (int, error) { return 0, ErrUnknown }
diff --git a/internal/console/apiinstancemove_offline.go b/internal/console/apiinstancemove_offline.go
index 32354ac35..4d0813f9a 100644
--- a/internal/console/apiinstancemove_offline.go
+++ b/internal/console/apiinstancemove_offline.go
@@ -21,8 +21,8 @@ import (
"github.com/carlosframework/platform/internal/hostops"
"github.com/carlosframework/platform/internal/moves"
"github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
"github.com/carlosframework/platform/internal/status"
- "github.com/carlosframework/platform/internal/svcrole"
)
// The offline operator move's doors (docs/superpowers/plans/2026-10-20-
@@ -57,7 +57,7 @@ const offlineMoveCapability = "offline-move-1"
const offlineReportFresh = 2 * time.Minute
// Every sentence a door speaks has its row in the copy record (MV-O20 to
-// MV-O31, MV-O40, MV-O41) or is an existing approved one, written out whole
+// MV-O31, MV-O50, MV-O51) or is an existing approved one, written out whole
// where it is used so the copy gate reads it.
// offlineName is a region code as the operator reads it: the home's
@@ -163,8 +163,13 @@ type offlineReport struct {
raw []byte
}
-// offlineEvidence is every report of one move, as read.
-type offlineEvidence struct{ reports []offlineReport }
+// offlineEvidence is every report of one move, as read, and its seal: the
+// immutable record of what the sealer decided (moves.OfflineSealKey), nil
+// when absent or not a valid seal of this move.
+type offlineEvidence struct {
+ reports []offlineReport
+ seal *offlineReport
+}
func (e offlineEvidence) get(role, box string) (offlineReport, bool) {
for _, r := range e.reports {
@@ -199,7 +204,11 @@ func (s *Server) readOfflineEvidence(ctx context.Context, host, id string) (offl
return offlineEvidence{}, err
}
var ev offlineEvidence
+ sealKey := moves.OfflineSealKey(host, id)
for _, key := range keys {
+ if key == sealKey {
+ continue // read below, as the seal and not as a report
+ }
raw, _, err := s.cfg.Releases.Get(ctx, key)
if errors.Is(err, os.ErrNotExist) {
continue
@@ -221,6 +230,19 @@ func (s *Server) readOfflineEvidence(ctx context.Context, host, id string) (offl
}
return ev.reports[i].Box < ev.reports[j].Box
})
+ raw, _, err := s.cfg.Releases.Get(ctx, sealKey)
+ switch {
+ case errors.Is(err, os.ErrNotExist):
+ case err != nil:
+ return offlineEvidence{}, err
+ default:
+ var rep moves.OfflineReport
+ if err := json.Unmarshal(raw, &rep); err != nil || moves.ValidateOfflineSeal(rep, host, id) != nil {
+ log.Printf("console: offline move %s: ignoring %s: not the seal of this move", host, sealKey)
+ } else {
+ ev.seal = &offlineReport{OfflineReport: rep, raw: raw}
+ }
+ }
return ev, nil
}
@@ -266,10 +288,12 @@ type offlineFleet struct {
func (s *Server) readOfflineFleet(ctx context.Context) (offlineFleet, bool) {
snap, err := s.liveEdges(ctx)
if err != nil || !snap.EvidenceComplete() {
+ log.Printf("console: offline move: the edges files are not complete evidence (error %v)", err)
return offlineFleet{}, false
}
boxes, complete, err := status.LoadBoxesComplete(ctx, s.cfg.Releases)
if err != nil || !complete {
+ log.Printf("console: offline move: the boxes' status reports are not complete evidence (complete %v, error %v)", complete, err)
return offlineFleet{}, false
}
f := offlineFleet{snap: snap, metrics: map[string]*status.BoxMetrics{}}
@@ -458,18 +482,6 @@ func (s *Server) handleAPIInstanceOfflineStart(w http.ResponseWriter, r *http.Re
writeRefusal(w, ref)
return
}
- all, unreadable, err := provision.ListAllRecordsStrict(ctx, s.cfg.Releases)
- if err != nil || len(unreadable) > 0 {
- log.Printf("console: offline move %s: listing records: %v (%d unreadable)", host, err, len(unreadable))
- jsonError(w, http.StatusBadGateway, "could not read the instance")
- return
- }
- for _, o := range all {
- if o.Host != host && o.Move.Offline() {
- jsonError(w, http.StatusConflict, fmt.Sprintf("%s is moving offline. move one instance at a time", o.Host))
- return
- }
- }
var req struct {
Region string `json:"region"`
}
@@ -501,9 +513,10 @@ func (s *Server) handleAPIInstanceOfflineStart(w http.ResponseWriter, r *http.Re
jsonError(w, http.StatusConflict, "the fleet's edges files could not all be read. try again")
return
}
- if to == "" {
+ if to == "" || rec.Region == "" {
// A box that sets CARLOS_REGION to the home's name is home to this
- // console and not to itself, so it cannot take the move.
+ // console and not to itself, so it cannot take a move into home, and
+ // out of home the placement it publishes is not the record's.
for _, f := range fleet.snap.Files {
if f.Placement != "" && homePlacementCode(s.moveHomeName(), f.Placement) == "" {
jsonError(w, http.StatusConflict, f.Box+" sets CARLOS_REGION, so it cannot take a move into the home region")
@@ -527,25 +540,155 @@ func (s *Server) handleAPIInstanceOfflineStart(w http.ResponseWriter, r *http.Re
jsonError(w, http.StatusConflict, offlineConflict(err))
return
}
- s.offlineAdvance(w, r, d, start)
+ // One offline move at a time: the lock is taken before the record is
+ // written, and released if the write fails.
+ if ref := s.offlineTakeLock(ctx, d, id); ref != nil {
+ writeRefusal(w, ref)
+ return
+ }
+ // The write is bounded well under offlineLockGrace, so the lock cannot be
+ // judged stale while it is still in flight.
+ wctx, cancel := context.WithTimeout(ctx, offlineStartWrite)
+ defer cancel()
+ got, err := s.offlineApply(wctx, d, start)
+ if err != nil {
+ // A failed write does not prove nothing was written (a commit whose
+ // reply was lost). The lock is never released here: if the record
+ // carries this move the start succeeded, and if it does not the
+ // lock goes stale and frees itself.
+ rctx, rcancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
+ defer rcancel()
+ if cur, rerr := provision.GetRecord(rctx, s.cfg.Releases, d.acct.Sqid, d.app, host); rerr == nil && cur.Move.Offline() && cur.Move.ID == id {
+ got, err = cur, nil
+ }
+ }
+ if err != nil {
+ s.offlineWriteFailed(w, d, err)
+ return
+ }
+ s.bumpSerial(ctx)
+ writeJSON(w, http.StatusAccepted, s.offlineViewOf(ctx, got))
}
-// offlineAdvance applies one transition and answers 202 with the status
-// view of the record it left.
-func (s *Server) offlineAdvance(w http.ResponseWriter, r *http.Request, d offlineDoor, mut func(*provision.Record) error) {
- ctx := r.Context()
- rec, err := provision.SetRecordMove(ctx, s.cfg.Releases, d.acct.Sqid, d.app, d.host, mut)
+// offlineStartWrite bounds the start door's record write.
+const offlineStartWrite = 30 * time.Second
+
+// offlineApply is one transition through the record writer.
+func (s *Server) offlineApply(ctx context.Context, d offlineDoor, mut func(*provision.Record) error) (*provision.Record, error) {
+ return provision.SetRecordMove(ctx, s.cfg.Releases, d.acct.Sqid, d.app, d.host, mut)
+}
+
+// offlineWriteFailed answers a transition's failure: a refusal with its
+// recorded sentence, anything else as unwritten.
+func (s *Server) offlineWriteFailed(w http.ResponseWriter, d offlineDoor, err error) {
if errors.Is(err, provision.ErrMoveConflict) {
jsonError(w, http.StatusConflict, offlineConflict(err))
return
}
+ log.Printf("console: offline move %s/%s/%s: %v", d.acct.Sqid, d.app, d.host, err)
+ jsonError(w, http.StatusBadGateway, "could not write the move. try again")
+}
+
+// offlineWrite applies one transition and bumps the serial, or answers the
+// refusal itself. When the move it leaves is cleared (finish, or the finish
+// that ends an abort) it releases the lock.
+func (s *Server) offlineWrite(w http.ResponseWriter, r *http.Request, d offlineDoor, mut func(*provision.Record) error) (*provision.Record, bool) {
+ ctx := r.Context()
+ rec, err := s.offlineApply(ctx, d, mut)
if err != nil {
- log.Printf("console: offline move %s/%s/%s: %v", d.acct.Sqid, d.app, d.host, err)
- jsonError(w, http.StatusBadGateway, "could not write the move. try again")
- return
+ s.offlineWriteFailed(w, d, err)
+ return nil, false
}
s.bumpSerial(ctx)
- writeJSON(w, http.StatusAccepted, s.offlineViewOf(ctx, rec))
+ if rec.Move == nil && d.rec.Move != nil {
+ s.offlineReleaseLock(ctx, d.host, d.rec.Move.ID)
+ }
+ return rec, true
+}
+
+// offlineAdvance applies one transition and answers 202 with the status
+// view of the record it left.
+func (s *Server) offlineAdvance(w http.ResponseWriter, r *http.Request, d offlineDoor, mut func(*provision.Record) error) {
+ if rec, ok := s.offlineWrite(w, r, d, mut); ok {
+ writeJSON(w, http.StatusAccepted, s.offlineViewOf(r.Context(), rec))
+ }
+}
+
+// offlineLockGrace is how long a lock is believed whatever its record says:
+// the start door takes the lock, then writes the record.
+const offlineLockGrace = 2 * time.Minute
+
+// offlineTakeLock takes control/offline-moves/active.json for this host and
+// move, create-only. A lock already there is live while the record it names
+// still carries that move, or is younger than offlineLockGrace (MV-O30 naming
+// its host); otherwise it is stale (a crashed start, a failed release) and is
+// replaced under its etag. A
+// lock that cannot be read is not guessed at.
+func (s *Server) offlineTakeLock(ctx context.Context, d offlineDoor, id string) *offlineRefusal {
+ unreadable := &offlineRefusal{status: http.StatusBadGateway, msg: "could not read the instance"}
+ body, err := json.Marshal(moves.OfflineActive{Host: d.host, Op: id, Account: d.acct.Sqid, App: d.app, At: time.Now().UTC()})
+ if err != nil {
+ return unreadable
+ }
+ _, err = s.cfg.Releases.Put(ctx, moves.OfflineActiveKey, body, "")
+ if err == nil {
+ return nil
+ }
+ if !errors.Is(err, release.ErrCASConflict) {
+ log.Printf("console: offline move %s: taking the lock: %v", d.host, err)
+ return unreadable
+ }
+ raw, etag, err := s.cfg.Releases.Get(ctx, moves.OfflineActiveKey)
+ var held moves.OfflineActive
+ if err != nil || json.Unmarshal(raw, &held) != nil || held.Host == "" {
+ log.Printf("console: offline move %s: reading the lock: %v", d.host, err)
+ return unreadable
+ }
+ busy := &offlineRefusal{status: http.StatusConflict, msg: fmt.Sprintf("%s is moving offline. move one instance at a time", held.Host)}
+ // A lock is taken before its record is written, so a record without the
+ // move is only a stale lock once the holder has had time to write it.
+ if time.Since(held.At) < offlineLockGrace {
+ return busy
+ }
+ other, err := provision.GetRecord(ctx, s.cfg.Releases, held.Account, held.App, held.Host)
+ switch {
+ case errors.Is(err, os.ErrNotExist):
+ case err != nil:
+ log.Printf("console: offline move %s: reading the lock holder %s: %v", d.host, held.Host, err)
+ return unreadable
+ case other.Move.Offline() && other.Move.ID == held.Op:
+ return busy
+ }
+ if _, err := s.cfg.Releases.Put(ctx, moves.OfflineActiveKey, body, etag); err != nil {
+ if errors.Is(err, release.ErrCASConflict) {
+ return busy
+ }
+ log.Printf("console: offline move %s: replacing a stale lock: %v", d.host, err)
+ return unreadable
+ }
+ return nil
+}
+
+// offlineReleaseLock deletes the lock under the etag it read, when it is
+// this move's. A failure is logged and left: the next start finds the lock
+// stale (its record no longer carries the move) and replaces it.
+func (s *Server) offlineReleaseLock(ctx context.Context, host, id string) {
+ ctx = context.WithoutCancel(ctx)
+ raw, etag, err := s.cfg.Releases.Get(ctx, moves.OfflineActiveKey)
+ if errors.Is(err, os.ErrNotExist) {
+ return
+ }
+ var held moves.OfflineActive
+ if err != nil || json.Unmarshal(raw, &held) != nil {
+ log.Printf("console: offline move %s: reading the lock to release it: %v", host, err)
+ return
+ }
+ if held.Host != host || held.Op != id {
+ return
+ }
+ if err := release.DeleteIfMatch(ctx, s.cfg.Releases, moves.OfflineActiveKey, etag); err != nil {
+ log.Printf("console: offline move %s: releasing the lock: %v", host, err)
+ }
}
// ---- the step each state waits for ----
@@ -678,15 +821,22 @@ func (s *Server) offlineSealCheck(f offlineFleet, ev offlineEvidence, mv *provis
return sealer, boxes, nil
}
-// offlineFlipCheck: the seal's report is the one the record names and every
-// item it owns is OK; every source box has stopped and left no process or
-// layout journal, on a report this fresh.
+// offlineSealHolds says the record's source.evidence names the seal on disk,
+// byte for byte. The seal is written once, so the hash holds for the whole
+// move whatever the sealer's living report does each pass.
+func offlineSealHolds(ev offlineEvidence, mv *provision.MoveState, host string) bool {
+ return ev.seal != nil && ev.seal.Box == mv.Sealer && mv.Source != nil &&
+ offlineEvidenceMatches(mv.Source.Evidence, moves.OfflineSealKey(host, mv.ID), ev.seal.raw)
+}
+
+// offlineFlipCheck: the seal is the one the record names and every item the
+// source side reports is OK in it; every source box has stopped and left no
+// process or layout journal, on a report this fresh.
func (s *Server) offlineFlipCheck(ev offlineEvidence, mv *provision.MoveState, host, from string, now time.Time) *offlineRefusal {
- seal, ok := ev.get(moves.OfflineRoleSealer, mv.Sealer)
- if !ok || mv.Source == nil || !offlineEvidenceMatches(mv.Source.Evidence, moves.OfflineReportKey(host, mv.ID, moves.OfflineRoleSealer, mv.Sealer), seal.raw) {
+ if !offlineSealHolds(ev, mv, host) {
return offlineStateRefusal(mv.State, host)
}
- if items, _ := offlineSealItems(seal.OfflineReport); !items {
+ if items, _ := offlineSealItems(ev.seal.OfflineReport); !items {
return offlineStateRefusal(mv.State, host)
}
for _, box := range mv.Boxes {
@@ -709,9 +859,9 @@ func processRefusal(box, host string) *offlineRefusal {
}
// offlineOpenCheck is the re-check of the stop: the target's landing report
-// has every landing item OK, the sealer's report is from this pass and holds
-// the tombstone for this move, and every source box is still held, fenced and
-// quiesced.
+// has every landing item OK, the seal is still the one the record names, the
+// sealer's living report is from this pass and holds the tombstone for this
+// move, and every source box is still held, fenced and quiesced.
func (s *Server) offlineOpenCheck(ev offlineEvidence, mv *provision.MoveState, host, from string, now time.Time) *offlineRefusal {
tgt := ev.byRole(moves.OfflineRoleTarget)
if len(tgt) != 1 {
@@ -720,6 +870,9 @@ func (s *Server) offlineOpenCheck(ev offlineEvidence, mv *provision.MoveState, h
if items, _ := offlineLandingItems(tgt[0].OfflineReport); !items {
return offlineStateRefusal(mv.State, host)
}
+ if !offlineSealHolds(ev, mv, host) {
+ return offlineStateRefusal(mv.State, host)
+ }
seal, ok := ev.get(moves.OfflineRoleSealer, mv.Sealer)
if !ok || !offlineFresh(seal.OfflineReport, now) || seal.TombstoneOwner != moves.TombstoneOwner(mv.ID) || !seal.TombstoneUntil.After(now) {
return offlineStateRefusal(mv.State, host)
@@ -757,13 +910,37 @@ func offlineFinishCheck(ev offlineEvidence, mv *provision.MoveState, host, targe
return refuse
}
for _, box := range boxes {
- if rep, ok := ev.get(moves.OfflineRoleSource, box); !ok || rep.Phase != sourcePhase {
+ rep, ok := ev.get(moves.OfflineRoleSource, box)
+ if !ok || rep.Phase != sourcePhase || !offlineRetiredTerminal(rep.Retired) {
return refuse
}
}
return nil
}
+// offlineRetiredTerminal says every path a parked report lists is settled:
+// parked, absent, or left with the reason it could not be parked (a
+// root-owned directory, a view mount). A left path is terminal; the move
+// goes on and the status view shows it. An unknown action, or a left with no
+// reason, is not a settled path.
+func offlineRetiredTerminal(retired []moves.RetiredPath) bool {
+ for _, p := range retired {
+ switch p.Action {
+ case moves.RetiredParked, moves.RetiredAbsent:
+ case moves.RetiredLeft:
+ if p.Reason == "" {
+ return false
+ }
+ default:
+ return false
+ }
+ if p.Path == "" {
+ return false
+ }
+ }
+ return true
+}
+
// offlineVerbDoor is the shape of seal, flip, open and finish: the gate, an
// offline move in flight, the state the verb follows, the evidence, the
// verb's checks, one transition.
@@ -827,6 +1004,14 @@ type offlineReportView struct {
Fresh bool `json:"fresh"`
}
+// offlineLeftView is a path a source box could not park and left in place,
+// with the reason it gave (the parked report's inventory).
+type offlineLeftView struct {
+ Box string `json:"box"`
+ Path string `json:"path"`
+ Reason string `json:"reason"`
+}
+
// offlineItemView is one row of the manifest with the latest check of it.
// Status is ok, waiting, failed or unchecked (no report decides the item).
type offlineItemView struct {
@@ -843,7 +1028,8 @@ type offlineItemView struct {
// State is "none" when no offline move is in flight (Last then carries the
// outcome of the one that ended). Next is the verb that moves the move on
// right now; empty while a box has work to do. Waiting is the sentence that
-// verb would be refused with. CanAbort says abort is legal.
+// verb would be refused with. CanAbort says abort is legal. Failed says the
+// sealer or the target reported it could not do its step: abort is the way on.
type offlineView struct {
Host string `json:"host"`
State string `json:"state"`
@@ -859,6 +1045,8 @@ type offlineView struct {
Next string `json:"next,omitempty"`
Waiting string `json:"waiting,omitempty"`
CanAbort bool `json:"can_abort,omitempty"`
+ Failed bool `json:"failed,omitempty"`
+ Left []offlineLeftView `json:"left,omitempty"`
Reports []offlineReportView `json:"reports,omitempty"`
Items []offlineItemView `json:"items,omitempty"`
Last *provision.MoveOutcome `json:"last,omitempty"`
@@ -886,7 +1074,7 @@ func (s *Server) offlineViewOf(ctx context.Context, rec *provision.Record) offli
}
ev, err := s.readOfflineEvidence(ctx, rec.Host, mv.ID)
if err != nil {
- log.Printf("console: offline move %s: reports: %v", rec.Host, err)
+ log.Printf("console: offline move %s: reading the reports: %v", rec.Host, err)
v.Waiting = "could not read the move's reports. try again"
return v
}
@@ -894,6 +1082,13 @@ func (s *Server) offlineViewOf(ctx context.Context, rec *provision.Record) offli
v.Reports = append(v.Reports, offlineReportView{OfflineReport: r.OfflineReport, Fresh: offlineFresh(r.OfflineReport, now)})
}
v.Items = offlineItems(ev, mv)
+ for _, r := range ev.byRole(moves.OfflineRoleSource) {
+ for _, p := range r.Retired {
+ if p.Action == moves.RetiredLeft {
+ v.Left = append(v.Left, offlineLeftView{Box: r.Box, Path: p.Path, Reason: p.Reason})
+ }
+ }
+ }
if verb := offlineVerbOf(mv.State); verb != "" {
d := offlineDoor{host: rec.Host}
if _, ref := s.offlineStepFor(ctx, d, mv, ev, now); ref != nil {
@@ -902,11 +1097,12 @@ func (s *Server) offlineViewOf(ctx context.Context, rec *provision.Record) offli
v.Next = verb
}
}
- // Landing waits for a box, unless the box says it cannot land.
- if mv.State == provision.OfflineLanding {
- for _, r := range ev.byRole(moves.OfflineRoleTarget) {
- if r.Phase == moves.OfflinePhaseFailed {
- v.Waiting = fmt.Sprintf("%s could not land %s. run carlos move offline abort --host %s", r.Box, rec.Host, rec.Host)
+ // A box that says it failed (the sealer, or the target that cannot
+ // land) means abort is the way on. The CLI says it with MV-O6.
+ if v.CanAbort {
+ for _, r := range ev.reports {
+ if r.Phase == moves.OfflinePhaseFailed && (r.Role == moves.OfflineRoleSealer || r.Role == moves.OfflineRoleTarget) {
+ v.Failed = true
}
}
}
@@ -924,6 +1120,11 @@ func offlineItems(ev offlineEvidence, mv *provision.MoveState) []offlineItemView
if r, ok := ev.get(moves.OfflineRoleSealer, mv.Sealer); ok {
owners[moves.OfflineOwnerSeal] = []offlineReport{r}
}
+ if ev.seal != nil {
+ // The seal is what the sealer decided; the living report is the
+ // latest word on the same items, and is asked first.
+ owners[moves.OfflineOwnerSeal] = append(owners[moves.OfflineOwnerSeal], *ev.seal)
+ }
out := make([]offlineItemView, 0, len(moves.OfflineManifest))
for _, it := range moves.OfflineManifest {
row := offlineItemView{ID: it.ID, Item: it.Item, Disposition: it.Disposition, Owner: it.Owner, Source: it.Source, Status: "waiting"}
@@ -960,18 +1161,3 @@ func (s *Server) handleAPIInstanceOfflineStatus(w http.ResponseWriter, r *http.R
}
writeJSON(w, http.StatusOK, s.offlineViewOf(r.Context(), d.rec))
}
-
-// registerOfflineMoveRoutes registers the offline move's doors. They are
-// always registered: a console on the claim path or with Phase 1 on answers
-// every one with MV-O20, after the operator gate.
-func (s *Server) registerOfflineMoveRoutes() {
- const base = "/api/cli/apps/{account}/{app}/instances/{host}/move/offline"
- inst := svcrole.Allow(svcrole.Instance)
- s.cliRoute("GET "+base, inst, s.handleAPIInstanceOfflineStatus)
- s.cliRoute("POST "+base, inst, s.handleAPIInstanceOfflineStart)
- s.cliRoute("POST "+base+"/seal", inst, s.offlineVerbDoor(offlineVerbSeal))
- s.cliRoute("POST "+base+"/flip", inst, s.offlineVerbDoor(offlineVerbFlip))
- s.cliRoute("POST "+base+"/open", inst, s.offlineVerbDoor(offlineVerbOpen))
- s.cliRoute("POST "+base+"/finish", inst, s.offlineVerbDoor(offlineVerbFinish))
- s.cliRoute("POST "+base+"/abort", inst, s.handleAPIInstanceOfflineAbort)
-}
diff --git a/internal/console/apiinstancemove_offline_test.go b/internal/console/apiinstancemove_offline_test.go
index 49a7776e8..6a991ff82 100644
--- a/internal/console/apiinstancemove_offline_test.go
+++ b/internal/console/apiinstancemove_offline_test.go
@@ -8,6 +8,7 @@ import (
"fmt"
"net/http"
"strings"
+ "sync"
"testing"
"time"
@@ -305,19 +306,38 @@ func sealerChecks() map[string]moves.Check {
return out
}
-// sealed plays the sealer: its report, and the transition that records
-// where it is.
+// putSeal writes seal.json as the sealer does: once and create-only when it
+// is not there, and replaced only by a test that tampers with it.
+func (w *offlineWorld) putSeal(sealer string, edit func(*moves.OfflineReport)) []byte {
+ w.t.Helper()
+ r := moves.OfflineReport{V: moves.OfflineReportVersion, Op: w.id, Host: offlineHost, Box: sealer, Role: moves.OfflineRoleSealer,
+ Phase: moves.OfflinePhaseSealed, At: time.Now().UTC(), Items: sealerChecks(), FinalTXID: 7, TombstoneOwner: moves.TombstoneOwner(w.id)}
+ if edit != nil {
+ edit(&r)
+ }
+ body, err := json.Marshal(r)
+ if err != nil {
+ w.t.Fatal(err)
+ }
+ key := moves.OfflineSealKey(offlineHost, w.id)
+ if _, _, err := w.fx.rel.Get(context.Background(), key); err != nil {
+ if _, err := w.fx.rel.Put(context.Background(), key, body, ""); err != nil {
+ w.t.Fatal(err)
+ }
+ } else {
+ offlinePut(w.t, w.fx.rel, key, body)
+ }
+ return body
+}
+
+// sealed plays the sealer: the seal, its living report, and the transition
+// that records where the seal is. It returns the seal's bytes.
func (w *offlineWorld) sealed(sealer string) []byte {
w.t.Helper()
- raw := w.report(moves.OfflineRoleSealer, sealer, moves.OfflinePhaseSealed, func(r *moves.OfflineReport) {
- r.Items = sealerChecks()
- r.TombstoneOwner = moves.TombstoneOwner(w.id)
- r.TombstoneUntil = time.Now().Add(moves.TombstoneTTL)
- })
- sum := sha256.Sum256(raw)
- key := moves.OfflineReportKey(offlineHost, w.id, moves.OfflineRoleSealer, sealer)
- w.edit(provision.OfflineSealedBy(w.id, sealer, provision.MoveSource{FinalTXID: 7, Evidence: key + "@" + hex.EncodeToString(sum[:])}))
- return raw
+ seal := w.putSeal(sealer, nil)
+ w.sealerFresh(sealer)
+ w.edit(provision.OfflineSealedBy(w.id, sealer, provision.MoveSource{FinalTXID: 7, Evidence: moves.OfflineSealEvidence(offlineHost, w.id, seal)}))
+ return seal
}
// landed plays the target: its landing report and the transition.
@@ -337,6 +357,9 @@ func (w *offlineWorld) toLanded() {
w.stopped(nil, "ie-box")
w.ok("/seal", "", provision.OfflineSealing)
w.sealed("ie-box")
+ // The sealer rewrites its living report on every pass, so by the time
+ // the operator flips it is not the bytes it was at the seal. The seal is.
+ w.sealerFresh("ie-box")
w.stopped(nil, "ie-box")
w.ok("/flip", "", provision.OfflineLanding)
w.landed("uk-box")
@@ -462,14 +485,11 @@ func TestOfflineStartRefusals(t *testing.T) {
}
const body = `{"region":"uk"}`
other := func(w *offlineWorld) {
- // A second instance, moving offline.
- u := instancesURL(w.fx, "bab", "titogo")
- if c, _, out := rawReq(t, http.MethodPost, u, w.token, `{"host":"beta.go.tito.io","region":"ie"}`); c != http.StatusCreated {
- t.Fatalf("create = %d %s", c, out)
- }
- if _, err := provision.SetRecordMove(context.Background(), w.fx.rel, "bab", "titogo", "beta.go.tito.io",
- provision.OfflineStart("mv_aaaaaaaaaaaaaaaaaaaaaaaaaa", "ie", "uk", "x", time.Now())); err != nil {
- t.Fatal(err)
+ // A second instance, moving offline: started through its own door, so
+ // it holds the lock.
+ v, code, out := w.startOther("beta.go.tito.io")
+ if code != http.StatusAccepted {
+ t.Fatalf("beta start = %d %s (%+v)", code, out, v)
}
}
cases := []tc{
@@ -725,23 +745,51 @@ func TestOfflineFlipDoor(t *testing.T) {
w.stopped(nil, "ie-box")
w.ok("/seal", "", provision.OfflineSealing)
w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO25(provision.OfflineSealing))
- // The sealer's report is not what the record names.
- w.sealed("ie-box")
+ seal := w.sealed("ie-box")
w.stopped(nil, "ie-box")
+ sealKey := moves.OfflineSealKey(offlineHost, w.id)
+ // seal.json is create-only: a second write does not replace it.
+ if _, err := w.fx.rel.Put(context.Background(), sealKey, []byte("{}"), ""); err == nil {
+ t.Fatal("the seal was written twice")
+ }
+ // The sealer's living report is rewritten on every pass and the seal is
+ // not: nothing the living report does moves the hash flip checks.
+ w.sealerFresh("ie-box")
w.report(moves.OfflineRoleSealer, "ie-box", moves.OfflinePhaseSealed, func(r *moves.OfflineReport) {
- r.Items = sealerChecks()
- r.FinalTXID = 99 // a different report from the one the record hashed
+ r.Items = map[string]moves.Check{} // the living report carries no items at all now
+ r.TombstoneOwner = moves.TombstoneOwner(w.id)
+ r.TombstoneUntil = time.Now().Add(time.Hour)
+ r.FinalTXID = 99
})
+ // A seal that is not the one the record names.
+ tampered := w.putSeal("ie-box", func(r *moves.OfflineReport) { r.FinalTXID = 99 })
+ if string(tampered) == string(seal) {
+ t.Fatal("the tampered seal is the seal")
+ }
w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO25(provision.OfflineSealed))
- // A seal item that is not OK.
- raw := w.report(moves.OfflineRoleSealer, "ie-box", moves.OfflinePhaseSealed, func(r *moves.OfflineReport) {
- r.Items = sealerChecks()
+ offlinePut(t, w.fx.rel, sealKey, seal)
+ // No seal at all.
+ if err := w.fx.rel.Delete(context.Background(), sealKey); err != nil {
+ t.Fatal(err)
+ }
+ w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO25(provision.OfflineSealed))
+ offlinePut(t, w.fx.rel, sealKey, seal)
+ // A seal item that is not OK, and a seal of another move's tombstone, each
+ // with a hash the record names.
+ bad := w.putSeal("ie-box", func(r *moves.OfflineReport) {
r.Items["files"] = moves.Check{OK: false, Reason: "a file the archive refuses"}
})
- w.rehash("ie-box", raw)
+ w.rehash(bad)
+ w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO25(provision.OfflineSealed))
+ bad = w.putSeal("ie-box", func(r *moves.OfflineReport) { r.TombstoneOwner = moves.TombstoneOwner("mv_other") })
+ w.rehash(bad)
+ w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO25(provision.OfflineSealed))
+ // A seal written by a box other than the recorded sealer.
+ bad = w.putSeal("ie-x", nil)
+ w.rehash(bad)
w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO25(provision.OfflineSealed))
- good := w.report(moves.OfflineRoleSealer, "ie-box", moves.OfflinePhaseSealed, func(r *moves.OfflineReport) { r.Items = sealerChecks() })
- w.rehash("ie-box", good)
+ offlinePut(t, w.fx.rel, sealKey, seal)
+ w.rehash(seal)
// A process left, a layout journal, a stale report.
w.stopped(func(_ string, r *moves.OfflineReport) { r.Procs = 1 }, "ie-box")
w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO31("ie-box"))
@@ -766,14 +814,12 @@ func TestOfflineFlipDoor(t *testing.T) {
}
}
-// rehash points the record's source evidence at a report's bytes, as the
-// sealer's transition does when it records its own report.
-func (w *offlineWorld) rehash(sealer string, raw []byte) {
+// rehash points the record's source evidence at a seal's bytes, as the
+// sealer's transition records its own seal.
+func (w *offlineWorld) rehash(seal []byte) {
w.t.Helper()
- sum := sha256.Sum256(raw)
- key := moves.OfflineReportKey(offlineHost, w.id, moves.OfflineRoleSealer, sealer)
w.edit(func(r *provision.Record) error {
- r.Move.Source.Evidence = key + "@" + hex.EncodeToString(sum[:])
+ r.Move.Source.Evidence = moves.OfflineSealEvidence(offlineHost, w.id, seal)
return nil
})
}
@@ -822,6 +868,15 @@ func TestOfflineOpenDoorRechecksTheStop(t *testing.T) {
w.stopped(func(_ string, r *moves.OfflineReport) { r.At = time.Now().Add(-10 * time.Minute) }, "ie-box")
w.refused(http.MethodPost, "/open", "", http.StatusConflict, msgMC10("ie-box"))
reset()
+ // The seal changed under the move.
+ sealKey := moves.OfflineSealKey(offlineHost, w.id)
+ sealRaw, _, err := w.fx.rel.Get(context.Background(), sealKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+ offlinePut(t, w.fx.rel, sealKey, append(append([]byte{}, sealRaw...), ' '))
+ w.refused(http.MethodPost, "/open", "", http.StatusConflict, msgO25(provision.OfflineLanded))
+ offlinePut(t, w.fx.rel, sealKey, sealRaw)
// A landing item that is not OK.
w.report(moves.OfflineRoleTarget, "uk-box", moves.OfflinePhaseLanded, func(r *moves.OfflineReport) {
r.Items = okChecks(moves.OfflineOwnerLanding)
@@ -860,7 +915,30 @@ func TestOfflineFinishDoor(t *testing.T) {
w.refused(http.MethodPost, "/finish", "", http.StatusConflict, msgO25(provision.OfflineOpen))
w.report(moves.OfflineRoleTarget, "uk-box", moves.OfflinePhaseOpened, func(r *moves.OfflineReport) { r.Items = okChecks(moves.OfflineOwnerLanding) })
w.refused(http.MethodPost, "/finish", "", http.StatusConflict, msgO25(provision.OfflineOpen))
- w.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseParked, nil)
+ // A parked report whose inventory is not settled is not parked.
+ for name, retired := range map[string][]moves.RetiredPath{
+ "an unknown action": {{Path: "/data/acme.go.tito.io", Action: "deleted"}},
+ "left, no reason": {{Path: "/opt/carlos/live/acme.go.tito.io", Action: moves.RetiredLeft}},
+ "no path": {{Action: moves.RetiredParked}},
+ } {
+ w.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseParked, func(r *moves.OfflineReport) { r.Retired = retired })
+ code, out := w.req(http.MethodPost, "/finish", "")
+ if code != http.StatusConflict || errorText(out) != msgO25(provision.OfflineOpen) {
+ t.Fatalf("%s: %d %s", name, code, out)
+ }
+ }
+ // A path the box could not park is terminal when it says why, and the
+ // status view shows it.
+ w.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseParked, func(r *moves.OfflineReport) {
+ r.Retired = []moves.RetiredPath{
+ {Path: "/data/acme.go.tito.io", Action: moves.RetiredParked},
+ {Path: "/data/acme.go.tito.io.env", Action: moves.RetiredAbsent},
+ {Path: "/opt/carlos/live/acme.go.tito.io", Action: moves.RetiredLeft, Reason: "root-owned"},
+ }
+ })
+ if v := w.status(); v.Next != offlineVerbFinish || len(v.Left) != 1 || v.Left[0] != (offlineLeftView{Box: "ie-box", Path: "/opt/carlos/live/acme.go.tito.io", Reason: "root-owned"}) {
+ t.Fatalf("status = next %q left %+v waiting %q", v.Next, v.Left, v.Waiting)
+ }
before := serialBytes(t, w.fx.rel)
w.ok("/finish", "", provision.OfflineFinishing)
if serialBytes(t, w.fx.rel) == before {
@@ -959,6 +1037,9 @@ func TestOfflineStatusView(t *testing.T) {
if v.State != provision.OfflineLanded || v.ID != w.id || v.Sealer != "ie-box" || v.From != "ie" || v.To != "uk" || !v.CanAbort || v.Next != offlineVerbOpen {
t.Fatalf("status = %+v", v)
}
+ if v.Failed {
+ t.Error("a healthy move is marked failed")
+ }
roles := map[string]bool{}
for _, r := range v.Reports {
roles[r.Role+"/"+r.Box] = true
@@ -992,6 +1073,9 @@ func TestOfflineStatusView(t *testing.T) {
delete(r.Items, "release")
r.Items["files"] = moves.Check{OK: false, Reason: "the digest differs"}
})
+ if !w.status().Failed {
+ t.Error("a sealer that reported failed is not marked failed in the view")
+ }
seen := map[string]offlineItemView{}
for _, it := range w.status().Items {
seen[it.ID] = it
@@ -1125,4 +1209,219 @@ func TestOfflineMovesOutOfAndIntoHome(t *testing.T) {
named := offlineBox{Box: "ie-named", Placement: "ie", Bucket: "data-ie"}
c := newOfflineWorldIn(t, "ie", "uk", homeBox, ukOff("uk-box"), named)
c.refused(http.MethodPost, "", `{"region":"ie"}`, http.StatusConflict, "ie-named sets CARLOS_REGION, so it cannot take a move into the home region")
+
+ // And out of home, where its published placement is not the record's.
+ d := newOfflineWorldIn(t, "ie", "ie", homeBox, ukOff("uk-box"), named)
+ d.refused(http.MethodPost, "", `{"region":"uk"}`, http.StatusConflict, "ie-named sets CARLOS_REGION, so it cannot take a move into the home region")
+}
+
+// startOther creates a second instance and starts it through its own door.
+func (w *offlineWorld) startOther(host string) (offlineView, int, string) {
+ w.t.Helper()
+ u := instancesURL(w.fx, "bab", "titogo")
+ if c, _, out := rawReq(w.t, http.MethodPost, u, w.token, `{"host":"`+host+`","region":"ie"}`); c != http.StatusCreated && c != http.StatusConflict {
+ w.t.Fatalf("create %s = %d %s", host, c, out)
+ }
+ code, _, out := rawReq(w.t, http.MethodPost, u+"/"+host+"/move/offline", w.token, `{"region":"uk"}`)
+ var v offlineView
+ _ = json.Unmarshal([]byte(out), &v)
+ return v, code, out
+}
+
+func (w *offlineWorld) lock() (moves.OfflineActive, bool) {
+ w.t.Helper()
+ raw, _, err := w.fx.rel.Get(context.Background(), moves.OfflineActiveKey)
+ if err != nil {
+ return moves.OfflineActive{}, false
+ }
+ var a moves.OfflineActive
+ if err := json.Unmarshal(raw, &a); err != nil {
+ w.t.Fatal(err)
+ }
+ return a, true
+}
+
+// Two starts at once for different hosts: exactly one wins the lock.
+func TestOfflineConcurrentStartsOneWins(t *testing.T) {
+ w := newOfflineWorld(t, oneEach...)
+ u := instancesURL(w.fx, "bab", "titogo")
+ if c, _, out := rawReq(t, http.MethodPost, u, w.token, `{"host":"beta.go.tito.io","region":"ie"}`); c != http.StatusCreated {
+ t.Fatalf("create = %d %s", c, out)
+ }
+ hosts := []string{offlineHost, "beta.go.tito.io"}
+ type answer struct {
+ code int
+ out string
+ }
+ got := make([]answer, len(hosts))
+ var wg sync.WaitGroup
+ gate := make(chan struct{})
+ for i, h := range hosts {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ <-gate
+ c, _, out := rawReq(t, http.MethodPost, u+"/"+h+"/move/offline", w.token, `{"region":"uk"}`)
+ got[i] = answer{c, out}
+ }()
+ }
+ close(gate)
+ wg.Wait()
+ won, lost := -1, -1
+ for i, a := range got {
+ switch a.code {
+ case http.StatusAccepted:
+ won = i
+ case http.StatusConflict:
+ lost = i
+ default:
+ t.Fatalf("start %s = %d %s", hosts[i], a.code, a.out)
+ }
+ }
+ if won < 0 || lost < 0 {
+ t.Fatalf("want one winner and one loser: %+v", got)
+ }
+ if want := hosts[won] + " is moving offline. move one instance at a time"; errorText(got[lost].out) != want {
+ t.Fatalf("the loser was told %q, want %q", errorText(got[lost].out), want)
+ }
+ if a, ok := w.lock(); !ok || a.Host != hosts[won] {
+ t.Fatalf("lock = %+v %v, want the winner %s", a, ok, hosts[won])
+ }
+ loser, err := provision.GetRecord(context.Background(), w.fx.rel, "bab", "titogo", hosts[lost])
+ if err != nil || loser.Move != nil {
+ t.Fatalf("the loser's record = %+v %v", loser.Move, err)
+ }
+}
+
+// The lock is released when the move ends, by finish and by the finish that
+// ends an abort, and a stale lock does not stop the next start.
+func TestOfflineLockLifecycle(t *testing.T) {
+ w := newOfflineWorld(t, oneEach...)
+ w.toLanded()
+ if a, ok := w.lock(); !ok || a.Host != offlineHost || a.Op != w.id || a.Account != "bab" || a.App != "titogo" {
+ t.Fatalf("lock = %+v %v", a, ok)
+ }
+ w.ok("/open", "", provision.OfflineOpen)
+ w.report(moves.OfflineRoleTarget, "uk-box", moves.OfflinePhaseOpened, func(r *moves.OfflineReport) { r.Items = okChecks(moves.OfflineOwnerLanding) })
+ w.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseParked, nil)
+ w.ok("/finish", "", provision.OfflineFinishing)
+ if _, ok := w.lock(); !ok {
+ t.Fatal("the lock was released before the move ended")
+ }
+ w.report(moves.OfflineRoleSealer, "ie-box", moves.OfflinePhaseLifted, nil)
+ w.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseGone, nil)
+ w.ok("/finish", "", "none")
+ if _, ok := w.lock(); ok {
+ t.Fatal("finish left the lock")
+ }
+
+ // An abort ends at finish too.
+ b := newOfflineWorld(t, oneEach...)
+ b.start()
+ b.ok("/abort", "", provision.OfflineReverting)
+ if _, ok := b.lock(); !ok {
+ t.Fatal("abort released the lock before the finish")
+ }
+ b.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseReverted, nil)
+ b.ok("/finish", "", "none")
+ if _, ok := b.lock(); ok {
+ t.Fatal("the finish after an abort left the lock")
+ }
+ // Another start now takes it.
+ b.start()
+ if a, ok := b.lock(); !ok || a.Op != b.id {
+ t.Fatalf("lock = %+v %v", a, ok)
+ }
+}
+
+func TestOfflineStaleLockIsReplaced(t *testing.T) {
+ put := func(w *offlineWorld, a moves.OfflineActive) {
+ body, _ := json.Marshal(a)
+ offlinePut(t, w.fx.rel, moves.OfflineActiveKey, body)
+ }
+ // A lock naming a host with no record.
+ w := newOfflineWorld(t, oneEach...)
+ put(w, moves.OfflineActive{Host: "gone.go.tito.io", Op: "mv_aaaaaaaaaaaaaaaaaaaaaaaaaa", Account: "bab", App: "titogo", At: time.Now().Add(-time.Hour)})
+ w.start()
+ if a, _ := w.lock(); a.Host != offlineHost || a.Op != w.id {
+ t.Fatalf("lock = %+v", a)
+ }
+ // A lock naming a host whose record carries no such move (a failed release).
+ x := newOfflineWorld(t, oneEach...)
+ if c, _, out := rawReq(t, http.MethodPost, instancesURL(x.fx, "bab", "titogo"), x.token, `{"host":"beta.go.tito.io","region":"ie"}`); c != http.StatusCreated {
+ t.Fatalf("create = %d %s", c, out)
+ }
+ put(x, moves.OfflineActive{Host: "beta.go.tito.io", Op: "mv_aaaaaaaaaaaaaaaaaaaaaaaaaa", Account: "bab", App: "titogo", At: time.Now().Add(-time.Hour)})
+ x.start()
+ // A lock whose record carries another move of that host is not this one's.
+ y := newOfflineWorld(t, oneEach...)
+ put(y, moves.OfflineActive{Host: offlineHost, Op: "mv_aaaaaaaaaaaaaaaaaaaaaaaaaa", Account: "bab", App: "titogo", At: time.Now().Add(-time.Hour)})
+ y.start()
+ // A young lock is believed even before its record carries the move: the
+ // start door takes the lock, then writes the record.
+ f := newOfflineWorld(t, oneEach...)
+ if c, _, out := rawReq(t, http.MethodPost, instancesURL(f.fx, "bab", "titogo"), f.token, `{"host":"beta.go.tito.io","region":"ie"}`); c != http.StatusCreated {
+ t.Fatalf("create = %d %s", c, out)
+ }
+ put(f, moves.OfflineActive{Host: "beta.go.tito.io", Op: "mv_aaaaaaaaaaaaaaaaaaaaaaaaaa", Account: "bab", App: "titogo", At: time.Now()})
+ f.refused(http.MethodPost, "", `{"region":"uk"}`, http.StatusConflict, "beta.go.tito.io is moving offline. move one instance at a time")
+ // An old lock is live while its record carries the move.
+ g := newOfflineWorld(t, oneEach...)
+ if _, code, out := g.startOther("beta.go.tito.io"); code != http.StatusAccepted {
+ t.Fatalf("beta start = %d %s", code, out)
+ }
+ a, _ := g.lock()
+ a.At = time.Now().Add(-24 * time.Hour)
+ put(g, a)
+ g.refused(http.MethodPost, "", `{"region":"uk"}`, http.StatusConflict, "beta.go.tito.io is moving offline. move one instance at a time")
+ // An unreadable lock is not guessed at.
+ z := newOfflineWorld(t, oneEach...)
+ offlinePut(t, z.fx.rel, moves.OfflineActiveKey, []byte("not json"))
+ z.refused(http.MethodPost, "", `{"region":"uk"}`, http.StatusBadGateway, "could not read the instance")
+}
+
+// A record nobody can read elsewhere does not block a start: the lock is the
+// gate, not a listing of every record.
+func TestOfflineStartIgnoresOtherRecords(t *testing.T) {
+ w := newOfflineWorld(t, oneEach...)
+ offlinePut(t, w.fx.rel, provision.RecordKey("bab", "titogo", "junk.go.tito.io"), []byte("{not a record"))
+ w.start()
+}
+
+// A failed record write does not prove nothing was written, so start never
+// releases its lock on one. A write that lands and then reports an error is a
+// start that succeeded; a write that does not land leaves the lock, which
+// goes stale by itself. Either way no other host starts meanwhile.
+func TestOfflineStartKeepsTheLockWhenTheRecordWriteFails(t *testing.T) {
+ for _, landed := range []bool{true, false} {
+ t.Run(fmt.Sprintf("landed=%v", landed), func(t *testing.T) {
+ w := newOfflineWorld(t, oneEach...)
+ if c, _, out := rawReq(t, http.MethodPost, instancesURL(w.fx, "bab", "titogo"), w.token, `{"host":"beta.go.tito.io","region":"ie"}`); c != http.StatusCreated {
+ t.Fatalf("create = %d %s", c, out)
+ }
+ w.fx.srv.cfg.Releases = &recordWriteFault{Store: w.fx.rel, key: provision.RecordKey("bab", "titogo", offlineHost), land: landed}
+ code, out := w.req(http.MethodPost, "", `{"region":"uk"}`)
+ if landed {
+ // The record carries the move: the start succeeded.
+ if code != http.StatusAccepted {
+ t.Fatalf("start = %d %s", code, out)
+ }
+ if v := w.view(code, out); v.State != provision.OfflineStopping || v.ID == "" {
+ t.Fatalf("view = %+v", v)
+ }
+ } else if code != http.StatusBadGateway || errorText(out) != "could not write the move. try again" {
+ t.Fatalf("start = %d %s", code, out)
+ }
+ a, ok := w.lock()
+ if !ok || a.Host != offlineHost {
+ t.Fatalf("lock = %+v %v, want it kept", a, ok)
+ }
+ // A second host's start is refused whichever way the write went.
+ bu := instancesURL(w.fx, "bab", "titogo") + "/beta.go.tito.io/move/offline"
+ c, _, o := rawReq(t, http.MethodPost, bu, w.token, `{"region":"uk"}`)
+ if c != http.StatusConflict || errorText(o) != offlineHost+" is moving offline. move one instance at a time" {
+ t.Fatalf("beta start = %d %s", c, o)
+ }
+ })
+ }
}
diff --git a/internal/console/authcli.go b/internal/console/authcli.go
index 8733e88f6..efcdb15d7 100644
--- a/internal/console/authcli.go
+++ b/internal/console/authcli.go
@@ -718,7 +718,13 @@ func (s *Server) registerCLIRoutes() {
}
// The offline operator move's doors: always registered, operator-only,
// and refused with MV-O20 on a console with host claims or Phase 1 on.
- s.registerOfflineMoveRoutes()
+ s.cliRoute("GET /api/cli/apps/{account}/{app}/instances/{host}/move/offline", svcrole.Allow(svcrole.Instance), s.handleAPIInstanceOfflineStatus)
+ s.cliRoute("POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline", svcrole.Allow(svcrole.Instance), s.handleAPIInstanceOfflineStart)
+ s.cliRoute("POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/seal", svcrole.Allow(svcrole.Instance), s.offlineVerbDoor(offlineVerbSeal))
+ s.cliRoute("POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/flip", svcrole.Allow(svcrole.Instance), s.offlineVerbDoor(offlineVerbFlip))
+ s.cliRoute("POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/open", svcrole.Allow(svcrole.Instance), s.offlineVerbDoor(offlineVerbOpen))
+ s.cliRoute("POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/finish", svcrole.Allow(svcrole.Instance), s.offlineVerbDoor(offlineVerbFinish))
+ s.cliRoute("POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/abort", svcrole.Allow(svcrole.Instance), s.handleAPIInstanceOfflineAbort)
s.cliRoute("DELETE /api/cli/apps/{account}/{app}/instances/{host}", svcrole.Allow(svcrole.Instance), s.handleAPIInstanceDelete)
// The import door (seeded-instances spec §5): the instance role, like
// the create door that begins it.
diff --git a/internal/console/svcroutes_test.go b/internal/console/svcroutes_test.go
index 1777ac98b..f46127714 100644
--- a/internal/console/svcroutes_test.go
+++ b/internal/console/svcroutes_test.go
@@ -529,11 +529,21 @@ var exemptFromLegacyPin = map[string]bool{
"POST /api/cli/apps/{account}/{app}/instances/{host}/move": true,
"POST /api/cli/apps/{account}/{app}/instances/{host}/move/cut": true,
"DELETE /api/cli/apps/{account}/{app}/instances/{host}/move": true,
- "GET /api/cli/apps/{account}/{app}/sidecars": true,
- "POST /api/cli/apps/{account}/{app}/sidecars": true,
- "GET /api/cli/apps/{account}/{app}/sidecars/{name}": true,
- "DELETE /api/cli/apps/{account}/{app}/sidecars/{name}": true,
- "POST /api/cli/apps/{account}/{app}/sidecars/{name}/wake": true,
+ // The offline operator move's doors: each is operator-only behind
+ // moveOperatorGate (a service token meets the family's 404), and
+ // refuses with MV-O20 on a console with host claims or Phase 1 on.
+ "GET /api/cli/apps/{account}/{app}/instances/{host}/move/offline": true,
+ "POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline": true,
+ "POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/seal": true,
+ "POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/flip": true,
+ "POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/open": true,
+ "POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/finish": true,
+ "POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/abort": true,
+ "GET /api/cli/apps/{account}/{app}/sidecars": true,
+ "POST /api/cli/apps/{account}/{app}/sidecars": true,
+ "GET /api/cli/apps/{account}/{app}/sidecars/{name}": true,
+ "DELETE /api/cli/apps/{account}/{app}/sidecars/{name}": true,
+ "POST /api/cli/apps/{account}/{app}/sidecars/{name}/wake": true,
// The platform's GeoIP lookup (geoip spec §11.1) is new access on the
// analytics read's footing: it did not exist before service principals,
// and the app itself is the consumer (a batch job or a webhook handler
diff --git a/internal/host/agent.go b/internal/host/agent.go
index 5ccaa3955..06c3f9c5e 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -1452,6 +1452,14 @@ var reconcileMu sync.Mutex
// not hide an instance-side one, and neither must stop the other running —
// each pass is independently safe to run and independently valuable.
func reconcileSerialized(ctx context.Context, s release.Store, reg *registry.Registry, cfg provision.ReconcileConfig) error {
+ return reconcileSerializedWith(ctx, s, reg, cfg, armedMovesWorker.Load())
+}
+
+// reconcileSerializedWith is reconcileSerialized with the moves worker whose
+// offline hooks the pass takes (movesoffline.go): the armed worker in the
+// agent, a box's own worker in a test. A nil worker, or one on a box that
+// cannot take part in an offline move, adds nothing.
+func reconcileSerializedWith(ctx context.Context, s release.Store, reg *registry.Registry, cfg provision.ReconcileConfig, w *movesWorker) error {
reconcileMu.Lock()
defer reconcileMu.Unlock()
// A fresh row is minted at the layout this box's intent names
@@ -1472,6 +1480,17 @@ func reconcileSerialized(ctx context.Context, s release.Store, reg *registry.Reg
// Move holds (moves spec §9.6): participant rows minted held and kept.
// Unchanged with the Phase 1 switch off.
cfg = withMovesHolds(cfg)
+ // Offline move holds: the hold from the record alone, the keep from the
+ // view, which is read here first if no listing has been read yet or the
+ // last one failed, so keep never fails closed on a box whose listing
+ // reads. Nothing on a box that cannot take part, and nothing where
+ // Phase 1's hooks are set.
+ if w != nil && w.cfg.OfflineCapable && w.cfg.Offline != nil {
+ if !w.cfg.Offline.built() || w.cfg.Offline.failed() {
+ refreshOfflineView(ctx, s, w.cfg.Offline)
+ }
+ cfg = withOfflineHolds(cfg, offlineHold(w.cfg), w.cfg.Offline)
+ }
// Runs ride this pass's record reads (run spec §5.2): the worker notes
// every in-scope record with a run, and acts after the pass.
cfg = withRunsHook(cfg)
@@ -2594,6 +2613,10 @@ func fastConvergeLoop(reg *registry.Registry, svc *activator.Service, sd *schedu
// runs before the first moves pass, and an unbuilt view freezes every
// host (moves spec §9.4). Nothing with the Phase 1 switch off.
refreshMovesHolds(ctx, store)
+ // The offline keep view, likewise before any reconcile pass: once it is
+ // read, keep answers false for every host with no offline move, so the
+ // first pass removes what it always has (movesoffline.go).
+ refreshOfflineView(ctx, store, armedOfflineView())
// carlos run's boot pass (run spec §5.2, index item 2): reconcile's
// first pass is a minute away and the CLI expires a run at 60 s.
runRunsBootPass(ctx)
diff --git a/internal/host/edgesfile.go b/internal/host/edgesfile.go
index c02520656..150f394a0 100644
--- a/internal/host/edgesfile.go
+++ b/internal/host/edgesfile.go
@@ -9,6 +9,7 @@ import (
"log"
"math/rand/v2"
"os"
+ "slices"
"strings"
"sync"
"time"
@@ -162,7 +163,8 @@ type edgeHeadroom struct {
const capRegionHA1 = "region-ha-1"
// edgesFileCapabilities publishes region-ha-1 only when CARLOS_REGION_HA is
-// set to something other than "" or "0".
+// set to something other than "" or "0". offline-move-1 is added per pass
+// by withOfflineCapability.
func edgesFileCapabilities(getenv func(string) string) []string {
if !regionHAEnabled(getenv) {
return nil
@@ -170,6 +172,28 @@ func edgesFileCapabilities(getenv func(string) string) []string {
return []string{capRegionHA1}
}
+// withOfflineCapability is caps with offline-move-1 when capable. The loop
+// passes armedOfflineCapable(): the armed moves worker's own answer, the one
+// the reconcile hooks act on, so a box never publishes the capability
+// without the hold that makes it safe (a target that published it with no
+// armed worker would mint a landing row ready). caps is not modified.
+func withOfflineCapability(caps []string, capable bool) []string {
+ if !capable {
+ return caps
+ }
+ return append(slices.Clone(caps), capOfflineMove1)
+}
+
+// headroomCapabilities is what headroomFor is asked about: region HA's
+// capability alone. offline-move-1 publishes no headroom, so a box that
+// gains only it publishes the file it always has, plus the one string.
+func headroomCapabilities(caps []string) []string {
+ if slices.Contains(caps, capRegionHA1) {
+ return caps
+ }
+ return nil
+}
+
// awakeCount is how many routes this box has running now. An error reads
// as zero: headroom is advisory.
func awakeCount(reg *registry.Registry) int {
@@ -815,6 +839,9 @@ func edgesFileLoop(reg *registry.Registry) {
BoxID: box,
Capabilities: edgesFileCapabilities(os.Getenv),
}
+ // The flag-derived capabilities; offline-move-1 joins them on each pass
+ // once the moves worker is armed and capable.
+ baseCaps := ident.Capabilities
dataDir := provisionTickConfig(os.Getenv).DataDir
optDir := release.OptDir()
// The local floor files this file publishes are refreshed hourly by
@@ -859,7 +886,8 @@ func edgesFileLoop(reg *registry.Registry) {
if state.timed() {
eip = readPublicIP()
}
- ident.Headroom = headroomFor(ident.Capabilities, dataDir, reg)
+ ident.Capabilities = withOfflineCapability(baseCaps, armedOfflineCapable())
+ ident.Headroom = headroomFor(headroomCapabilities(ident.Capabilities), dataDir, reg)
ident = withMovesFields(ident, dataDir, optDir)
ident = withSeedsFields(ident, dataDir, optDir)
err = edgesFileWriteClaims(ctx, store, reg, ident, eip, &last, state.timed(), claims)
diff --git a/internal/host/movesoffline.go b/internal/host/movesoffline.go
new file mode 100644
index 000000000..9b1832ef6
--- /dev/null
+++ b/internal/host/movesoffline.go
@@ -0,0 +1,976 @@
+package host
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log"
+ "os"
+ "path"
+ "path/filepath"
+ "sort"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "syscall"
+ "time"
+
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/hostwrite"
+ "github.com/carlosframework/platform/internal/instancectl"
+ "github.com/carlosframework/platform/internal/layout"
+ "github.com/carlosframework/platform/internal/lease"
+ "github.com/carlosframework/platform/internal/moves"
+ "github.com/carlosframework/platform/internal/park"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// The offline operator move on a box (docs/superpowers/plans/2026-10-20-
+// offline-operator-move.md, Task 3): the hold reconcile asks, the keep view,
+// the capability, the moves pass's dispatch, and the source boxes' steps.
+// The sealer's step is movesoffline_seal.go (Task 4) and the target's is
+// movesoffline_land.go (Task 5).
+//
+// The stop has three layers. This file holds the first two on every
+// source-region box: reconcile holds the row from the record alone
+// (offlineHold), and each pass fences the activator and writes the box's
+// source report. The third, the tombstone lease, is the sealer's.
+//
+// Dormant, with no offline record anywhere, nothing here acts: the hold
+// answers false for every record, keep answers false once the startup
+// listing is read, the dispatch matches no record, and the edges file
+// carries one more capability string.
+
+// capOfflineMove1 is the edges-file capability a box publishes when it can
+// take part in an offline move (offlineCapable). The console's start door
+// refuses a region holding a box that does not publish it.
+const capOfflineMove1 = "offline-move-1"
+
+// offlineCapable is the box's own witness that it can take part in an
+// offline move: hibernation is configured and neither CARLOS_HOST_CLAIM nor
+// CARLOS_MOVES_PHASE1 is set to anything at all. The guard mode is not read
+// (platform-1e's ruling of 2026-10-20, the plan's Global Constraints): the
+// target's build stages through ordinary adoption, which root's own floor
+// check already covers on a guarded box. A box that is not capable publishes
+// no capability and runs no offline step.
+func offlineCapable(getenv func(string) string, hibernation bool) bool {
+ if !hibernation || hostwrite.Enabled(getenv) {
+ return false
+ }
+ return strings.TrimSpace(getenv(moves.SwitchEnv)) == "" && !moves.Enabled(getenv)
+}
+
+// hibernationConfigured is the edges file's HibernationConfigured, read the
+// same way, so the capability and the field beside it never disagree.
+func hibernationConfigured(getenv func(string) string) bool {
+ return strings.TrimSpace(getenv("CARLOS_HIBERNATE_BUCKET")) != ""
+}
+
+// offlineFenceOwner is the activator fence owner of one offline move.
+func offlineFenceOwner(id string) string { return "offline:" + id }
+
+// offlineHold is reconcile's Hold for offline moves: a pure function of the
+// record it is handed and this box's placement, never of a snapshot (Fable's
+// B1). A target box that restarts after the flip reconciles before its first
+// moves pass, and must mint the landing row held even then.
+//
+// - A source-region box (placement equals move.from) holds the host's row
+// "moving" under the move id from offline-stopping through
+// offline-sealed. After the flip the record names the target, the row
+// is out of this box's scope and offlineView.keep keeps it, held as it
+// is, until offline-finishing.
+// - A target-region box (placement equals move.to) holds it "landing"
+// while the record names the target and the move is offline-landing or
+// offline-landed. offline-open is the target's to release (Task 5).
+//
+// Placements are compared as movesNamesBox compares them, by
+// park.PlacementMatches under the verified home binding, so the home's ""
+// and its roster name are one.
+func offlineHold(box moveCfg) func(rec provision.Record) (status, op string, ok bool) {
+ home, here := box.ParkHomeRegion, box.parkRegion()
+ return func(rec provision.Record) (string, string, bool) {
+ m := rec.Move
+ if !m.Offline() || m.ID == "" || here == "" {
+ return "", "", false
+ }
+ switch m.State {
+ case provision.OfflineStopping, provision.OfflineSealing, provision.OfflineSealed:
+ if park.PlacementMatches(home, m.From, here) {
+ return "moving", m.ID, true
+ }
+ case provision.OfflineLanding, provision.OfflineLanded:
+ if rec.Region == m.To && park.PlacementMatches(home, m.To, here) {
+ return "landing", m.ID, true
+ }
+ }
+ return "", "", false
+ }
+}
+
+// offlineView is what reconcile's KeepParticipant asks about offline moves:
+// the offline moves in flight, as the last listing read them. Only keep
+// reads it; the hold never does (offlineHold).
+//
+// A source box's view has seen the move before the record flips: the seal
+// door needs this box's report for the move, and the pass that writes it
+// refreshes the view first; an agent restart reads the listing again at
+// start. From then until finishing every listing carries the move.
+//
+// It is refreshed from every moves pass's record list and once at agent
+// start, before the reconcile loops (refreshOfflineView). keep fails closed,
+// keeping every row, only while no listing has been read (reconcileSerialized-
+// With reads one first when the view is unbuilt or its last listing failed,
+// so a failed startup listing heals on the next reconcile), and for a host
+// whose record does not decode. Once a listing is read, a host with no
+// offline move is not kept, so removals happen on the first pass as they
+// always have.
+type offlineView struct {
+ home, region string
+ // written is reconcile's HoldWritten for the offline hooks: the
+ // stage-requested marker after a hold write (registry.SetHold's
+ // contract, G10). Nil writes nothing.
+ written func(host string)
+ snap atomic.Pointer[offlineSnapshot]
+ logged sync.Map // host -> time.Time: the last unreadable-record line
+}
+
+type offlineSnapshot struct {
+ moves map[string]provision.MoveState // host -> its offline move
+ failed bool // the listing failed: keep everything
+ // unreadable are the hosts whose record read but did not decode, or
+ // disagreed with its path (provision.ListAllRecordsStrict): their move
+ // cannot be known, so their rows are kept.
+ unreadable map[string]bool
+}
+
+// newOfflineView builds the view for the box cfg describes. Unbuilt until
+// its first refresh.
+func newOfflineView(cfg moveCfg, written func(host string)) *offlineView {
+ return &offlineView{home: cfg.ParkHomeRegion, region: cfg.parkRegion(), written: written}
+}
+
+// refresh rebuilds the view from a record list that was read, with the
+// hosts provision.ListAllRecordsStrict could not decode: those are kept
+// (fail closed for keep only; the hold never reads the view) and logged at
+// most once a minute each.
+func (v *offlineView) refresh(recs []provision.Record, unreadable map[string]bool) {
+ next := &offlineSnapshot{moves: map[string]provision.MoveState{}, unreadable: map[string]bool{}}
+ for _, rec := range recs {
+ if rec.Move.Offline() {
+ next.moves[rec.Host] = *rec.Move
+ }
+ }
+ now := time.Now()
+ for host := range unreadable {
+ next.unreadable[host] = true
+ if last, ok := v.logged.Load(host); !ok || now.Sub(last.(time.Time)) >= time.Minute {
+ v.logged.Store(host, now)
+ log.Printf("moves: offline view: %s: the record does not decode; its row is kept until it does", host)
+ }
+ }
+ v.snap.Store(next)
+}
+
+// fail records a listing that could not be read, before any was: keep
+// answers true for every host until a listing is read.
+func (v *offlineView) fail() { v.snap.Store(&offlineSnapshot{failed: true}) }
+
+// built reports whether any listing, read or failed, has been recorded.
+func (v *offlineView) built() bool { return v.snap.Load() != nil }
+
+// failed reports whether the last listing recorded failed: keep answers true
+// for every host until one reads.
+func (v *offlineView) failed() bool {
+ snap := v.snap.Load()
+ return snap != nil && snap.failed
+}
+
+// keep is reconcile's KeepParticipant: the row of a host whose offline move
+// names this box's region stays, whatever the record's region says. The
+// source keeps it until offline-finishing (when it is removed, as the plan's
+// finishing step relies on); the target keeps it until the move is cleared
+// (the target's revert deletes it itself). An unknown state keeps.
+func (v *offlineView) keep(host string) bool {
+ snap := v.snap.Load()
+ if snap == nil || snap.failed || snap.unreadable[host] {
+ return true
+ }
+ m, ok := snap.moves[host]
+ if !ok || v.region == "" {
+ return false
+ }
+ if park.PlacementMatches(v.home, m.To, v.region) {
+ return true
+ }
+ if park.PlacementMatches(v.home, m.From, v.region) {
+ return m.State != provision.OfflineFinishing
+ }
+ return false
+}
+
+// refreshOfflineView reads the record list once, outside a moves pass: at
+// agent start, before the reconcile loops, next to refreshMovesHolds. A
+// failed listing leaves keep failing closed until a listing is read.
+func refreshOfflineView(ctx context.Context, s release.Store, v *offlineView) {
+ if v == nil || s == nil {
+ return
+ }
+ recs, unreadable, err := provision.ListAllRecordsStrict(ctx, s)
+ if err != nil {
+ log.Printf("moves: offline view: list records: %v; every row is kept until a listing reads", err)
+ v.fail()
+ return
+ }
+ v.refresh(recs, unreadable)
+}
+
+// armedOfflineView is the armed worker's view, nil when no worker is armed
+// or the box cannot take part in an offline move.
+func armedOfflineView() *offlineView {
+ w := armedMovesWorker.Load()
+ if w == nil || !w.cfg.OfflineCapable {
+ return nil
+ }
+ return w.cfg.Offline
+}
+
+// armedOfflineCapable is whether this box publishes offline-move-1: the
+// armed worker is capable and has its keep view, exactly the condition
+// under which reconcileSerializedWith installs the offline hooks.
+func armedOfflineCapable() bool { return armedOfflineView() != nil }
+
+// withOfflineHolds gives a reconcile pass the offline hooks, only when the
+// caller and Phase 1's view set none (withMovesHolds runs first): the two
+// are never armed together, since a box with Phase 1 on is not capable.
+func withOfflineHolds(cfg provision.ReconcileConfig, hold func(provision.Record) (string, string, bool), v *offlineView) provision.ReconcileConfig {
+ if hold == nil || v == nil || cfg.Hold != nil || cfg.KeepParticipant != nil {
+ return cfg
+ }
+ cfg.Hold, cfg.KeepParticipant = hold, v.keep
+ if cfg.HoldWritten == nil && v.written != nil {
+ cfg.HoldWritten = v.written
+ }
+ return cfg
+}
+
+// newMovesWorker is the worker for cfg: armMovesPass's, and each test box's.
+func newMovesWorker(cfg moveCfg) *movesWorker {
+ return &movesWorker{cfg: cfg, mirrors: map[string]lease.MirrorStore{}}
+}
+
+// fenceAsWith is moveCfg.FenceAs over an activator: FenceAndStopAs under a
+// short deadline, so one slow stop does not hold up the pass.
+// ErrStopUnproven still means the fence is installed, which is all the step
+// asks of it; Quiesced answers the rest.
+func fenceAsWith(svc *activator.Service) func(ctx context.Context, host string, generation uint64, owner string) error {
+ return func(ctx context.Context, host string, generation uint64, owner string) error {
+ fctx, cancel := context.WithTimeout(ctx, 2*time.Second)
+ defer cancel()
+ if _, err := svc.FenceAndStopAs(fctx, host, generation, owner); err != nil && !errors.Is(err, activator.ErrStopUnproven) {
+ return err
+ }
+ return nil
+ }
+}
+
+// hostSliceCgroup is the absolute cgroup directory of a nested host's
+// systemd slice under cgroupRoot (the cgroup v2 mount): every role and
+// generation unit of the host runs in it.
+func hostSliceCgroup(cgroupRoot, host string) string {
+ return filepath.Join(cgroupRoot, filepath.FromSlash(path.Dir(instancectl.ClaimUnitControlGroup(host))))
+}
+
+// offlineRound is the moves pass's round for a record with an offline move,
+// reached before every other branch of the pass (the Phase 1 skip and the
+// lost-box gate included). A box that is not capable logs one notice per
+// host and does nothing. Otherwise it runs the source step when its
+// placement is move.from, the sealer's when it is move.sealer, and the
+// target's when its placement is move.to, in that order, so the sealer
+// reads the report this box has just written.
+func (w *movesWorker) offlineRound(ctx context.Context, s release.Store, rec provision.Record) error {
+ cfg := w.cfg
+ m := rec.Move
+ if !m.Offline() {
+ return nil
+ }
+ here := cfg.parkRegion()
+ source := here != "" && park.PlacementMatches(cfg.ParkHomeRegion, m.From, here)
+ target := here != "" && park.PlacementMatches(cfg.ParkHomeRegion, m.To, here)
+ sealer := cfg.Box != "" && cfg.Box == m.Sealer
+ if !cfg.OfflineCapable {
+ if source || target || sealer {
+ w.notice(rec.Host+"#offline", "moves: %s: offline move %s waits: this box cannot take part in an offline move (it needs hibernation configured, and %s and %s unset)", rec.Host, m.ID, "CARLOS_HOST_CLAIM", moves.SwitchEnv)
+ }
+ return nil
+ }
+ var errs []error
+ if source {
+ errs = append(errs, w.offlineSourceStep(ctx, s, rec))
+ }
+ if sealer {
+ errs = append(errs, w.offlineSealerStep(ctx, s, rec))
+ }
+ if target {
+ errs = append(errs, w.offlineTargetStep(ctx, s, rec))
+ }
+ return errors.Join(errs...)
+}
+
+// now is cfg.Now, or the wall clock when a test left it unset.
+func (c moveCfg) now() time.Time {
+ if c.Now != nil {
+ return c.Now()
+ }
+ return time.Now()
+}
+
+// writeOfflineReport writes r at its key (moves.OfflineReportKey), stamped
+// with the report version, after the shape check every reader makes. The
+// source, sealer and target steps all write through it.
+func writeOfflineReport(ctx context.Context, s release.Store, r moves.OfflineReport) error {
+ r.V = moves.OfflineReportVersion
+ key := moves.OfflineReportKey(r.Host, r.Op, r.Role, r.Box)
+ if key == "" {
+ return fmt.Errorf("offline report: host %q, op %q, role %q or box %q is not usable", r.Host, r.Op, r.Role, r.Box)
+ }
+ if err := moves.ValidateOfflineReport(r); err != nil {
+ return err
+ }
+ body, err := json.Marshal(r)
+ if err != nil {
+ return err
+ }
+ // A report is rewritten every pass: read its etag and replace it
+ // (release.Store.Put with no etag is create-only).
+ return moves.PutReplace(ctx, s, key, body)
+}
+
+// putOfflineCreateOnly writes body at key only if nothing is there yet:
+// the immutable seal record (control/offline-moves/<host>/<id>/seal.json,
+// moves.OfflineSealKey, written once by the sealer and never rewritten).
+// It is an ensure: when the key already exists it writes nothing and
+// returns what is stored, with created false, so a sealer re-running after
+// a crash compares rather than overwrites, and a reader (flip, the landing)
+// always sees the first write. key must be non-empty.
+func putOfflineCreateOnly(ctx context.Context, s release.Store, key string, body []byte) (stored []byte, created bool, err error) {
+ if key == "" {
+ return nil, false, errors.New("offline move: no key to write")
+ }
+ _, err = s.Put(ctx, key, body, "")
+ if err == nil {
+ return body, true, nil
+ }
+ if !errors.Is(err, release.ErrCASConflict) {
+ return nil, false, fmt.Errorf("write %s: %w", key, err)
+ }
+ stored, _, err = s.Get(ctx, key)
+ if err != nil {
+ return nil, false, fmt.Errorf("read %s after a create-only write found it: %w", key, err)
+ }
+ return stored, false, nil
+}
+
+// readOfflineReports reads every report of one role for one move, keyed by
+// box. A report that does not validate is an error, never skipped.
+func readOfflineReports(ctx context.Context, s release.Store, host, op, role string) (map[string]moves.OfflineReport, error) {
+ prefix := moves.OfflineReportPrefix(host, op)
+ if prefix == "" {
+ return nil, fmt.Errorf("offline report: host %q or op %q is not usable", host, op)
+ }
+ prefix += role + "/"
+ keys, err := s.List(ctx, prefix)
+ if err != nil {
+ return nil, err
+ }
+ out := map[string]moves.OfflineReport{}
+ for _, key := range keys {
+ if !strings.HasSuffix(key, ".json") {
+ continue
+ }
+ body, _, err := s.Get(ctx, key)
+ if err != nil {
+ return nil, err
+ }
+ var r moves.OfflineReport
+ if err := json.Unmarshal(body, &r); err != nil {
+ return nil, fmt.Errorf("%s: %w", key, err)
+ }
+ if err := moves.ValidateOfflineReport(r); err != nil {
+ return nil, fmt.Errorf("%s: %w", key, err)
+ }
+ if moves.OfflineReportKey(r.Host, r.Op, r.Role, r.Box) != key {
+ return nil, fmt.Errorf("%s: the report names %s/%s/%s/%s", key, r.Host, r.Op, r.Role, r.Box)
+ }
+ out[r.Box] = r
+ }
+ return out, nil
+}
+
+// offlineSourceStep is a source-region box's part of an offline move. Every
+// state's step is an ensure: it reads what is on disk, in the registry and
+// in the activator, does only what is missing, and reports what it found.
+func (w *movesWorker) offlineSourceStep(ctx context.Context, s release.Store, rec provision.Record) error {
+ if w.cfg.Box == "" {
+ return errors.New("offline move: this box has no label to report under")
+ }
+ switch rec.Move.State {
+ case provision.OfflineStopping, provision.OfflineSealing, provision.OfflineSealed, provision.OfflineLanding, provision.OfflineLanded:
+ return w.offlineSourceStop(ctx, s, rec)
+ case provision.OfflineOpen:
+ return w.offlineSourcePark(ctx, s, rec)
+ case provision.OfflineFinishing:
+ return w.offlineSourceGone(ctx, s, rec)
+ case provision.OfflineReverting:
+ return w.offlineSourceRevert(ctx, s, rec)
+ }
+ return nil
+}
+
+// offlineFence installs, or meets again, this move's activator fence for
+// host. Fences live in the activator's memory, so every pass installs it
+// again and an agent restart regains it on its next pass. True when the
+// fence is this move's now.
+func (w *movesWorker) offlineFence(ctx context.Context, host string, m *provision.MoveState) bool {
+ if w.cfg.FenceAs == nil {
+ return false
+ }
+ if err := w.cfg.FenceAs(ctx, host, moveFenceGeneration(m), offlineFenceOwner(m.ID)); err != nil {
+ w.notice(host+"#offline-fence", "moves: %s: offline move %s: the wake fence is not installed: %v", host, m.ID, err)
+ return false
+ }
+ return true
+}
+
+// offlineLiftFence lifts this move's fence, and only it.
+func (w *movesWorker) offlineLiftFence(host string, m *provision.MoveState) {
+ if w.cfg.LiftFenceAs != nil && w.cfg.LiftFenceAs(host, moveFenceGeneration(m), offlineFenceOwner(m.ID)) {
+ log.Printf("moves: %s: offline move %s: wake fence lifted", host, m.ID)
+ }
+}
+
+// offlineRow reads the host's row on this box; nil when it has none.
+func (w *movesWorker) offlineRow(host string) (*registry.Route, error) {
+ if w.cfg.Route == nil {
+ return nil, errors.New("offline move: no registry on this box")
+ }
+ return w.cfg.Route(host)
+}
+
+// heldBy reports whether rt is held by op. An absent row counts: nothing
+// can serve or wake a host this box has no row for.
+func heldBy(rt *registry.Route, op string) bool {
+ return rt == nil || (rt.HoldOp == op && registry.IsHold(rt.Status))
+}
+
+// offlineLocal is what a box keeps on disk for a host, judged by lstat and
+// directory listings only, never by following a link. A row's own layout is
+// read at the row's paths; a box with no row, or a leftover of the other
+// layout, is read at both.
+type offlineLocal struct {
+ LocalDB, Files, Env, Sidecar bool
+ LocalTXID uint64
+}
+
+// flatDBPath is where a flat row's database is: the row's own path when it
+// has one and is flat, else the default.
+func (c moveCfg) flatDBPath(host string, rt *registry.Route) string {
+ if rt != nil && rt.DBPath != "" && !provision.NestedPaths(c.DataDir, *rt) {
+ return rt.DBPath
+ }
+ return provision.DefaultDBPath(c.DataDir, host)
+}
+
+// activatorMetaDir is the activator's litestream meta directory beside a
+// database (localLTXPosition reads it).
+func activatorMetaDir(dbPath string) string {
+ return filepath.Join(filepath.Dir(dbPath), ".activator-"+filepath.Base(dbPath)+"-litestream")
+}
+
+func lexists(p string) bool {
+ _, err := os.Lstat(p)
+ return err == nil
+}
+
+func isDir(p string) bool {
+ st, err := os.Lstat(p)
+ return err == nil && st.IsDir()
+}
+
+// offlinePlatformNames are a nested instance directory's platform entries:
+// everything else in it is the app's files.
+func offlinePlatformNames(host string) map[string]bool {
+ db := host + ".db"
+ return map[string]bool{
+ db: true, db + "-wal": true, db + "-shm": true, db + "-litestream": true,
+ ".activator-" + db + "-litestream": true, db + ".corrupt": true, db + ".gen": true,
+ host + ".env": true, host + ".config.env": true, host + "-sidecar.env": true,
+ host + ".instance-key-adoption.json": true, ".files-gen": true,
+ }
+}
+
+// localState reads what this box keeps for host: the DB file or either
+// litestream meta dir (S8: a meta dir alone is local state, since
+// localLTXPosition reads it), the files directory, the env file and a
+// sidecar env file, under the row's layout and any leftover of the other.
+func (w *movesWorker) localState(host string, rt *registry.Route) (offlineLocal, error) {
+ cfg := w.cfg
+ var out offlineLocal
+ nestedRow := rt != nil && provision.NestedPaths(cfg.DataDir, *rt)
+ // Flat: a flat row's own paths, or a nested row's flat leftovers.
+ {
+ db := cfg.flatDBPath(host, rt)
+ out.LocalDB = lexists(db) || lexists(db+"-litestream") || lexists(activatorMetaDir(db))
+ out.Files = isDir(filepath.Join(cfg.DataDir, host))
+ out.Env = lexists(filepath.Join(cfg.DataDir, host+".env"))
+ out.Sidecar = lexists(filepath.Join(cfg.DataDir, host+"-sidecar.env"))
+ pos, err := localLTXPosition(db)
+ if err != nil {
+ return out, err
+ }
+ out.LocalTXID = pos
+ }
+ // Nested, read through the raw directory (layout.RawInstanceDir).
+ raw := layout.RawInstanceDir(cfg.DataDir, host)
+ if nestedRow || isDir(raw) {
+ db := filepath.Join(raw, host+".db")
+ out.LocalDB = out.LocalDB || lexists(db) || lexists(db+"-litestream") || lexists(activatorMetaDir(db))
+ out.Env = out.Env || lexists(filepath.Join(raw, host+".env"))
+ out.Sidecar = out.Sidecar || lexists(filepath.Join(raw, host+"-sidecar.env"))
+ ents, err := os.ReadDir(raw)
+ if err != nil && !errors.Is(err, os.ErrNotExist) {
+ return out, err
+ }
+ platform := offlinePlatformNames(host)
+ for _, e := range ents {
+ if !platform[e.Name()] {
+ out.Files = true
+ break
+ }
+ }
+ dbPath := db
+ if nestedRow && rt.DBPath != "" {
+ dbPath = rt.DBPath
+ }
+ pos, err := localLTXPosition(dbPath)
+ if err != nil {
+ return out, err
+ }
+ out.LocalTXID = max(out.LocalTXID, pos)
+ }
+ return out, nil
+}
+
+// procs is the process count for the report: -1 when it cannot be read,
+// which the flip door reads as not stopped.
+func (w *movesWorker) procs(host string) int {
+ if w.cfg.Procs == nil {
+ return -1
+ }
+ n, err := w.cfg.Procs(host)
+ if err != nil {
+ w.notice(host+"#offline-procs", "moves: %s: the host's processes cannot be counted on this box: %v", host, err)
+ return -1
+ }
+ return n
+}
+
+// sourceReport is the source report's evidence as it stands now.
+func (w *movesWorker) sourceReport(host string, m *provision.MoveState, rt *registry.Route, fenced bool, phase string) (moves.OfflineReport, error) {
+ cfg := w.cfg
+ local, err := w.localState(host, rt)
+ if err != nil {
+ return moves.OfflineReport{}, err
+ }
+ quiesced := cfg.Quiesced != nil && cfg.Quiesced(host)
+ return moves.OfflineReport{
+ Op: m.ID, Host: host, Box: cfg.Box, Role: moves.OfflineRoleSource, Phase: phase, At: cfg.now().UTC(),
+ Held: heldBy(rt, m.ID), Fenced: fenced, Quiesced: quiesced, RowPresent: rt != nil,
+ LocalDB: local.LocalDB, Files: local.Files, Env: local.Env, Sidecar: local.Sidecar, LocalTXID: local.LocalTXID,
+ Procs: w.procs(host), LayoutJournal: layout.JournalPresent(cfg.DataDir, host),
+ }, nil
+}
+
+// offlineSourceStop is the source step from offline-stopping through
+// offline-landed: install the fence, read the row and the local state, and
+// rewrite this box's source report with At on every pass (the open door
+// needs it current). The row's hold is reconcile's (offlineHold) and,
+// after the flip, its keep (offlineView.keep).
+func (w *movesWorker) offlineSourceStop(ctx context.Context, s release.Store, rec provision.Record) error {
+ m := rec.Move
+ fenced := w.offlineFence(ctx, rec.Host, m)
+ rt, err := w.offlineRow(rec.Host)
+ if err != nil {
+ return err
+ }
+ r, err := w.sourceReport(rec.Host, m, rt, fenced, moves.OfflinePhaseStopped)
+ if err != nil {
+ return err
+ }
+ return writeOfflineReport(ctx, s, r)
+}
+
+// retirePath is one path of the manifest's source-retire row on this box:
+// where it is, its name in the park directory, and, for a path the agent
+// never renames, why.
+type retirePath struct {
+ path, name, left string
+}
+
+// Why a path is left where it is.
+const (
+ leftRootOwned = "root-owned: listed for the census, not renamed"
+ leftView = "the mount view of the instance directory: it stays, empty"
+ leftInstance = "the instance directory stays, empty"
+)
+
+// nestedParkPrefix is where a nested instance directory's entries go inside
+// the park directory, apart from the flat names.
+const nestedParkPrefix = "instance"
+
+// retirePaths is the manifest's source-retire row as paths on this box, in
+// a fixed order: the row's layout in full, the other layout's paths only
+// where they exist, the app's own entries of a nested instance directory as
+// found there and in the park directory, then what is never renamed.
+func (w *movesWorker) retirePaths(host, parkDir string, rt *registry.Route) ([]retirePath, error) {
+ cfg := w.cfg
+ nestedRow := rt != nil && provision.NestedPaths(cfg.DataDir, *rt)
+ raw := layout.RawInstanceDir(cfg.DataDir, host)
+ var out []retirePath
+ add := func(p, name string, always bool) {
+ if always || lexists(p) {
+ out = append(out, retirePath{path: p, name: name})
+ }
+ }
+ flatDB := cfg.flatDBPath(host, rt)
+ flat := !nestedRow
+ add(flatDB, filepath.Base(flatDB), flat)
+ add(flatDB+"-wal", filepath.Base(flatDB)+"-wal", flat)
+ add(flatDB+"-shm", filepath.Base(flatDB)+"-shm", flat)
+ add(flatDB+"-litestream", filepath.Base(flatDB)+"-litestream", flat)
+ add(activatorMetaDir(flatDB), filepath.Base(activatorMetaDir(flatDB)), flat)
+ add(filepath.Join(cfg.DataDir, host), host, flat)
+ add(filepath.Join(cfg.DataDir, host+".env"), host+".env", flat)
+ add(filepath.Join(cfg.DataDir, host+".config.env"), host+".config.env", flat)
+
+ nested := nestedRow || isDir(raw)
+ if nested {
+ named := map[string]bool{}
+ for _, n := range []string{host + ".db", host + ".db-wal", host + ".db-shm", host + ".db-litestream", ".activator-" + host + ".db-litestream", host + ".env", host + ".config.env"} {
+ named[n] = true
+ add(filepath.Join(raw, n), nestedParkPrefix+"/"+n, true)
+ }
+ // The app's files, and every other entry the directory holds: what
+ // is still there, and what an earlier pass already parked.
+ // A numbered copy in the park directory (parkName) belongs to the
+ // entry it is a copy of, not to the app.
+ others := map[string]bool{}
+ listed := map[string][]string{}
+ for _, dir := range []string{raw, filepath.Join(parkDir, nestedParkPrefix)} {
+ ents, err := os.ReadDir(dir)
+ if err != nil && !errors.Is(err, os.ErrNotExist) {
+ return nil, err
+ }
+ for _, e := range ents {
+ listed[dir] = append(listed[dir], e.Name())
+ }
+ }
+ known := map[string]bool{}
+ for n := range named {
+ known[n] = true
+ }
+ for _, n := range listed[raw] {
+ known[n] = true
+ }
+ for _, n := range listed[filepath.Join(parkDir, nestedParkPrefix)] {
+ known[n] = true
+ }
+ for _, n := range listed[raw] {
+ if !named[n] {
+ others[n] = true
+ }
+ }
+ for _, n := range listed[filepath.Join(parkDir, nestedParkPrefix)] {
+ if !named[n] && !isNumberedCopy(n, known) {
+ others[n] = true
+ }
+ }
+ names := make([]string, 0, len(others))
+ for n := range others {
+ names = append(names, n)
+ }
+ sort.Strings(names)
+ for _, n := range names {
+ add(filepath.Join(raw, n), nestedParkPrefix+"/"+n, true)
+ }
+ }
+ add(filepath.Join(cfg.DataDir, "programs", host+".json"), host+".json", true)
+ if nested {
+ out = append(out, retirePath{path: provision.InstanceDir(cfg.DataDir, host), left: leftView})
+ out = append(out, retirePath{path: raw, left: leftInstance})
+ }
+ optDir := cfg.OptDir
+ if optDir == "" {
+ optDir = release.OptDir()
+ }
+ out = append(out, retirePath{path: filepath.Join(optDir, "live", host), left: leftRootOwned})
+ return out, nil
+}
+
+// parkCopySep separates a name from its copy number in the park directory
+// (parkName). Chosen to be unlikely in an app's own file names.
+const parkCopySep = ".parked-"
+
+// isNumberedCopy reports whether n is a known entry's numbered park copy
+// (parkName's "<name>.parked-<n>"), which belongs to that entry.
+func isNumberedCopy(n string, known map[string]bool) bool {
+ i := strings.LastIndex(n, parkCopySep)
+ if i <= 0 || !known[n[:i]] || i+len(parkCopySep) == len(n) {
+ return false
+ }
+ for _, c := range n[i+len(parkCopySep):] {
+ if c < '0' || c > '9' {
+ return false
+ }
+ }
+ return true
+}
+
+// parkName is where name goes in dir: name itself, or name.parked-<n> for
+// the first free n when something of that name was parked before (the
+// source path came back after an earlier pass parked it). The park
+// directory is the move's own, so the first park of every path lands at its
+// plain name and a later pass reports it from there.
+func parkName(dir, name string) string {
+ dst := filepath.Join(dir, name)
+ if !lexists(dst) {
+ return dst
+ }
+ for n := 1; ; n++ {
+ if c := dst + parkCopySep + fmt.Sprint(n); !lexists(c) {
+ return c
+ }
+ }
+}
+
+// Why a rename the park attempted did not happen. A path whose rename fails
+// with an error a later pass would meet again (a mount, EXDEV, EBUSY; a path
+// the agent may not rename, EPERM or EACCES; a read-only filesystem) is
+// left for good: reported once, with leftCannot and the error, and never
+// tried again for this move, so a pass does not re-fence, fail and lift
+// over it. Any other error is leftNotYet, and the next pass tries again.
+const (
+ leftCannot = "cannot be renamed: "
+ leftNotYet = "not renamed yet: "
+)
+
+// renameTerminal reports whether a rename error is one a later pass would
+// meet again.
+func renameTerminal(err error) bool {
+ return errors.Is(err, os.ErrPermission) || errors.Is(err, syscall.EXDEV) ||
+ errors.Is(err, syscall.EBUSY) || errors.Is(err, syscall.EROFS)
+}
+
+// retire reports one path's action from what is on disk, renaming it into
+// the park directory first when park is true and it is there to rename.
+// cannot is the reason an earlier pass recorded for a path it found it can
+// never rename ("" when none): such a path is reported left again, as it
+// is, and not tried. Nothing is ever deleted.
+func retire(p retirePath, parkDir string, park bool, cannot string) moves.RetiredPath {
+ present := lexists(p.path)
+ switch {
+ case p.left != "" && present:
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredLeft, Reason: p.left}
+ case p.left != "":
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredAbsent}
+ case present && cannot != "":
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredLeft, Reason: cannot}
+ case present && park:
+ dst := parkName(parkDir, p.name)
+ err := os.MkdirAll(filepath.Dir(dst), 0o700)
+ if err == nil {
+ err = os.Rename(p.path, dst)
+ }
+ switch {
+ case err == nil:
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredParked}
+ case errors.Is(err, os.ErrNotExist) && !lexists(p.path):
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredAbsent}
+ case renameTerminal(err):
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredLeft, Reason: leftCannot + err.Error()}
+ }
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredLeft, Reason: leftNotYet + err.Error()}
+ case present:
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredLeft, Reason: leftNotYet + "the stop could not be confirmed this pass"}
+ case lexists(filepath.Join(parkDir, p.name)):
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredParked}
+ }
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredAbsent}
+}
+
+// cannotRename reads this box's own last report for the move and returns
+// the paths it recorded as never renameable (leftCannot), with their
+// reasons. Reading them from the report, not from memory, keeps the answer
+// across an agent restart.
+func (w *movesWorker) cannotRename(ctx context.Context, s release.Store, host, op string) (map[string]string, error) {
+ out := map[string]string{}
+ body, _, err := s.Get(ctx, moves.OfflineReportKey(host, op, moves.OfflineRoleSource, w.cfg.Box))
+ if errors.Is(err, os.ErrNotExist) {
+ return out, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+ var r moves.OfflineReport
+ if err := json.Unmarshal(body, &r); err != nil {
+ return nil, err
+ }
+ for _, rp := range r.Retired {
+ if rp.Action == moves.RetiredLeft && strings.HasPrefix(rp.Reason, leftCannot) {
+ out[rp.Path] = rp.Reason
+ }
+ }
+ return out, nil
+}
+
+// offlineSourcePark is the source step at offline-open: with the fence held,
+// the row held (or absent), the host quiesced, no process left and no layout
+// journal, rename every path of the source-retire row that is still here
+// into the park directory; then lift the fence and report parked with the
+// inventory. A pass that finds nothing left to rename does not fence again:
+// it reports from what is parked. A path that can never be renamed
+// (leftCannot) is reported left with its reason and does not count as left
+// to rename, so it neither blocks the parked report nor brings the fence
+// back on every pass. A rename that may yet succeed (leftNotYet) reports
+// failed, and the next pass tries again. Nothing is deleted.
+func (w *movesWorker) offlineSourcePark(ctx context.Context, s release.Store, rec provision.Record) error {
+ cfg := w.cfg
+ m, host := rec.Move, rec.Host
+ parkDir := moves.OfflineParkDir(cfg.DataDir, host, m.ID, moves.OfflineRoleSource)
+ if parkDir == "" {
+ return fmt.Errorf("offline move %s: no park directory for %s under %q", m.ID, host, cfg.DataDir)
+ }
+ rt, err := w.offlineRow(host)
+ if err != nil {
+ return err
+ }
+ paths, err := w.retirePaths(host, parkDir, rt)
+ if err != nil {
+ return err
+ }
+ cannot, err := w.cannotRename(ctx, s, host, m.ID)
+ if err != nil {
+ return err
+ }
+ pending := false
+ for _, p := range paths {
+ if p.left == "" && cannot[p.path] == "" && lexists(p.path) {
+ pending = true
+ break
+ }
+ }
+ if pending {
+ fenced := w.offlineFence(ctx, host, m)
+ quiesced := cfg.Quiesced != nil && cfg.Quiesced(host)
+ procs := w.procs(host)
+ journal := layout.JournalPresent(cfg.DataDir, host)
+ if !fenced || !heldBy(rt, m.ID) || !quiesced || procs != 0 || journal {
+ return fmt.Errorf("offline move %s: not parking %s yet: fenced %v, held %v, quiesced %v, processes %d, layout journal %v",
+ m.ID, host, fenced, heldBy(rt, m.ID), quiesced, procs, journal)
+ }
+ if err := os.MkdirAll(parkDir, 0o700); err != nil {
+ return err
+ }
+ }
+ retired := make([]moves.RetiredPath, 0, len(paths))
+ retry := false
+ for _, p := range paths {
+ rp := retire(p, parkDir, pending, cannot[p.path])
+ if rp.Action == moves.RetiredLeft && strings.HasPrefix(rp.Reason, leftNotYet) {
+ retry = true
+ }
+ retired = append(retired, rp)
+ }
+ w.offlineLiftFence(host, m)
+ phase := moves.OfflinePhaseParked
+ if retry {
+ phase = moves.OfflinePhaseFailed
+ }
+ r, err := w.sourceReport(host, m, rt, false, phase)
+ if err != nil {
+ return err
+ }
+ r.Retired = retired
+ if err := writeOfflineReport(ctx, s, r); err != nil {
+ return err
+ }
+ if retry {
+ return fmt.Errorf("offline move %s: %s: some paths were not renamed this pass; the next pass tries again", m.ID, host)
+ }
+ return nil
+}
+
+// offlineSourceGone is the source step at offline-finishing: keep no longer
+// holds the row, so reconcile removes it; once it is gone, report gone. The
+// fence was lifted at open and is lifted again, should a restart have
+// re-fenced in between.
+func (w *movesWorker) offlineSourceGone(ctx context.Context, s release.Store, rec provision.Record) error {
+ m, host := rec.Move, rec.Host
+ rt, err := w.offlineRow(host)
+ if err != nil {
+ return err
+ }
+ w.offlineLiftFence(host, m)
+ if rt != nil {
+ w.notice(host+"#offline-gone", "moves: %s: offline move %s: waiting for reconcile to remove the row", host, m.ID)
+ return nil
+ }
+ r, err := w.sourceReport(host, m, nil, false, moves.OfflinePhaseGone)
+ if err != nil {
+ return err
+ }
+ return writeOfflineReport(ctx, s, r)
+}
+
+// offlineSourceRevert is every source box's part of offline-reverting: the
+// row leaves this move's hold (ReleaseHold under the host's hold lock), the
+// fence is lifted, and the box reports reverted. The sealer's extra part is
+// its own step's (Task 4).
+//
+// The release also holds reconcileMu: a reconcile pass that listed the
+// records before the abort would otherwise mint a missing row held again
+// from its stale listing after the release, and a row held by a move that
+// has finished is never released. A pass that starts after the release
+// lists the record at offline-reverting, which holds nothing.
+func (w *movesWorker) offlineSourceRevert(ctx context.Context, s release.Store, rec provision.Record) error {
+ cfg := w.cfg
+ m, host := rec.Move, rec.Host
+ if cfg.ReleaseHold == nil {
+ return errors.New("offline move: no registry to release the hold through")
+ }
+ var relErr error
+ reconcileMu.Lock()
+ provision.WithHostHoldLock(host, func() {
+ _, relErr = cfg.ReleaseHold(host, m.ID)
+ })
+ reconcileMu.Unlock()
+ if relErr != nil {
+ return relErr
+ }
+ w.offlineLiftFence(host, m)
+ rt, err := w.offlineRow(host)
+ if err != nil {
+ return err
+ }
+ if rt != nil && rt.HoldOp == m.ID {
+ return fmt.Errorf("offline move %s: %s is still held by the move", m.ID, host)
+ }
+ r, err := w.sourceReport(host, m, rt, false, moves.OfflinePhaseReverted)
+ if err != nil {
+ return err
+ }
+ r.Held = false
+ return writeOfflineReport(ctx, s, r)
+}
diff --git a/internal/host/movesoffline_land.go b/internal/host/movesoffline_land.go
new file mode 100644
index 000000000..d3ea217f5
--- /dev/null
+++ b/internal/host/movesoffline_land.go
@@ -0,0 +1,30 @@
+package host
+
+import (
+ "context"
+
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// offlineTargetStep is the target box's part of an offline move: the
+// landing, open and the target's revert
+// (docs/superpowers/plans/2026-10-20-offline-operator-move.md, Task 5, which
+// owns this file). offlineRound calls it on a box whose placement equals
+// rec.Move.To, in every offline state, after the source and sealer steps.
+// rec carries an offline move (rec.Move.Offline()) and the box is capable
+// (moveCfg.OfflineCapable). Reconcile mints the row held "landing" from the
+// record alone (offlineHold) and keeps it until the move is cleared
+// (offlineView.keep); ending that hold (open, revert) must take
+// provision.WithHostHoldLock around the release. Every step must be an
+// ensure; an error is logged by the pass, throttled per host, and the next
+// pass runs the step again.
+//
+// The landing compares against the immutable seal record
+// (moves.OfflineSealKey(host, id), added on Task 6's branch), read with
+// s.Get; never against the sealer's per-pass report, which keeps changing.
+//
+// Task 3 leaves it a no-op.
+func (w *movesWorker) offlineTargetStep(ctx context.Context, s release.Store, rec provision.Record) error {
+ return nil
+}
diff --git a/internal/host/movesoffline_seal.go b/internal/host/movesoffline_seal.go
new file mode 100644
index 000000000..e13489c57
--- /dev/null
+++ b/internal/host/movesoffline_seal.go
@@ -0,0 +1,28 @@
+package host
+
+import (
+ "context"
+
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// offlineSealerStep is the sealer's part of an offline move
+// (docs/superpowers/plans/2026-10-20-offline-operator-move.md, Task 4, which
+// owns this file). offlineRound calls it on the box whose moveCfg.Box equals
+// rec.Move.Sealer, in every offline state, after this box's source step has
+// written its report for the pass. rec carries an offline move
+// (rec.Move.Offline()) and the box is capable (moveCfg.OfflineCapable).
+// Every step must be an ensure; an error is logged by the pass, throttled
+// per host, and the next pass runs the step again.
+//
+// The immutable seal record (moves.OfflineSealKey(host, id), added on
+// Task 6's branch) is written once through putOfflineCreateOnly, never
+// rewritten: on a re-run it returns the stored body with created false, and
+// the step compares instead of overwriting. The per-pass sealer report goes
+// through writeOfflineReport, which replaces.
+//
+// Task 3 leaves it a no-op.
+func (w *movesWorker) offlineSealerStep(ctx context.Context, s release.Store, rec provision.Record) error {
+ return nil
+}
diff --git a/internal/host/movesoffline_test.go b/internal/host/movesoffline_test.go
new file mode 100644
index 000000000..b1b8da306
--- /dev/null
+++ b/internal/host/movesoffline_test.go
@@ -0,0 +1,1283 @@
+package host
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "os"
+ "path/filepath"
+ "reflect"
+ "slices"
+ "sort"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/cgroup"
+ "github.com/carlosframework/platform/internal/domains"
+ "github.com/carlosframework/platform/internal/hostwrite"
+ "github.com/carlosframework/platform/internal/instancectl"
+ "github.com/carlosframework/platform/internal/layout"
+ "github.com/carlosframework/platform/internal/moves"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// The offline operator move's box side (Task 3 of
+// docs/superpowers/plans/2026-10-20-offline-operator-move.md), tested
+// through the functions the agent calls: newMovesWorker, armOffline,
+// reconcileSerializedWith, the production edges-file writer and movesPass,
+// over one DirStore, real registries and real activator services (the
+// package's fake wake backend, which only the fence needs).
+
+const (
+ offHost = "shop.example.com"
+ offID = "mv_zyxwvutsrqponmlkjihgfedcba"
+ offID2 = "mv_abcdefghijklmnopqrstuvwxy2"
+)
+
+type offWorld struct {
+ t *testing.T
+ s release.Store
+ acct string
+ mu sync.Mutex
+ now time.Time
+}
+
+func (wd *offWorld) clock() time.Time {
+ wd.mu.Lock()
+ defer wd.mu.Unlock()
+ return wd.now
+}
+
+func (wd *offWorld) advance(d time.Duration) {
+ wd.mu.Lock()
+ wd.now = wd.now.Add(d)
+ wd.mu.Unlock()
+}
+
+// newOffWorld is a deployment with one account, one app and one instance
+// record for offHost at the home placement ("").
+func newOffWorld(t *testing.T) *offWorld {
+ t.Helper()
+ t.Setenv("CARLOS_HOST_CLAIM", "") // the appconfig and provision guards read the process env
+ ctx := context.Background()
+ wd := &offWorld{t: t, s: release.NewDirStore(t.TempDir()), acct: domains.Sqid(2), now: time.Date(2026, 10, 20, 9, 0, 0, 0, time.UTC)}
+ manifest, err := json.Marshal(map[string]any{"id": 2, "sqid": wd.acct, "name": "fixture", "created_at": wd.now})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := wd.s.Put(ctx, "accounts/"+wd.acct+"/account.json", manifest, ""); err != nil {
+ t.Fatal(err)
+ }
+ if err := provision.PutManifest(ctx, wd.s, wd.acct, "app", provision.Manifest{Schema: 1, App: "app", Domain: "example.com"}); err != nil {
+ t.Fatal(err)
+ }
+ wd.create(offHost)
+ return wd
+}
+
+func (wd *offWorld) create(host string) {
+ wd.t.Helper()
+ rec := provision.Record{Schema: 1, Host: host, Slug: strings.SplitN(host, ".", 2)[0], Account: wd.acct, App: "app", Region: "", Channel: "stable", CreatedAt: wd.now}
+ if err := provision.CreateRecord(context.Background(), wd.s, rec); err != nil {
+ wd.t.Fatal(err)
+ }
+}
+
+// move applies one offline transition to offHost's record.
+func (wd *offWorld) move(mut func(*provision.Record) error) provision.Record {
+ wd.t.Helper()
+ return wd.moveHost(offHost, mut)
+}
+
+func (wd *offWorld) moveHost(host string, mut func(*provision.Record) error) provision.Record {
+ wd.t.Helper()
+ rec, err := provision.SetRecordMove(context.Background(), wd.s, wd.acct, "app", host, mut)
+ if err != nil {
+ wd.t.Fatal(err)
+ }
+ return *rec
+}
+
+func (wd *offWorld) record() provision.Record {
+ wd.t.Helper()
+ rec, err := provision.GetRecord(context.Background(), wd.s, wd.acct, "app", offHost)
+ if err != nil {
+ wd.t.Fatal(err)
+ }
+ return *rec
+}
+
+// The legal path to a state, from the record at rest.
+func (wd *offWorld) start() {
+ wd.move(provision.OfflineStart(offID, "", "uk", "op@example.com", wd.clock()))
+}
+func (wd *offWorld) seal() { wd.move(provision.OfflineSeal(offID, "ie-a", []string{"ie-a", "ie-b"})) }
+func (wd *offWorld) sealed() {
+ wd.move(provision.OfflineSealedBy(offID, "ie-a", provision.MoveSource{}))
+}
+func (wd *offWorld) flip() { wd.move(provision.OfflineFlip(offID)) }
+func (wd *offWorld) landed() { wd.move(provision.OfflineLandedBy(offID, "uk")) }
+func (wd *offWorld) open() { wd.move(provision.OfflineOpenTransition(offID, wd.clock())) }
+func (wd *offWorld) finishing() {
+ wd.move(provision.OfflineFinishingTransition(offID))
+}
+func (wd *offWorld) abort() { wd.move(provision.OfflineAbort(offID)) }
+
+// report reads one box's report for offID, or fails.
+func (wd *offWorld) report(role, box string) moves.OfflineReport {
+ wd.t.Helper()
+ r, ok := wd.reportIf(role, box)
+ if !ok {
+ wd.t.Fatalf("no %s report from %s", role, box)
+ }
+ return r
+}
+
+func (wd *offWorld) reportIf(role, box string) (moves.OfflineReport, bool) {
+ wd.t.Helper()
+ body, _, err := wd.s.Get(context.Background(), moves.OfflineReportKey(offHost, offID, role, box))
+ if errors.Is(err, os.ErrNotExist) {
+ return moves.OfflineReport{}, false
+ }
+ if err != nil {
+ wd.t.Fatal(err)
+ }
+ var r moves.OfflineReport
+ if err := json.Unmarshal(body, &r); err != nil {
+ wd.t.Fatal(err)
+ }
+ if err := moves.ValidateOfflineReport(r); err != nil {
+ wd.t.Fatal(err)
+ }
+ return r, true
+}
+
+// offBox is one box: its own data directory, registry, activator and moves
+// worker, armed as armMovesPass arms it (armOffline).
+type offBox struct {
+ wd *offWorld
+ name, region string
+ dd, opt, cg string
+ env map[string]string
+ reg *registry.Registry
+ svc *activator.Service
+ w *movesWorker
+ nested bool
+ last []byte
+}
+
+func (wd *offWorld) box(name, region string, env map[string]string) *offBox {
+ wd.t.Helper()
+ b := &offBox{wd: wd, name: name, region: region, dd: wd.t.TempDir(), opt: wd.t.TempDir(), cg: wd.t.TempDir(), env: env}
+ reg, err := registry.Open(filepath.Join(b.dd, "registry.db"))
+ if err != nil {
+ wd.t.Fatal(err)
+ }
+ wd.t.Cleanup(func() { reg.Close() })
+ b.reg = reg
+ b.restart()
+ return b
+}
+
+func (b *offBox) getenv(k string) string {
+ if v, ok := b.env[k]; ok {
+ return v
+ }
+ if k == "CARLOS_HIBERNATE_BUCKET" {
+ return "data"
+ }
+ return ""
+}
+
+// restart is an agent restart: a fresh activator (no fences) and a fresh
+// worker over the same registry and data directory.
+func (b *offBox) restart() {
+ b.svc = activator.New(activator.Config{Bucket: "data", Idle: time.Hour}, &fakeWakeBackend{}, b.reg)
+ cfg := moveCfg{Region: b.region, Box: b.name, DataDir: b.dd, OptDir: b.opt, Quiesced: b.svc.Quiesced, Route: b.reg.Get, Now: b.wd.clock}
+ armOffline(&cfg, b.dd, b.reg, b.svc, nil, hostwrite.Config{}, b.getenv)
+ // The stage-requested marker is root's business, and its throttled
+ // timer would outlive the test's directories.
+ cfg.Offline = newOfflineView(cfg, nil)
+ cfg.ReleaseHold = releaseHeld(b.reg, nil)
+ cfg.Procs = func(host string) (int, error) { return cgroup.HostProcs(b.cg, host, "") }
+ b.w = newMovesWorker(cfg)
+}
+
+func (b *offBox) reconcile() {
+ b.wd.t.Helper()
+ cfg := provision.ReconcileConfig{Region: b.region, DataDir: b.dd}
+ if b.nested {
+ cfg.Layout = func(context.Context, release.Store) (string, error) { return release.LayoutModeNested, nil }
+ } else {
+ cfg.Layout = func(context.Context, release.Store) (string, error) { return release.LayoutModeFlat, nil }
+ }
+ if err := reconcileSerializedWith(context.Background(), b.wd.s, b.reg, cfg, b.w); err != nil {
+ b.wd.t.Fatal(err)
+ }
+}
+
+func (b *offBox) edges() {
+ b.wd.t.Helper()
+ if err := edgesFileWrite(context.Background(), b.wd.s, b.reg, edgesFile{Box: b.name, Placement: b.region}, "", &b.last, false); err != nil {
+ b.wd.t.Fatal(err)
+ }
+}
+
+func (b *offBox) pass() {
+ b.wd.t.Helper()
+ if err := movesPass(context.Background(), b.wd.s, b.w); err != nil {
+ b.wd.t.Fatal(err)
+ }
+}
+
+// tick is the agent's order on a serial bump: reconcile, the edges file,
+// the moves pass.
+func (b *offBox) tick() {
+ b.wd.t.Helper()
+ b.reconcile()
+ b.edges()
+ b.pass()
+}
+
+func (b *offBox) row(host string) *registry.Route {
+ b.wd.t.Helper()
+ rt, err := b.reg.Get(host)
+ if err != nil {
+ b.wd.t.Fatal(err)
+ }
+ return rt
+}
+
+// lists reports whether this box's published edges file lists host.
+func (b *offBox) lists(host string) bool {
+ b.wd.t.Helper()
+ body, _, err := b.wd.s.Get(context.Background(), edgesFileKey(b.name))
+ if err != nil {
+ b.wd.t.Fatal(err)
+ }
+ var f edgesFile
+ if err := json.Unmarshal(body, &f); err != nil {
+ b.wd.t.Fatal(err)
+ }
+ _, ok := f.Hosts[host]
+ return ok
+}
+
+func writeFile(t *testing.T, p, body string) {
+ t.Helper()
+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// --- the hold ---------------------------------------------------------------
+
+// offlineHold's table: a pure function of the record and the box's
+// placement. The home's "" and a named region; every state.
+func TestOfflineHoldIsAFunctionOfTheRecord(t *testing.T) {
+ home := moveCfg{Region: ""}
+ uk := moveCfg{Region: "uk"}
+ fr := moveCfg{Region: "fr"}
+ rec := func(region, state string) provision.Record {
+ return provision.Record{Host: offHost, Region: region, Move: &provision.MoveState{Kind: provision.MoveKindOffline, ID: offID, From: "", To: "uk", State: state}}
+ }
+ cases := []struct {
+ name string
+ box moveCfg
+ rec provision.Record
+ wantStatus, wantOp string
+ }{
+ {"source stopping", home, rec("", provision.OfflineStopping), "moving", offID},
+ {"source sealing", home, rec("", provision.OfflineSealing), "moving", offID},
+ {"source sealed", home, rec("", provision.OfflineSealed), "moving", offID},
+ {"source reverting", home, rec("", provision.OfflineReverting), "", ""},
+ {"source after the flip", home, rec("uk", provision.OfflineLanding), "", ""},
+ {"target before the flip", uk, rec("", provision.OfflineSealed), "", ""},
+ {"target landing", uk, rec("uk", provision.OfflineLanding), "landing", offID},
+ {"target landed", uk, rec("uk", provision.OfflineLanded), "landing", offID},
+ {"target landing, region not yet the target's", uk, rec("", provision.OfflineLanding), "", ""},
+ {"target open", uk, rec("uk", provision.OfflineOpen), "", ""},
+ {"target finishing", uk, rec("uk", provision.OfflineFinishing), "", ""},
+ {"target reverting", uk, rec("", provision.OfflineReverting), "", ""},
+ {"a third region", fr, rec("", provision.OfflineStopping), "", ""},
+ {"a third region after the flip", fr, rec("uk", provision.OfflineLanding), "", ""},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ status, op, ok := offlineHold(c.box)(c.rec)
+ if status != c.wantStatus || op != c.wantOp || ok != (c.wantStatus != "") {
+ t.Fatalf("hold = (%q, %q, %v), want (%q, %q)", status, op, ok, c.wantStatus, c.wantOp)
+ }
+ })
+ }
+ // A record at rest, a legacy move and a Phase 1 move are never held.
+ for _, m := range []*provision.MoveState{nil, {To: "uk", State: provision.MoveCutting}, {ID: offID, From: "", To: "uk", State: provision.OfflineStopping}} {
+ if _, _, ok := offlineHold(home)(provision.Record{Host: offHost, Move: m}); ok {
+ t.Fatalf("held: %+v", m)
+ }
+ }
+}
+
+// B1, the pinning test: a fresh target worker whose keep view has never
+// seen the move (built from a list with no offline record in it, as a
+// box's view is before the flip) mints the row held "landing" on its first
+// reconcile after the flip. And with the view never built at all, reconcile
+// through the offline hooks alone does the same.
+func TestOfflineFreshTargetMintsTheLandingRowHeld(t *testing.T) {
+ wd := newOffWorld(t)
+ wd.start()
+ wd.seal()
+ wd.sealed()
+ wd.flip()
+
+ uk := wd.box("uk-box", "uk", nil)
+ uk.w.cfg.Offline.refresh(nil, nil) // stale: no offline move in it
+ uk.reconcile()
+ rt := uk.row(offHost)
+ if rt == nil || rt.Status != "landing" || rt.HoldOp != offID {
+ t.Fatalf("after the flip the target minted %+v, want held landing by %s", rt, offID)
+ }
+ uk.edges()
+ if uk.lists(offHost) {
+ t.Fatal("the target's edges file lists a held landing row")
+ }
+
+ // The hooks with a view that was never built, straight into
+ // provision.Reconcile: the hold needs no view.
+ fresh := wd.box("uk-box-2", "uk", nil)
+ if fresh.w.cfg.Offline.built() {
+ t.Fatal("fixture: the view is built")
+ }
+ cfg := withOfflineHolds(provision.ReconcileConfig{Region: "uk", DataDir: fresh.dd}, offlineHold(fresh.w.cfg), fresh.w.cfg.Offline)
+ if err := provision.Reconcile(context.Background(), wd.s, fresh.reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ if rt := fresh.row(offHost); rt == nil || rt.Status != "landing" || rt.HoldOp != offID {
+ t.Fatalf("an unbuilt view: minted %+v", rt)
+ }
+}
+
+// --- the source boxes -------------------------------------------------------
+
+// start holds the row on both source boxes (one minting it held, one
+// holding the row it has, keeping its other columns), both report held,
+// fenced and quiesced, and neither edges file lists the host. An agent
+// restart re-fences and re-reports. After the flip the rows stay, kept.
+func TestOfflineStartHoldsEverySourceRow(t *testing.T) {
+ wd := newOffWorld(t)
+ a := wd.box("ie-a", "", nil)
+ b := wd.box("ie-b", "", nil)
+ uk := wd.box("uk-box", "uk", nil)
+ a.tick() // ie-a serves the host before the move; ie-b has no row yet
+ if rt := a.row(offHost); rt == nil || rt.Status != "ready" || !a.lists(offHost) {
+ t.Fatalf("fixture: ie-a does not serve the host: %+v", rt)
+ }
+ if err := a.reg.SetReplication(offHost, "daily"); err != nil {
+ t.Fatal(err)
+ }
+ if err := a.reg.SetNextDue(offHost, 1234); err != nil {
+ t.Fatal(err)
+ }
+ before := *a.row(offHost)
+
+ wd.start()
+ a.tick()
+ b.tick()
+ uk.tick()
+ for _, bx := range []*offBox{a, b} {
+ rt := bx.row(offHost)
+ if rt == nil || rt.Status != "moving" || rt.HoldOp != offID {
+ t.Fatalf("%s: row %+v, want held moving by the move", bx.name, rt)
+ }
+ if bx.lists(offHost) {
+ t.Fatalf("%s's edges file lists a held host", bx.name)
+ }
+ if !bx.svc.Fenced(offHost) {
+ t.Fatalf("%s: no wake fence", bx.name)
+ }
+ r := wd.report(moves.OfflineRoleSource, bx.name)
+ if !r.Held || !r.Fenced || !r.Quiesced || !r.RowPresent || r.Procs != 0 || r.LayoutJournal || r.Phase != moves.OfflinePhaseStopped || !r.At.Equal(wd.clock()) {
+ t.Fatalf("%s: report %+v", bx.name, r)
+ }
+ }
+ after := *a.row(offHost)
+ after.Status, after.HoldOp = before.Status, before.HoldOp
+ if !reflect.DeepEqual(after, before) {
+ t.Fatalf("holding the existing row changed its other columns:\n before %+v\n after %+v", before, after)
+ }
+ if uk.row(offHost) != nil {
+ t.Fatal("the target minted a row before the flip")
+ }
+ if _, ok := wd.reportIf(moves.OfflineRoleSource, "uk-box"); ok {
+ t.Fatal("the target wrote a source report")
+ }
+
+ // An agent restart: the fence is gone with the process, and the next
+ // pass installs it again and rewrites the report.
+ a.restart()
+ if a.svc.Fenced(offHost) {
+ t.Fatal("fixture: a fresh activator is fenced")
+ }
+ wd.advance(time.Minute)
+ a.tick()
+ if !a.svc.Fenced(offHost) {
+ t.Fatal("after a restart the pass did not fence again")
+ }
+ if r := wd.report(moves.OfflineRoleSource, "ie-a"); !r.Fenced || !r.At.Equal(wd.clock()) {
+ t.Fatalf("after a restart: %+v", r)
+ }
+
+ // The flip: the record names uk; both source rows stay, held, and keep
+ // reporting; the target mints its row held landing.
+ wd.seal()
+ wd.sealed()
+ wd.flip()
+ wd.advance(time.Minute)
+ for _, bx := range []*offBox{a, b, uk} {
+ bx.tick()
+ }
+ for _, bx := range []*offBox{a, b} {
+ if rt := bx.row(offHost); rt == nil || rt.Status != "moving" || rt.HoldOp != offID {
+ t.Fatalf("%s after the flip: %+v, want the row kept, held", bx.name, rt)
+ }
+ if r := wd.report(moves.OfflineRoleSource, bx.name); !r.Held || !r.Fenced || !r.At.Equal(wd.clock()) {
+ t.Fatalf("%s after the flip: %+v", bx.name, r)
+ }
+ }
+ if rt := uk.row(offHost); rt == nil || rt.Status != "landing" || rt.HoldOp != offID {
+ t.Fatalf("target after the flip: %+v", rt)
+ }
+ if uk.lists(offHost) {
+ t.Fatal("the target lists the landing row")
+ }
+}
+
+// What the report reads from disk: a box with only a meta dir has local
+// state; a process left in the host's cgroup and a layout journal show.
+func TestOfflineSourceReportReadsTheBox(t *testing.T) {
+ wd := newOffWorld(t)
+ a := wd.box("ie-a", "", nil)
+ b := wd.box("ie-b", "", nil)
+ a.tick()
+ wd.start()
+ // ie-b has no DB file, only the activator's meta dir with one LTX file.
+ meta := activatorMetaDir(provision.DefaultDBPath(b.dd, offHost))
+ writeFile(t, filepath.Join(meta, "ltx", "0", "0000000000000001-0000000000000007.ltx"), "x")
+ // ie-a: files, env, a process in the host's exec cgroup, a journal.
+ writeFile(t, filepath.Join(a.dd, offHost, "instance.key"), "k")
+ writeFile(t, filepath.Join(a.dd, offHost+".env"), "CARLOS_INSTANCE_KEY=k\n")
+ writeFile(t, filepath.Join(a.cg, cgroup.ExecName(offHost), "cgroup.procs"), "4242\n")
+ writeFile(t, provision.LayoutJournalPath(a.dd, offHost), "{}")
+ a.tick()
+ b.tick()
+ rb := wd.report(moves.OfflineRoleSource, "ie-b")
+ if !rb.LocalDB || rb.LocalTXID != 7 || rb.Files || rb.Env || rb.Sidecar {
+ t.Fatalf("a meta dir alone: %+v", rb)
+ }
+ ra := wd.report(moves.OfflineRoleSource, "ie-a")
+ if ra.LocalDB || !ra.Files || !ra.Env || ra.Procs != 1 || !ra.LayoutJournal {
+ t.Fatalf("ie-a: %+v", ra)
+ }
+ // A box that cannot count its processes reports -1, never 0.
+ a.w.cfg.Procs = func(string) (int, error) { return 0, cgroup.ErrUnknown }
+ a.pass()
+ if r := wd.report(moves.OfflineRoleSource, "ie-a"); r.Procs != -1 {
+ t.Fatalf("unknown processes reported as %d", r.Procs)
+ }
+ // A sidecar file shows.
+ writeFile(t, filepath.Join(b.dd, offHost+"-sidecar.env"), "X=1\n")
+ b.pass()
+ if r := wd.report(moves.OfflineRoleSource, "ie-b"); !r.Sidecar {
+ t.Fatalf("sidecar not seen: %+v", r)
+ }
+}
+
+// offline-reverting, before and after the flip: every source row is
+// released and served again, the fence is lifted, the boxes report
+// reverted, and the next reconcile does not hold the rows again.
+func TestOfflineRevertReleasesEverySourceRow(t *testing.T) {
+ for _, afterFlip := range []bool{false, true} {
+ t.Run(map[bool]string{false: "before the flip", true: "after the flip"}[afterFlip], func(t *testing.T) {
+ wd := newOffWorld(t)
+ a := wd.box("ie-a", "", nil)
+ b := wd.box("ie-b", "", nil)
+ uk := wd.box("uk-box", "uk", nil)
+ a.tick()
+ b.tick()
+ wd.start()
+ if afterFlip {
+ wd.seal()
+ wd.sealed()
+ wd.flip()
+ }
+ for _, bx := range []*offBox{a, b, uk} {
+ bx.tick()
+ }
+ wd.abort()
+ for i := 0; i < 2; i++ {
+ for _, bx := range []*offBox{a, b, uk} {
+ bx.tick()
+ }
+ }
+ for _, bx := range []*offBox{a, b} {
+ rt := bx.row(offHost)
+ if rt == nil || rt.Status != "ready" || rt.HoldOp != "" {
+ t.Fatalf("%s: %+v, want ready and unheld", bx.name, rt)
+ }
+ if !bx.lists(offHost) {
+ t.Fatalf("%s does not serve the host again", bx.name)
+ }
+ if bx.svc.Fenced(offHost) {
+ t.Fatalf("%s: the fence is still up", bx.name)
+ }
+ if r := wd.report(moves.OfflineRoleSource, bx.name); r.Phase != moves.OfflinePhaseReverted || r.Held {
+ t.Fatalf("%s: %+v", bx.name, r)
+ }
+ }
+ // The move finishes; nothing holds the rows again.
+ wd.move(provision.OfflineFinish(offID, wd.clock()))
+ for _, bx := range []*offBox{a, b} {
+ bx.tick()
+ if rt := bx.row(offHost); rt == nil || rt.Status != "ready" || rt.HoldOp != "" {
+ t.Fatalf("%s after finish: %+v", bx.name, rt)
+ }
+ }
+ })
+ }
+}
+
+// offline-open on a flat source: every path of the source-retire row that
+// is there is renamed into the park directory byte for byte, nothing is
+// deleted, the fence is lifted, and the parked report lists every path with
+// its action. A process left in the host's cgroup stops the park first.
+// offline-finishing then removes the row and the box reports gone.
+func TestOfflineOpenParksTheFlatSource(t *testing.T) {
+ wd := newOffWorld(t)
+ a := wd.box("ie-a", "", nil)
+ uk := wd.box("uk-box", "uk", nil)
+ a.tick()
+ db := a.row(offHost).DBPath
+ contents := map[string]string{
+ db: "sqlite",
+ db + "-wal": "wal",
+ filepath.Join(activatorMetaDir(db), "ltx", "0", "0000000000000001-0000000000000003.ltx"): "ltx",
+ filepath.Join(a.dd, offHost, "instance.key"): "key",
+ filepath.Join(a.dd, offHost+".env"): "CARLOS_INSTANCE_KEY=k\n",
+ filepath.Join(a.dd, offHost+".config.env"): "SECRET=s\n",
+ filepath.Join(a.dd, "programs", offHost+".json"): `{"programs":[]}`,
+ filepath.Join(a.opt, "live", offHost, "app"): "binary",
+ filepath.Join(a.dd, "neighbour.example.com.db"): "not ours",
+ }
+ for p, body := range contents {
+ writeFile(t, p, body)
+ }
+ wd.start()
+ a.tick() // the source reports before the seal, as the seal door requires
+ wd.seal()
+ wd.sealed()
+ wd.flip()
+ wd.landed()
+ a.tick()
+ uk.tick()
+ wd.open()
+
+ // A process still in the host's cgroup: nothing is renamed.
+ procs := filepath.Join(a.cg, cgroup.ExecName(offHost), "cgroup.procs")
+ writeFile(t, procs, "4242\n")
+ a.tick()
+ if _, err := os.Stat(db); err != nil {
+ t.Fatalf("parked with a process left: %v", err)
+ }
+ if _, ok := wd.reportIf(moves.OfflineRoleSource, "ie-a"); !ok {
+ t.Fatal("fixture: no report")
+ }
+ if r := wd.report(moves.OfflineRoleSource, "ie-a"); r.Phase == moves.OfflinePhaseParked {
+ t.Fatalf("reported parked with a process left: %+v", r)
+ }
+ writeFile(t, procs, "")
+
+ a.tick()
+ park := moves.OfflineParkDir(a.dd, offHost, offID, moves.OfflineRoleSource)
+ for p, body := range contents {
+ switch {
+ case strings.HasPrefix(p, a.opt), strings.Contains(p, "neighbour"):
+ if got, err := os.ReadFile(p); err != nil || string(got) != body {
+ t.Fatalf("%s was touched: %q %v", p, got, err)
+ }
+ continue
+ }
+ if _, err := os.Lstat(p); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("%s is still in place: %v", p, err)
+ }
+ }
+ for rel, body := range map[string]string{
+ filepath.Base(db): "sqlite",
+ filepath.Base(db) + "-wal": "wal",
+ filepath.Join(filepath.Base(activatorMetaDir(db)), "ltx", "0", "0000000000000001-0000000000000003.ltx"): "ltx",
+ filepath.Join(offHost, "instance.key"): "key",
+ offHost + ".env": "CARLOS_INSTANCE_KEY=k\n",
+ offHost + ".config.env": "SECRET=s\n",
+ offHost + ".json": `{"programs":[]}`,
+ } {
+ if got, err := os.ReadFile(filepath.Join(park, rel)); err != nil || string(got) != body {
+ t.Fatalf("park/%s = %q %v, want %q", rel, got, err, body)
+ }
+ }
+ r := wd.report(moves.OfflineRoleSource, "ie-a")
+ if r.Phase != moves.OfflinePhaseParked || r.LocalDB || r.Files || r.Env || r.Fenced {
+ t.Fatalf("parked report: %+v", r)
+ }
+ if a.svc.Fenced(offHost) {
+ t.Fatal("the fence is still up after the park")
+ }
+ want := map[string]string{
+ db: moves.RetiredParked, db + "-wal": moves.RetiredParked, db + "-shm": moves.RetiredAbsent,
+ db + "-litestream": moves.RetiredAbsent, activatorMetaDir(db): moves.RetiredParked,
+ filepath.Join(a.dd, offHost): moves.RetiredParked, filepath.Join(a.dd, offHost+".env"): moves.RetiredParked,
+ filepath.Join(a.dd, offHost+".config.env"): moves.RetiredParked, filepath.Join(a.dd, "programs", offHost+".json"): moves.RetiredParked,
+ filepath.Join(a.opt, "live", offHost): moves.RetiredLeft,
+ }
+ assertRetired(t, r.Retired, want)
+
+ // A second pass changes nothing and reports the same inventory.
+ before := r.Retired
+ a.tick()
+ if r := wd.report(moves.OfflineRoleSource, "ie-a"); !reflect.DeepEqual(r.Retired, before) || r.Phase != moves.OfflinePhaseParked {
+ t.Fatalf("second pass: %+v", r.Retired)
+ }
+ // The row stays held through open.
+ if rt := a.row(offHost); rt == nil || rt.HoldOp != offID {
+ t.Fatalf("row at open: %+v", rt)
+ }
+
+ // finishing: keep no longer holds the row; reconcile removes it and
+ // the box reports gone. The parked state stays.
+ // The view the reconcile reads is the last moves pass's, so the row
+ // goes on the pass after the one that first sees finishing.
+ wd.finishing()
+ a.tick()
+ a.tick()
+ if rt := a.row(offHost); rt != nil {
+ t.Fatalf("row at finishing: %+v", rt)
+ }
+ a.pass()
+ if r := wd.report(moves.OfflineRoleSource, "ie-a"); r.Phase != moves.OfflinePhaseGone || r.RowPresent || r.LocalDB {
+ t.Fatalf("gone report: %+v", r)
+ }
+ if got, err := os.ReadFile(filepath.Join(park, filepath.Base(db))); err != nil || string(got) != "sqlite" {
+ t.Fatalf("the parked DB: %q %v", got, err)
+ }
+}
+
+func assertRetired(t *testing.T, got []moves.RetiredPath, want map[string]string) {
+ t.Helper()
+ seen := map[string]string{}
+ for _, rp := range got {
+ if _, dup := seen[rp.Path]; dup {
+ t.Fatalf("%s listed twice", rp.Path)
+ }
+ seen[rp.Path] = rp.Action
+ if rp.Action == moves.RetiredLeft && rp.Reason == "" {
+ t.Fatalf("%s left with no reason", rp.Path)
+ }
+ }
+ for p, action := range want {
+ if seen[p] != action {
+ t.Fatalf("%s: %q, want %q (inventory %+v)", p, seen[p], action, got)
+ }
+ }
+ if len(seen) != len(want) {
+ var extra []string
+ for p := range seen {
+ if _, ok := want[p]; !ok {
+ extra = append(extra, p+"="+seen[p])
+ }
+ }
+ sort.Strings(extra)
+ t.Fatalf("unexpected entries: %v", extra)
+ }
+}
+
+// M3: a path the agent can never rename (here a file in a directory it may
+// not write, as a root-owned /data/programs is) is reported left once, with
+// its reason, and is not tried again: later passes, an agent restart
+// included, neither fence nor rename, and the report stays parked.
+func TestOfflineParkLeavesWhatCannotBeRenamedOnce(t *testing.T) {
+ if os.Geteuid() == 0 {
+ t.Skip("root can rename anything")
+ }
+ wd := newOffWorld(t)
+ a := wd.box("ie-a", "", nil)
+ a.tick()
+ db := a.row(offHost).DBPath
+ writeFile(t, db, "sqlite")
+ programs := filepath.Join(a.dd, "programs")
+ writeFile(t, filepath.Join(programs, offHost+".json"), "{}")
+ if err := os.Chmod(programs, 0o555); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { os.Chmod(programs, 0o755) })
+ fences := 0
+ count := func() {
+ inner := a.w.cfg.FenceAs
+ a.w.cfg.FenceAs = func(ctx context.Context, host string, gen uint64, owner string) error {
+ fences++
+ return inner(ctx, host, gen, owner)
+ }
+ }
+ wd.start()
+ a.tick()
+ wd.seal()
+ wd.sealed()
+ wd.flip()
+ wd.landed()
+ a.tick()
+ wd.open()
+ count()
+ before := fences
+ a.tick()
+ if fences != before+1 {
+ t.Fatalf("the first park fenced %d times", fences-before)
+ }
+ r := wd.report(moves.OfflineRoleSource, "ie-a")
+ if r.Phase != moves.OfflinePhaseParked {
+ t.Fatalf("phase %s", r.Phase)
+ }
+ var left moves.RetiredPath
+ for _, rp := range r.Retired {
+ if rp.Path == filepath.Join(programs, offHost+".json") {
+ left = rp
+ }
+ if rp.Path == db && rp.Action != moves.RetiredParked {
+ t.Fatalf("the DB: %+v", rp)
+ }
+ }
+ if left.Action != moves.RetiredLeft || !strings.HasPrefix(left.Reason, leftCannot) {
+ t.Fatalf("the unrenameable path: %+v", left)
+ }
+ for i := 0; i < 2; i++ {
+ before = fences
+ a.tick()
+ if fences != before {
+ t.Fatal("a later pass fenced again for a path it cannot rename")
+ }
+ if again := wd.report(moves.OfflineRoleSource, "ie-a"); again.Phase != moves.OfflinePhaseParked || !reflect.DeepEqual(again.Retired, r.Retired) {
+ t.Fatalf("pass %d: %+v", i, again)
+ }
+ }
+ a.restart()
+ count()
+ before = fences
+ a.tick()
+ if fences != before || a.svc.Fenced(offHost) {
+ t.Fatal("after a restart the pass fenced again for a path it cannot rename")
+ }
+ if again := wd.report(moves.OfflineRoleSource, "ie-a"); !reflect.DeepEqual(again.Retired, r.Retired) {
+ t.Fatalf("after a restart: %+v", again.Retired)
+ }
+ if _, err := os.Stat(filepath.Join(programs, offHost+".json")); err != nil {
+ t.Fatalf("the left path: %v", err)
+ }
+}
+
+// offline-open on a nested source: the contents of the raw instance
+// directory are renamed into the park directory, the directory and its
+// view stay, empty, and the inventory names each path.
+func TestOfflineOpenParksTheNestedSource(t *testing.T) {
+ wd := newOffWorld(t)
+ a := wd.box("ie-a", "", nil)
+ a.nested = true
+ raw := layout.RawInstanceDir(a.dd, offHost)
+ if err := os.MkdirAll(raw, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ // The bind mount at <dd>/instances, as a link in the fixture.
+ if err := os.Symlink(filepath.Base(filepath.Dir(raw)), filepath.Join(a.dd, "instances")); err != nil {
+ t.Fatal(err)
+ }
+ a.tick()
+ rt := a.row(offHost)
+ if rt == nil || !provision.NestedPaths(a.dd, *rt) {
+ t.Fatalf("fixture: not a nested row: %+v", rt)
+ }
+ contents := map[string]string{
+ offHost + ".db": "sqlite",
+ offHost + ".db-shm": "shm",
+ offHost + ".env": "CARLOS_INSTANCE_KEY=k\n",
+ "instance.key": "key",
+ "uploads/a.png": "png",
+ }
+ for rel, body := range contents {
+ writeFile(t, filepath.Join(raw, rel), body)
+ }
+ wd.start()
+ a.tick()
+ if r := wd.report(moves.OfflineRoleSource, "ie-a"); !r.LocalDB || !r.Files || !r.Env {
+ t.Fatalf("nested report: %+v", r)
+ }
+ wd.seal()
+ wd.sealed()
+ wd.flip()
+ wd.landed()
+ wd.open()
+ a.tick()
+ park := moves.OfflineParkDir(a.dd, offHost, offID, moves.OfflineRoleSource)
+ for rel, body := range contents {
+ if got, err := os.ReadFile(filepath.Join(park, nestedParkPrefix, rel)); err != nil || string(got) != body {
+ t.Fatalf("park/instance/%s = %q %v", rel, got, err)
+ }
+ }
+ ents, err := os.ReadDir(raw)
+ if err != nil || len(ents) != 0 {
+ t.Fatalf("the instance directory: %v %v, want it there and empty", ents, err)
+ }
+ r := wd.report(moves.OfflineRoleSource, "ie-a")
+ if r.Phase != moves.OfflinePhaseParked || r.LocalDB || r.Files || r.Env {
+ t.Fatalf("parked report: %+v", r)
+ }
+ want := map[string]string{
+ filepath.Join(raw, offHost+".db"): moves.RetiredParked,
+ filepath.Join(raw, offHost+".db-wal"): moves.RetiredAbsent,
+ filepath.Join(raw, offHost+".db-shm"): moves.RetiredParked,
+ filepath.Join(raw, offHost+".db-litestream"): moves.RetiredAbsent,
+ filepath.Join(raw, ".activator-"+offHost+".db-litestream"): moves.RetiredAbsent,
+ filepath.Join(raw, offHost+".env"): moves.RetiredParked,
+ filepath.Join(raw, offHost+".config.env"): moves.RetiredAbsent,
+ filepath.Join(raw, "instance.key"): moves.RetiredParked,
+ filepath.Join(raw, "uploads"): moves.RetiredParked,
+ filepath.Join(a.dd, "programs", offHost+".json"): moves.RetiredAbsent,
+ provision.InstanceDir(a.dd, offHost): moves.RetiredLeft,
+ raw: moves.RetiredLeft,
+ filepath.Join(a.opt, "live", offHost): moves.RetiredAbsent,
+ }
+ assertRetired(t, r.Retired, want)
+ // Re-run: the same inventory, from what is parked.
+ a.tick()
+ if again := wd.report(moves.OfflineRoleSource, "ie-a"); !reflect.DeepEqual(again.Retired, r.Retired) {
+ t.Fatalf("second pass:\n %+v\n %+v", again.Retired, r.Retired)
+ }
+}
+
+// --- the dispatcher and the capability --------------------------------------
+
+// An offline record reaches offlineRound with Phase 1 off, ahead of the
+// Phase 1 skip and the lost-box gate: no host claim is read. A
+// Phase-1-shaped record with Phase 1 off is still logged and skipped.
+func TestOfflineDispatchPrecedesThePhase1Skip(t *testing.T) {
+ wd := newOffWorld(t)
+ wd.create("other.example.com")
+ a := wd.box("ie-a", "", nil)
+ a.tick()
+ wd.start()
+ // A Phase 1 move on the other host (its id is what makes it Phase 1).
+ if _, err := provision.SetRecordMove(context.Background(), wd.s, wd.acct, "app", "other.example.com", func(rec *provision.Record) error {
+ rec.Move = &provision.MoveState{ID: offID2, From: "", To: "uk", State: provision.MovePreparing, RequestedAt: wd.clock()}
+ return nil
+ }); err != nil {
+ t.Fatal(err)
+ }
+ a.tick()
+ if _, ok := wd.reportIf(moves.OfflineRoleSource, "ie-a"); !ok {
+ t.Fatal("the offline record never reached offlineRound")
+ }
+ if a.w.lost != nil {
+ t.Fatalf("the lost-box gate ran: %v", a.w.lost)
+ }
+ if _, ok := a.w.notices.Load("other.example.com#switch"); !ok {
+ t.Fatal("the Phase 1 record was not logged as waiting on the switch")
+ }
+ if rt := a.row("other.example.com"); rt == nil || rt.HoldOp != "" || rt.Status != "ready" {
+ t.Fatalf("the Phase 1 record's row was touched: %+v", rt)
+ }
+ keys, err := wd.s.List(context.Background(), "control/offline-moves/other.example.com/")
+ if err != nil || len(keys) != 0 {
+ t.Fatalf("the Phase 1 record was reported as offline: %v %v", keys, err)
+ }
+}
+
+// The capability is the box's own witness: published with hibernation
+// configured and both switches unset, withheld otherwise. A box without it
+// runs no offline step and installs no offline hook: its row is never
+// held, nothing is reported, and one notice is logged. The guard mode is
+// not an input (platform-1e's ruling of 2026-10-20).
+func TestOfflineCapabilityAndAnIncapableBox(t *testing.T) {
+ env := func(kv ...string) func(string) string {
+ m := map[string]string{"CARLOS_HIBERNATE_BUCKET": "data"}
+ for i := 0; i+1 < len(kv); i += 2 {
+ m[kv[i]] = kv[i+1]
+ }
+ return func(k string) string { return m[k] }
+ }
+ // The published capability is the armed worker's own answer (M5).
+ prev := armedMovesWorker.Load()
+ t.Cleanup(func() { armedMovesWorker.Store(prev) })
+ armedMovesWorker.Store(nil)
+ if armedOfflineCapable() {
+ t.Fatal("capable with no worker armed")
+ }
+ if got := withOfflineCapability(edgesFileCapabilities(env()), armedOfflineCapable()); len(got) != 0 {
+ t.Fatalf("no worker armed: %v", got)
+ }
+ wdCap := newOffWorld(t)
+ armed := wdCap.box("armed", "", nil)
+ armedMovesWorker.Store(armed.w)
+ if !armedOfflineCapable() {
+ t.Fatal("a capable armed worker does not publish")
+ }
+ if got := withOfflineCapability(edgesFileCapabilities(env()), armedOfflineCapable()); !slices.Equal(got, []string{capOfflineMove1}) {
+ t.Fatalf("capable box: %v", got)
+ }
+ base := edgesFileCapabilities(env("CARLOS_REGION_HA", "1"))
+ if got := withOfflineCapability(base, true); !slices.Equal(got, []string{capRegionHA1, capOfflineMove1}) || !slices.Equal(base, []string{capRegionHA1}) {
+ t.Fatalf("region HA too: %v (base %v)", got, base)
+ }
+ for _, kv := range [][]string{
+ {"CARLOS_HIBERNATE_BUCKET", ""},
+ {"CARLOS_HOST_CLAIM", "1"},
+ {"CARLOS_HOST_CLAIM", "0"}, // set at all
+ {moves.SwitchEnv, "on"},
+ {moves.SwitchEnv, "off"}, // set at all
+ } {
+ if offlineCapable(env(kv...), hibernationConfigured(env(kv...))) {
+ t.Fatalf("%s=%q is capable", kv[0], kv[1])
+ }
+ incapable := wdCap.box("incapable", "", map[string]string{kv[0]: kv[1]})
+ armedMovesWorker.Store(incapable.w)
+ if got := withOfflineCapability(edgesFileCapabilities(env(kv...)), armedOfflineCapable()); slices.Contains(got, capOfflineMove1) {
+ t.Fatalf("%s=%q publishes: %v", kv[0], kv[1], got)
+ }
+ }
+ armedMovesWorker.Store(prev)
+
+ wd := newOffWorld(t)
+ a := wd.box("ie-a", "", map[string]string{"CARLOS_HOST_CLAIM": "1"})
+ if a.w.cfg.OfflineCapable {
+ t.Fatal("armOffline: a claims box is capable")
+ }
+ a.tick()
+ wd.start()
+ a.tick()
+ if rt := a.row(offHost); rt == nil || rt.Status != "ready" || rt.HoldOp != "" {
+ t.Fatalf("an incapable box held the row: %+v", rt)
+ }
+ if a.svc.Fenced(offHost) {
+ t.Fatal("an incapable box fenced")
+ }
+ if _, ok := wd.reportIf(moves.OfflineRoleSource, "ie-a"); ok {
+ t.Fatal("an incapable box reported")
+ }
+ if _, ok := a.w.notices.Load(offHost + "#offline"); !ok {
+ t.Fatal("no notice from an incapable box")
+ }
+}
+
+// --- dormant ---------------------------------------------------------------
+
+// flatLayout is a reconcile config's layout read, always flat.
+func flatLayout(context.Context, release.Store) (string, error) { return release.LayoutModeFlat, nil }
+
+// dormantWorld is a deployment with no offline record anywhere: offHost at
+// rest, gone.example.com (whose record the test deletes), a legacy move and
+// a Phase 1 move on records of their own.
+func dormantWorld(t *testing.T) *offWorld {
+ t.Helper()
+ wd := newOffWorld(t)
+ for _, h := range []string{"gone.example.com", "legacy.example.com", "phase1.example.com"} {
+ wd.create(h)
+ }
+ wd.moveHost("legacy.example.com", func(rec *provision.Record) error {
+ rec.Move = &provision.MoveState{To: "uk", State: provision.MovePreparing}
+ return nil
+ })
+ wd.moveHost("phase1.example.com", func(rec *provision.Record) error {
+ rec.Move = &provision.MoveState{ID: offID2, From: "", To: "uk", State: provision.MovePreparing, RequestedAt: wd.clock()}
+ return nil
+ })
+ return wd
+}
+
+// dormantBoxes are two home boxes minted from the same records with no
+// hooks at all, so their registries start equal: the plain one stands for
+// main, the hooked one has a capable worker whose view no listing has read.
+func dormantBoxes(t *testing.T, wd *offWorld) (plain, hooked *offBox) {
+ t.Helper()
+ plain, hooked = wd.box("main", "", nil), wd.box("hooked", "", nil)
+ for _, b := range []*offBox{plain, hooked} {
+ if err := reconcileSerializedWith(context.Background(), wd.s, b.reg, provision.ReconcileConfig{DataDir: b.dd, Layout: flatLayout}, nil); err != nil {
+ t.Fatal(err)
+ }
+ if b.row("gone.example.com") == nil {
+ t.Fatalf("fixture: %s has no row to retire", b.name)
+ }
+ }
+ if hooked.w.cfg.Offline.built() || !hooked.w.cfg.OfflineCapable {
+ t.Fatal("fixture: the hooked box's view is built, or the box is not capable")
+ }
+ return plain, hooked
+}
+
+// sameRegistries fails unless the two boxes' registries match, paths and
+// creation times aside (each box has its own data directory and clock).
+func sameRegistries(t *testing.T, a, b *offBox) {
+ t.Helper()
+ strip := func(rs []registry.Route) []registry.Route {
+ out := append([]registry.Route(nil), rs...)
+ for i := range out {
+ out[i].DBPath, out[i].Socket, out[i].CreatedAt = "", "", 0
+ }
+ return out
+ }
+ ar, err := a.reg.List()
+ if err != nil {
+ t.Fatal(err)
+ }
+ br, err := b.reg.List()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !reflect.DeepEqual(strip(ar), strip(br)) {
+ t.Fatalf("the registries differ:\n %s %+v\n %s %+v", a.name, ar, b.name, br)
+ }
+}
+
+// With no offline record anywhere (a legacy and a Phase 1 move included):
+// the very first reconcile on a box whose view no listing has read removes
+// a retired row exactly as main does; after it the hold answers false for
+// every record and keep false for every host; and the edges file differs
+// by the one capability string and nothing else.
+func TestOfflineDormantIsMain(t *testing.T) {
+ ctx := context.Background()
+ wd := dormantWorld(t)
+ plain, hooked := dormantBoxes(t, wd)
+ if err := wd.s.Delete(ctx, provision.RecordKey(wd.acct, "app", "gone.example.com")); err != nil {
+ t.Fatal(err)
+ }
+ for _, b := range []*offBox{plain, hooked} {
+ w := b.w
+ if b == plain {
+ w = nil
+ }
+ if err := reconcileSerializedWith(ctx, wd.s, b.reg, provision.ReconcileConfig{DataDir: b.dd, Layout: flatLayout}, w); err != nil {
+ t.Fatal(err)
+ }
+ if rt := b.row("gone.example.com"); rt != nil {
+ t.Fatalf("%s kept a retired row on its first pass: %+v", b.name, rt)
+ }
+ }
+ sameRegistries(t, plain, hooked)
+ v := hooked.w.cfg.Offline
+ if !v.built() || v.failed() {
+ t.Fatal("the first reconcile did not read the view")
+ }
+ for _, h := range []string{offHost, "gone.example.com", "legacy.example.com", "phase1.example.com", "never.example.com"} {
+ if v.keep(h) {
+ t.Fatalf("keep(%s)", h)
+ }
+ }
+ recs, err := provision.ListAllRecords(ctx, wd.s)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(recs) != 3 {
+ t.Fatalf("fixture: %d records", len(recs))
+ }
+ for _, rec := range recs {
+ if _, _, ok := offlineHold(hooked.w.cfg)(rec); ok {
+ t.Fatalf("held %s", rec.Host)
+ }
+ }
+ // A further pass changes nothing on either box.
+ for _, b := range []*offBox{plain, hooked} {
+ w := b.w
+ if b == plain {
+ w = nil
+ }
+ if err := reconcileSerializedWith(ctx, wd.s, b.reg, provision.ReconcileConfig{DataDir: b.dd, Layout: flatLayout}, w); err != nil {
+ t.Fatal(err)
+ }
+ }
+ sameRegistries(t, plain, hooked)
+
+ // The edges file: the one string, and no headroom with it.
+ var last1, last2 []byte
+ none := func(string) string { return "" }
+ plainFile := edgesFile{Box: "plain", Placement: "", Capabilities: withOfflineCapability(edgesFileCapabilities(none), false)}
+ capable := edgesFile{Box: "plain", Placement: "", Capabilities: withOfflineCapability(edgesFileCapabilities(none), true)}
+ if headroomCapabilities(capable.Capabilities) != nil {
+ t.Fatal("offline-move-1 asks for headroom")
+ }
+ s1, s2 := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir())
+ if err := edgesFileWrite(ctx, s1, plain.reg, plainFile, "1.2.3.4", &last1, false); err != nil {
+ t.Fatal(err)
+ }
+ if err := edgesFileWrite(ctx, s2, plain.reg, capable, "1.2.3.4", &last2, false); err != nil {
+ t.Fatal(err)
+ }
+ b1, _, _ := s1.Get(ctx, edgesFileKey("plain"))
+ b2, _, _ := s2.Get(ctx, edgesFileKey("plain"))
+ var m1, m2 map[string]any
+ if json.Unmarshal(b1, &m1) != nil || json.Unmarshal(b2, &m2) != nil {
+ t.Fatal("edges files do not parse")
+ }
+ if caps, _ := m2["capabilities"].([]any); len(caps) != 1 || caps[0] != capOfflineMove1 {
+ t.Fatalf("capabilities: %v", m2["capabilities"])
+ }
+ delete(m2, "capabilities")
+ delete(m1, "generation")
+ delete(m2, "generation")
+ if !reflect.DeepEqual(m1, m2) {
+ t.Fatalf("the edges file differs beyond the capability:\n %s\n %s", b1, b2)
+ }
+ if bytes.Contains(b1, []byte("capabilities")) {
+ t.Fatal("fixture: the plain file has capabilities")
+ }
+}
+
+// M1: a failed startup listing keeps every row only until the next
+// reconcile, which reads the listing again: the retired row then goes as on
+// main, without waiting for a moves pass.
+func TestOfflineFailedListingHealsOnTheNextReconcile(t *testing.T) {
+ ctx := context.Background()
+ wd := dormantWorld(t)
+ plain, hooked := dormantBoxes(t, wd)
+ v := hooked.w.cfg.Offline
+ refreshOfflineView(ctx, failingListStore{Store: wd.s, prefix: "apps/", err: errors.New("listing refused")}, v)
+ if !v.built() || !v.failed() || !v.keep(offHost) {
+ t.Fatal("a failed startup listing does not keep")
+ }
+ if err := wd.s.Delete(ctx, provision.RecordKey(wd.acct, "app", "gone.example.com")); err != nil {
+ t.Fatal(err)
+ }
+ for _, b := range []*offBox{plain, hooked} {
+ w := b.w
+ if b == plain {
+ w = nil
+ }
+ if err := reconcileSerializedWith(ctx, wd.s, b.reg, provision.ReconcileConfig{DataDir: b.dd, Layout: flatLayout}, w); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if v.failed() || v.keep("gone.example.com") {
+ t.Fatal("the reconcile did not read the listing again")
+ }
+ if rt := hooked.row("gone.example.com"); rt != nil {
+ t.Fatalf("the retired row stayed after the view healed: %+v", rt)
+ }
+ sameRegistries(t, plain, hooked)
+ // While the listing keeps failing the view stays closed.
+ refreshOfflineView(ctx, failingListStore{Store: wd.s, prefix: "apps/", err: errors.New("listing refused")}, v)
+ if !v.keep(offHost) {
+ t.Fatal("a failed listing does not keep")
+ }
+ unbuilt := newOfflineView(moveCfg{Region: ""}, nil)
+ if !unbuilt.keep(offHost) {
+ t.Fatal("an unbuilt view keeps nothing")
+ }
+}
+
+// M4: a record that does not decode is kept by keep alone (its move cannot
+// be known), logged, and every other host is judged as usual. The moves
+// pass's listing carries the same answer.
+func TestOfflineUndecodableRecordIsKept(t *testing.T) {
+ ctx := context.Background()
+ wd := newOffWorld(t)
+ wd.create("broken.example.com")
+ if _, err := wd.s.Put(ctx, provision.RecordKey(wd.acct, "app", "broken.example.com"), []byte("{not json"), mustETag(t, wd.s, provision.RecordKey(wd.acct, "app", "broken.example.com"))); err != nil {
+ t.Fatal(err)
+ }
+ a := wd.box("ie-a", "", nil)
+ refreshOfflineView(ctx, wd.s, a.w.cfg.Offline)
+ v := a.w.cfg.Offline
+ if v.failed() || !v.keep("broken.example.com") || v.keep(offHost) {
+ t.Fatalf("startup view: failed %v, keep(broken) %v, keep(%s) %v", v.failed(), v.keep("broken.example.com"), offHost, v.keep(offHost))
+ }
+ fresh := newOfflineView(a.w.cfg, nil)
+ a.w.cfg.Offline = fresh
+ a.pass()
+ if !fresh.keep("broken.example.com") || fresh.keep(offHost) {
+ t.Fatal("the moves pass's listing does not keep the undecodable host")
+ }
+ if _, _, ok := offlineHold(a.w.cfg)(provision.Record{Host: "broken.example.com"}); ok {
+ t.Fatal("held a record with no move")
+ }
+}
+
+func mustETag(t *testing.T, s release.Store, key string) string {
+ t.Helper()
+ _, etag, err := s.Get(context.Background(), key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return etag
+}
+
+// hostSliceCgroup is the slice instancectl's units run in.
+func TestOfflineHostSliceIsInstancectlsSlice(t *testing.T) {
+ got := hostSliceCgroup("/sys/fs/cgroup", offHost)
+ want := filepath.Join("/sys/fs/cgroup", "carlos.slice", "carlos-instances.slice", instancectl.SliceName(offHost))
+ if got != want {
+ t.Fatalf("hostSliceCgroup = %s, want %s", got, want)
+ }
+ if filepath.Dir(filepath.Join("/sys/fs/cgroup", filepath.FromSlash(instancectl.ClaimUnitControlGroup(offHost)))) != got {
+ t.Fatal("the run unit is not in the slice")
+ }
+}
+
+// The seal record's writer is create-only: the first body stays, a re-run
+// gets it back and writes nothing, and the report writer beside it replaces.
+func TestOfflineCreateOnlyKeepsTheFirstWrite(t *testing.T) {
+ ctx := context.Background()
+ s := release.NewDirStore(t.TempDir())
+ key := moves.OfflineReportPrefix(offHost, offID) + "seal.json"
+ got, created, err := putOfflineCreateOnly(ctx, s, key, []byte(`{"n":1}`))
+ if err != nil || !created || string(got) != `{"n":1}` {
+ t.Fatalf("first write: %q %v %v", got, created, err)
+ }
+ got, created, err = putOfflineCreateOnly(ctx, s, key, []byte(`{"n":2}`))
+ if err != nil || created || string(got) != `{"n":1}` {
+ t.Fatalf("second write: %q %v %v, want the first body back, nothing written", got, created, err)
+ }
+ if body, _, err := s.Get(ctx, key); err != nil || string(body) != `{"n":1}` {
+ t.Fatalf("stored: %q %v", body, err)
+ }
+ if _, _, err := putOfflineCreateOnly(ctx, s, "", nil); err == nil {
+ t.Fatal("an empty key was written")
+ }
+}
+
+// keep's table: the source keeps until finishing, the target until the
+// move is cleared, a third region never.
+func TestOfflineKeep(t *testing.T) {
+ rec := func(region, state string) provision.Record {
+ return provision.Record{Host: offHost, Region: region, Move: &provision.MoveState{Kind: provision.MoveKindOffline, ID: offID, From: "", To: "uk", State: state}}
+ }
+ states := []string{provision.OfflineStopping, provision.OfflineSealing, provision.OfflineSealed, provision.OfflineLanding,
+ provision.OfflineLanded, provision.OfflineOpen, provision.OfflineFinishing, provision.OfflineReverting}
+ for _, st := range states {
+ recs := []provision.Record{rec("uk", st)}
+ src := newOfflineView(moveCfg{Region: ""}, nil)
+ src.refresh(recs, nil)
+ if got, want := src.keep(offHost), st != provision.OfflineFinishing; got != want {
+ t.Fatalf("source at %s: keep %v", st, got)
+ }
+ tgt := newOfflineView(moveCfg{Region: "uk"}, nil)
+ tgt.refresh(recs, nil)
+ if !tgt.keep(offHost) {
+ t.Fatalf("target at %s: not kept", st)
+ }
+ other := newOfflineView(moveCfg{Region: "fr"}, nil)
+ other.refresh(recs, nil)
+ if other.keep(offHost) {
+ t.Fatalf("a third region keeps at %s", st)
+ }
+ }
+}
diff --git a/internal/host/movespass.go b/internal/host/movespass.go
index 5ca6fce15..16efd306e 100644
--- a/internal/host/movespass.go
+++ b/internal/host/movespass.go
@@ -3,6 +3,7 @@ package host
import (
"context"
"crypto/ecdh"
+ "crypto/ed25519"
"encoding/json"
"errors"
"fmt"
@@ -17,6 +18,7 @@ import (
"github.com/carlosframework/platform/internal/activator"
"github.com/carlosframework/platform/internal/appconfig"
+ "github.com/carlosframework/platform/internal/cgroup"
"github.com/carlosframework/platform/internal/controlsock"
"github.com/carlosframework/platform/internal/dbverify"
"github.com/carlosframework/platform/internal/hostbox"
@@ -174,7 +176,7 @@ func (w *movesWorker) setStrict(recs []provision.Record) {
// rule (park.PlacementMatches under the verified home binding,
// ParkHomeRegion), so the home's roster name and its "" code are one.
func movesNamesBox(cfg moveCfg, rec provision.Record) bool {
- if rec.Move == nil || rec.Move.Legacy() {
+ if rec.Move == nil || rec.Move.Legacy() || rec.Move.Offline() {
return false
}
here := cfg.parkRegion()
@@ -321,7 +323,12 @@ func armMovesPass(hib activator.Config, dataDir string, reg *registry.Registry,
Events: events,
LogEvent: moveLogWriter(events),
}
- w := &movesWorker{mirrors: map[string]lease.MirrorStore{}}
+ // The edges box label and the version store, whatever the switch says:
+ // the offline move reports under the label (movesoffline.go). armPhase1
+ // sets the same two values.
+ cfg.Box = metricsBoxLabel(os.Getenv, readMachineID())
+ cfg.OptDir = release.OptDir()
+ w := newMovesWorker(cfg)
if cfg.Enabled {
armPhase1(&cfg, w, hib, dataDir, reg, svc, eb, writes, os.Getenv)
if writes.Enabled && providers.guard.FenceView != nil {
@@ -338,10 +345,86 @@ func armMovesPass(hib activator.Config, dataDir string, reg *registry.Registry,
cancel()
}
}
+ armOffline(&cfg, dataDir, reg, svc, eb, writes, os.Getenv)
w.cfg = cfg
armedMovesWorker.Store(w)
}
+// armOffline sets every moveCfg field the offline operator move reads
+// (movesoffline.go and Tasks 4 and 5 of docs/superpowers/plans/2026-10-20-
+// offline-operator-move.md), whatever the Phase 1 switch says, after
+// armPhase1 so a hook it already set is kept. Nothing here acts: the fields
+// are read only by offline steps, which run only on a capable box and only
+// for a record with an offline move.
+func armOffline(cfg *moveCfg, dataDir string, reg *registry.Registry, svc *activator.Service, eb *activator.ExecBackend, writes hostwrite.Config, getenv func(string) string) {
+ cfg.OfflineCapable = offlineCapable(getenv, hibernationConfigured(getenv))
+ cfg.Offline = newOfflineView(*cfg, holdStageRequester(dataDir))
+ if svc != nil {
+ cfg.FenceAs = fenceAsWith(svc)
+ cfg.LiftFenceAs = svc.LiftFenceAs
+ }
+ if reg != nil {
+ cfg.Registry = reg
+ // The caller holds provision.WithHostHoldLock; a release that
+ // landed touches the stage-requested marker, so root re-judges the
+ // now-ready row (registry.SetHold's contract, G10).
+ cfg.ReleaseHold = releaseHeld(reg, holdStageRequester(dataDir))
+ // The row and the timers the move landed on it, in one
+ // transaction, only while op holds the row.
+ cfg.DeleteIfHold = reg.DeleteIfHoldWithTimers
+ }
+ if eb != nil {
+ cfg.LaunchLayers = eb.LaunchLayers
+ cgRoot := eb.CgroupRoot
+ if cgRoot == "" {
+ cgRoot = "/sys/fs/cgroup"
+ }
+ execRoot := eb.Cgroups.Root()
+ cfg.Procs = func(host string) (int, error) {
+ return cgroup.HostProcs(execRoot, host, hostSliceCgroup(cgRoot, host))
+ }
+ }
+ if raw := strings.TrimSpace(getenv("CARLOS_SECRETS_KEY")); raw != "" {
+ if priv, err := appconfig.ParsePrivateKey(raw, "CARLOS_SECRETS_KEY"); err == nil {
+ cfg.ConfigPriv, cfg.ConfigPub = priv, priv.PublicKey()
+ if cfg.SecretsKey == "" {
+ cfg.SecretsKey = raw
+ }
+ }
+ }
+ if cfg.RenderHeld == nil {
+ cfg.RenderHeld = heldRenderer(writes, cfg.ConfigPriv, dataDir, cfg.Box)
+ }
+ optDir := cfg.OptDir
+ cfg.LiveVersion = func(host string) (string, error) { return release.LiveVersion(optDir, host) }
+ pub, perr := release.ParsePublicKey(getenv("CARLOS_RELEASE_PUBKEY"), "CARLOS_RELEASE_PUBKEY")
+ cfg.ChannelVersion = func(ctx context.Context, s release.Store, rt registry.Route) (string, error) {
+ if perr != nil {
+ return "", perr
+ }
+ return channelVersion(ctx, s, pub, rt)
+ }
+ if cfg.Now == nil {
+ cfg.Now = time.Now
+ }
+}
+
+// channelVersion is the build the row's channel points at, as adoption
+// resolves it: the account-qualified pointer of the row's account, app and
+// channel ("stable" when the row names none), verified under the release
+// key.
+func channelVersion(ctx context.Context, s release.Store, pub ed25519.PublicKey, rt registry.Route) (string, error) {
+ channel := rt.Channel
+ if channel == "" {
+ channel = "stable"
+ }
+ ptr, err := release.Channel(ctx, s, pub, release.RouteAccount(rt), rt.App, channel)
+ if err != nil {
+ return "", err
+ }
+ return ptr.Version, nil
+}
+
// movesDataDir is the data directory the moves worker is built with,
// resolved once to an absolute, cleaned path (final re-review 1, I2): the
// outboxes under it sync every directory from an entry's own up to it, and
@@ -892,6 +975,55 @@ type moveCfg struct {
// laneParkTarget's, armed as the revert's (revertLaneArmed). abFault is
// its test seam, nil in production.
abFault func(point string) error
+
+ // The offline operator move (movesoffline.go; docs/superpowers/plans/
+ // 2026-10-20-offline-operator-move.md, Tasks 3 to 5). armOffline sets
+ // every field below whatever the Phase 1 switch says, and so Box,
+ // OptDir, SecretsKey, RenderHeld and Now above; tests set them by hand.
+ // The sealer (Task 4) and the target (Task 5) read only these, plus the
+ // hibernate coordinates, DataDir, Region, Mirror, Quiesced, Route,
+ // Checkpoint and CarryRequest above.
+ //
+ // OfflineCapable is offlineCapable for this box: false, and every
+ // offline step logs one notice per host and does nothing. Offline is the
+ // keep view reconcile reads (offlineView.keep); the hold needs none.
+ OfflineCapable bool
+ Offline *offlineView
+ // FenceAs installs, or meets again, an owned activator fence
+ // (activator.Service.FenceAndStopAs under a 2 s deadline,
+ // ErrStopUnproven counted as installed: fenceAsWith); LiftFenceAs lifts
+ // exactly that fence. Nil (no activator): never fenced.
+ FenceAs func(ctx context.Context, host string, generation uint64, owner string) error
+ LiftFenceAs func(host string, generation uint64, owner string) bool
+ // ReleaseHold opens a row held by op (registry.ReleaseHold, touching the
+ // stage-requested marker when it landed); DeleteIfHold deletes a row
+ // held by op, with its timers rows, in one transaction
+ // (registry.DeleteIfHoldWithTimers). Both answer false when op does not
+ // hold the row. The caller holds provision.WithHostHoldLock.
+ ReleaseHold func(host, op string) (bool, error)
+ DeleteIfHold func(host, op string) (bool, error)
+ // Registry is this box's registry: TimersFor, ReplaceTimers,
+ // SetNextDue, SetReplication and Get.
+ Registry *registry.Registry
+ // LaunchLayers is the exec backend's composed launch environment for a
+ // row (activator.ExecBackend.LaunchLayers).
+ LaunchLayers func(rt registry.Route) (env map[string]string, instance map[string]bool, shadowed []string, err error)
+ // ConfigPriv and ConfigPub are the deployment's config keypair, parsed
+ // from CARLOS_SECRETS_KEY: ConfigPub seals the files archive, ConfigPriv
+ // opens it and renders the held config. Nil without the key.
+ ConfigPriv *ecdh.PrivateKey
+ ConfigPub *ecdh.PublicKey
+ // LiveVersion is the build a row's live link names on this box
+ // (release.LiveVersion over OptDir). ChannelVersion is the build the
+ // row's channel points at, as adoption resolves it (channelVersion,
+ // verified under CARLOS_RELEASE_PUBKEY; an error when that key does
+ // not parse).
+ LiveVersion func(host string) (string, error)
+ ChannelVersion func(ctx context.Context, s release.Store, rt registry.Route) (string, error)
+ // Procs counts the processes left for a host in its exec cgroup and its
+ // nested slice (cgroup.HostProcs). Nil, or an error: unknown, which a
+ // source report records as -1, not stopped.
+ Procs func(host string) (int, error)
}
// defaultMoveReadyDelta is the readiness bar when
@@ -942,7 +1074,9 @@ func movesPass(ctx context.Context, s release.Store, w *movesWorker) error {
w.flushPageOutbox(ctx)
w.flushLogOutbox(ctx, s)
}
- recs, err := provision.ListAllRecords(ctx, s)
+ // Strict returns exactly ListAllRecords' records and errors, plus the
+ // hosts whose record did not decode, which the offline keep view keeps.
+ recs, unreadable, err := provision.ListAllRecordsStrict(ctx, s)
if err != nil {
return fmt.Errorf("moves: list records: %w", err)
}
@@ -966,7 +1100,23 @@ func movesPass(ctx context.Context, s release.Store, w *movesWorker) error {
w.holds.refresh(ctx, s, recs)
}
}
+ // The offline keep view, from the list this pass read (movesoffline.go).
+ if cfg.Offline != nil {
+ cfg.Offline.refresh(recs, unreadable)
+ }
for _, rec := range recs {
+ // An offline operator move is routed first, ahead of the Phase 1
+ // skip and the lost-box gate below: its id makes it look like a
+ // Phase 1 record, which with the switch off is skipped, and it
+ // never reads a host claim (movesoffline.go).
+ if rec.Move.Offline() {
+ if err := w.offlineRound(ctx, s, rec); err != nil {
+ w.logTrouble(rec.Host, err)
+ continue
+ }
+ w.trouble.Delete(rec.Host)
+ continue
+ }
// A move closed or replaced by another writer is forgotten by the
// move log's memory (movesevents.go). With the switch off the pass
// touches no outbox, index or memory of the move log at all (final
@@ -1134,7 +1284,7 @@ const legacyMoveSentence = "this move was started by an older console and has no
// hasPhase1Into reports whether any Phase 1 move targets region.
func hasPhase1Into(recs []provision.Record, region string) bool {
for _, rec := range recs {
- if rec.Move != nil && !rec.Move.Legacy() && rec.Move.To == region {
+ if rec.Move != nil && !rec.Move.Legacy() && !rec.Move.Offline() && rec.Move.To == region {
return true
}
}
diff --git a/internal/layout/extras.go b/internal/layout/extras.go
index 5fef52ff6..329fdf930 100644
--- a/internal/layout/extras.go
+++ b/internal/layout/extras.go
@@ -321,6 +321,17 @@ func FlatLeftovers(dataDir, host string, hosts []string) []string {
return ours
}
+// RawInstanceDir is a nested host's instance directory through the raw
+// directory behind the /data/instances bind mount, for a caller outside this
+// package that renames entries out of it. The offline operator move parks a
+// nested source's contents this way (docs/superpowers/plans/2026-10-20-
+// offline-operator-move.md, Task 3): a rename through the mount answers
+// EXDEV, as the pass's own renames would (Artifact). rename(2) and lstat(2)
+// act on names, never on what a planted link points at; a caller uses only
+// those and a listing of the directory itself, and never follows a link
+// beneath it.
+func RawInstanceDir(dataDir, host string) string { return rawInstanceDir(dataDir, host) }
+
// platformNames are the entries the platform itself keeps directly under
// the data directory. UnattributedEntries leaves them out of its report.
var platformNames = map[string]bool{
@@ -328,6 +339,10 @@ var platformNames = map[string]bool{
"instances": true, ".instances": true, "layout": true, "system": true,
"moves": true, ".moves": true, ".hostexec": true, "litestream.env": true,
"lost+found": true,
+ // The offline operator move's park directory: local state it renames
+ // aside, never deletes (docs/superpowers/plans/2026-10-20-offline-
+ // operator-move.md).
+ ".offline-moves": true,
}
// UnattributedEntries lists the entries directly under dataDir that no
diff --git a/internal/layout/extras_test.go b/internal/layout/extras_test.go
index 42b105af4..4737ca688 100644
--- a/internal/layout/extras_test.go
+++ b/internal/layout/extras_test.go
@@ -88,7 +88,7 @@ func TestValidateJudgesTheJournalsExtraNames(t *testing.T) {
// wrote, which no move can take.
func TestUnattributedEntriesSkipsHostsAndPlatformNames(t *testing.T) {
dd := t.TempDir()
- for _, name := range []string{"blobs", "registry.db", "layout", ".instances", "instances", host, host + "-blobs", "cache.tmp"} {
+ for _, name := range []string{"blobs", "registry.db", "layout", ".instances", "instances", ".offline-moves", host, host + "-blobs", "cache.tmp"} {
if err := os.MkdirAll(filepath.Join(dd, name), 0o700); err != nil {
t.Fatal(err)
}
diff --git a/internal/lease/leasetest/s3server.go b/internal/lease/leasetest/s3server.go
index 0b07ae29d..051b9f391 100644
--- a/internal/lease/leasetest/s3server.go
+++ b/internal/lease/leasetest/s3server.go
@@ -217,6 +217,17 @@ func NewS3Server(f *VersionedBucket, serves ...string) (http.Handler, func() int
}
w.WriteHeader(http.StatusNoContent)
case r.Method == http.MethodDelete && key != "":
+ if match := r.Header.Get("If-Match"); match != "" { // as S3 enforces it on DeleteObject
+ current := f.buckets[bucket][key]
+ if len(current) == 0 || current[0].deleteMarker {
+ s3err(w, http.StatusNotFound, "NoSuchKey")
+ return
+ }
+ if current[0].etag != strings.Trim(match, `"`) {
+ s3err(w, http.StatusPreconditionFailed, "PreconditionFailed")
+ return
+ }
+ }
id, err := f.bareDeleteLocked(bucket, key) // versioning on: a delete marker
if err != nil {
writeErr(w, err)
diff --git a/internal/lease/leasetest/s3server_test.go b/internal/lease/leasetest/s3server_test.go
index b4546f938..3b3581c2b 100644
--- a/internal/lease/leasetest/s3server_test.go
+++ b/internal/lease/leasetest/s3server_test.go
@@ -255,3 +255,35 @@ func TestS3ServerConditionalReplaceUsesReadETag(t *testing.T) {
t.Fatal("CAS over delete marker accepted", got.Code)
}
}
+
+// DeleteObject honours If-Match as S3 does: a stale ETag is a 412 and removes
+// nothing, a missing key is a 404, the current ETag deletes, and a plain
+// delete (no header) is unchanged.
+func TestS3ServerConditionalDeleteUsesReadETag(t *testing.T) {
+ m := NewVersionedBucket()
+ m.AddBucket("data")
+ h, _ := NewS3Server(m)
+ do(t, h, http.MethodPut, "/data/lock.json", "one", map[string]string{"If-None-Match": "*"})
+ stale := do(t, h, http.MethodGet, "/data/lock.json", "", nil).Header().Get("ETag")
+ do(t, h, http.MethodPut, "/data/lock.json", "two", map[string]string{"If-Match": stale})
+ current := do(t, h, http.MethodGet, "/data/lock.json", "", nil).Header().Get("ETag")
+ before := m.Snapshot()
+ if got := do(t, h, http.MethodDelete, "/data/lock.json", "", map[string]string{"If-Match": stale}); got.Code != http.StatusPreconditionFailed {
+ t.Fatal("stale conditional delete accepted", got.Code)
+ }
+ if m.Snapshot() != before {
+ t.Fatal("stale conditional delete mutated versions")
+ }
+ if got := do(t, h, http.MethodDelete, "/data/lock.json", "", map[string]string{"If-Match": current}); got.Code != http.StatusNoContent {
+ t.Fatal("current conditional delete refused", got.Code)
+ }
+ if got := do(t, h, http.MethodDelete, "/data/lock.json", "", map[string]string{"If-Match": current}); got.Code != http.StatusNotFound {
+ t.Fatal("conditional delete of a deleted key", got.Code)
+ }
+ if got := do(t, h, http.MethodDelete, "/data/never", "", map[string]string{"If-Match": current}); got.Code != http.StatusNotFound {
+ t.Fatal("conditional delete of a missing key", got.Code)
+ }
+ if got := do(t, h, http.MethodDelete, "/data/never", "", nil); got.Code != http.StatusNoContent {
+ t.Fatal("plain delete changed", got.Code)
+ }
+}
diff --git a/internal/lease/tombstone.go b/internal/lease/tombstone.go
new file mode 100644
index 000000000..c0054208f
--- /dev/null
+++ b/internal/lease/tombstone.go
@@ -0,0 +1,308 @@
+package lease
+
+// The tombstone lease: the offline operator move's linearization point.
+//
+// A move that has stopped a host on its source box writes a lease at the
+// host's own lock.json under the owner "moved:<id>" and a long expiry. From
+// then on lease.Acquire on that location fails with a *HeldError naming the
+// move, so no box can wake the host and write to the replica behind the
+// move's back. Three operations manage it, each a conditional write over the
+// client AcquireTTL builds (IfNoneMatch "*" on create, IfMatch on replace
+// or delete) and none of them starts a renewer: the move's sealer re-asserts
+// the tombstone itself on every pass.
+//
+// - Tombstone writes it. It never replaces a live holder's lease.
+// - Reassert pushes the expiry out again, only on a lease the same move
+// owns, with the ETag it just read.
+// - LiftTombstone lifts it, only when the owner is a moved:<id> lease the
+// caller claims as its own, with the ETag it just read. It lifts by a
+// conditional PUT that rewrites the lease as already expired, never by a
+// DELETE: Ceph RGW and MinIO are not known to enforce If-Match on a
+// DELETE, and the conditional PUT is the primitive the rest of this
+// package relies on. IsLiftedTombstone reads that state back.
+//
+// "Free" is exactly what Acquire means by it (leaseFree): the lease's expiry
+// has passed. There is no RenewMargin here. A tombstone that is free to a
+// wake is free to the next Tombstone, and the two cannot drift apart.
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/benbjohnson/litestream"
+ lss3 "github.com/benbjohnson/litestream/s3"
+)
+
+// TombstonePrefix starts the owner of every tombstone lease. The three
+// functions here refuse any other owner.
+const TombstonePrefix = "moved:"
+
+// MaxTombstoneTTL is the longest a tombstone is ever written for. The moves
+// package's TombstoneTTL equals it; this package cannot import that one.
+const MaxTombstoneTTL = 7 * 24 * time.Hour
+
+var (
+ // ErrTombstoneOwner refuses an owner that is not moved:<id>.
+ ErrTombstoneOwner = errors.New("lease: a tombstone owner is moved:<id>")
+ // ErrTombstoneTTL refuses a TTL that is not positive or exceeds
+ // MaxTombstoneTTL.
+ ErrTombstoneTTL = errors.New("lease: a tombstone lives for at most 7 days")
+ // ErrTombstoneChanged reports that the lease was rewritten or removed
+ // between the read and the conditional write. Nothing was overwritten;
+ // the caller's next pass reads it afresh.
+ ErrTombstoneChanged = errors.New("lease: the lock changed under the tombstone write")
+)
+
+// leaseFree is the predicate litestream's AcquireLease applies to the lease
+// it reads (Lease.IsExpired: now is after the expiry), which is what decides
+// whether lease.Acquire takes a lock over. Tombstone, Reassert and
+// IsLiftedTombstone use it so a lease is free to them exactly when it is free
+// to a wake. TestTombstoneAndAcquireAgreeOnFree pins the two together.
+func leaseFree(expires, now time.Time) bool { return now.After(expires) }
+
+// liftedExpiry is the expiry LiftTombstone writes: long past, so every
+// reading (leaseFree, and LockState's RenewMargin) agrees the lease is free.
+var liftedExpiry = time.Unix(0, 0).UTC()
+
+// IsLiftedTombstone reports whether a lock read as owner and expires (from
+// LockState) is a lifted tombstone: a moved:<id> lease that has expired. That
+// is what LiftTombstone leaves behind, and it is free to every Acquire (which
+// already treats any expired lease as free) and to the next Tombstone. It is
+// false for an ordinary process's lease, expired or not, and for a tombstone
+// that is still unexpired. An absent lock is not "lifted" here: callers read
+// absence from LockState's zero owner and expiry and treat it the same way.
+func IsLiftedTombstone(owner string, expires time.Time) bool {
+ return tombstoneOwnerOK(owner) && !expires.IsZero() && leaseFree(expires, time.Now())
+}
+
+func tombstoneOwnerOK(owner string) bool {
+ return strings.HasPrefix(owner, TombstonePrefix) && len(owner) > len(TombstonePrefix)
+}
+
+func tombstoneTTLOK(ttl time.Duration) bool { return ttl > 0 && ttl <= MaxTombstoneTTL }
+
+// tombstoneLeaser is the leaser a tombstone call writes with, built over the
+// client AcquireTTL builds, and the lock's key. ttl is zero for calls that
+// write no expiry (LiftTombstone).
+func tombstoneLeaser(ctx context.Context, bucket, location, region, owner string, ttl time.Duration) (*lss3.Leaser, string, error) {
+ key, err := lockKey(bucket, location)
+ if err != nil {
+ return nil, "", err
+ }
+ client, _, err := newLeaserClient(ctx, region)
+ if err != nil {
+ return nil, "", err
+ }
+ leaser := lss3.NewLeaser()
+ leaser.SetClient(client)
+ leaser.Bucket = bucket
+ leaser.Path = strings.TrimSuffix(location, "/")
+ leaser.TTL = ttl
+ leaser.Owner = owner
+ return leaser, key, nil
+}
+
+// readLock reads the lock with the ETag it was read at. Only S3's explicit
+// NoSuchKey is absence (lockAbsent); a garbled body, an oversized one or a
+// lock with no expiry is an error, never "free".
+func readLock(ctx context.Context, leaser *lss3.Leaser, key string) (lock *litestream.Lease, absent bool, err error) {
+ out, err := leaser.Client().GetObject(ctx, &awss3.GetObjectInput{Bucket: aws.String(leaser.Bucket), Key: aws.String(key)})
+ if err != nil {
+ if lockAbsent(err) {
+ return nil, true, nil
+ }
+ return nil, false, fmt.Errorf("lease: read %s: %w", key, err)
+ }
+ defer out.Body.Close()
+ body, err := io.ReadAll(io.LimitReader(out.Body, lockBodyLimit+1))
+ if err != nil {
+ return nil, false, fmt.Errorf("lease: read %s: %w", key, err)
+ }
+ if len(body) > lockBodyLimit {
+ return nil, false, fmt.Errorf("lease: %s is larger than a lock", key)
+ }
+ var l litestream.Lease
+ if err := json.Unmarshal(body, &l); err != nil {
+ return nil, false, fmt.Errorf("lease: decode %s: %w", key, err)
+ }
+ if l.ExpiresAt.IsZero() {
+ return nil, false, fmt.Errorf("lease: %s has no expiry", key)
+ }
+ if out.ETag == nil || *out.ETag == "" {
+ return nil, false, fmt.Errorf("lease: %s came back without an ETag", key)
+ }
+ l.ETag = *out.ETag
+ return &l, false, nil
+}
+
+func heldErr(l *litestream.Lease) error {
+ return fmt.Errorf("acquire lease: %w", &HeldError{Owner: l.Owner, ExpiresAt: l.ExpiresAt})
+}
+
+// Tombstone writes the lease at s3://bucket/<location>/lock.json owned by
+// owner (moved:<id>), expiring ttl from now (at most MaxTombstoneTTL).
+//
+// - absent, or expired (as Acquire reads it): created (IfNoneMatch "*") or
+// replaced (IfMatch on the expired lease's ETag);
+// - live and held by owner: nil and no write; the caller re-asserts with
+// Reassert;
+// - live and held by anyone else: a *HeldError, nothing written;
+// - the lock changed under the write and cannot be read back:
+// ErrTombstoneChanged.
+//
+// A live holder is never replaced: the replace is conditional on the ETag of
+// the lease that was read as expired, so a holder that renews in between
+// wins and Tombstone reports it as held.
+func Tombstone(ctx context.Context, bucket, location, region, owner string, ttl time.Duration) error {
+ if !tombstoneOwnerOK(owner) {
+ return ErrTombstoneOwner
+ }
+ if !tombstoneTTLOK(ttl) {
+ return ErrTombstoneTTL
+ }
+ leaser, key, err := tombstoneLeaser(ctx, bucket, location, region, owner, ttl)
+ if err != nil {
+ return err
+ }
+ cur, absent, err := readLock(ctx, leaser, key)
+ if err != nil {
+ return err
+ }
+ if !absent && !leaseFree(cur.ExpiresAt, time.Now()) {
+ if cur.Owner == owner {
+ return nil
+ }
+ return heldErr(cur)
+ }
+ // Absent or expired. AcquireLease reads again and writes conditionally:
+ // IfNoneMatch "*" when absent, IfMatch on the ETag it reads otherwise.
+ // If a holder appeared since our read it answers LeaseExistsError.
+ if _, err := leaser.AcquireLease(ctx); err != nil {
+ var lexists *litestream.LeaseExistsError
+ if errors.As(err, &lexists) {
+ switch lexists.Owner {
+ case "":
+ // The write lost its condition and AcquireLease could not
+ // read the winner back (it was removed again, or the read
+ // failed). There is no holder to name.
+ return fmt.Errorf("tombstone lease: %w", ErrTombstoneChanged)
+ case owner:
+ return nil
+ }
+ return fmt.Errorf("acquire lease: %w", &HeldError{Owner: lexists.Owner, ExpiresAt: lexists.ExpiresAt})
+ }
+ return fmt.Errorf("tombstone lease: %w", err)
+ }
+ return nil
+}
+
+// Reassert moves the expiry of a tombstone owner already holds to now+ttl:
+//
+// - live and held by owner: rewritten with IfMatch on the ETag just read.
+// If the lock changed since, ErrTombstoneChanged and nothing is written;
+// - absent, or expired (including lifted): Tombstone, which writes a new
+// one;
+// - live and held by anyone else: a *HeldError, nothing written.
+//
+// Callers must read LockState first and treat absent or expired as a lost
+// stop; never Reassert after Lift. Reassert cannot tell a tombstone that was
+// kept from one that lapsed and was re-created, and in between a wake may
+// have acquired, written and released the lease.
+func Reassert(ctx context.Context, bucket, location, region, owner string, ttl time.Duration) error {
+ if !tombstoneOwnerOK(owner) {
+ return ErrTombstoneOwner
+ }
+ if !tombstoneTTLOK(ttl) {
+ return ErrTombstoneTTL
+ }
+ leaser, key, err := tombstoneLeaser(ctx, bucket, location, region, owner, ttl)
+ if err != nil {
+ return err
+ }
+ cur, absent, err := readLock(ctx, leaser, key)
+ if err != nil {
+ return err
+ }
+ if absent || leaseFree(cur.ExpiresAt, time.Now()) {
+ return Tombstone(ctx, bucket, location, region, owner, ttl)
+ }
+ if cur.Owner != owner {
+ return heldErr(cur)
+ }
+ if _, err := leaser.RenewLease(ctx, cur); err != nil {
+ if errors.Is(err, litestream.ErrLeaseNotHeld) {
+ return fmt.Errorf("reassert tombstone: %w", ErrTombstoneChanged)
+ }
+ return fmt.Errorf("reassert tombstone: %w", err)
+ }
+ return nil
+}
+
+// LiftTombstone lifts the tombstone at s3://bucket/<location>/lock.json by
+// rewriting it as an expired lease of the same owner and generation, with
+// IfMatch on the ETag just read. It never deletes the object.
+//
+// - absent: (false, nil), and mine is not consulted;
+// - owner is not a moved:<id> that mine accepts: (false, *HeldError),
+// nothing written. This holds for an expired lease too: only the move
+// that wrote a tombstone lifts it;
+// - mine and already lifted (expired, IsLiftedTombstone): (false, nil),
+// nothing written;
+// - mine and live: rewritten, (true, nil). Acquire takes the lease from
+// here, as it takes any expired lease, and so does Tombstone;
+// - the lock was rewritten since the read: (false, ErrTombstoneChanged),
+// and the rewrite stands. One that was removed since is (false, nil).
+//
+// After a lift, LockState reports held=false with the move's owner and a
+// past expiry; IsLiftedTombstone recognises that. Never Reassert after it.
+func LiftTombstone(ctx context.Context, bucket, location, region string, mine func(owner string) bool) (lifted bool, err error) {
+ leaser, key, err := tombstoneLeaser(ctx, bucket, location, region, "", 0)
+ if err != nil {
+ return false, err
+ }
+ cur, absent, err := readLock(ctx, leaser, key)
+ if err != nil {
+ return false, err
+ }
+ if absent {
+ return false, nil
+ }
+ if !tombstoneOwnerOK(cur.Owner) || mine == nil || !mine(cur.Owner) {
+ return false, heldErr(cur)
+ }
+ if IsLiftedTombstone(cur.Owner, cur.ExpiresAt) {
+ return false, nil
+ }
+ body, err := json.Marshal(&litestream.Lease{Generation: cur.Generation, ExpiresAt: liftedExpiry, Owner: cur.Owner})
+ if err != nil {
+ return false, fmt.Errorf("lift tombstone: %w", err)
+ }
+ _, err = leaser.Client().PutObject(ctx, &awss3.PutObjectInput{
+ Bucket: aws.String(leaser.Bucket),
+ Key: aws.String(key),
+ Body: bytes.NewReader(body),
+ ContentType: aws.String("application/json"),
+ IfMatch: aws.String(cur.ETag),
+ })
+ if err == nil {
+ return true, nil
+ }
+ if code, status := apiCode(err); code == "PreconditionFailed" || status == http.StatusPreconditionFailed || code == "NoSuchKey" || status == http.StatusNotFound {
+ // Rewritten, or gone (AWS answers a conditional PUT on a missing key
+ // 404, some stores 412). Read it back to tell which.
+ if _, gone, rerr := readLock(ctx, leaser, key); rerr == nil && gone {
+ return false, nil
+ }
+ return false, fmt.Errorf("lift tombstone: %w", ErrTombstoneChanged)
+ }
+ return false, fmt.Errorf("lift tombstone: %w", err)
+}
diff --git a/internal/lease/tombstone_test.go b/internal/lease/tombstone_test.go
new file mode 100644
index 000000000..9dc522871
--- /dev/null
+++ b/internal/lease/tombstone_test.go
@@ -0,0 +1,784 @@
+package lease_test
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/lease"
+ "github.com/carlosframework/platform/internal/lease/leasetest"
+)
+
+const (
+ tombLoc = "instances/h.example.com"
+ tombMove = "moved:mv_01AAAAAAAAAAAAAAAAAAAAAAAA"
+ tombOther = "moved:mv_01BBBBBBBBBBBBBBBBBBBBBBBB"
+ tombRegion = "test-region"
+)
+
+// tombStore is leasetest's S3 server behind a wrapper that runs a hook in the
+// window between the code under test's read of lock.json and its conditional
+// write, so a race is a request ordering, not a sleep. The conditions
+// themselves (If-Match, If-None-Match) are leasetest's own.
+type tombStore struct {
+ *leasetest.VersionedBucket
+ mu sync.Mutex
+ before func(method string)
+ writes []string // "PUT" and "DELETE" requests that reached lock.json, in order
+}
+
+// beforeWrite arms hook for the next write to lock.json, once.
+func (s *tombStore) beforeWrite(hook func(method string)) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.before = hook
+}
+
+func tombFixture(t *testing.T) *tombStore {
+ t.Helper()
+ bucket := leasetest.NewVersionedBucket()
+ bucket.AddBucket("data")
+ inner, _ := leasetest.NewS3Server(bucket)
+ s := &tombStore{VersionedBucket: bucket}
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ key := strings.TrimPrefix(r.URL.Path, "/data/")
+ if strings.HasSuffix(key, "lock.json") && (r.Method == http.MethodPut || r.Method == http.MethodDelete) {
+ s.mu.Lock()
+ hook := s.before
+ s.before = nil
+ s.writes = append(s.writes, r.Method)
+ s.mu.Unlock()
+ if hook != nil {
+ hook(r.Method)
+ }
+ }
+ inner.ServeHTTP(w, r)
+ }))
+ t.Cleanup(srv.Close)
+ t.Setenv("AWS_ACCESS_KEY_ID", "test-key")
+ t.Setenv("AWS_SECRET_ACCESS_KEY", "test-secret")
+ t.Setenv("AWS_SESSION_TOKEN", "")
+ t.Setenv("AWS_EC2_METADATA_DISABLED", "true")
+ t.Setenv("AWS_CONFIG_FILE", t.TempDir()+"/absent")
+ t.Setenv("AWS_SHARED_CREDENTIALS_FILE", t.TempDir()+"/absent")
+ t.Setenv("CARLOS_S3_ENDPOINT", srv.URL)
+ return s
+}
+
+func (s *tombStore) methods() []string {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ return append([]string(nil), s.writes...)
+}
+
+func (s *tombStore) writeCount() int {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ return len(s.writes)
+}
+
+type tombLock struct {
+ Generation int64 `json:"generation"`
+ ExpiresAt time.Time `json:"expires_at"`
+ Owner string `json:"owner"`
+}
+
+func (s *tombStore) putLock(t *testing.T, loc, owner string, gen int64, expires time.Time) {
+ t.Helper()
+ body, err := json.Marshal(tombLock{Generation: gen, ExpiresAt: expires.UTC(), Owner: owner})
+ if err != nil {
+ t.Fatal(err)
+ }
+ s.Put("data", loc+"/lock.json", body)
+}
+
+func (s *tombStore) lock(t *testing.T, loc string) (tombLock, []byte, bool) {
+ t.Helper()
+ body, ok := s.Current("data", loc+"/lock.json")
+ if !ok {
+ return tombLock{}, nil, false
+ }
+ var l tombLock
+ if err := json.Unmarshal(body, &l); err != nil {
+ t.Fatalf("lock.json is not a lock: %v", err)
+ }
+ return l, body, true
+}
+
+func wantHeld(t *testing.T, err error, owner string) {
+ t.Helper()
+ var held *lease.HeldError
+ if !errors.As(err, &held) {
+ t.Fatalf("err = %v, want a *HeldError", err)
+ }
+ if held.Owner != owner {
+ t.Fatalf("held by %q, want %q", held.Owner, owner)
+ }
+}
+
+func near(t *testing.T, got time.Time, want time.Duration) {
+ t.Helper()
+ if d := time.Until(got) - want; d < -time.Minute || d > time.Minute {
+ t.Fatalf("expiry %s is %s from now, want about %s", got, time.Until(got), want)
+ }
+}
+
+func isMine(owner string) func(string) bool {
+ return func(o string) bool { return o == owner }
+}
+
+// A tombstone is a lease like any other: Acquire on the host's location
+// fails with a HeldError that names the move.
+func TestTombstoneBlocksAcquire(t *testing.T) {
+ s := tombFixture(t)
+ ctx := context.Background()
+ if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, 2*time.Hour); err != nil {
+ t.Fatal(err)
+ }
+ l, _, ok := s.lock(t, tombLoc)
+ if !ok || l.Owner != tombMove {
+ t.Fatalf("lock = %+v present %v, want owner %s", l, ok, tombMove)
+ }
+ near(t, l.ExpiresAt, 2*time.Hour)
+ _, err := lease.Acquire(ctx, "data", tombLoc, tombRegion)
+ wantHeld(t, err, tombMove)
+ var held *lease.HeldError
+ errors.As(err, &held)
+ near(t, held.ExpiresAt, 2*time.Hour)
+ // The trailing-slash form ReplicaSource hands over reads the same lock.
+ got, owner, _, err := lease.LockState(ctx, "data", tombLoc+"/", tombRegion)
+ if err != nil || !got || owner != tombMove {
+ t.Fatalf("LockState = held %v owner %q err %v", got, owner, err)
+ }
+}
+
+func TestTombstoneNeverReplacesALiveHolder(t *testing.T) {
+ s := tombFixture(t)
+ ctx := context.Background()
+ held, err := lease.Acquire(ctx, "data", tombLoc, tombRegion)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = held.Release(context.Background()) })
+ _, before, _ := s.lock(t, tombLoc)
+ writes := s.writeCount()
+ err = lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour)
+ var he *lease.HeldError
+ if !errors.As(err, &he) || strings.HasPrefix(he.Owner, "moved:") {
+ t.Fatalf("Tombstone over a live holder = %v, want a *HeldError naming the holder", err)
+ }
+ if _, after, _ := s.lock(t, tombLoc); string(after) != string(before) {
+ t.Fatalf("lock.json changed: %s -> %s", before, after)
+ }
+ if s.writeCount() != writes {
+ t.Fatal("Tombstone wrote to a live holder's lock")
+ }
+ if err := held.Err(); err != nil {
+ t.Fatalf("the holder lost its lease: %v", err)
+ }
+ // Reassert is no way round it either.
+ err = lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour)
+ if !errors.As(err, &he) {
+ t.Fatalf("Reassert over a live holder = %v, want a *HeldError", err)
+ }
+ if s.writeCount() != writes {
+ t.Fatal("Reassert wrote to a live holder's lock")
+ }
+}
+
+// Free means what Acquire means by it: the expiry has passed. There is no
+// RenewMargin grace, so a lease that lapsed a moment ago is taken by Tombstone
+// exactly as lease.Acquire would take it. The two are run side by side on
+// identical locks so they cannot drift.
+func TestTombstoneAndAcquireAgreeOnFree(t *testing.T) {
+ ctx := context.Background()
+ for name, tc := range map[string]struct {
+ expires time.Time
+ free bool
+ }{
+ "expired a second ago": {time.Now().Add(-time.Second), true},
+ "expired within the margin": {time.Now().Add(-lease.RenewMargin / 2), true},
+ "expires in a minute": {time.Now().Add(time.Minute), false},
+ } {
+ t.Run(name, func(t *testing.T) {
+ s := tombFixture(t)
+ s.putLock(t, tombLoc, "box-a:1", 3, tc.expires)
+ _, before, _ := s.lock(t, tombLoc)
+ acq, acqErr := lease.Acquire(ctx, "data", tombLoc, tombRegion)
+ if acqErr == nil {
+ _ = acq.Release(ctx)
+ }
+ // Put the same lock back for Tombstone.
+ s.putLock(t, tombLoc, "box-a:1", 3, tc.expires)
+ tombErr := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour)
+ if (acqErr == nil) != tc.free || (tombErr == nil) != tc.free {
+ t.Fatalf("free %v: Acquire err %v, Tombstone err %v", tc.free, acqErr, tombErr)
+ }
+ if !tc.free {
+ wantHeld(t, tombErr, "box-a:1")
+ if _, after, _ := s.lock(t, tombLoc); string(after) != string(before) {
+ t.Fatal("a live lease was replaced")
+ }
+ }
+ })
+ }
+}
+
+// An own tombstone that has expired is free like any other: Tombstone writes
+// a fresh one rather than reporting the dead one held.
+func TestTombstoneRewritesItsOwnExpiredTombstone(t *testing.T) {
+ s := tombFixture(t)
+ s.putLock(t, tombLoc, tombMove, 4, time.Now().Add(-time.Second))
+ if err := lease.Tombstone(context.Background(), "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
+ t.Fatal(err)
+ }
+ l, _, _ := s.lock(t, tombLoc)
+ if l.Owner != tombMove || l.Generation != 5 {
+ t.Fatalf("lock = %+v, want owner %s at generation 5", l, tombMove)
+ }
+ near(t, l.ExpiresAt, time.Hour)
+}
+
+// The write loses its condition and AcquireLease cannot read a winner back:
+// there is no holder to name, so the answer is ErrTombstoneChanged, never a
+// HeldError with an empty owner.
+func TestTombstoneWriteLostThenReadLostIsChanged(t *testing.T) {
+ s := tombFixture(t)
+ s.putLock(t, tombLoc, "box-a:1", 3, time.Now().Add(-time.Hour))
+ s.beforeWrite(func(string) { s.BareDelete("data", tombLoc+"/lock.json") })
+ err := lease.Tombstone(context.Background(), "data", tombLoc, tombRegion, tombMove, time.Hour)
+ var he *lease.HeldError
+ if !errors.Is(err, lease.ErrTombstoneChanged) || errors.As(err, &he) {
+ t.Fatalf("err = %v, want ErrTombstoneChanged and no HeldError", err)
+ }
+ if _, _, ok := s.lock(t, tombLoc); ok {
+ t.Fatal("Tombstone wrote over the vanished lock")
+ }
+}
+
+// The write loses its condition and the read-back shows this very owner
+// (another pass of the same move got there first): nothing to do.
+func TestTombstoneWriteLostToItsOwnOwnerIsNil(t *testing.T) {
+ s := tombFixture(t)
+ s.putLock(t, tombLoc, "box-a:1", 3, time.Now().Add(-time.Hour))
+ var theirs []byte
+ s.beforeWrite(func(string) {
+ s.putLock(t, tombLoc, tombMove, 4, time.Now().Add(2*time.Hour))
+ _, theirs, _ = s.lock(t, tombLoc)
+ })
+ if err := lease.Tombstone(context.Background(), "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
+ t.Fatalf("err = %v, want nil", err)
+ }
+ if _, now, _ := s.lock(t, tombLoc); string(now) != string(theirs) {
+ t.Fatalf("the winner's lock was overwritten: %s -> %s", theirs, now)
+ }
+}
+
+func TestTombstoneReplacesAnExpiredLease(t *testing.T) {
+ s := tombFixture(t)
+ s.putLock(t, tombLoc, "box-a:1", 3, time.Now().Add(-time.Hour))
+ if err := lease.Tombstone(context.Background(), "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
+ t.Fatal(err)
+ }
+ l, _, _ := s.lock(t, tombLoc)
+ if l.Owner != tombMove || l.Generation != 4 {
+ t.Fatalf("lock = %+v, want owner %s at generation 4", l, tombMove)
+ }
+ near(t, l.ExpiresAt, time.Hour)
+}
+
+func TestTombstoneByTheSameOwnerIsANoOp(t *testing.T) {
+ s := tombFixture(t)
+ ctx := context.Background()
+ if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
+ t.Fatal(err)
+ }
+ _, first, _ := s.lock(t, tombLoc)
+ writes := s.writeCount()
+ // A longer ttl on the repeat changes nothing: Reassert is the call that moves it.
+ if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, 48*time.Hour); err != nil {
+ t.Fatal(err)
+ }
+ if _, second, _ := s.lock(t, tombLoc); string(second) != string(first) || s.writeCount() != writes {
+ t.Fatalf("the repeat wrote: %s -> %s", first, second)
+ }
+}
+
+func TestTombstoneRefusesAnotherMove(t *testing.T) {
+ s := tombFixture(t)
+ ctx := context.Background()
+ if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
+ t.Fatal(err)
+ }
+ _, before, _ := s.lock(t, tombLoc)
+ wantHeld(t, lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombOther, time.Hour), tombMove)
+ wantHeld(t, lease.Reassert(ctx, "data", tombLoc, tombRegion, tombOther, time.Hour), tombMove)
+ if _, after, _ := s.lock(t, tombLoc); string(after) != string(before) {
+ t.Fatalf("another move's tombstone was overwritten: %s -> %s", before, after)
+ }
+}
+
+func TestTombstoneRefusesWhatItShouldNotWrite(t *testing.T) {
+ s := tombFixture(t)
+ ctx := context.Background()
+ for _, owner := range []string{"", "box-a:1", "moved:", "move:mv_1", "Moved:mv_1"} {
+ if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, owner, time.Hour); !errors.Is(err, lease.ErrTombstoneOwner) {
+ t.Fatalf("Tombstone owner %q = %v, want ErrTombstoneOwner", owner, err)
+ }
+ if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, owner, time.Hour); !errors.Is(err, lease.ErrTombstoneOwner) {
+ t.Fatalf("Reassert owner %q = %v, want ErrTombstoneOwner", owner, err)
+ }
+ }
+ for _, ttl := range []time.Duration{0, -time.Hour, lease.MaxTombstoneTTL + time.Second, 30 * 24 * time.Hour} {
+ if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, ttl); !errors.Is(err, lease.ErrTombstoneTTL) {
+ t.Fatalf("Tombstone ttl %s = %v, want ErrTombstoneTTL", ttl, err)
+ }
+ if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, ttl); !errors.Is(err, lease.ErrTombstoneTTL) {
+ t.Fatalf("Reassert ttl %s = %v, want ErrTombstoneTTL", ttl, err)
+ }
+ }
+ for _, loc := range []string{"", "/abs", "../x", "a/../b", `a\b`} {
+ if err := lease.Tombstone(ctx, "data", loc, tombRegion, tombMove, time.Hour); !errors.Is(err, lease.ErrLockLocation) {
+ t.Fatalf("Tombstone location %q = %v, want ErrLockLocation", loc, err)
+ }
+ }
+ if s.writeCount() != 0 {
+ t.Fatal("a refused call wrote")
+ }
+ // Seven days exactly is the longest there is.
+ if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, lease.MaxTombstoneTTL); err != nil {
+ t.Fatal(err)
+ }
+ l, _, _ := s.lock(t, tombLoc)
+ near(t, l.ExpiresAt, lease.MaxTombstoneTTL)
+ if time.Until(l.ExpiresAt) > lease.MaxTombstoneTTL {
+ t.Fatalf("expiry %s is beyond 7 days", l.ExpiresAt)
+ }
+}
+
+func TestReassertMovesTheExpiry(t *testing.T) {
+ s := tombFixture(t)
+ ctx := context.Background()
+ if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
+ t.Fatal(err)
+ }
+ first, _, _ := s.lock(t, tombLoc)
+ if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, 48*time.Hour); err != nil {
+ t.Fatal(err)
+ }
+ second, _, _ := s.lock(t, tombLoc)
+ if second.Owner != tombMove || second.Generation != first.Generation {
+ t.Fatalf("lock = %+v, want the same owner and generation as %+v", second, first)
+ }
+ near(t, second.ExpiresAt, 48*time.Hour)
+ // And it can be pulled in again: the expiry is what the caller says.
+ if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
+ t.Fatal(err)
+ }
+ third, _, _ := s.lock(t, tombLoc)
+ near(t, third.ExpiresAt, time.Hour)
+ _, err := lease.Acquire(ctx, "data", tombLoc, tombRegion)
+ wantHeld(t, err, tombMove)
+}
+
+func TestReassertWithNothingToReassertTombstones(t *testing.T) {
+ s := tombFixture(t)
+ ctx := context.Background()
+ if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
+ t.Fatalf("absent: %v", err)
+ }
+ if l, _, ok := s.lock(t, tombLoc); !ok || l.Owner != tombMove {
+ t.Fatalf("absent: lock = %+v present %v", l, ok)
+ }
+ // An expired tombstone of its own is replaced and live again.
+ s.putLock(t, tombLoc, tombMove, 5, time.Now().Add(-time.Hour))
+ if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
+ t.Fatalf("expired: %v", err)
+ }
+ l, _, _ := s.lock(t, tombLoc)
+ if l.Owner != tombMove || l.Generation != 6 {
+ t.Fatalf("expired: lock = %+v, want owner %s at generation 6", l, tombMove)
+ }
+ near(t, l.ExpiresAt, time.Hour)
+ // An expired lease of someone else is replaced the same way.
+ s.putLock(t, tombLoc, tombOther, 2, time.Now().Add(-time.Hour))
+ if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
+ t.Fatalf("expired other: %v", err)
+ }
+ if l, _, _ := s.lock(t, tombLoc); l.Owner != tombMove {
+ t.Fatalf("expired other: lock = %+v", l)
+ }
+}
+
+func TestReassertRefusesAHolderThatIsNotTheMove(t *testing.T) {
+ s := tombFixture(t)
+ s.putLock(t, tombLoc, "box-a:1", 3, time.Now().Add(time.Hour))
+ _, before, _ := s.lock(t, tombLoc)
+ wantHeld(t, lease.Reassert(context.Background(), "data", tombLoc, tombRegion, tombMove, time.Hour), "box-a:1")
+ if _, after, _ := s.lock(t, tombLoc); string(after) != string(before) {
+ t.Fatal("Reassert overwrote a live holder")
+ }
+}
+
+// The lock is rewritten between Reassert's read and its write: the write
+// carries the stale ETag, S3 refuses it, and the rewrite stands.
+func TestReassertFailsWhenTheLockChangesUnderIt(t *testing.T) {
+ for name, rewrite := range map[string]func(s *tombStore, t *testing.T){
+ "another move takes it": func(s *tombStore, t *testing.T) {
+ s.putLock(t, tombLoc, tombOther, 9, time.Now().Add(3*time.Hour))
+ },
+ "the same move rewrites it": func(s *tombStore, t *testing.T) {
+ s.putLock(t, tombLoc, tombMove, 9, time.Now().Add(3*time.Hour))
+ },
+ "it is removed": func(s *tombStore, t *testing.T) {
+ s.BareDelete("data", tombLoc+"/lock.json")
+ },
+ } {
+ t.Run(name, func(t *testing.T) {
+ s := tombFixture(t)
+ ctx := context.Background()
+ if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
+ t.Fatal(err)
+ }
+ var after []byte
+ var present bool
+ s.beforeWrite(func(method string) {
+ if method != http.MethodPut {
+ t.Errorf("Reassert sent a %s", method)
+ }
+ rewrite(s, t)
+ _, after, present = s.lock(t, tombLoc)
+ })
+ err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, 48*time.Hour)
+ if !errors.Is(err, lease.ErrTombstoneChanged) {
+ t.Fatalf("err = %v, want ErrTombstoneChanged", err)
+ }
+ _, now, ok := s.lock(t, tombLoc)
+ if ok != present || string(now) != string(after) {
+ t.Fatalf("Reassert overwrote the rewrite: %s -> %s", after, now)
+ }
+ })
+ }
+}
+
+// Tombstone's own windows: the lock is absent or expired when it reads, and a
+// live holder is there when it writes.
+func TestTombstoneLosesToAHolderThatArrivesFirst(t *testing.T) {
+ for name, setup := range map[string]func(s *tombStore, t *testing.T){
+ "absent": func(s *tombStore, t *testing.T) {},
+ "expired": func(s *tombStore, t *testing.T) { s.putLock(t, tombLoc, "box-a:1", 3, time.Now().Add(-time.Hour)) },
+ } {
+ t.Run(name, func(t *testing.T) {
+ s := tombFixture(t)
+ setup(s, t)
+ var holder []byte
+ s.beforeWrite(func(string) {
+ s.putLock(t, tombLoc, "box-b:2", 7, time.Now().Add(15*time.Minute))
+ _, holder, _ = s.lock(t, tombLoc)
+ })
+ err := lease.Tombstone(context.Background(), "data", tombLoc, tombRegion, tombMove, time.Hour)
+ wantHeld(t, err, "box-b:2")
+ if _, now, _ := s.lock(t, tombLoc); string(now) != string(holder) {
+ t.Fatalf("Tombstone replaced the holder that arrived first: %s -> %s", holder, now)
+ }
+ })
+ }
+}
+
+func TestLiftTombstone(t *testing.T) {
+ s := tombFixture(t)
+ ctx := context.Background()
+ called := ""
+ mine := func(o string) bool { called = o; return o == tombMove }
+
+ // Absent: nothing to lift, and mine is not consulted.
+ if lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, mine); lifted || err != nil || called != "" {
+ t.Fatalf("absent: lifted %v err %v consulted %q", lifted, err, called)
+ }
+
+ if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
+ t.Fatal(err)
+ }
+ live, before, _ := s.lock(t, tombLoc)
+
+ // Another move's tombstone is not ours to lift.
+ lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombOther))
+ wantHeld(t, err, tombMove)
+ if lifted {
+ t.Fatal("lifted another move's tombstone")
+ }
+ if _, after, _ := s.lock(t, tombLoc); string(after) != string(before) {
+ t.Fatal("the tombstone changed")
+ }
+ // No ownership test at all lifts nothing.
+ if lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, nil); lifted || err == nil {
+ t.Fatalf("nil mine: lifted %v err %v", lifted, err)
+ }
+
+ // Ours lifts: by a conditional PUT, leaving the same owner and generation
+ // with a long-past expiry, never by a DELETE.
+ writes := s.writeCount()
+ lifted, err = lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, mine)
+ if err != nil || !lifted || called != tombMove {
+ t.Fatalf("lift: lifted %v err %v consulted %q", lifted, err, called)
+ }
+ if got := s.methods()[writes:]; len(got) != 1 || got[0] != http.MethodPut {
+ t.Fatalf("lift wrote %v, want one PUT", got)
+ }
+ l, _, ok := s.lock(t, tombLoc)
+ if !ok || l.Owner != tombMove || l.Generation != live.Generation || !l.ExpiresAt.Before(time.Now().Add(-lease.RenewMargin)) {
+ t.Fatalf("after the lift: %+v present %v, want %s at generation %d, long expired", l, ok, tombMove, live.Generation)
+ }
+ if held, owner, expires, err := lease.LockState(ctx, "data", tombLoc, tombRegion); err != nil || held || !lease.IsLiftedTombstone(owner, expires) {
+ t.Fatalf("LockState after the lift: held %v owner %q expires %v err %v", held, owner, expires, err)
+ }
+ // The host can be acquired again, and so can a new tombstone.
+ held, err := lease.Acquire(ctx, "data", tombLoc, tombRegion)
+ if err != nil {
+ t.Fatalf("Acquire after the lift: %v", err)
+ }
+ if err := held.Release(ctx); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestLiftedTombstoneIsFreeToTheNextMove(t *testing.T) {
+ s := tombFixture(t)
+ ctx := context.Background()
+ if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
+ t.Fatal(err)
+ }
+ if lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombMove)); err != nil || !lifted {
+ t.Fatalf("lifted %v err %v", lifted, err)
+ }
+ if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombOther, time.Hour); err != nil {
+ t.Fatalf("Tombstone over a lifted one: %v", err)
+ }
+ if l, _, _ := s.lock(t, tombLoc); l.Owner != tombOther {
+ t.Fatalf("lock = %+v", l)
+ }
+}
+
+// Already lifted, however it came about, is (false, nil) with nothing written.
+func TestLiftTombstoneAlreadyLifted(t *testing.T) {
+ s := tombFixture(t)
+ ctx := context.Background()
+ if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
+ t.Fatal(err)
+ }
+ if lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombMove)); err != nil || !lifted {
+ t.Fatalf("first lift: %v %v", lifted, err)
+ }
+ writes := s.writeCount()
+ for i := 0; i < 2; i++ {
+ if lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombMove)); lifted || err != nil {
+ t.Fatalf("already lifted: lifted %v err %v", lifted, err)
+ }
+ }
+ // An own tombstone that simply ran out is lifted too.
+ s.putLock(t, tombLoc, tombMove, 4, time.Now().Add(-48*time.Hour))
+ writes = s.writeCount()
+ if lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombMove)); lifted || err != nil {
+ t.Fatalf("expired own: lifted %v err %v", lifted, err)
+ }
+ if s.writeCount() != writes {
+ t.Fatal("lifting an already-lifted tombstone wrote")
+ }
+ // And so is an absent one.
+ s.BareDelete("data", tombLoc+"/lock.json")
+ if lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombMove)); lifted || err != nil {
+ t.Fatalf("absent: lifted %v err %v", lifted, err)
+ }
+}
+
+func TestIsLiftedTombstone(t *testing.T) {
+ past, future := time.Now().Add(-time.Second), time.Now().Add(time.Hour)
+ for name, tc := range map[string]struct {
+ owner string
+ expires time.Time
+ want bool
+ }{
+ "expired move": {tombMove, past, true},
+ "long expired move": {tombMove, time.Unix(0, 0), true},
+ "live move": {tombMove, future, false},
+ "expired ordinary lease": {"box-a:1", past, false},
+ "live ordinary lease": {"box-a:1", future, false},
+ "absent (zero owner/exp)": {"", time.Time{}, false},
+ "bare prefix": {"moved:", past, false},
+ "zero expiry": {tombMove, time.Time{}, false},
+ } {
+ if got := lease.IsLiftedTombstone(tc.owner, tc.expires); got != tc.want {
+ t.Errorf("%s: IsLiftedTombstone(%q, %v) = %v, want %v", name, tc.owner, tc.expires, got, tc.want)
+ }
+ }
+}
+
+// Whatever mine says, only a moved:<id> lease is lifted: a live process's
+// lease is never this function's to touch.
+func TestLiftTombstoneLeavesOrdinaryLeasesAlone(t *testing.T) {
+ s := tombFixture(t)
+ ctx := context.Background()
+ for name, expires := range map[string]time.Time{"live": time.Now().Add(time.Hour), "expired": time.Now().Add(-time.Hour)} {
+ s.putLock(t, tombLoc, "box-a:1", 3, expires)
+ _, before, _ := s.lock(t, tombLoc)
+ lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, func(string) bool { return true })
+ wantHeld(t, err, "box-a:1")
+ if lifted {
+ t.Fatalf("%s: lifted an ordinary lease", name)
+ }
+ if _, after, _ := s.lock(t, tombLoc); string(after) != string(before) {
+ t.Fatalf("%s: the lease changed", name)
+ }
+ }
+ if s.writeCount() != 0 { // putLock writes the bucket directly; only requests count
+ t.Fatal("a refused lift wrote")
+ }
+}
+
+func TestLiftTombstoneFailsWhenTheLockChangesUnderIt(t *testing.T) {
+ ctx := context.Background()
+ t.Run("rewritten", func(t *testing.T) {
+ s := tombFixture(t)
+ if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
+ t.Fatal(err)
+ }
+ var after []byte
+ s.beforeWrite(func(method string) {
+ if method != http.MethodPut {
+ t.Errorf("LiftTombstone sent a %s", method)
+ }
+ // Still ours by owner, but a different lock: Reassert beat the lift.
+ s.putLock(t, tombLoc, tombMove, 8, time.Now().Add(72*time.Hour))
+ _, after, _ = s.lock(t, tombLoc)
+ })
+ lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombMove))
+ if lifted || !errors.Is(err, lease.ErrTombstoneChanged) {
+ t.Fatalf("lifted %v err %v, want (false, ErrTombstoneChanged)", lifted, err)
+ }
+ if _, now, ok := s.lock(t, tombLoc); !ok || string(now) != string(after) {
+ t.Fatalf("the lift overwrote the rewrite: %s", now)
+ }
+ })
+ t.Run("taken over by a holder", func(t *testing.T) {
+ s := tombFixture(t)
+ // Live tombstone read; before the write a holder cannot legitimately
+ // appear, but a rewrite by anything must still stand.
+ if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
+ t.Fatal(err)
+ }
+ var after []byte
+ s.beforeWrite(func(string) {
+ s.putLock(t, tombLoc, "box-b:2", 5, time.Now().Add(15*time.Minute))
+ _, after, _ = s.lock(t, tombLoc)
+ })
+ lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombMove))
+ if lifted || !errors.Is(err, lease.ErrTombstoneChanged) {
+ t.Fatalf("lifted %v err %v, want (false, ErrTombstoneChanged)", lifted, err)
+ }
+ if _, now, _ := s.lock(t, tombLoc); string(now) != string(after) {
+ t.Fatalf("the lift overwrote a live holder's lock: %s", now)
+ }
+ })
+ t.Run("removed", func(t *testing.T) {
+ s := tombFixture(t)
+ if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
+ t.Fatal(err)
+ }
+ s.beforeWrite(func(string) { s.BareDelete("data", tombLoc+"/lock.json") })
+ if lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombMove)); lifted || err != nil {
+ t.Fatalf("lifted %v err %v, want (false, nil): already gone", lifted, err)
+ }
+ if _, _, ok := s.lock(t, tombLoc); ok {
+ t.Fatal("the lift re-created a removed lock")
+ }
+ })
+}
+
+// The tombstone is a lock.json like any other, and MirrorPrefix, which copies
+// a host's replica to a staging prefix, leaves lock.json alone both ways.
+func TestMirrorPrefixStillSkipsTheTombstone(t *testing.T) {
+ s := tombFixture(t)
+ ctx := context.Background()
+ s.Put("data", "instances/h.example.com/generations/a/snapshot", []byte("snap"))
+ if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
+ t.Fatal(err)
+ }
+ // A staging lease of the target's own, under the destination prefix.
+ s.putLock(t, "moves/h.example.com", "box-z:9", 1, time.Now().Add(time.Hour))
+ _, staged, _ := s.lock(t, "moves/h.example.com")
+
+ m := bucketMirror{s.VersionedBucket}
+ r, err := lease.MirrorPrefix(ctx, m, "data", tombLoc+"/", "data", "moves/h.example.com/", lease.Receipts{})
+ if err != nil || r.Changed != 1 {
+ t.Fatalf("round = %+v, %v; want only the snapshot to move", r, err)
+ }
+ if _, now, _ := s.lock(t, "moves/h.example.com"); string(now) != string(staged) {
+ t.Fatal("the staging lease was touched")
+ }
+ if l, _, ok := s.lock(t, tombLoc); !ok || l.Owner != tombMove {
+ t.Fatalf("the tombstone was touched: %+v", l)
+ }
+ // Reasserting the tombstone is churn the next round does not see.
+ if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, 24*time.Hour); err != nil {
+ t.Fatal(err)
+ }
+ if r, err := lease.MirrorPrefix(ctx, m, "data", tombLoc+"/", "data", "moves/h.example.com/", lease.Receipts{}); err != nil || r.Changed != 0 {
+ t.Fatalf("caught-up round = %+v, %v; want 0 despite the reassert", r, err)
+ }
+}
+
+// bucketMirror is a MirrorStore over the versioned bucket the S3 server also
+// serves, so MirrorPrefix runs over the very lock.json objects the tombstone
+// calls wrote (the S3 server does not speak ListObjectsV2).
+type bucketMirror struct{ b *leasetest.VersionedBucket }
+
+func (m bucketMirror) List(ctx context.Context, bucket, prefix string) (map[string]lease.MirrorObject, error) {
+ vs, err := m.b.ListVersions(ctx, bucket, prefix)
+ if err != nil {
+ return nil, err
+ }
+ out := map[string]lease.MirrorObject{}
+ for _, v := range vs {
+ if v.IsLatest && !v.DeleteMarker {
+ out[v.Key] = lease.MirrorObject{ETag: v.ETag, Size: v.Size}
+ }
+ }
+ return out, nil
+}
+
+func (m bucketMirror) Copy(ctx context.Context, srcBucket, srcKey, srcETag, dstBucket, dstKey string) (string, error) {
+ vs, err := m.b.ListVersions(ctx, srcBucket, srcKey)
+ if err != nil {
+ return "", err
+ }
+ for _, v := range vs {
+ if v.Key == srcKey && v.IsLatest && !v.DeleteMarker && v.ETag == srcETag {
+ _, etag, err := m.b.CopyVersion(ctx, srcBucket, srcKey, v.VersionID, dstBucket, dstKey, false)
+ return etag, err
+ }
+ }
+ return "", lease.ErrMirrorNotFound
+}
+
+func (m bucketMirror) Delete(_ context.Context, bucket, key string) error {
+ m.b.BareDelete(bucket, key)
+ return nil
+}
+
+func (m bucketMirror) Get(_ context.Context, bucket, key string) ([]byte, error) {
+ if body, ok := m.b.Current(bucket, key); ok {
+ return body, nil
+ }
+ return nil, lease.ErrMirrorNotFound
+}
+
+func (m bucketMirror) Put(_ context.Context, bucket, key string, body []byte) error {
+ m.b.Put(bucket, key, body)
+ return nil
+}
diff --git a/internal/moves/offline.go b/internal/moves/offline.go
index a51bdbbe1..ea6f56fd4 100644
--- a/internal/moves/offline.go
+++ b/internal/moves/offline.go
@@ -1,6 +1,8 @@
package moves
import (
+ "crypto/sha256"
+ "encoding/hex"
"fmt"
"path/filepath"
"time"
@@ -217,9 +219,6 @@ var (
offlineRoles = map[string]bool{OfflineRoleSource: true, OfflineRoleSealer: true, OfflineRoleTarget: true}
offlinePhases = map[string]bool{OfflinePhaseStopped: true, OfflinePhaseSealed: true, OfflinePhaseLanded: true, OfflinePhaseOpened: true,
OfflinePhaseParked: true, OfflinePhaseGone: true, OfflinePhaseLifted: true, OfflinePhaseReverted: true, OfflinePhaseFailed: true}
- // The Checks a role's report may carry: the source side's (the sealer
- // and the stopped source boxes) carry items with Source "seal", the target
- // carries items it owns ("landing").
)
// offlineCarries reports whether a role's report may carry the item.
@@ -261,8 +260,8 @@ func ValidateOfflineReport(r OfflineReport) error {
// ItemsOK says whether the report holds an OK Check for every manifest
// item the owner decides, and names the ones that are missing or not OK, in
-// manifest order. The doors use it: flip needs the sealer's report OK for
-// owner seal, open the target's for owner landing.
+// manifest order. The open door uses it for owner landing; flip checks the
+// sealer's source values with SourceItemsOK.
func (r OfflineReport) ItemsOK(owner string) (ok bool, bad []string) {
for _, it := range OfflineOwned(owner) {
if c, found := r.Items[it.ID]; !found || !c.OK {
@@ -323,6 +322,68 @@ func OfflineReportPrefix(host, op string) string {
return "control/offline-moves/" + host + "/" + op + "/"
}
+// OfflineSealKey is where the sealer writes the seal, once and create-only:
+// control/offline-moves/<host>/<op>/seal.json. Unlike a report it is never
+// rewritten. It is an OfflineReport (role sealer, phase sealed) holding what
+// the sealer decided and recorded at the seal: the sealed position
+// (FinalTXID, Staged), the files and env digests, the env pin, the release,
+// the timers digest, the replication rung and next due, TombstoneOwner
+// (moved:<op>), and Items, an OK Check for every manifest item with Source
+// "seal". At is when it was sealed; TombstoneUntil is zero (the live
+// expiry is the sealer's per-pass report). The record's source.evidence is
+// this key and the sha256 of its bytes (OfflineSealEvidence). "" for a host
+// that fails provision.ValidHostShape or an unusable op.
+func OfflineSealKey(host, op string) string {
+ if !provision.ValidHostShape(host) || !offlineSegment(op) {
+ return ""
+ }
+ return "control/offline-moves/" + host + "/" + op + "/seal.json"
+}
+
+// OfflineSealEvidence is source.evidence for a seal written as raw: the seal
+// key, "@", and the hex sha256 of the bytes. "" for a bad host or op.
+func OfflineSealEvidence(host, op string, raw []byte) string {
+ key := OfflineSealKey(host, op)
+ if key == "" {
+ return ""
+ }
+ sum := sha256.Sum256(raw)
+ return key + "@" + hex.EncodeToString(sum[:])
+}
+
+// ValidateOfflineSeal reports why a seal cannot be trusted as the seal of op
+// on host: it is a valid sealer report of that move, phase sealed, that names
+// the move's tombstone owner and carries an OK Check for every item the
+// source side reports.
+func ValidateOfflineSeal(r OfflineReport, host, op string) error {
+ if err := ValidateOfflineReport(r); err != nil {
+ return err
+ }
+ if r.Host != host || r.Op != op || r.Role != OfflineRoleSealer || r.Phase != OfflinePhaseSealed {
+ return fmt.Errorf("offline seal: not the sealed report of this move")
+ }
+ if r.TombstoneOwner != TombstoneOwner(op) {
+ return fmt.Errorf("offline seal: tombstone owner %q, want %q", r.TombstoneOwner, TombstoneOwner(op))
+ }
+ if ok, bad := r.SourceItemsOK(); !ok {
+ return fmt.Errorf("offline seal: items not OK: %v", bad)
+ }
+ return nil
+}
+
+// OfflineActiveKey is the one lock of the offline move: while it names a
+// host whose record still carries that move, no other offline move starts.
+const OfflineActiveKey = "control/offline-moves/active.json"
+
+// OfflineActive is the lock's body.
+type OfflineActive struct {
+ Host string `json:"host"`
+ Op string `json:"op"`
+ Account string `json:"account"`
+ App string `json:"app"`
+ At time.Time `json:"at"`
+}
+
// OfflineArchivePrefix is where the move's sealed files archive and its
// timers snapshot live in the target bucket: moves/@offline/<host>/<op>/. The
// "@" cannot start a host, so it never collides with a host's own staging
diff --git a/internal/moves/offline_test.go b/internal/moves/offline_test.go
index 635eb9e23..84a0e4050 100644
--- a/internal/moves/offline_test.go
+++ b/internal/moves/offline_test.go
@@ -441,3 +441,55 @@ func TestOfflineStateAndPhaseNamesAreDistinct(t *testing.T) {
t.Fatalf("kind %q", provision.MoveKindOffline)
}
}
+
+func TestOfflineSealKeyAndValidation(t *testing.T) {
+ const host, op = "acme.go.tito.io", "mv_aaaaaaaaaaaaaaaaaaaaaaaaaa"
+ if got := OfflineSealKey(host, op); got != "control/offline-moves/"+host+"/"+op+"/seal.json" {
+ t.Fatalf("seal key = %q", got)
+ }
+ for _, bad := range [][2]string{{"", op}, {"not a host", op}, {host, ""}, {host, "../x"}, {host, "a/b"}} {
+ if OfflineSealKey(bad[0], bad[1]) != "" || OfflineSealEvidence(bad[0], bad[1], []byte("x")) != "" {
+ t.Errorf("key for %q accepted", bad)
+ }
+ }
+ // The seal key is not a report key and sits under the report prefix.
+ if !strings.HasPrefix(OfflineSealKey(host, op), OfflineReportPrefix(host, op)) {
+ t.Error("the seal is outside the move's prefix")
+ }
+ items := map[string]Check{}
+ for _, it := range OfflineSourced() {
+ items[it.ID] = Check{OK: true}
+ }
+ good := OfflineReport{V: OfflineReportVersion, Op: op, Host: host, Box: "ie-box", Role: OfflineRoleSealer, Phase: OfflinePhaseSealed,
+ Items: items, TombstoneOwner: TombstoneOwner(op), At: time.Now()}
+ if err := ValidateOfflineSeal(good, host, op); err != nil {
+ t.Fatal(err)
+ }
+ for name, edit := range map[string]func(*OfflineReport){
+ "another host": func(r *OfflineReport) { r.Host = "other.go.tito.io" },
+ "another op": func(r *OfflineReport) { r.Op = "mv_bbbbbbbbbbbbbbbbbbbbbbbbbb" },
+ "a source role": func(r *OfflineReport) { r.Role = OfflineRoleSource },
+ "not sealed": func(r *OfflineReport) { r.Phase = OfflinePhaseStopped },
+ "no tombstone": func(r *OfflineReport) { r.TombstoneOwner = "" },
+ "an item not OK": func(r *OfflineReport) {
+ c := map[string]Check{}
+ for k, v := range items {
+ c[k] = v
+ }
+ c["files"] = Check{}
+ r.Items = c
+ },
+ "a missing item": func(r *OfflineReport) { r.Items = map[string]Check{} },
+ } {
+ r := good
+ edit(&r)
+ if ValidateOfflineSeal(r, host, op) == nil {
+ t.Errorf("%s: accepted", name)
+ }
+ }
+ // The evidence is the key and the hash of the bytes.
+ ev := OfflineSealEvidence(host, op, []byte("x"))
+ if !strings.HasPrefix(ev, OfflineSealKey(host, op)+"@") || len(ev) != len(OfflineSealKey(host, op))+1+64 {
+ t.Errorf("evidence = %q", ev)
+ }
+}