amadan

carlos / platform Public

offline-move: hold, keep, capability, dispatcher and source steps

The box side of the offline operator move (Task 3 of
docs/superpowers/plans/2026-10-20-offline-operator-move.md):

- offlineHold: reconcile's Hold, a pure function of the record and the
  box's placement (Fable B1). A fresh target mints the landing row held
  with no view built.
- offlineView.keep: reconcile's KeepParticipant from the last listing,
  refreshed at agent start and every moves pass; fails closed only
  while no listing has been read.
- offline-move-1 in the edges file when hibernation is configured and
  CARLOS_HOST_CLAIM and CARLOS_MOVES_PHASE1 are unset (guard mode not
  read, per the 2026-10-20 ruling). It asks for no headroom.
- movesPass routes an offline record to offlineRound first, ahead of
  the Phase 1 skip and the lost-box gate.
- Source steps, each an ensure: fence and report (stopping to landed),
  park by rename with a per-path inventory (open), report gone once
  reconcile removes the row (finishing), release the hold and lift the
  fence (reverting).
- reconcileSerializedWith, armOffline and every moveCfg field Tasks 4
  and 5 read; movesoffline_seal.go and movesoffline_land.go are no-op
  stubs those tasks own.

Dormant (no offline record): the hold answers false, keep is false
after the startup listing, removals happen on the first pass, and the
edges file gains one string (TestOfflineDormantIsMain).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude c03598e127792e41e45e78016106514c67eeebcb parent 1b561a9
7 files changed, +2128 −8
  • internal/host/agent.go +22 −0
  • internal/host/edgesfile.go +22 −5
  • internal/host/movesoffline.go +844 −0
  • internal/host/movesoffline_land.go +26 −0
  • internal/host/movesoffline_seal.go +22 −0
  • internal/host/movesoffline_test.go +1041 −0
  • internal/host/movespass.go +151 −3
diff --git a/internal/host/agent.go b/internal/host/agent.go
index 5ccaa3955..0faf0cd02 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -1452,6 +1452,14 @@ var reconcileMu sync.Mutex
// not hide an instance-side one, and neither must stop the other running —
// each pass is independently safe to run and independently valuable.
func reconcileSerialized(ctx context.Context, s release.Store, reg *registry.Registry, cfg provision.ReconcileConfig) error {
+ return reconcileSerializedWith(ctx, s, reg, cfg, armedMovesWorker.Load())
+}
+
+// reconcileSerializedWith is reconcileSerialized with the moves worker whose
+// offline hooks the pass takes (movesoffline.go): the armed worker in the
+// agent, a box's own worker in a test. A nil worker, or one on a box that
+// cannot take part in an offline move, adds nothing.
+func reconcileSerializedWith(ctx context.Context, s release.Store, reg *registry.Registry, cfg provision.ReconcileConfig, w *movesWorker) error {
reconcileMu.Lock()
defer reconcileMu.Unlock()
// A fresh row is minted at the layout this box's intent names
@@ -1472,6 +1480,16 @@ func reconcileSerialized(ctx context.Context, s release.Store, reg *registry.Reg
// Move holds (moves spec §9.6): participant rows minted held and kept.
// Unchanged with the Phase 1 switch off.
cfg = withMovesHolds(cfg)
+ // Offline move holds: the hold from the record alone, the keep from the
+ // view, which is read here first if nothing has read it yet, so keep
+ // never fails closed on a box whose listing reads. Nothing on a box that
+ // cannot take part, and nothing where Phase 1's hooks are set.
+ if w != nil && w.cfg.OfflineCapable && w.cfg.Offline != nil {
+ if !w.cfg.Offline.built() {
+ refreshOfflineView(ctx, s, w.cfg.Offline)
+ }
+ cfg = withOfflineHolds(cfg, offlineHold(w.cfg), w.cfg.Offline)
+ }
// Runs ride this pass's record reads (run spec §5.2): the worker notes
// every in-scope record with a run, and acts after the pass.
cfg = withRunsHook(cfg)
@@ -2594,6 +2612,10 @@ func fastConvergeLoop(reg *registry.Registry, svc *activator.Service, sd *schedu
// runs before the first moves pass, and an unbuilt view freezes every
// host (moves spec §9.4). Nothing with the Phase 1 switch off.
refreshMovesHolds(ctx, store)
+ // The offline keep view, likewise before any reconcile pass: once it is
+ // read, keep answers false for every host with no offline move, so the
+ // first pass removes what it always has (movesoffline.go).
+ refreshOfflineView(ctx, store, armedOfflineView())
// carlos run's boot pass (run spec §5.2, index item 2): reconcile's
// first pass is a minute away and the CLI expires a run at 60 s.
runRunsBootPass(ctx)
diff --git a/internal/host/edgesfile.go b/internal/host/edgesfile.go
index c02520656..d39f3f6be 100644
--- a/internal/host/edgesfile.go
+++ b/internal/host/edgesfile.go
@@ -9,6 +9,7 @@ import (
"log"
"math/rand/v2"
"os"
+ "slices"
"strings"
"sync"
"time"
@@ -162,12 +163,28 @@ type edgeHeadroom struct {
const capRegionHA1 = "region-ha-1"
// edgesFileCapabilities publishes region-ha-1 only when CARLOS_REGION_HA is
-// set to something other than "" or "0".
+// set to something other than "" or "0", and offline-move-1 only when the
+// box can take part in an offline operator move (offlineCapable:
+// hibernation configured, CARLOS_HOST_CLAIM and CARLOS_MOVES_PHASE1 unset).
func edgesFileCapabilities(getenv func(string) string) []string {
- if !regionHAEnabled(getenv) {
- return nil
+ var caps []string
+ if regionHAEnabled(getenv) {
+ caps = append(caps, capRegionHA1)
+ }
+ if offlineCapable(getenv, hibernationConfigured(getenv)) {
+ caps = append(caps, capOfflineMove1)
+ }
+ return caps
+}
+
+// headroomCapabilities is what headroomFor is asked about: region HA's
+// capability alone. offline-move-1 publishes no headroom, so a box that
+// gains only it publishes the file it always has, plus the one string.
+func headroomCapabilities(caps []string) []string {
+ if slices.Contains(caps, capRegionHA1) {
+ return caps
}
- return []string{capRegionHA1}
+ return nil
}
// awakeCount is how many routes this box has running now. An error reads
@@ -859,7 +876,7 @@ func edgesFileLoop(reg *registry.Registry) {
if state.timed() {
eip = readPublicIP()
}
- ident.Headroom = headroomFor(ident.Capabilities, dataDir, reg)
+ ident.Headroom = headroomFor(headroomCapabilities(ident.Capabilities), dataDir, reg)
ident = withMovesFields(ident, dataDir, optDir)
ident = withSeedsFields(ident, dataDir, optDir)
err = edgesFileWriteClaims(ctx, store, reg, ident, eip, &last, state.timed(), claims)
diff --git a/internal/host/movesoffline.go b/internal/host/movesoffline.go
new file mode 100644
index 000000000..47bb7e23c
--- /dev/null
+++ b/internal/host/movesoffline.go
@@ -0,0 +1,844 @@
+package host
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log"
+ "os"
+ "path"
+ "path/filepath"
+ "sort"
+ "strings"
+ "sync/atomic"
+ "time"
+
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/hostwrite"
+ "github.com/carlosframework/platform/internal/instancectl"
+ "github.com/carlosframework/platform/internal/layout"
+ "github.com/carlosframework/platform/internal/lease"
+ "github.com/carlosframework/platform/internal/moves"
+ "github.com/carlosframework/platform/internal/park"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// The offline operator move on a box (docs/superpowers/plans/2026-10-20-
+// offline-operator-move.md, Task 3): the hold reconcile asks, the keep view,
+// the capability, the moves pass's dispatch, and the source boxes' steps.
+// The sealer's step is movesoffline_seal.go (Task 4) and the target's is
+// movesoffline_land.go (Task 5).
+//
+// The stop has three layers. This file holds the first two on every
+// source-region box: reconcile holds the row from the record alone
+// (offlineHold), and each pass fences the activator and writes the box's
+// source report. The third, the tombstone lease, is the sealer's.
+//
+// Dormant, with no offline record anywhere, nothing here acts: the hold
+// answers false for every record, keep answers false once the startup
+// listing is read, the dispatch matches no record, and the edges file
+// carries one more capability string.
+
+// capOfflineMove1 is the edges-file capability a box publishes when it can
+// take part in an offline move (offlineCapable). The console's start door
+// refuses a region holding a box that does not publish it.
+const capOfflineMove1 = "offline-move-1"
+
+// offlineCapable is the box's own witness that it can take part in an
+// offline move: hibernation is configured and neither CARLOS_HOST_CLAIM nor
+// CARLOS_MOVES_PHASE1 is set to anything at all. The guard mode is not read
+// (platform-1e's ruling of 2026-10-20, the plan's Global Constraints): the
+// target's build stages through ordinary adoption, which root's own floor
+// check already covers on a guarded box. A box that is not capable publishes
+// no capability and runs no offline step.
+func offlineCapable(getenv func(string) string, hibernation bool) bool {
+ if !hibernation || hostwrite.Enabled(getenv) {
+ return false
+ }
+ return strings.TrimSpace(getenv(moves.SwitchEnv)) == "" && !moves.Enabled(getenv)
+}
+
+// hibernationConfigured is the edges file's HibernationConfigured, read the
+// same way, so the capability and the field beside it never disagree.
+func hibernationConfigured(getenv func(string) string) bool {
+ return strings.TrimSpace(getenv("CARLOS_HIBERNATE_BUCKET")) != ""
+}
+
+// offlineFenceOwner is the activator fence owner of one offline move.
+func offlineFenceOwner(id string) string { return "offline:" + id }
+
+// offlineHold is reconcile's Hold for offline moves: a pure function of the
+// record it is handed and this box's placement, never of a snapshot (Fable's
+// B1). A target box that restarts after the flip reconciles before its first
+// moves pass, and must mint the landing row held even then.
+//
+// - A source-region box (placement equals move.from) holds the host's row
+// "moving" under the move id from offline-stopping through
+// offline-sealed. After the flip the record names the target, the row
+// is out of this box's scope and offlineView.keep keeps it, held as it
+// is, until offline-finishing.
+// - A target-region box (placement equals move.to) holds it "landing"
+// while the record names the target and the move is offline-landing or
+// offline-landed. offline-open is the target's to release (Task 5).
+//
+// Placements are compared as movesNamesBox compares them, by
+// park.PlacementMatches under the verified home binding, so the home's ""
+// and its roster name are one.
+func offlineHold(box moveCfg) func(rec provision.Record) (status, op string, ok bool) {
+ home, here := box.ParkHomeRegion, box.parkRegion()
+ return func(rec provision.Record) (string, string, bool) {
+ m := rec.Move
+ if !m.Offline() || m.ID == "" || here == "" {
+ return "", "", false
+ }
+ switch m.State {
+ case provision.OfflineStopping, provision.OfflineSealing, provision.OfflineSealed:
+ if park.PlacementMatches(home, m.From, here) {
+ return "moving", m.ID, true
+ }
+ case provision.OfflineLanding, provision.OfflineLanded:
+ if rec.Region == m.To && park.PlacementMatches(home, m.To, here) {
+ return "landing", m.ID, true
+ }
+ }
+ return "", "", false
+ }
+}
+
+// offlineView is what reconcile's KeepParticipant asks about offline moves:
+// the offline moves in flight, as the last listing read them. Only keep
+// reads it; the hold never does (offlineHold).
+//
+// A source box's view has seen the move before the record flips: the seal
+// door needs this box's report for the move, and the pass that writes it
+// refreshes the view first; an agent restart reads the listing again at
+// start. From then until finishing every listing carries the move.
+//
+// It is refreshed from every moves pass's record list and once at agent
+// start, before the reconcile loops (refreshOfflineView). keep fails closed,
+// keeping every row, only while no listing has been read: when the startup
+// listing failed, and in the moment before it (reconcileSerializedWith reads
+// one first when the view is unbuilt). Once a listing is read, a host with
+// no offline move is not kept, so removals happen on the first pass as they
+// always have.
+type offlineView struct {
+ home, region string
+ // written is reconcile's HoldWritten for the offline hooks: the
+ // stage-requested marker after a hold write (registry.SetHold's
+ // contract, G10). Nil writes nothing.
+ written func(host string)
+ snap atomic.Pointer[offlineSnapshot]
+}
+
+type offlineSnapshot struct {
+ moves map[string]provision.MoveState // host -> its offline move
+ failed bool // the listing failed: keep everything
+}
+
+// newOfflineView builds the view for the box cfg describes. Unbuilt until
+// its first refresh.
+func newOfflineView(cfg moveCfg, written func(host string)) *offlineView {
+ return &offlineView{home: cfg.ParkHomeRegion, region: cfg.parkRegion(), written: written}
+}
+
+// refresh rebuilds the view from a record list that was read.
+func (v *offlineView) refresh(recs []provision.Record) {
+ next := &offlineSnapshot{moves: map[string]provision.MoveState{}}
+ for _, rec := range recs {
+ if rec.Move.Offline() {
+ next.moves[rec.Host] = *rec.Move
+ }
+ }
+ v.snap.Store(next)
+}
+
+// fail records a listing that could not be read, before any was: keep
+// answers true for every host until a listing is read.
+func (v *offlineView) fail() { v.snap.Store(&offlineSnapshot{failed: true}) }
+
+// built reports whether any listing, read or failed, has been recorded.
+func (v *offlineView) built() bool { return v.snap.Load() != nil }
+
+// keep is reconcile's KeepParticipant: the row of a host whose offline move
+// names this box's region stays, whatever the record's region says. The
+// source keeps it until offline-finishing (when it is removed, as the plan's
+// finishing step relies on); the target keeps it until the move is cleared
+// (the target's revert deletes it itself). An unknown state keeps.
+func (v *offlineView) keep(host string) bool {
+ snap := v.snap.Load()
+ if snap == nil || snap.failed {
+ return true
+ }
+ m, ok := snap.moves[host]
+ if !ok || v.region == "" {
+ return false
+ }
+ if park.PlacementMatches(v.home, m.To, v.region) {
+ return true
+ }
+ if park.PlacementMatches(v.home, m.From, v.region) {
+ return m.State != provision.OfflineFinishing
+ }
+ return false
+}
+
+// refreshOfflineView reads the record list once, outside a moves pass: at
+// agent start, before the reconcile loops, next to refreshMovesHolds. A
+// failed listing leaves keep failing closed until a listing is read.
+func refreshOfflineView(ctx context.Context, s release.Store, v *offlineView) {
+ if v == nil || s == nil {
+ return
+ }
+ recs, err := provision.ListAllRecords(ctx, s)
+ if err != nil {
+ log.Printf("moves: offline view: list records: %v; every row is kept until a listing reads", err)
+ v.fail()
+ return
+ }
+ v.refresh(recs)
+}
+
+// armedOfflineView is the armed worker's view, nil when no worker is armed
+// or the box cannot take part in an offline move.
+func armedOfflineView() *offlineView {
+ w := armedMovesWorker.Load()
+ if w == nil || !w.cfg.OfflineCapable {
+ return nil
+ }
+ return w.cfg.Offline
+}
+
+// withOfflineHolds gives a reconcile pass the offline hooks, only when the
+// caller and Phase 1's view set none (withMovesHolds runs first): the two
+// are never armed together, since a box with Phase 1 on is not capable.
+func withOfflineHolds(cfg provision.ReconcileConfig, hold func(provision.Record) (string, string, bool), v *offlineView) provision.ReconcileConfig {
+ if hold == nil || v == nil || cfg.Hold != nil || cfg.KeepParticipant != nil {
+ return cfg
+ }
+ cfg.Hold, cfg.KeepParticipant = hold, v.keep
+ if cfg.HoldWritten == nil && v.written != nil {
+ cfg.HoldWritten = v.written
+ }
+ return cfg
+}
+
+// newMovesWorker is the worker for cfg: armMovesPass's, and each test box's.
+func newMovesWorker(cfg moveCfg) *movesWorker {
+ return &movesWorker{cfg: cfg, mirrors: map[string]lease.MirrorStore{}}
+}
+
+// fenceAsWith is moveCfg.FenceAs over an activator: FenceAndStopAs under a
+// short deadline, so one slow stop does not hold up the pass.
+// ErrStopUnproven still means the fence is installed, which is all the step
+// asks of it; Quiesced answers the rest.
+func fenceAsWith(svc *activator.Service) func(ctx context.Context, host string, generation uint64, owner string) error {
+ return func(ctx context.Context, host string, generation uint64, owner string) error {
+ fctx, cancel := context.WithTimeout(ctx, 2*time.Second)
+ defer cancel()
+ if _, err := svc.FenceAndStopAs(fctx, host, generation, owner); err != nil && !errors.Is(err, activator.ErrStopUnproven) {
+ return err
+ }
+ return nil
+ }
+}
+
+// hostSliceCgroup is the absolute cgroup directory of a nested host's
+// systemd slice under cgroupRoot (the cgroup v2 mount): every role and
+// generation unit of the host runs in it.
+func hostSliceCgroup(cgroupRoot, host string) string {
+ return filepath.Join(cgroupRoot, filepath.FromSlash(path.Dir(instancectl.ClaimUnitControlGroup(host))))
+}
+
+// offlineRound is the moves pass's round for a record with an offline move,
+// reached before every other branch of the pass (the Phase 1 skip and the
+// lost-box gate included). A box that is not capable logs one notice per
+// host and does nothing. Otherwise it runs the source step when its
+// placement is move.from, the sealer's when it is move.sealer, and the
+// target's when its placement is move.to, in that order, so the sealer
+// reads the report this box has just written.
+func (w *movesWorker) offlineRound(ctx context.Context, s release.Store, rec provision.Record) error {
+ cfg := w.cfg
+ m := rec.Move
+ if !m.Offline() {
+ return nil
+ }
+ here := cfg.parkRegion()
+ source := here != "" && park.PlacementMatches(cfg.ParkHomeRegion, m.From, here)
+ target := here != "" && park.PlacementMatches(cfg.ParkHomeRegion, m.To, here)
+ sealer := cfg.Box != "" && cfg.Box == m.Sealer
+ if !cfg.OfflineCapable {
+ if source || target || sealer {
+ w.notice(rec.Host+"#offline", "moves: %s: offline move %s waits: this box cannot take part in an offline move (it needs hibernation configured, and %s and %s unset)", rec.Host, m.ID, "CARLOS_HOST_CLAIM", moves.SwitchEnv)
+ }
+ return nil
+ }
+ var errs []error
+ if source {
+ errs = append(errs, w.offlineSourceStep(ctx, s, rec))
+ }
+ if sealer {
+ errs = append(errs, w.offlineSealerStep(ctx, s, rec))
+ }
+ if target {
+ errs = append(errs, w.offlineTargetStep(ctx, s, rec))
+ }
+ return errors.Join(errs...)
+}
+
+// now is cfg.Now, or the wall clock when a test left it unset.
+func (c moveCfg) now() time.Time {
+ if c.Now != nil {
+ return c.Now()
+ }
+ return time.Now()
+}
+
+// writeOfflineReport writes r at its key (moves.OfflineReportKey), stamped
+// with the report version, after the shape check every reader makes. The
+// source, sealer and target steps all write through it.
+func writeOfflineReport(ctx context.Context, s release.Store, r moves.OfflineReport) error {
+ r.V = moves.OfflineReportVersion
+ key := moves.OfflineReportKey(r.Host, r.Op, r.Role, r.Box)
+ if key == "" {
+ return fmt.Errorf("offline report: host %q, op %q, role %q or box %q is not usable", r.Host, r.Op, r.Role, r.Box)
+ }
+ if err := moves.ValidateOfflineReport(r); err != nil {
+ return err
+ }
+ body, err := json.Marshal(r)
+ if err != nil {
+ return err
+ }
+ // A report is rewritten every pass: read its etag and replace it
+ // (release.Store.Put with no etag is create-only).
+ return moves.PutReplace(ctx, s, key, body)
+}
+
+// readOfflineReports reads every report of one role for one move, keyed by
+// box. A report that does not validate is an error, never skipped.
+func readOfflineReports(ctx context.Context, s release.Store, host, op, role string) (map[string]moves.OfflineReport, error) {
+ prefix := moves.OfflineReportPrefix(host, op)
+ if prefix == "" {
+ return nil, fmt.Errorf("offline report: host %q or op %q is not usable", host, op)
+ }
+ prefix += role + "/"
+ keys, err := s.List(ctx, prefix)
+ if err != nil {
+ return nil, err
+ }
+ out := map[string]moves.OfflineReport{}
+ for _, key := range keys {
+ if !strings.HasSuffix(key, ".json") {
+ continue
+ }
+ body, _, err := s.Get(ctx, key)
+ if err != nil {
+ return nil, err
+ }
+ var r moves.OfflineReport
+ if err := json.Unmarshal(body, &r); err != nil {
+ return nil, fmt.Errorf("%s: %w", key, err)
+ }
+ if err := moves.ValidateOfflineReport(r); err != nil {
+ return nil, fmt.Errorf("%s: %w", key, err)
+ }
+ if moves.OfflineReportKey(r.Host, r.Op, r.Role, r.Box) != key {
+ return nil, fmt.Errorf("%s: the report names %s/%s/%s/%s", key, r.Host, r.Op, r.Role, r.Box)
+ }
+ out[r.Box] = r
+ }
+ return out, nil
+}
+
+// offlineSourceStep is a source-region box's part of an offline move. Every
+// state's step is an ensure: it reads what is on disk, in the registry and
+// in the activator, does only what is missing, and reports what it found.
+func (w *movesWorker) offlineSourceStep(ctx context.Context, s release.Store, rec provision.Record) error {
+ if w.cfg.Box == "" {
+ return errors.New("offline move: this box has no label to report under")
+ }
+ switch rec.Move.State {
+ case provision.OfflineStopping, provision.OfflineSealing, provision.OfflineSealed, provision.OfflineLanding, provision.OfflineLanded:
+ return w.offlineSourceStop(ctx, s, rec)
+ case provision.OfflineOpen:
+ return w.offlineSourcePark(ctx, s, rec)
+ case provision.OfflineFinishing:
+ return w.offlineSourceGone(ctx, s, rec)
+ case provision.OfflineReverting:
+ return w.offlineSourceRevert(ctx, s, rec)
+ }
+ return nil
+}
+
+// offlineFence installs, or meets again, this move's activator fence for
+// host. Fences live in the activator's memory, so every pass installs it
+// again and an agent restart regains it on its next pass. True when the
+// fence is this move's now.
+func (w *movesWorker) offlineFence(ctx context.Context, host string, m *provision.MoveState) bool {
+ if w.cfg.FenceAs == nil {
+ return false
+ }
+ if err := w.cfg.FenceAs(ctx, host, moveFenceGeneration(m), offlineFenceOwner(m.ID)); err != nil {
+ w.notice(host+"#offline-fence", "moves: %s: offline move %s: the wake fence is not installed: %v", host, m.ID, err)
+ return false
+ }
+ return true
+}
+
+// offlineLiftFence lifts this move's fence, and only it.
+func (w *movesWorker) offlineLiftFence(host string, m *provision.MoveState) {
+ if w.cfg.LiftFenceAs != nil && w.cfg.LiftFenceAs(host, moveFenceGeneration(m), offlineFenceOwner(m.ID)) {
+ log.Printf("moves: %s: offline move %s: wake fence lifted", host, m.ID)
+ }
+}
+
+// offlineRow reads the host's row on this box; nil when it has none.
+func (w *movesWorker) offlineRow(host string) (*registry.Route, error) {
+ if w.cfg.Route == nil {
+ return nil, errors.New("offline move: no registry on this box")
+ }
+ return w.cfg.Route(host)
+}
+
+// heldBy reports whether rt is held by op. An absent row counts: nothing
+// can serve or wake a host this box has no row for.
+func heldBy(rt *registry.Route, op string) bool {
+ return rt == nil || (rt.HoldOp == op && registry.IsHold(rt.Status))
+}
+
+// offlineLocal is what a box keeps on disk for a host, judged by lstat and
+// directory listings only, never by following a link. A row's own layout is
+// read at the row's paths; a box with no row, or a leftover of the other
+// layout, is read at both.
+type offlineLocal struct {
+ LocalDB, Files, Env, Sidecar bool
+ LocalTXID uint64
+}
+
+// flatDBPath is where a flat row's database is: the row's own path when it
+// has one and is flat, else the default.
+func (c moveCfg) flatDBPath(host string, rt *registry.Route) string {
+ if rt != nil && rt.DBPath != "" && !provision.NestedPaths(c.DataDir, *rt) {
+ return rt.DBPath
+ }
+ return provision.DefaultDBPath(c.DataDir, host)
+}
+
+// activatorMetaDir is the activator's litestream meta directory beside a
+// database (localLTXPosition reads it).
+func activatorMetaDir(dbPath string) string {
+ return filepath.Join(filepath.Dir(dbPath), ".activator-"+filepath.Base(dbPath)+"-litestream")
+}
+
+func lexists(p string) bool {
+ _, err := os.Lstat(p)
+ return err == nil
+}
+
+func isDir(p string) bool {
+ st, err := os.Lstat(p)
+ return err == nil && st.IsDir()
+}
+
+// offlinePlatformNames are a nested instance directory's platform entries:
+// everything else in it is the app's files.
+func offlinePlatformNames(host string) map[string]bool {
+ db := host + ".db"
+ return map[string]bool{
+ db: true, db + "-wal": true, db + "-shm": true, db + "-litestream": true,
+ ".activator-" + db + "-litestream": true, db + ".corrupt": true, db + ".gen": true,
+ host + ".env": true, host + ".config.env": true, host + "-sidecar.env": true,
+ host + ".instance-key-adoption.json": true, ".files-gen": true,
+ }
+}
+
+// localState reads what this box keeps for host: the DB file or either
+// litestream meta dir (S8: a meta dir alone is local state, since
+// localLTXPosition reads it), the files directory, the env file and a
+// sidecar env file, under the row's layout and any leftover of the other.
+func (w *movesWorker) localState(host string, rt *registry.Route) (offlineLocal, error) {
+ cfg := w.cfg
+ var out offlineLocal
+ nestedRow := rt != nil && provision.NestedPaths(cfg.DataDir, *rt)
+ // Flat: a flat row's own paths, or a nested row's flat leftovers.
+ {
+ db := cfg.flatDBPath(host, rt)
+ out.LocalDB = lexists(db) || lexists(db+"-litestream") || lexists(activatorMetaDir(db))
+ out.Files = isDir(filepath.Join(cfg.DataDir, host))
+ out.Env = lexists(filepath.Join(cfg.DataDir, host+".env"))
+ out.Sidecar = lexists(filepath.Join(cfg.DataDir, host+"-sidecar.env"))
+ pos, err := localLTXPosition(db)
+ if err != nil {
+ return out, err
+ }
+ out.LocalTXID = pos
+ }
+ // Nested, read through the raw directory (layout.RawInstanceDir).
+ raw := layout.RawInstanceDir(cfg.DataDir, host)
+ if nestedRow || isDir(raw) {
+ db := filepath.Join(raw, host+".db")
+ out.LocalDB = out.LocalDB || lexists(db) || lexists(db+"-litestream") || lexists(activatorMetaDir(db))
+ out.Env = out.Env || lexists(filepath.Join(raw, host+".env"))
+ out.Sidecar = out.Sidecar || lexists(filepath.Join(raw, host+"-sidecar.env"))
+ ents, err := os.ReadDir(raw)
+ if err != nil && !errors.Is(err, os.ErrNotExist) {
+ return out, err
+ }
+ platform := offlinePlatformNames(host)
+ for _, e := range ents {
+ if !platform[e.Name()] {
+ out.Files = true
+ break
+ }
+ }
+ dbPath := db
+ if nestedRow && rt.DBPath != "" {
+ dbPath = rt.DBPath
+ }
+ pos, err := localLTXPosition(dbPath)
+ if err != nil {
+ return out, err
+ }
+ out.LocalTXID = max(out.LocalTXID, pos)
+ }
+ return out, nil
+}
+
+// procs is the process count for the report: -1 when it cannot be read,
+// which the flip door reads as not stopped.
+func (w *movesWorker) procs(host string) int {
+ if w.cfg.Procs == nil {
+ return -1
+ }
+ n, err := w.cfg.Procs(host)
+ if err != nil {
+ w.notice(host+"#offline-procs", "moves: %s: the host's processes cannot be counted on this box: %v", host, err)
+ return -1
+ }
+ return n
+}
+
+// sourceReport is the source report's evidence as it stands now.
+func (w *movesWorker) sourceReport(host string, m *provision.MoveState, rt *registry.Route, fenced bool, phase string) (moves.OfflineReport, error) {
+ cfg := w.cfg
+ local, err := w.localState(host, rt)
+ if err != nil {
+ return moves.OfflineReport{}, err
+ }
+ quiesced := cfg.Quiesced != nil && cfg.Quiesced(host)
+ return moves.OfflineReport{
+ Op: m.ID, Host: host, Box: cfg.Box, Role: moves.OfflineRoleSource, Phase: phase, At: cfg.now().UTC(),
+ Held: heldBy(rt, m.ID), Fenced: fenced, Quiesced: quiesced, RowPresent: rt != nil,
+ LocalDB: local.LocalDB, Files: local.Files, Env: local.Env, Sidecar: local.Sidecar, LocalTXID: local.LocalTXID,
+ Procs: w.procs(host), LayoutJournal: layout.JournalPresent(cfg.DataDir, host),
+ }, nil
+}
+
+// offlineSourceStop is the source step from offline-stopping through
+// offline-landed: install the fence, read the row and the local state, and
+// rewrite this box's source report with At on every pass (the open door
+// needs it current). The row's hold is reconcile's (offlineHold) and,
+// after the flip, its keep (offlineView.keep).
+func (w *movesWorker) offlineSourceStop(ctx context.Context, s release.Store, rec provision.Record) error {
+ m := rec.Move
+ fenced := w.offlineFence(ctx, rec.Host, m)
+ rt, err := w.offlineRow(rec.Host)
+ if err != nil {
+ return err
+ }
+ r, err := w.sourceReport(rec.Host, m, rt, fenced, moves.OfflinePhaseStopped)
+ if err != nil {
+ return err
+ }
+ return writeOfflineReport(ctx, s, r)
+}
+
+// retirePath is one path of the manifest's source-retire row on this box:
+// where it is, its name in the park directory, and, for a path the agent
+// never renames, why.
+type retirePath struct {
+ path, name, left string
+}
+
+// Why a path is left where it is.
+const (
+ leftRootOwned = "root-owned: listed for the census, not renamed"
+ leftView = "the mount view of the instance directory: it stays, empty"
+ leftInstance = "the instance directory stays, empty"
+)
+
+// nestedParkPrefix is where a nested instance directory's entries go inside
+// the park directory, apart from the flat names.
+const nestedParkPrefix = "instance"
+
+// retirePaths is the manifest's source-retire row as paths on this box, in
+// a fixed order: the row's layout in full, the other layout's paths only
+// where they exist, the app's own entries of a nested instance directory as
+// found there and in the park directory, then what is never renamed.
+func (w *movesWorker) retirePaths(host, parkDir string, rt *registry.Route) ([]retirePath, error) {
+ cfg := w.cfg
+ nestedRow := rt != nil && provision.NestedPaths(cfg.DataDir, *rt)
+ raw := layout.RawInstanceDir(cfg.DataDir, host)
+ var out []retirePath
+ add := func(p, name string, always bool) {
+ if always || lexists(p) {
+ out = append(out, retirePath{path: p, name: name})
+ }
+ }
+ flatDB := cfg.flatDBPath(host, rt)
+ flat := !nestedRow
+ add(flatDB, filepath.Base(flatDB), flat)
+ add(flatDB+"-wal", filepath.Base(flatDB)+"-wal", flat)
+ add(flatDB+"-shm", filepath.Base(flatDB)+"-shm", flat)
+ add(flatDB+"-litestream", filepath.Base(flatDB)+"-litestream", flat)
+ add(activatorMetaDir(flatDB), filepath.Base(activatorMetaDir(flatDB)), flat)
+ add(filepath.Join(cfg.DataDir, host), host, flat)
+ add(filepath.Join(cfg.DataDir, host+".env"), host+".env", flat)
+ add(filepath.Join(cfg.DataDir, host+".config.env"), host+".config.env", flat)
+
+ nested := nestedRow || isDir(raw)
+ if nested {
+ named := map[string]bool{}
+ for _, n := range []string{host + ".db", host + ".db-wal", host + ".db-shm", host + ".db-litestream", ".activator-" + host + ".db-litestream", host + ".env", host + ".config.env"} {
+ named[n] = true
+ add(filepath.Join(raw, n), nestedParkPrefix+"/"+n, true)
+ }
+ // The app's files, and every other entry the directory holds: what
+ // is still there, and what an earlier pass already parked.
+ // A numbered copy in the park directory (parkName) belongs to the
+ // entry it is a copy of, not to the app.
+ others := map[string]bool{}
+ listed := map[string][]string{}
+ for _, dir := range []string{raw, filepath.Join(parkDir, nestedParkPrefix)} {
+ ents, err := os.ReadDir(dir)
+ if err != nil && !errors.Is(err, os.ErrNotExist) {
+ return nil, err
+ }
+ for _, e := range ents {
+ listed[dir] = append(listed[dir], e.Name())
+ }
+ }
+ known := map[string]bool{}
+ for n := range named {
+ known[n] = true
+ }
+ for _, n := range listed[raw] {
+ known[n] = true
+ }
+ for _, n := range listed[filepath.Join(parkDir, nestedParkPrefix)] {
+ known[n] = true
+ }
+ for _, n := range listed[raw] {
+ if !named[n] {
+ others[n] = true
+ }
+ }
+ for _, n := range listed[filepath.Join(parkDir, nestedParkPrefix)] {
+ if !named[n] && !isNumberedCopy(n, known) {
+ others[n] = true
+ }
+ }
+ names := make([]string, 0, len(others))
+ for n := range others {
+ names = append(names, n)
+ }
+ sort.Strings(names)
+ for _, n := range names {
+ add(filepath.Join(raw, n), nestedParkPrefix+"/"+n, true)
+ }
+ }
+ add(filepath.Join(cfg.DataDir, "programs", host+".json"), host+".json", true)
+ if nested {
+ out = append(out, retirePath{path: provision.InstanceDir(cfg.DataDir, host), left: leftView})
+ out = append(out, retirePath{path: raw, left: leftInstance})
+ }
+ optDir := cfg.OptDir
+ if optDir == "" {
+ optDir = release.OptDir()
+ }
+ out = append(out, retirePath{path: filepath.Join(optDir, "live", host), left: leftRootOwned})
+ return out, nil
+}
+
+// parkCopySep separates a name from its copy number in the park directory
+// (parkName). Chosen to be unlikely in an app's own file names.
+const parkCopySep = ".parked-"
+
+// isNumberedCopy reports whether n is a known entry's numbered park copy
+// (parkName's "<name>.parked-<n>"), which belongs to that entry.
+func isNumberedCopy(n string, known map[string]bool) bool {
+ i := strings.LastIndex(n, parkCopySep)
+ if i <= 0 || !known[n[:i]] || i+len(parkCopySep) == len(n) {
+ return false
+ }
+ for _, c := range n[i+len(parkCopySep):] {
+ if c < '0' || c > '9' {
+ return false
+ }
+ }
+ return true
+}
+
+// parkName is where name goes in dir: name itself, or name.parked-<n> for
+// the first free n when something of that name was parked before (the
+// source path came back after an earlier pass parked it). The park
+// directory is the move's own, so the first park of every path lands at its
+// plain name and a later pass reports it from there.
+func parkName(dir, name string) string {
+ dst := filepath.Join(dir, name)
+ if !lexists(dst) {
+ return dst
+ }
+ for n := 1; ; n++ {
+ if c := dst + parkCopySep + fmt.Sprint(n); !lexists(c) {
+ return c
+ }
+ }
+}
+
+// retire reports one path's action from what is on disk, renaming it into
+// the park directory first when park is true and it is there to rename.
+// Nothing is ever deleted.
+func retire(p retirePath, parkDir string, park bool) moves.RetiredPath {
+ present := lexists(p.path)
+ switch {
+ case p.left != "" && present:
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredLeft, Reason: p.left}
+ case p.left != "":
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredAbsent}
+ case present && park:
+ dst := parkName(parkDir, p.name)
+ if err := os.MkdirAll(filepath.Dir(dst), 0o700); err != nil {
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredLeft, Reason: "not renamed: " + err.Error()}
+ }
+ if err := os.Rename(p.path, dst); err != nil {
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredLeft, Reason: "not renamed: " + err.Error()}
+ }
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredParked}
+ case present:
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredLeft, Reason: "not renamed: the stop could not be confirmed this pass"}
+ case lexists(filepath.Join(parkDir, p.name)):
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredParked}
+ }
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredAbsent}
+}
+
+// offlineSourcePark is the source step at offline-open: with the fence held,
+// the row held (or absent), the host quiesced, no process left and no layout
+// journal, rename every path of the source-retire row that is still here
+// into the park directory; then lift the fence and report parked with the
+// inventory. A pass that finds nothing left to rename does not fence again:
+// it reports from what is parked. Nothing is deleted.
+func (w *movesWorker) offlineSourcePark(ctx context.Context, s release.Store, rec provision.Record) error {
+ cfg := w.cfg
+ m, host := rec.Move, rec.Host
+ parkDir := moves.OfflineParkDir(cfg.DataDir, host, m.ID, moves.OfflineRoleSource)
+ if parkDir == "" {
+ return fmt.Errorf("offline move %s: no park directory for %s under %q", m.ID, host, cfg.DataDir)
+ }
+ rt, err := w.offlineRow(host)
+ if err != nil {
+ return err
+ }
+ paths, err := w.retirePaths(host, parkDir, rt)
+ if err != nil {
+ return err
+ }
+ pending := false
+ for _, p := range paths {
+ if p.left == "" && lexists(p.path) {
+ pending = true
+ break
+ }
+ }
+ if pending {
+ fenced := w.offlineFence(ctx, host, m)
+ quiesced := cfg.Quiesced != nil && cfg.Quiesced(host)
+ procs := w.procs(host)
+ journal := layout.JournalPresent(cfg.DataDir, host)
+ if !fenced || !heldBy(rt, m.ID) || !quiesced || procs != 0 || journal {
+ return fmt.Errorf("offline move %s: not parking %s yet: fenced %v, held %v, quiesced %v, processes %d, layout journal %v",
+ m.ID, host, fenced, heldBy(rt, m.ID), quiesced, procs, journal)
+ }
+ if err := os.MkdirAll(parkDir, 0o700); err != nil {
+ return err
+ }
+ }
+ retired := make([]moves.RetiredPath, 0, len(paths))
+ for _, p := range paths {
+ retired = append(retired, retire(p, parkDir, pending))
+ }
+ w.offlineLiftFence(host, m)
+ r, err := w.sourceReport(host, m, rt, false, moves.OfflinePhaseParked)
+ if err != nil {
+ return err
+ }
+ r.Retired = retired
+ return writeOfflineReport(ctx, s, r)
+}
+
+// offlineSourceGone is the source step at offline-finishing: keep no longer
+// holds the row, so reconcile removes it; once it is gone, report gone. The
+// fence was lifted at open and is lifted again, should a restart have
+// re-fenced in between.
+func (w *movesWorker) offlineSourceGone(ctx context.Context, s release.Store, rec provision.Record) error {
+ m, host := rec.Move, rec.Host
+ rt, err := w.offlineRow(host)
+ if err != nil {
+ return err
+ }
+ w.offlineLiftFence(host, m)
+ if rt != nil {
+ w.notice(host+"#offline-gone", "moves: %s: offline move %s: waiting for reconcile to remove the row", host, m.ID)
+ return nil
+ }
+ r, err := w.sourceReport(host, m, nil, false, moves.OfflinePhaseGone)
+ if err != nil {
+ return err
+ }
+ return writeOfflineReport(ctx, s, r)
+}
+
+// offlineSourceRevert is every source box's part of offline-reverting: the
+// row leaves this move's hold (ReleaseHold under the host's hold lock), the
+// fence is lifted, and the box reports reverted. The sealer's extra part is
+// its own step's (Task 4).
+//
+// The release also holds reconcileMu: a reconcile pass that listed the
+// records before the abort would otherwise mint a missing row held again
+// from its stale listing after the release, and a row held by a move that
+// has finished is never released. A pass that starts after the release
+// lists the record at offline-reverting, which holds nothing.
+func (w *movesWorker) offlineSourceRevert(ctx context.Context, s release.Store, rec provision.Record) error {
+ cfg := w.cfg
+ m, host := rec.Move, rec.Host
+ if cfg.ReleaseHold == nil {
+ return errors.New("offline move: no registry to release the hold through")
+ }
+ var relErr error
+ reconcileMu.Lock()
+ provision.WithHostHoldLock(host, func() {
+ _, relErr = cfg.ReleaseHold(host, m.ID)
+ })
+ reconcileMu.Unlock()
+ if relErr != nil {
+ return relErr
+ }
+ w.offlineLiftFence(host, m)
+ rt, err := w.offlineRow(host)
+ if err != nil {
+ return err
+ }
+ if rt != nil && rt.HoldOp == m.ID {
+ return fmt.Errorf("offline move %s: %s is still held by the move", m.ID, host)
+ }
+ r, err := w.sourceReport(host, m, rt, false, moves.OfflinePhaseReverted)
+ if err != nil {
+ return err
+ }
+ r.Held = false
+ return writeOfflineReport(ctx, s, r)
+}
diff --git a/internal/host/movesoffline_land.go b/internal/host/movesoffline_land.go
new file mode 100644
index 000000000..b4c2687ce
--- /dev/null
+++ b/internal/host/movesoffline_land.go
@@ -0,0 +1,26 @@
+package host
+
+import (
+ "context"
+
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// offlineTargetStep is the target box's part of an offline move: the
+// landing, open and the target's revert
+// (docs/superpowers/plans/2026-10-20-offline-operator-move.md, Task 5, which
+// owns this file). offlineRound calls it on a box whose placement equals
+// rec.Move.To, in every offline state, after the source and sealer steps.
+// rec carries an offline move (rec.Move.Offline()) and the box is capable
+// (moveCfg.OfflineCapable). Reconcile mints the row held "landing" from the
+// record alone (offlineHold) and keeps it until the move is cleared
+// (offlineView.keep); ending that hold (open, revert) must take
+// provision.WithHostHoldLock around the release. Every step must be an
+// ensure; an error is logged by the pass, throttled per host, and the next
+// pass runs the step again.
+//
+// Task 3 leaves it a no-op.
+func (w *movesWorker) offlineTargetStep(ctx context.Context, s release.Store, rec provision.Record) error {
+ return nil
+}
diff --git a/internal/host/movesoffline_seal.go b/internal/host/movesoffline_seal.go
new file mode 100644
index 000000000..2e3b2d6af
--- /dev/null
+++ b/internal/host/movesoffline_seal.go
@@ -0,0 +1,22 @@
+package host
+
+import (
+ "context"
+
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// offlineSealerStep is the sealer's part of an offline move
+// (docs/superpowers/plans/2026-10-20-offline-operator-move.md, Task 4, which
+// owns this file). offlineRound calls it on the box whose moveCfg.Box equals
+// rec.Move.Sealer, in every offline state, after this box's source step has
+// written its report for the pass. rec carries an offline move
+// (rec.Move.Offline()) and the box is capable (moveCfg.OfflineCapable).
+// Every step must be an ensure; an error is logged by the pass, throttled
+// per host, and the next pass runs the step again.
+//
+// Task 3 leaves it a no-op.
+func (w *movesWorker) offlineSealerStep(ctx context.Context, s release.Store, rec provision.Record) error {
+ return nil
+}
diff --git a/internal/host/movesoffline_test.go b/internal/host/movesoffline_test.go
new file mode 100644
index 000000000..6b767b38f
--- /dev/null
+++ b/internal/host/movesoffline_test.go
@@ -0,0 +1,1041 @@
+package host
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "os"
+ "path/filepath"
+ "reflect"
+ "slices"
+ "sort"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/cgroup"
+ "github.com/carlosframework/platform/internal/domains"
+ "github.com/carlosframework/platform/internal/hostwrite"
+ "github.com/carlosframework/platform/internal/layout"
+ "github.com/carlosframework/platform/internal/moves"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// The offline operator move's box side (Task 3 of
+// docs/superpowers/plans/2026-10-20-offline-operator-move.md), tested
+// through the functions the agent calls: newMovesWorker, armOffline,
+// reconcileSerializedWith, the production edges-file writer and movesPass,
+// over one DirStore, real registries and real activator services (the
+// package's fake wake backend, which only the fence needs).
+
+const (
+ offHost = "shop.example.com"
+ offID = "mv_zyxwvutsrqponmlkjihgfedcba"
+ offID2 = "mv_abcdefghijklmnopqrstuvwxy2"
+)
+
+type offWorld struct {
+ t *testing.T
+ s release.Store
+ acct string
+ mu sync.Mutex
+ now time.Time
+}
+
+func (wd *offWorld) clock() time.Time {
+ wd.mu.Lock()
+ defer wd.mu.Unlock()
+ return wd.now
+}
+
+func (wd *offWorld) advance(d time.Duration) {
+ wd.mu.Lock()
+ wd.now = wd.now.Add(d)
+ wd.mu.Unlock()
+}
+
+// newOffWorld is a deployment with one account, one app and one instance
+// record for offHost at the home placement ("").
+func newOffWorld(t *testing.T) *offWorld {
+ t.Helper()
+ t.Setenv("CARLOS_HOST_CLAIM", "") // the appconfig and provision guards read the process env
+ ctx := context.Background()
+ wd := &offWorld{t: t, s: release.NewDirStore(t.TempDir()), acct: domains.Sqid(2), now: time.Date(2026, 10, 20, 9, 0, 0, 0, time.UTC)}
+ manifest, err := json.Marshal(map[string]any{"id": 2, "sqid": wd.acct, "name": "fixture", "created_at": wd.now})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := wd.s.Put(ctx, "accounts/"+wd.acct+"/account.json", manifest, ""); err != nil {
+ t.Fatal(err)
+ }
+ if err := provision.PutManifest(ctx, wd.s, wd.acct, "app", provision.Manifest{Schema: 1, App: "app", Domain: "example.com"}); err != nil {
+ t.Fatal(err)
+ }
+ wd.create(offHost)
+ return wd
+}
+
+func (wd *offWorld) create(host string) {
+ wd.t.Helper()
+ rec := provision.Record{Schema: 1, Host: host, Slug: strings.SplitN(host, ".", 2)[0], Account: wd.acct, App: "app", Region: "", Channel: "stable", CreatedAt: wd.now}
+ if err := provision.CreateRecord(context.Background(), wd.s, rec); err != nil {
+ wd.t.Fatal(err)
+ }
+}
+
+// move applies one offline transition to offHost's record.
+func (wd *offWorld) move(mut func(*provision.Record) error) provision.Record {
+ wd.t.Helper()
+ return wd.moveHost(offHost, mut)
+}
+
+func (wd *offWorld) moveHost(host string, mut func(*provision.Record) error) provision.Record {
+ wd.t.Helper()
+ rec, err := provision.SetRecordMove(context.Background(), wd.s, wd.acct, "app", host, mut)
+ if err != nil {
+ wd.t.Fatal(err)
+ }
+ return *rec
+}
+
+func (wd *offWorld) record() provision.Record {
+ wd.t.Helper()
+ rec, err := provision.GetRecord(context.Background(), wd.s, wd.acct, "app", offHost)
+ if err != nil {
+ wd.t.Fatal(err)
+ }
+ return *rec
+}
+
+// The legal path to a state, from the record at rest.
+func (wd *offWorld) start() {
+ wd.move(provision.OfflineStart(offID, "", "uk", "op@example.com", wd.clock()))
+}
+func (wd *offWorld) seal() { wd.move(provision.OfflineSeal(offID, "ie-a", []string{"ie-a", "ie-b"})) }
+func (wd *offWorld) sealed() {
+ wd.move(provision.OfflineSealedBy(offID, "ie-a", provision.MoveSource{}))
+}
+func (wd *offWorld) flip() { wd.move(provision.OfflineFlip(offID)) }
+func (wd *offWorld) landed() { wd.move(provision.OfflineLandedBy(offID, "uk")) }
+func (wd *offWorld) open() { wd.move(provision.OfflineOpenTransition(offID, wd.clock())) }
+func (wd *offWorld) finishing() {
+ wd.move(provision.OfflineFinishingTransition(offID))
+}
+func (wd *offWorld) abort() { wd.move(provision.OfflineAbort(offID)) }
+
+// report reads one box's report for offID, or fails.
+func (wd *offWorld) report(role, box string) moves.OfflineReport {
+ wd.t.Helper()
+ r, ok := wd.reportIf(role, box)
+ if !ok {
+ wd.t.Fatalf("no %s report from %s", role, box)
+ }
+ return r
+}
+
+func (wd *offWorld) reportIf(role, box string) (moves.OfflineReport, bool) {
+ wd.t.Helper()
+ body, _, err := wd.s.Get(context.Background(), moves.OfflineReportKey(offHost, offID, role, box))
+ if errors.Is(err, os.ErrNotExist) {
+ return moves.OfflineReport{}, false
+ }
+ if err != nil {
+ wd.t.Fatal(err)
+ }
+ var r moves.OfflineReport
+ if err := json.Unmarshal(body, &r); err != nil {
+ wd.t.Fatal(err)
+ }
+ if err := moves.ValidateOfflineReport(r); err != nil {
+ wd.t.Fatal(err)
+ }
+ return r, true
+}
+
+// offBox is one box: its own data directory, registry, activator and moves
+// worker, armed as armMovesPass arms it (armOffline).
+type offBox struct {
+ wd *offWorld
+ name, region string
+ dd, opt, cg string
+ env map[string]string
+ reg *registry.Registry
+ svc *activator.Service
+ w *movesWorker
+ nested bool
+ last []byte
+}
+
+func (wd *offWorld) box(name, region string, env map[string]string) *offBox {
+ wd.t.Helper()
+ b := &offBox{wd: wd, name: name, region: region, dd: wd.t.TempDir(), opt: wd.t.TempDir(), cg: wd.t.TempDir(), env: env}
+ reg, err := registry.Open(filepath.Join(b.dd, "registry.db"))
+ if err != nil {
+ wd.t.Fatal(err)
+ }
+ wd.t.Cleanup(func() { reg.Close() })
+ b.reg = reg
+ b.restart()
+ return b
+}
+
+func (b *offBox) getenv(k string) string {
+ if v, ok := b.env[k]; ok {
+ return v
+ }
+ if k == "CARLOS_HIBERNATE_BUCKET" {
+ return "data"
+ }
+ return ""
+}
+
+// restart is an agent restart: a fresh activator (no fences) and a fresh
+// worker over the same registry and data directory.
+func (b *offBox) restart() {
+ b.svc = activator.New(activator.Config{Bucket: "data", Idle: time.Hour}, &fakeWakeBackend{}, b.reg)
+ cfg := moveCfg{Region: b.region, Box: b.name, DataDir: b.dd, OptDir: b.opt, Quiesced: b.svc.Quiesced, Route: b.reg.Get, Now: b.wd.clock}
+ armOffline(&cfg, b.dd, b.reg, b.svc, nil, hostwrite.Config{}, b.getenv)
+ // The stage-requested marker is root's business, and its throttled
+ // timer would outlive the test's directories.
+ cfg.Offline = newOfflineView(cfg, nil)
+ cfg.ReleaseHold = releaseHeld(b.reg, nil)
+ cfg.Procs = func(host string) (int, error) { return cgroup.HostProcs(b.cg, host, "") }
+ b.w = newMovesWorker(cfg)
+}
+
+func (b *offBox) reconcile() {
+ b.wd.t.Helper()
+ cfg := provision.ReconcileConfig{Region: b.region, DataDir: b.dd}
+ if b.nested {
+ cfg.Layout = func(context.Context, release.Store) (string, error) { return release.LayoutModeNested, nil }
+ } else {
+ cfg.Layout = func(context.Context, release.Store) (string, error) { return release.LayoutModeFlat, nil }
+ }
+ if err := reconcileSerializedWith(context.Background(), b.wd.s, b.reg, cfg, b.w); err != nil {
+ b.wd.t.Fatal(err)
+ }
+}
+
+func (b *offBox) edges() {
+ b.wd.t.Helper()
+ if err := edgesFileWrite(context.Background(), b.wd.s, b.reg, edgesFile{Box: b.name, Placement: b.region}, "", &b.last, false); err != nil {
+ b.wd.t.Fatal(err)
+ }
+}
+
+func (b *offBox) pass() {
+ b.wd.t.Helper()
+ if err := movesPass(context.Background(), b.wd.s, b.w); err != nil {
+ b.wd.t.Fatal(err)
+ }
+}
+
+// tick is the agent's order on a serial bump: reconcile, the edges file,
+// the moves pass.
+func (b *offBox) tick() {
+ b.wd.t.Helper()
+ b.reconcile()
+ b.edges()
+ b.pass()
+}
+
+func (b *offBox) row(host string) *registry.Route {
+ b.wd.t.Helper()
+ rt, err := b.reg.Get(host)
+ if err != nil {
+ b.wd.t.Fatal(err)
+ }
+ return rt
+}
+
+// lists reports whether this box's published edges file lists host.
+func (b *offBox) lists(host string) bool {
+ b.wd.t.Helper()
+ body, _, err := b.wd.s.Get(context.Background(), edgesFileKey(b.name))
+ if err != nil {
+ b.wd.t.Fatal(err)
+ }
+ var f edgesFile
+ if err := json.Unmarshal(body, &f); err != nil {
+ b.wd.t.Fatal(err)
+ }
+ _, ok := f.Hosts[host]
+ return ok
+}
+
+func writeFile(t *testing.T, p, body string) {
+ t.Helper()
+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// --- the hold ---------------------------------------------------------------
+
+// offlineHold's table: a pure function of the record and the box's
+// placement. The home's "" and a named region; every state.
+func TestOfflineHoldIsAFunctionOfTheRecord(t *testing.T) {
+ home := moveCfg{Region: ""}
+ uk := moveCfg{Region: "uk"}
+ fr := moveCfg{Region: "fr"}
+ rec := func(region, state string) provision.Record {
+ return provision.Record{Host: offHost, Region: region, Move: &provision.MoveState{Kind: provision.MoveKindOffline, ID: offID, From: "", To: "uk", State: state}}
+ }
+ cases := []struct {
+ name string
+ box moveCfg
+ rec provision.Record
+ wantStatus, wantOp string
+ }{
+ {"source stopping", home, rec("", provision.OfflineStopping), "moving", offID},
+ {"source sealing", home, rec("", provision.OfflineSealing), "moving", offID},
+ {"source sealed", home, rec("", provision.OfflineSealed), "moving", offID},
+ {"source reverting", home, rec("", provision.OfflineReverting), "", ""},
+ {"source after the flip", home, rec("uk", provision.OfflineLanding), "", ""},
+ {"target before the flip", uk, rec("", provision.OfflineSealed), "", ""},
+ {"target landing", uk, rec("uk", provision.OfflineLanding), "landing", offID},
+ {"target landed", uk, rec("uk", provision.OfflineLanded), "landing", offID},
+ {"target landing, region not yet the target's", uk, rec("", provision.OfflineLanding), "", ""},
+ {"target open", uk, rec("uk", provision.OfflineOpen), "", ""},
+ {"target finishing", uk, rec("uk", provision.OfflineFinishing), "", ""},
+ {"target reverting", uk, rec("", provision.OfflineReverting), "", ""},
+ {"a third region", fr, rec("", provision.OfflineStopping), "", ""},
+ {"a third region after the flip", fr, rec("uk", provision.OfflineLanding), "", ""},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ status, op, ok := offlineHold(c.box)(c.rec)
+ if status != c.wantStatus || op != c.wantOp || ok != (c.wantStatus != "") {
+ t.Fatalf("hold = (%q, %q, %v), want (%q, %q)", status, op, ok, c.wantStatus, c.wantOp)
+ }
+ })
+ }
+ // A record at rest, a legacy move and a Phase 1 move are never held.
+ for _, m := range []*provision.MoveState{nil, {To: "uk", State: provision.MoveCutting}, {ID: offID, From: "", To: "uk", State: provision.OfflineStopping}} {
+ if _, _, ok := offlineHold(home)(provision.Record{Host: offHost, Move: m}); ok {
+ t.Fatalf("held: %+v", m)
+ }
+ }
+}
+
+// B1, the pinning test: a fresh target worker whose keep view has never
+// seen the move (built from a list with no offline record in it, as a
+// box's view is before the flip) mints the row held "landing" on its first
+// reconcile after the flip. And with the view never built at all, reconcile
+// through the offline hooks alone does the same.
+func TestOfflineFreshTargetMintsTheLandingRowHeld(t *testing.T) {
+ wd := newOffWorld(t)
+ wd.start()
+ wd.seal()
+ wd.sealed()
+ wd.flip()
+
+ uk := wd.box("uk-box", "uk", nil)
+ uk.w.cfg.Offline.refresh(nil) // stale: no offline move in it
+ uk.reconcile()
+ rt := uk.row(offHost)
+ if rt == nil || rt.Status != "landing" || rt.HoldOp != offID {
+ t.Fatalf("after the flip the target minted %+v, want held landing by %s", rt, offID)
+ }
+ uk.edges()
+ if uk.lists(offHost) {
+ t.Fatal("the target's edges file lists a held landing row")
+ }
+
+ // The hooks with a view that was never built, straight into
+ // provision.Reconcile: the hold needs no view.
+ fresh := wd.box("uk-box-2", "uk", nil)
+ if fresh.w.cfg.Offline.built() {
+ t.Fatal("fixture: the view is built")
+ }
+ cfg := withOfflineHolds(provision.ReconcileConfig{Region: "uk", DataDir: fresh.dd}, offlineHold(fresh.w.cfg), fresh.w.cfg.Offline)
+ if err := provision.Reconcile(context.Background(), wd.s, fresh.reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ if rt := fresh.row(offHost); rt == nil || rt.Status != "landing" || rt.HoldOp != offID {
+ t.Fatalf("an unbuilt view: minted %+v", rt)
+ }
+}
+
+// --- the source boxes -------------------------------------------------------
+
+// start holds the row on both source boxes (one minting it held, one
+// holding the row it has, keeping its other columns), both report held,
+// fenced and quiesced, and neither edges file lists the host. An agent
+// restart re-fences and re-reports. After the flip the rows stay, kept.
+func TestOfflineStartHoldsEverySourceRow(t *testing.T) {
+ wd := newOffWorld(t)
+ a := wd.box("ie-a", "", nil)
+ b := wd.box("ie-b", "", nil)
+ uk := wd.box("uk-box", "uk", nil)
+ a.tick() // ie-a serves the host before the move; ie-b has no row yet
+ if rt := a.row(offHost); rt == nil || rt.Status != "ready" || !a.lists(offHost) {
+ t.Fatalf("fixture: ie-a does not serve the host: %+v", rt)
+ }
+ if err := a.reg.SetReplication(offHost, "daily"); err != nil {
+ t.Fatal(err)
+ }
+ if err := a.reg.SetNextDue(offHost, 1234); err != nil {
+ t.Fatal(err)
+ }
+ before := *a.row(offHost)
+
+ wd.start()
+ a.tick()
+ b.tick()
+ uk.tick()
+ for _, bx := range []*offBox{a, b} {
+ rt := bx.row(offHost)
+ if rt == nil || rt.Status != "moving" || rt.HoldOp != offID {
+ t.Fatalf("%s: row %+v, want held moving by the move", bx.name, rt)
+ }
+ if bx.lists(offHost) {
+ t.Fatalf("%s's edges file lists a held host", bx.name)
+ }
+ if !bx.svc.Fenced(offHost) {
+ t.Fatalf("%s: no wake fence", bx.name)
+ }
+ r := wd.report(moves.OfflineRoleSource, bx.name)
+ if !r.Held || !r.Fenced || !r.Quiesced || !r.RowPresent || r.Procs != 0 || r.LayoutJournal || r.Phase != moves.OfflinePhaseStopped || !r.At.Equal(wd.clock()) {
+ t.Fatalf("%s: report %+v", bx.name, r)
+ }
+ }
+ after := *a.row(offHost)
+ after.Status, after.HoldOp = before.Status, before.HoldOp
+ if !reflect.DeepEqual(after, before) {
+ t.Fatalf("holding the existing row changed its other columns:\n before %+v\n after %+v", before, after)
+ }
+ if uk.row(offHost) != nil {
+ t.Fatal("the target minted a row before the flip")
+ }
+ if _, ok := wd.reportIf(moves.OfflineRoleSource, "uk-box"); ok {
+ t.Fatal("the target wrote a source report")
+ }
+
+ // An agent restart: the fence is gone with the process, and the next
+ // pass installs it again and rewrites the report.
+ a.restart()
+ if a.svc.Fenced(offHost) {
+ t.Fatal("fixture: a fresh activator is fenced")
+ }
+ wd.advance(time.Minute)
+ a.tick()
+ if !a.svc.Fenced(offHost) {
+ t.Fatal("after a restart the pass did not fence again")
+ }
+ if r := wd.report(moves.OfflineRoleSource, "ie-a"); !r.Fenced || !r.At.Equal(wd.clock()) {
+ t.Fatalf("after a restart: %+v", r)
+ }
+
+ // The flip: the record names uk; both source rows stay, held, and keep
+ // reporting; the target mints its row held landing.
+ wd.seal()
+ wd.sealed()
+ wd.flip()
+ wd.advance(time.Minute)
+ for _, bx := range []*offBox{a, b, uk} {
+ bx.tick()
+ }
+ for _, bx := range []*offBox{a, b} {
+ if rt := bx.row(offHost); rt == nil || rt.Status != "moving" || rt.HoldOp != offID {
+ t.Fatalf("%s after the flip: %+v, want the row kept, held", bx.name, rt)
+ }
+ if r := wd.report(moves.OfflineRoleSource, bx.name); !r.Held || !r.Fenced || !r.At.Equal(wd.clock()) {
+ t.Fatalf("%s after the flip: %+v", bx.name, r)
+ }
+ }
+ if rt := uk.row(offHost); rt == nil || rt.Status != "landing" || rt.HoldOp != offID {
+ t.Fatalf("target after the flip: %+v", rt)
+ }
+ if uk.lists(offHost) {
+ t.Fatal("the target lists the landing row")
+ }
+}
+
+// What the report reads from disk: a box with only a meta dir has local
+// state; a process left in the host's cgroup and a layout journal show.
+func TestOfflineSourceReportReadsTheBox(t *testing.T) {
+ wd := newOffWorld(t)
+ a := wd.box("ie-a", "", nil)
+ b := wd.box("ie-b", "", nil)
+ a.tick()
+ wd.start()
+ // ie-b has no DB file, only the activator's meta dir with one LTX file.
+ meta := activatorMetaDir(provision.DefaultDBPath(b.dd, offHost))
+ writeFile(t, filepath.Join(meta, "ltx", "0", "0000000000000001-0000000000000007.ltx"), "x")
+ // ie-a: files, env, a process in the host's exec cgroup, a journal.
+ writeFile(t, filepath.Join(a.dd, offHost, "instance.key"), "k")
+ writeFile(t, filepath.Join(a.dd, offHost+".env"), "CARLOS_INSTANCE_KEY=k\n")
+ writeFile(t, filepath.Join(a.cg, cgroup.ExecName(offHost), "cgroup.procs"), "4242\n")
+ writeFile(t, provision.LayoutJournalPath(a.dd, offHost), "{}")
+ a.tick()
+ b.tick()
+ rb := wd.report(moves.OfflineRoleSource, "ie-b")
+ if !rb.LocalDB || rb.LocalTXID != 7 || rb.Files || rb.Env || rb.Sidecar {
+ t.Fatalf("a meta dir alone: %+v", rb)
+ }
+ ra := wd.report(moves.OfflineRoleSource, "ie-a")
+ if ra.LocalDB || !ra.Files || !ra.Env || ra.Procs != 1 || !ra.LayoutJournal {
+ t.Fatalf("ie-a: %+v", ra)
+ }
+ // A box that cannot count its processes reports -1, never 0.
+ a.w.cfg.Procs = func(string) (int, error) { return 0, cgroup.ErrUnknown }
+ a.pass()
+ if r := wd.report(moves.OfflineRoleSource, "ie-a"); r.Procs != -1 {
+ t.Fatalf("unknown processes reported as %d", r.Procs)
+ }
+ // A sidecar file shows.
+ writeFile(t, filepath.Join(b.dd, offHost+"-sidecar.env"), "X=1\n")
+ b.pass()
+ if r := wd.report(moves.OfflineRoleSource, "ie-b"); !r.Sidecar {
+ t.Fatalf("sidecar not seen: %+v", r)
+ }
+}
+
+// offline-reverting, before and after the flip: every source row is
+// released and served again, the fence is lifted, the boxes report
+// reverted, and the next reconcile does not hold the rows again.
+func TestOfflineRevertReleasesEverySourceRow(t *testing.T) {
+ for _, afterFlip := range []bool{false, true} {
+ t.Run(map[bool]string{false: "before the flip", true: "after the flip"}[afterFlip], func(t *testing.T) {
+ wd := newOffWorld(t)
+ a := wd.box("ie-a", "", nil)
+ b := wd.box("ie-b", "", nil)
+ uk := wd.box("uk-box", "uk", nil)
+ a.tick()
+ b.tick()
+ wd.start()
+ if afterFlip {
+ wd.seal()
+ wd.sealed()
+ wd.flip()
+ }
+ for _, bx := range []*offBox{a, b, uk} {
+ bx.tick()
+ }
+ wd.abort()
+ for i := 0; i < 2; i++ {
+ for _, bx := range []*offBox{a, b, uk} {
+ bx.tick()
+ }
+ }
+ for _, bx := range []*offBox{a, b} {
+ rt := bx.row(offHost)
+ if rt == nil || rt.Status != "ready" || rt.HoldOp != "" {
+ t.Fatalf("%s: %+v, want ready and unheld", bx.name, rt)
+ }
+ if !bx.lists(offHost) {
+ t.Fatalf("%s does not serve the host again", bx.name)
+ }
+ if bx.svc.Fenced(offHost) {
+ t.Fatalf("%s: the fence is still up", bx.name)
+ }
+ if r := wd.report(moves.OfflineRoleSource, bx.name); r.Phase != moves.OfflinePhaseReverted || r.Held {
+ t.Fatalf("%s: %+v", bx.name, r)
+ }
+ }
+ // The move finishes; nothing holds the rows again.
+ wd.move(provision.OfflineFinish(offID, wd.clock()))
+ for _, bx := range []*offBox{a, b} {
+ bx.tick()
+ if rt := bx.row(offHost); rt == nil || rt.Status != "ready" || rt.HoldOp != "" {
+ t.Fatalf("%s after finish: %+v", bx.name, rt)
+ }
+ }
+ })
+ }
+}
+
+// offline-open on a flat source: every path of the source-retire row that
+// is there is renamed into the park directory byte for byte, nothing is
+// deleted, the fence is lifted, and the parked report lists every path with
+// its action. A process left in the host's cgroup stops the park first.
+// offline-finishing then removes the row and the box reports gone.
+func TestOfflineOpenParksTheFlatSource(t *testing.T) {
+ wd := newOffWorld(t)
+ a := wd.box("ie-a", "", nil)
+ uk := wd.box("uk-box", "uk", nil)
+ a.tick()
+ db := a.row(offHost).DBPath
+ contents := map[string]string{
+ db: "sqlite",
+ db + "-wal": "wal",
+ filepath.Join(activatorMetaDir(db), "ltx", "0", "0000000000000001-0000000000000003.ltx"): "ltx",
+ filepath.Join(a.dd, offHost, "instance.key"): "key",
+ filepath.Join(a.dd, offHost+".env"): "CARLOS_INSTANCE_KEY=k\n",
+ filepath.Join(a.dd, offHost+".config.env"): "SECRET=s\n",
+ filepath.Join(a.dd, "programs", offHost+".json"): `{"programs":[]}`,
+ filepath.Join(a.opt, "live", offHost, "app"): "binary",
+ filepath.Join(a.dd, "neighbour.example.com.db"): "not ours",
+ }
+ for p, body := range contents {
+ writeFile(t, p, body)
+ }
+ wd.start()
+ a.tick() // the source reports before the seal, as the seal door requires
+ wd.seal()
+ wd.sealed()
+ wd.flip()
+ wd.landed()
+ a.tick()
+ uk.tick()
+ wd.open()
+
+ // A process still in the host's cgroup: nothing is renamed.
+ procs := filepath.Join(a.cg, cgroup.ExecName(offHost), "cgroup.procs")
+ writeFile(t, procs, "4242\n")
+ a.tick()
+ if _, err := os.Stat(db); err != nil {
+ t.Fatalf("parked with a process left: %v", err)
+ }
+ if _, ok := wd.reportIf(moves.OfflineRoleSource, "ie-a"); !ok {
+ t.Fatal("fixture: no report")
+ }
+ if r := wd.report(moves.OfflineRoleSource, "ie-a"); r.Phase == moves.OfflinePhaseParked {
+ t.Fatalf("reported parked with a process left: %+v", r)
+ }
+ writeFile(t, procs, "")
+
+ a.tick()
+ park := moves.OfflineParkDir(a.dd, offHost, offID, moves.OfflineRoleSource)
+ for p, body := range contents {
+ switch {
+ case strings.HasPrefix(p, a.opt), strings.Contains(p, "neighbour"):
+ if got, err := os.ReadFile(p); err != nil || string(got) != body {
+ t.Fatalf("%s was touched: %q %v", p, got, err)
+ }
+ continue
+ }
+ if _, err := os.Lstat(p); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("%s is still in place: %v", p, err)
+ }
+ }
+ for rel, body := range map[string]string{
+ filepath.Base(db): "sqlite",
+ filepath.Base(db) + "-wal": "wal",
+ filepath.Join(filepath.Base(activatorMetaDir(db)), "ltx", "0", "0000000000000001-0000000000000003.ltx"): "ltx",
+ filepath.Join(offHost, "instance.key"): "key",
+ offHost + ".env": "CARLOS_INSTANCE_KEY=k\n",
+ offHost + ".config.env": "SECRET=s\n",
+ offHost + ".json": `{"programs":[]}`,
+ } {
+ if got, err := os.ReadFile(filepath.Join(park, rel)); err != nil || string(got) != body {
+ t.Fatalf("park/%s = %q %v, want %q", rel, got, err, body)
+ }
+ }
+ r := wd.report(moves.OfflineRoleSource, "ie-a")
+ if r.Phase != moves.OfflinePhaseParked || r.LocalDB || r.Files || r.Env || r.Fenced {
+ t.Fatalf("parked report: %+v", r)
+ }
+ if a.svc.Fenced(offHost) {
+ t.Fatal("the fence is still up after the park")
+ }
+ want := map[string]string{
+ db: moves.RetiredParked, db + "-wal": moves.RetiredParked, db + "-shm": moves.RetiredAbsent,
+ db + "-litestream": moves.RetiredAbsent, activatorMetaDir(db): moves.RetiredParked,
+ filepath.Join(a.dd, offHost): moves.RetiredParked, filepath.Join(a.dd, offHost+".env"): moves.RetiredParked,
+ filepath.Join(a.dd, offHost+".config.env"): moves.RetiredParked, filepath.Join(a.dd, "programs", offHost+".json"): moves.RetiredParked,
+ filepath.Join(a.opt, "live", offHost): moves.RetiredLeft,
+ }
+ assertRetired(t, r.Retired, want)
+
+ // A second pass changes nothing and reports the same inventory.
+ before := r.Retired
+ a.tick()
+ if r := wd.report(moves.OfflineRoleSource, "ie-a"); !reflect.DeepEqual(r.Retired, before) || r.Phase != moves.OfflinePhaseParked {
+ t.Fatalf("second pass: %+v", r.Retired)
+ }
+ // The row stays held through open.
+ if rt := a.row(offHost); rt == nil || rt.HoldOp != offID {
+ t.Fatalf("row at open: %+v", rt)
+ }
+
+ // finishing: keep no longer holds the row; reconcile removes it and
+ // the box reports gone. The parked state stays.
+ // The view the reconcile reads is the last moves pass's, so the row
+ // goes on the pass after the one that first sees finishing.
+ wd.finishing()
+ a.tick()
+ a.tick()
+ if rt := a.row(offHost); rt != nil {
+ t.Fatalf("row at finishing: %+v", rt)
+ }
+ a.pass()
+ if r := wd.report(moves.OfflineRoleSource, "ie-a"); r.Phase != moves.OfflinePhaseGone || r.RowPresent || r.LocalDB {
+ t.Fatalf("gone report: %+v", r)
+ }
+ if got, err := os.ReadFile(filepath.Join(park, filepath.Base(db))); err != nil || string(got) != "sqlite" {
+ t.Fatalf("the parked DB: %q %v", got, err)
+ }
+}
+
+func assertRetired(t *testing.T, got []moves.RetiredPath, want map[string]string) {
+ t.Helper()
+ seen := map[string]string{}
+ for _, rp := range got {
+ if _, dup := seen[rp.Path]; dup {
+ t.Fatalf("%s listed twice", rp.Path)
+ }
+ seen[rp.Path] = rp.Action
+ if rp.Action == moves.RetiredLeft && rp.Reason == "" {
+ t.Fatalf("%s left with no reason", rp.Path)
+ }
+ }
+ for p, action := range want {
+ if seen[p] != action {
+ t.Fatalf("%s: %q, want %q (inventory %+v)", p, seen[p], action, got)
+ }
+ }
+ if len(seen) != len(want) {
+ var extra []string
+ for p := range seen {
+ if _, ok := want[p]; !ok {
+ extra = append(extra, p+"="+seen[p])
+ }
+ }
+ sort.Strings(extra)
+ t.Fatalf("unexpected entries: %v", extra)
+ }
+}
+
+// offline-open on a nested source: the contents of the raw instance
+// directory are renamed into the park directory, the directory and its
+// view stay, empty, and the inventory names each path.
+func TestOfflineOpenParksTheNestedSource(t *testing.T) {
+ wd := newOffWorld(t)
+ a := wd.box("ie-a", "", nil)
+ a.nested = true
+ raw := layout.RawInstanceDir(a.dd, offHost)
+ if err := os.MkdirAll(raw, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ // The bind mount at <dd>/instances, as a link in the fixture.
+ if err := os.Symlink(filepath.Base(filepath.Dir(raw)), filepath.Join(a.dd, "instances")); err != nil {
+ t.Fatal(err)
+ }
+ a.tick()
+ rt := a.row(offHost)
+ if rt == nil || !provision.NestedPaths(a.dd, *rt) {
+ t.Fatalf("fixture: not a nested row: %+v", rt)
+ }
+ contents := map[string]string{
+ offHost + ".db": "sqlite",
+ offHost + ".db-shm": "shm",
+ offHost + ".env": "CARLOS_INSTANCE_KEY=k\n",
+ "instance.key": "key",
+ "uploads/a.png": "png",
+ }
+ for rel, body := range contents {
+ writeFile(t, filepath.Join(raw, rel), body)
+ }
+ wd.start()
+ a.tick()
+ if r := wd.report(moves.OfflineRoleSource, "ie-a"); !r.LocalDB || !r.Files || !r.Env {
+ t.Fatalf("nested report: %+v", r)
+ }
+ wd.seal()
+ wd.sealed()
+ wd.flip()
+ wd.landed()
+ wd.open()
+ a.tick()
+ park := moves.OfflineParkDir(a.dd, offHost, offID, moves.OfflineRoleSource)
+ for rel, body := range contents {
+ if got, err := os.ReadFile(filepath.Join(park, nestedParkPrefix, rel)); err != nil || string(got) != body {
+ t.Fatalf("park/instance/%s = %q %v", rel, got, err)
+ }
+ }
+ ents, err := os.ReadDir(raw)
+ if err != nil || len(ents) != 0 {
+ t.Fatalf("the instance directory: %v %v, want it there and empty", ents, err)
+ }
+ r := wd.report(moves.OfflineRoleSource, "ie-a")
+ if r.Phase != moves.OfflinePhaseParked || r.LocalDB || r.Files || r.Env {
+ t.Fatalf("parked report: %+v", r)
+ }
+ want := map[string]string{
+ filepath.Join(raw, offHost+".db"): moves.RetiredParked,
+ filepath.Join(raw, offHost+".db-wal"): moves.RetiredAbsent,
+ filepath.Join(raw, offHost+".db-shm"): moves.RetiredParked,
+ filepath.Join(raw, offHost+".db-litestream"): moves.RetiredAbsent,
+ filepath.Join(raw, ".activator-"+offHost+".db-litestream"): moves.RetiredAbsent,
+ filepath.Join(raw, offHost+".env"): moves.RetiredParked,
+ filepath.Join(raw, offHost+".config.env"): moves.RetiredAbsent,
+ filepath.Join(raw, "instance.key"): moves.RetiredParked,
+ filepath.Join(raw, "uploads"): moves.RetiredParked,
+ filepath.Join(a.dd, "programs", offHost+".json"): moves.RetiredAbsent,
+ provision.InstanceDir(a.dd, offHost): moves.RetiredLeft,
+ raw: moves.RetiredLeft,
+ filepath.Join(a.opt, "live", offHost): moves.RetiredAbsent,
+ }
+ assertRetired(t, r.Retired, want)
+ // Re-run: the same inventory, from what is parked.
+ a.tick()
+ if again := wd.report(moves.OfflineRoleSource, "ie-a"); !reflect.DeepEqual(again.Retired, r.Retired) {
+ t.Fatalf("second pass:\n %+v\n %+v", again.Retired, r.Retired)
+ }
+}
+
+// --- the dispatcher and the capability --------------------------------------
+
+// An offline record reaches offlineRound with Phase 1 off, ahead of the
+// Phase 1 skip and the lost-box gate: no host claim is read. A
+// Phase-1-shaped record with Phase 1 off is still logged and skipped.
+func TestOfflineDispatchPrecedesThePhase1Skip(t *testing.T) {
+ wd := newOffWorld(t)
+ wd.create("other.example.com")
+ a := wd.box("ie-a", "", nil)
+ a.tick()
+ wd.start()
+ // A Phase 1 move on the other host (its id is what makes it Phase 1).
+ if _, err := provision.SetRecordMove(context.Background(), wd.s, wd.acct, "app", "other.example.com", func(rec *provision.Record) error {
+ rec.Move = &provision.MoveState{ID: offID2, From: "", To: "uk", State: provision.MovePreparing, RequestedAt: wd.clock()}
+ return nil
+ }); err != nil {
+ t.Fatal(err)
+ }
+ a.tick()
+ if _, ok := wd.reportIf(moves.OfflineRoleSource, "ie-a"); !ok {
+ t.Fatal("the offline record never reached offlineRound")
+ }
+ if a.w.lost != nil {
+ t.Fatalf("the lost-box gate ran: %v", a.w.lost)
+ }
+ if _, ok := a.w.notices.Load("other.example.com#switch"); !ok {
+ t.Fatal("the Phase 1 record was not logged as waiting on the switch")
+ }
+ if rt := a.row("other.example.com"); rt == nil || rt.HoldOp != "" || rt.Status != "ready" {
+ t.Fatalf("the Phase 1 record's row was touched: %+v", rt)
+ }
+ keys, err := wd.s.List(context.Background(), "control/offline-moves/other.example.com/")
+ if err != nil || len(keys) != 0 {
+ t.Fatalf("the Phase 1 record was reported as offline: %v %v", keys, err)
+ }
+}
+
+// The capability is the box's own witness: published with hibernation
+// configured and both switches unset, withheld otherwise. A box without it
+// runs no offline step and installs no offline hook: its row is never
+// held, nothing is reported, and one notice is logged. The guard mode is
+// not an input (platform-1e's ruling of 2026-10-20).
+func TestOfflineCapabilityAndAnIncapableBox(t *testing.T) {
+ env := func(kv ...string) func(string) string {
+ m := map[string]string{"CARLOS_HIBERNATE_BUCKET": "data"}
+ for i := 0; i+1 < len(kv); i += 2 {
+ m[kv[i]] = kv[i+1]
+ }
+ return func(k string) string { return m[k] }
+ }
+ if got := edgesFileCapabilities(env()); !slices.Equal(got, []string{capOfflineMove1}) {
+ t.Fatalf("capable box: %v", got)
+ }
+ if got := edgesFileCapabilities(env("CARLOS_REGION_HA", "1")); !slices.Equal(got, []string{capRegionHA1, capOfflineMove1}) {
+ t.Fatalf("region HA too: %v", got)
+ }
+ for _, kv := range [][]string{
+ {"CARLOS_HIBERNATE_BUCKET", ""},
+ {"CARLOS_HOST_CLAIM", "1"},
+ {"CARLOS_HOST_CLAIM", "0"}, // set at all
+ {moves.SwitchEnv, "on"},
+ {moves.SwitchEnv, "off"}, // set at all
+ } {
+ if got := edgesFileCapabilities(env(kv...)); slices.Contains(got, capOfflineMove1) {
+ t.Fatalf("%s=%q: %v", kv[0], kv[1], got)
+ }
+ if offlineCapable(env(kv...), hibernationConfigured(env(kv...))) {
+ t.Fatalf("%s=%q is capable", kv[0], kv[1])
+ }
+ }
+
+ wd := newOffWorld(t)
+ a := wd.box("ie-a", "", map[string]string{"CARLOS_HOST_CLAIM": "1"})
+ if a.w.cfg.OfflineCapable {
+ t.Fatal("armOffline: a claims box is capable")
+ }
+ a.tick()
+ wd.start()
+ a.tick()
+ if rt := a.row(offHost); rt == nil || rt.Status != "ready" || rt.HoldOp != "" {
+ t.Fatalf("an incapable box held the row: %+v", rt)
+ }
+ if a.svc.Fenced(offHost) {
+ t.Fatal("an incapable box fenced")
+ }
+ if _, ok := wd.reportIf(moves.OfflineRoleSource, "ie-a"); ok {
+ t.Fatal("an incapable box reported")
+ }
+ if _, ok := a.w.notices.Load(offHost + "#offline"); !ok {
+ t.Fatal("no notice from an incapable box")
+ }
+}
+
+// --- dormant ----------------------------------------------------------------
+
+// With no offline record anywhere: the hold answers false for every record,
+// keep answers false once the startup listing is read (and true only while
+// none has been, or when it failed), a retired row is removed on the first
+// pass exactly as with no hooks at all, and the edges file differs by the
+// one capability string and nothing else.
+func TestOfflineDormantIsMain(t *testing.T) {
+ ctx := context.Background()
+ wd := newOffWorld(t)
+ wd.create("gone.example.com")
+ // A Phase 1 and a legacy move on records of their own: no offline hook
+ // answers for them either.
+ wd.create("legacy.example.com")
+ wd.moveHost("legacy.example.com", func(rec *provision.Record) error {
+ rec.Move = &provision.MoveState{To: "uk", State: provision.MovePreparing}
+ return nil
+ })
+
+ run := func(withHooks bool) *offBox {
+ b := wd.box(map[bool]string{false: "main", true: "hooked"}[withHooks], "", nil)
+ w := b.w
+ if !withHooks {
+ w = nil
+ }
+ cfg := provision.ReconcileConfig{Region: "", DataDir: b.dd, Layout: func(context.Context, release.Store) (string, error) { return release.LayoutModeFlat, nil }}
+ if err := reconcileSerializedWith(ctx, wd.s, b.reg, cfg, w); err != nil {
+ t.Fatal(err)
+ }
+ return b
+ }
+ plainBox := run(false)
+ hooked := run(true)
+
+ // The startup refresh: keep is false for every host.
+ refreshOfflineView(ctx, wd.s, hooked.w.cfg.Offline)
+ for _, h := range []string{offHost, "gone.example.com", "legacy.example.com", "never.example.com"} {
+ if hooked.w.cfg.Offline.keep(h) {
+ t.Fatalf("keep(%s) after the startup refresh", h)
+ }
+ }
+ recs, err := provision.ListAllRecords(ctx, wd.s)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, rec := range recs {
+ if _, _, ok := offlineHold(hooked.w.cfg)(rec); ok {
+ t.Fatalf("held %s", rec.Host)
+ }
+ }
+
+ // The record of gone.example.com goes; the first pass removes its row
+ // on both boxes alike.
+ if err := wd.s.Delete(ctx, provision.RecordKey(wd.acct, "app", "gone.example.com")); err != nil {
+ t.Fatal(err)
+ }
+ for _, b := range []*offBox{plainBox, hooked} {
+ w := b.w
+ if b == plainBox {
+ w = nil
+ }
+ cfg := provision.ReconcileConfig{Region: "", DataDir: b.dd, Layout: func(context.Context, release.Store) (string, error) { return release.LayoutModeFlat, nil }}
+ if err := reconcileSerializedWith(ctx, wd.s, b.reg, cfg, w); err != nil {
+ t.Fatal(err)
+ }
+ if rt := b.row("gone.example.com"); rt != nil {
+ t.Fatalf("%s kept a retired row: %+v", b.name, rt)
+ }
+ }
+ strip := func(rs []registry.Route) []registry.Route {
+ out := append([]registry.Route(nil), rs...)
+ for i := range out {
+ out[i].DBPath, out[i].Socket, out[i].CreatedAt = "", "", 0
+ }
+ return out
+ }
+ mr, _ := plainBox.reg.List()
+ hr, _ := hooked.reg.List()
+ if !reflect.DeepEqual(strip(mr), strip(hr)) {
+ t.Fatalf("the registries differ:\n main %+v\n hooked %+v", mr, hr)
+ }
+
+ // keep fails closed only with no listing read: never built, or failed.
+ unbuilt := newOfflineView(moveCfg{Region: ""}, nil)
+ if !unbuilt.keep(offHost) {
+ t.Fatal("an unbuilt view keeps nothing")
+ }
+ refreshOfflineView(ctx, failingListStore{Store: wd.s, prefix: "apps/", err: errors.New("listing refused")}, unbuilt)
+ if !unbuilt.built() || !unbuilt.keep(offHost) {
+ t.Fatal("a failed startup listing does not keep")
+ }
+ unbuilt.refresh(recs)
+ if unbuilt.keep(offHost) {
+ t.Fatal("a listing read after a failure still keeps")
+ }
+
+ // The edges file: the one string, and no headroom with it.
+ var last1, last2 []byte
+ plain := edgesFile{Box: "plain", Placement: ""}
+ capable := edgesFile{Box: "plain", Placement: "", Capabilities: edgesFileCapabilities(func(k string) string {
+ if k == "CARLOS_HIBERNATE_BUCKET" {
+ return "data"
+ }
+ return ""
+ })}
+ if headroomCapabilities(capable.Capabilities) != nil {
+ t.Fatal("offline-move-1 asks for headroom")
+ }
+ s1, s2 := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir())
+ if err := edgesFileWrite(ctx, s1, plainBox.reg, plain, "1.2.3.4", &last1, false); err != nil {
+ t.Fatal(err)
+ }
+ if err := edgesFileWrite(ctx, s2, plainBox.reg, capable, "1.2.3.4", &last2, false); err != nil {
+ t.Fatal(err)
+ }
+ b1, _, _ := s1.Get(ctx, edgesFileKey("plain"))
+ b2, _, _ := s2.Get(ctx, edgesFileKey("plain"))
+ var m1, m2 map[string]any
+ if json.Unmarshal(b1, &m1) != nil || json.Unmarshal(b2, &m2) != nil {
+ t.Fatal("edges files do not parse")
+ }
+ if caps, _ := m2["capabilities"].([]any); len(caps) != 1 || caps[0] != capOfflineMove1 {
+ t.Fatalf("capabilities: %v", m2["capabilities"])
+ }
+ delete(m2, "capabilities")
+ delete(m1, "generation")
+ delete(m2, "generation")
+ if !reflect.DeepEqual(m1, m2) {
+ t.Fatalf("the edges file differs beyond the capability:\n %s\n %s", b1, b2)
+ }
+ if bytes.Contains(b1, []byte("capabilities")) {
+ t.Fatal("fixture: the plain file has capabilities")
+ }
+}
+
+// keep's table: the source keeps until finishing, the target until the
+// move is cleared, a third region never.
+func TestOfflineKeep(t *testing.T) {
+ rec := func(region, state string) provision.Record {
+ return provision.Record{Host: offHost, Region: region, Move: &provision.MoveState{Kind: provision.MoveKindOffline, ID: offID, From: "", To: "uk", State: state}}
+ }
+ states := []string{provision.OfflineStopping, provision.OfflineSealing, provision.OfflineSealed, provision.OfflineLanding,
+ provision.OfflineLanded, provision.OfflineOpen, provision.OfflineFinishing, provision.OfflineReverting}
+ for _, st := range states {
+ recs := []provision.Record{rec("uk", st)}
+ src := newOfflineView(moveCfg{Region: ""}, nil)
+ src.refresh(recs)
+ if got, want := src.keep(offHost), st != provision.OfflineFinishing; got != want {
+ t.Fatalf("source at %s: keep %v", st, got)
+ }
+ tgt := newOfflineView(moveCfg{Region: "uk"}, nil)
+ tgt.refresh(recs)
+ if !tgt.keep(offHost) {
+ t.Fatalf("target at %s: not kept", st)
+ }
+ other := newOfflineView(moveCfg{Region: "fr"}, nil)
+ other.refresh(recs)
+ if other.keep(offHost) {
+ t.Fatalf("a third region keeps at %s", st)
+ }
+ }
+}
diff --git a/internal/host/movespass.go b/internal/host/movespass.go
index 5ca6fce15..b28e99d94 100644
--- a/internal/host/movespass.go
+++ b/internal/host/movespass.go
@@ -3,6 +3,7 @@ package host
import (
"context"
"crypto/ecdh"
+ "crypto/ed25519"
"encoding/json"
"errors"
"fmt"
@@ -17,6 +18,7 @@ import (
"github.com/carlosframework/platform/internal/activator"
"github.com/carlosframework/platform/internal/appconfig"
+ "github.com/carlosframework/platform/internal/cgroup"
"github.com/carlosframework/platform/internal/controlsock"
"github.com/carlosframework/platform/internal/dbverify"
"github.com/carlosframework/platform/internal/hostbox"
@@ -174,7 +176,7 @@ func (w *movesWorker) setStrict(recs []provision.Record) {
// rule (park.PlacementMatches under the verified home binding,
// ParkHomeRegion), so the home's roster name and its "" code are one.
func movesNamesBox(cfg moveCfg, rec provision.Record) bool {
- if rec.Move == nil || rec.Move.Legacy() {
+ if rec.Move == nil || rec.Move.Legacy() || rec.Move.Offline() {
return false
}
here := cfg.parkRegion()
@@ -321,7 +323,12 @@ func armMovesPass(hib activator.Config, dataDir string, reg *registry.Registry,
Events: events,
LogEvent: moveLogWriter(events),
}
- w := &movesWorker{mirrors: map[string]lease.MirrorStore{}}
+ // The edges box label and the version store, whatever the switch says:
+ // the offline move reports under the label (movesoffline.go). armPhase1
+ // sets the same two values.
+ cfg.Box = metricsBoxLabel(os.Getenv, readMachineID())
+ cfg.OptDir = release.OptDir()
+ w := newMovesWorker(cfg)
if cfg.Enabled {
armPhase1(&cfg, w, hib, dataDir, reg, svc, eb, writes, os.Getenv)
if writes.Enabled && providers.guard.FenceView != nil {
@@ -338,10 +345,86 @@ func armMovesPass(hib activator.Config, dataDir string, reg *registry.Registry,
cancel()
}
}
+ armOffline(&cfg, dataDir, reg, svc, eb, writes, os.Getenv)
w.cfg = cfg
armedMovesWorker.Store(w)
}
+// armOffline sets every moveCfg field the offline operator move reads
+// (movesoffline.go and Tasks 4 and 5 of docs/superpowers/plans/2026-10-20-
+// offline-operator-move.md), whatever the Phase 1 switch says, after
+// armPhase1 so a hook it already set is kept. Nothing here acts: the fields
+// are read only by offline steps, which run only on a capable box and only
+// for a record with an offline move.
+func armOffline(cfg *moveCfg, dataDir string, reg *registry.Registry, svc *activator.Service, eb *activator.ExecBackend, writes hostwrite.Config, getenv func(string) string) {
+ cfg.OfflineCapable = offlineCapable(getenv, hibernationConfigured(getenv))
+ cfg.Offline = newOfflineView(*cfg, holdStageRequester(dataDir))
+ if svc != nil {
+ cfg.FenceAs = fenceAsWith(svc)
+ cfg.LiftFenceAs = svc.LiftFenceAs
+ }
+ if reg != nil {
+ cfg.Registry = reg
+ // The caller holds provision.WithHostHoldLock; a release that
+ // landed touches the stage-requested marker, so root re-judges the
+ // now-ready row (registry.SetHold's contract, G10).
+ cfg.ReleaseHold = releaseHeld(reg, holdStageRequester(dataDir))
+ // The row and the timers the move landed on it, in one
+ // transaction, only while op holds the row.
+ cfg.DeleteIfHold = reg.DeleteIfHoldWithTimers
+ }
+ if eb != nil {
+ cfg.LaunchLayers = eb.LaunchLayers
+ cgRoot := eb.CgroupRoot
+ if cgRoot == "" {
+ cgRoot = "/sys/fs/cgroup"
+ }
+ execRoot := eb.Cgroups.Root()
+ cfg.Procs = func(host string) (int, error) {
+ return cgroup.HostProcs(execRoot, host, hostSliceCgroup(cgRoot, host))
+ }
+ }
+ if raw := strings.TrimSpace(getenv("CARLOS_SECRETS_KEY")); raw != "" {
+ if priv, err := appconfig.ParsePrivateKey(raw, "CARLOS_SECRETS_KEY"); err == nil {
+ cfg.ConfigPriv, cfg.ConfigPub = priv, priv.PublicKey()
+ if cfg.SecretsKey == "" {
+ cfg.SecretsKey = raw
+ }
+ }
+ }
+ if cfg.RenderHeld == nil {
+ cfg.RenderHeld = heldRenderer(writes, cfg.ConfigPriv, dataDir, cfg.Box)
+ }
+ optDir := cfg.OptDir
+ cfg.LiveVersion = func(host string) (string, error) { return release.LiveVersion(optDir, host) }
+ pub, perr := release.ParsePublicKey(getenv("CARLOS_RELEASE_PUBKEY"), "CARLOS_RELEASE_PUBKEY")
+ cfg.ChannelVersion = func(ctx context.Context, s release.Store, rt registry.Route) (string, error) {
+ if perr != nil {
+ return "", perr
+ }
+ return channelVersion(ctx, s, pub, rt)
+ }
+ if cfg.Now == nil {
+ cfg.Now = time.Now
+ }
+}
+
+// channelVersion is the build the row's channel points at, as adoption
+// resolves it: the account-qualified pointer of the row's account, app and
+// channel ("stable" when the row names none), verified under the release
+// key.
+func channelVersion(ctx context.Context, s release.Store, pub ed25519.PublicKey, rt registry.Route) (string, error) {
+ channel := rt.Channel
+ if channel == "" {
+ channel = "stable"
+ }
+ ptr, err := release.Channel(ctx, s, pub, release.RouteAccount(rt), rt.App, channel)
+ if err != nil {
+ return "", err
+ }
+ return ptr.Version, nil
+}
+
// movesDataDir is the data directory the moves worker is built with,
// resolved once to an absolute, cleaned path (final re-review 1, I2): the
// outboxes under it sync every directory from an entry's own up to it, and
@@ -892,6 +975,55 @@ type moveCfg struct {
// laneParkTarget's, armed as the revert's (revertLaneArmed). abFault is
// its test seam, nil in production.
abFault func(point string) error
+
+ // The offline operator move (movesoffline.go; docs/superpowers/plans/
+ // 2026-10-20-offline-operator-move.md, Tasks 3 to 5). armOffline sets
+ // every field below whatever the Phase 1 switch says, and so Box,
+ // OptDir, SecretsKey, RenderHeld and Now above; tests set them by hand.
+ // The sealer (Task 4) and the target (Task 5) read only these, plus the
+ // hibernate coordinates, DataDir, Region, Mirror, Quiesced, Route,
+ // Checkpoint and CarryRequest above.
+ //
+ // OfflineCapable is offlineCapable for this box: false, and every
+ // offline step logs one notice per host and does nothing. Offline is the
+ // keep view reconcile reads (offlineView.keep); the hold needs none.
+ OfflineCapable bool
+ Offline *offlineView
+ // FenceAs installs, or meets again, an owned activator fence
+ // (activator.Service.FenceAndStopAs under a 2 s deadline,
+ // ErrStopUnproven counted as installed: fenceAsWith); LiftFenceAs lifts
+ // exactly that fence. Nil (no activator): never fenced.
+ FenceAs func(ctx context.Context, host string, generation uint64, owner string) error
+ LiftFenceAs func(host string, generation uint64, owner string) bool
+ // ReleaseHold opens a row held by op (registry.ReleaseHold, touching the
+ // stage-requested marker when it landed); DeleteIfHold deletes a row
+ // held by op, with its timers rows, in one transaction
+ // (registry.DeleteIfHoldWithTimers). Both answer false when op does not
+ // hold the row. The caller holds provision.WithHostHoldLock.
+ ReleaseHold func(host, op string) (bool, error)
+ DeleteIfHold func(host, op string) (bool, error)
+ // Registry is this box's registry: TimersFor, ReplaceTimers,
+ // SetNextDue, SetReplication and Get.
+ Registry *registry.Registry
+ // LaunchLayers is the exec backend's composed launch environment for a
+ // row (activator.ExecBackend.LaunchLayers).
+ LaunchLayers func(rt registry.Route) (env map[string]string, instance map[string]bool, shadowed []string, err error)
+ // ConfigPriv and ConfigPub are the deployment's config keypair, parsed
+ // from CARLOS_SECRETS_KEY: ConfigPub seals the files archive, ConfigPriv
+ // opens it and renders the held config. Nil without the key.
+ ConfigPriv *ecdh.PrivateKey
+ ConfigPub *ecdh.PublicKey
+ // LiveVersion is the build a row's live link names on this box
+ // (release.LiveVersion over OptDir). ChannelVersion is the build the
+ // row's channel points at, as adoption resolves it (channelVersion,
+ // verified under CARLOS_RELEASE_PUBKEY; an error when that key does
+ // not parse).
+ LiveVersion func(host string) (string, error)
+ ChannelVersion func(ctx context.Context, s release.Store, rt registry.Route) (string, error)
+ // Procs counts the processes left for a host in its exec cgroup and its
+ // nested slice (cgroup.HostProcs). Nil, or an error: unknown, which a
+ // source report records as -1, not stopped.
+ Procs func(host string) (int, error)
}
// defaultMoveReadyDelta is the readiness bar when
@@ -966,7 +1098,23 @@ func movesPass(ctx context.Context, s release.Store, w *movesWorker) error {
w.holds.refresh(ctx, s, recs)
}
}
+ // The offline keep view, from the list this pass read (movesoffline.go).
+ if cfg.Offline != nil {
+ cfg.Offline.refresh(recs)
+ }
for _, rec := range recs {
+ // An offline operator move is routed first, ahead of the Phase 1
+ // skip and the lost-box gate below: its id makes it look like a
+ // Phase 1 record, which with the switch off is skipped, and it
+ // never reads a host claim (movesoffline.go).
+ if rec.Move.Offline() {
+ if err := w.offlineRound(ctx, s, rec); err != nil {
+ w.logTrouble(rec.Host, err)
+ continue
+ }
+ w.trouble.Delete(rec.Host)
+ continue
+ }
// A move closed or replaced by another writer is forgotten by the
// move log's memory (movesevents.go). With the switch off the pass
// touches no outbox, index or memory of the move log at all (final
@@ -1134,7 +1282,7 @@ const legacyMoveSentence = "this move was started by an older console and has no
// hasPhase1Into reports whether any Phase 1 move targets region.
func hasPhase1Into(recs []provision.Record, region string) bool {
for _, rec := range recs {
- if rec.Move != nil && !rec.Move.Legacy() && rec.Move.To == region {
+ if rec.Move != nil && !rec.Move.Legacy() && !rec.Move.Offline() && rec.Move.To == region {
return true
}
}