carlos / platform Public

Merge branch 'moves-offline' into moves-offline-t5

claude 4 days ago df3c30b51ebebf982ddfc8d7f15bee6e0ad5d63a parent c1f27caparent 4927cdf

22 files changed, +5105 −86

approved_moves_copy_scan_test.go +25 −1
approved_moves_copy_test.go +58 −1
docs/site/cli.md +14 −0
docs/superpowers/plans/2026-10-06-moves-phase1-copy.md +67 −0
internal/console/apiinstancemove_offline.go +41 −11
internal/console/apiinstancemove_offline_test.go +42 −0
internal/host/movesoffline.go +7 −13
internal/host/movesoffline_seal.go +1159 −14
internal/host/movesoffline_seal_test.go +1405 −0
internal/layout/plan.go +77 −0
internal/layout/plan_test.go +63 −0
internal/lease/leasetest/s3server.go +126 −1
internal/lease/leasetest/s3server_test.go +132 −0
internal/lease/tombstone.go +15 −8
internal/lease/tombstone_test.go +29 −22
internal/moves/archive/archive.go +31 −5
internal/moves/archive/archive_test.go +131 −0
internal/moves/archive/archive_unix_test.go +21 −0
internal/moves/archive/limits.go +6 −1
move.go +726 −9
move_test.go +712 −0
offline_move_e2e_test.go +218 −0

diff --git a/approved_moves_copy_scan_test.go b/approved_moves_copy_scan_test.go

index 1de9a25b0..2de558d4d 100644

--- a/approved_moves_copy_scan_test.go

+++ b/approved_moves_copy_scan_test.go

@@ -373,6 +373,24 @@ type movesUniverse struct {

paramSinks map[string]bool // functions whose parameter reaches a sink

files []string

steps int

+ // diagSkipped counts, per movesDiagnosticWrites entry, the writes the

+ // universe left out of the field's writes.

+ diagSkipped map[string]int

+// callsError reports whether e calls an Error method: the write carries an

+// error's text. Only such writes can be listed in movesDiagnosticWrites.

+func callsError(e ast.Expr) bool {

+ found := false

+ ast.Inspect(e, func(n ast.Node) bool {

+ if c, ok := n.(*ast.CallExpr); ok {

+ if sel, ok := c.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "Error" {

+ found = true

+ }

+ return !found

+ })

+ return found

}

func (u *movesUniverse) at(p string) string {

@@ -386,7 +404,7 @@ func newUniverse(t *testing.T, root string, files []string) *movesUniverse {

t.Helper()

u := &movesUniverse{t: t, root: root, fieldWrites: map[string][]*fieldWrite{}, fieldTypes: map[string]map[string]bool{},

structFields: map[string][]string{}, fieldTypeOf: map[string]map[string]ast.Expr{}, results: map[string]*fieldResult{}, fieldTexts: map[string]string{}, used: map[string]bool{},

- paramSinks: map[string]bool{}, files: files}

+ paramSinks: map[string]bool{}, files: files, diagSkipped: map[string]int{}}

parsed := map[string]*ast.File{}

for _, f := range files {

af, err := parser.ParseFile(movesFset, u.at(f), nil, 0)

@@ -468,6 +486,12 @@ func newUniverse(t *testing.T, root string, files []string) *movesUniverse {

for i, el := range n.Elts {

if kv, ok := el.(*ast.KeyValueExpr); ok {

if k, ok := kv.Key.(*ast.Ident); ok {

+ if fd != nil {

+ if key := fd.Name.Name + "|" + ctype + "." + k.Name; movesDiagnosticWrites[f][key].Why != "" && callsError(kv.Value) {

+ u.diagSkipped[f+"|"+key]++

+ continue

+ }

u.fieldWrites[k.Name] = append(u.fieldWrites[k.Name], &fieldWrite{file: f, fd: fd, expr: kv.Value, ctype: ctype})

}

} else if i < len(order) {

diff --git a/approved_moves_copy_test.go b/approved_moves_copy_test.go

index 4716b4dfe..ab3f90371 100644

--- a/approved_moves_copy_test.go

+++ b/approved_moves_copy_test.go

@@ -96,7 +96,10 @@ var movesCopyIDs = []string{

"MV-O20", "MV-O21", "MV-O22", "MV-O23", "MV-O24", "MV-O25", "MV-O26", "MV-O27", "MV-O28", "MV-O29", "MV-O30", "MV-O31",

"MV-O32", "MV-O33",

"MV-O34", "MV-O35", "MV-O36", "MV-O37", "MV-O38", "MV-O39", "MV-O40", "MV-O41", "MV-O42", "MV-O43",

- "MV-O50", "MV-O51",

+ "MV-O50", "MV-O51", "MV-O52", "MV-O53", "MV-O54", "MV-O55", "MV-O56", "MV-O57", "MV-O58", "MV-O59", "MV-O60", "MV-O61", "MV-O62", "MV-O63", "MV-O64", "MV-O65",

+ "MV-O66", "MV-O67", "MV-O68", "MV-O69", "MV-O70", "MV-O71", "MV-O72", "MV-O73", "MV-O74", "MV-O75", "MV-O76", "MV-O77", "MV-O78", "MV-O79", "MV-O80", "MV-O81",

+ "MV-O82", "MV-O83", "MV-O84", "MV-O85", "MV-O86", "MV-O87", "MV-O88", "MV-O89", "MV-O90", "MV-O91", "MV-O92", "MV-O93", "MV-O94", "MV-O95", "MV-O96", "MV-O97",

+ "MV-O98",

}

// copyRow is one row of the record. The File cell is the file the sentence is

@@ -1261,6 +1264,8 @@ var movesScanFiles = []string{

"internal/host/movesparks.go",

"internal/host/movespaging.go",

"internal/host/movespass.go",

+ "internal/host/movesoffline.go",

+ "internal/host/movesoffline_seal.go",

"internal/host/movesphase1.go",

"internal/host/movespolicy.go",

"internal/host/movesprobe.go",

@@ -1446,6 +1451,9 @@ var movesReturnedSentences = map[string]map[string]bool{

// The box's move pages (Plan 6 Task 5, MV-N385 to MV-N390) and M-C20,

// the operators' sentence for a stray process (the stray page's body).

"internal/host/movespaging.go": {"rolledBackPage": true, "strayPage": true, "strayLine": true, "startRefusedPage": true},

+ // The offline sealer's lost-stop page, its two causes and the lease

+ // states the first names (offline move Task 4, MV-O66 to MV-O75).

+ "internal/host/movesoffline_seal.go": {"offlineLostStopPage": true, "offlineLeaseCause": true, "offlinePastSealCause": true, "offlineLeaseText": true},

// The console's move pass pages (Plan 5 Task 7, MV-N325 to MV-N334), and

// the dirty fenced target's page (moves after claim Plan 12, Task 5,

// MV-N450 and MV-N451).

@@ -1524,6 +1532,39 @@ var movesSinkAllow = map[string]map[string]string{

}

+// movesDiagnosticWrites are field writes the scan leaves out of a field's

+// writes because what they write is an OS diagnostic for the operator's

+// eye, not copy. Paul's rule (2026-10-16): internal error fragments and

+// diagnostics stay out of copy review. The key is "<function>|<struct>.<field>".

+// An entry leaves out only the composite-literal writes of that field in that

+// function of that file whose value calls Error(), and says how many there are

+// (Count), so the field's other writes in the same function stay scanned, and

+// so do its other writers (the Phase 1 Reason reads in move.go among them).

+// TestMovesDiagnosticWritesAreReal holds each entry to exactly that many.

+type movesDiagnosticWrite struct {

+ Why string

+ Count int

+var movesDiagnosticWrites = map[string]map[string]movesDiagnosticWrite{

+ "internal/host/movesoffline.go": {

+ "retire|RetiredPath.Reason": {

+ Why: "the parked report's reason for a path left in place ends in the OS error of the rename (leftCannot or leftNotYet plus err.Error()), a box diagnostic that `carlos move offline status` shows as MV-O58's <reason>; it is not copy",

+ Count: 2,

+ },

+ "internal/host/movesoffline_seal.go": {

+ "offlineTakeSeal|sealRefusal.Reason": {

+ Why: "the env-pin refusal for a CARLOS_SECRETS_KEY that does not decode is moves.ReportKeyFromSecrets' own error text (err.Error()), a box diagnostic that `carlos move offline status` shows as MV-O55's <detail>; it is not copy",

+ Count: 1,

+ },

+ "offlineArchive|sealRefusal.Reason": {

+ Why: "the files refusal for an archive Build refuses is archive.Build's own error text (err.Error()), whose sentinels are already rows MV-N77 to MV-N81 and MV-N86; a box diagnostic that `carlos move offline status` shows as MV-O55's <detail>",

+ Count: 1,

+ },

// movesServerText names, per file, the type that carries text the console

// wrote (fix round 2 of Plan 6 Task 1a, ruling 1). It is made in one function

// that reads an HTTP response body, and said through one sink; the gate

@@ -2471,3 +2512,19 @@ func TestMovesServerTextRefusesForgedValues(t *testing.T) {

}

+// Every movesDiagnosticWrites entry leaves out exactly the writes it counts,

+// and the reason it gives is a sentence.

+func TestMovesDiagnosticWritesAreReal(t *testing.T) {

+ u := newUniverse(t, ".", movesScanFiles)

+ for f, keys := range movesDiagnosticWrites {

+ for key, w := range keys {

+ if len(w.Why) < 40 {

+ t.Errorf("%s %s: the reason is too short to be one", f, key)

+ }

+ if got := u.diagSkipped[f+"|"+key]; got != w.Count {

+ t.Errorf("%s %s leaves out %d writes, want exactly %d", f, key, got, w.Count)

+ }

diff --git a/docs/site/cli.md b/docs/site/cli.md

index 22601a4d6..194ae0780 100644

--- a/docs/site/cli.md

+++ b/docs/site/cli.md

@@ -2347,6 +2347,20 @@ to confirm.

Operators who are members of the instance's account.

+`carlos move offline` moves one instance with planned downtime, one step at a time. It is for operators, works with host claims and Phase 1 moves off, and never acts on its own: each verb does one step and says which verb comes next.

+```sh

+carlos move offline start --host <host> --to <region>

+carlos move offline seal --host <host>

+carlos move offline flip --host <host>

+carlos move offline open --host <host>

+carlos move offline finish --host <host>

+carlos move offline abort --host <host>

+carlos move offline status --host <host>

+```

+The instance is offline from `start` until `open`. Until `open`, `abort` puts it back where it was with nothing lost. After `open`, moving it back is a new move.

## On the host

These are what systemd starts. You will rarely type them, but knowing what

diff --git a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md

index 6715204bc..f6db016aa 100644

--- a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md

+++ b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md

@@ -542,3 +542,70 @@ Two more refusals of the console doors (Task 6), for cases MV-O20 to MV-O31 do n

|---|---|---|---|

+Seven more lines of `carlos move offline` (`move.go`, Task 7), for places MV-O1 to MV-O10 do not reach: what `abort` says once every box has put the instance back (the plan's runbook has the operator run `finish` next, and no row said so), the lines `status` prints for each box and for each manifest item (`<status>` is ok, waiting, failed or unchecked; `<detail>` and `<phase>` are box diagnostics, not copy; an item with no detail uses the shorter line), the usage line, and the line `status` prints for a path a source box could not park and left where it was (MV-O58; `<reason>` is the box's own reason, not copy). A box whose report is not current says so: `flip` and `open` refuse on a stale report, so `status` has to show it. MV-O56 follows MV-O57 in the table only by number.

+|---|---|---|---|

+The source box's reasons for a path it left in place (`internal/host/movesoffline.go`, `retire`), which `carlos move offline status` shows as MV-O58's `<reason>`. They are human text the box writes into its parked report. The two that end in the box's own OS error (`<error>`, a diagnostic, not copy) are recorded for their fixed beginning.

+|---|---|---|---|

+The sealer's lost-stop page (`internal/host/movesoffline_seal.go`, Task 4 fix round 1, review I2 and M2). After the seal the stop is kept, never taken again; when it no longer holds, the sealer pages an operator once per move. The body splits by state: before `open`, `abort` puts the instance back with nothing lost; from `open` on, `abort` is refused (MV-O40) and the move is fixed forward by hand. `<state>` is the record's offline state (`offline-sealed`, `offline-open` and so on). `<cause>` is one of the two causes below, and the failed item's reason (MV-O71) carries the same cause. `<lease>` is one of the four lease states MV-O72 to MV-O75; `<owner>` is a lease owner (`moved:<id>` or a box's lease id) and `<time>` an RFC 3339 time.

+|---|---|---|---|

+| MV-O67 | internal/host/movesoffline_seal.go | The offline move <id> of <host> is <state>, and its stop no longer holds: <cause>. The move cannot go on. Read carlos move offline status --host <host> and abort the move. | pending |

+| MV-O68 | internal/host/movesoffline_seal.go | The offline move <id> of <host> is <state>, and its stop no longer holds: <cause>. It cannot be aborted once it is open. Read carlos move offline status --host <host> and follow the offline move runbook. | pending |

+The sealer's item reasons, which `carlos move offline status` shows as MV-O55's `<detail>`. The copy gate reads every Check reason, so each has a row. MV-O76 is the `files` refusal for an entry the app keeps beside its files directory (review I3): the move cannot carry it, and moving it into the files directory or removing it would lose the app's data, so the row offers neither and points to the runbook; `<paths>` are the entries' paths on the box. MV-O77 to MV-O84 are the other refusals. Two refusals carry an error's own text and are box diagnostics, not rows (`movesDiagnosticWrites`): the archive's own refusal (its sentinels are MV-N77 to MV-N81 and MV-N86) and a secrets key that does not decode. MV-O85 to MV-O98 are the reasons of the items that passed; `<rung>` is the replication rung, quoted.

+|---|---|---|---|

diff --git a/internal/console/apiinstancemove_offline.go b/internal/console/apiinstancemove_offline.go

index 033ceb635..4d0813f9a 100644

--- a/internal/console/apiinstancemove_offline.go

+++ b/internal/console/apiinstancemove_offline.go

@@ -546,27 +546,57 @@ func (s *Server) handleAPIInstanceOfflineStart(w http.ResponseWriter, r *http.Re

writeRefusal(w, ref)

return

}

- got, ok := s.offlineWrite(w, r, d, start)

- if !ok {

- s.offlineReleaseLock(ctx, host, id)

+ // The write is bounded well under offlineLockGrace, so the lock cannot be

+ // judged stale while it is still in flight.

+ wctx, cancel := context.WithTimeout(ctx, offlineStartWrite)

+ defer cancel()

+ got, err := s.offlineApply(wctx, d, start)

+ if err != nil {

+ // A failed write does not prove nothing was written (a commit whose

+ // reply was lost). The lock is never released here: if the record

+ // carries this move the start succeeded, and if it does not the

+ // lock goes stale and frees itself.

+ rctx, rcancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)

+ defer rcancel()

+ if cur, rerr := provision.GetRecord(rctx, s.cfg.Releases, d.acct.Sqid, d.app, host); rerr == nil && cur.Move.Offline() && cur.Move.ID == id {

+ got, err = cur, nil

+ }

+ if err != nil {

+ s.offlineWriteFailed(w, d, err)

return

}

+ s.bumpSerial(ctx)

writeJSON(w, http.StatusAccepted, s.offlineViewOf(ctx, got))

}

+// offlineStartWrite bounds the start door's record write.

+const offlineStartWrite = 30 * time.Second

+// offlineApply is one transition through the record writer.

+func (s *Server) offlineApply(ctx context.Context, d offlineDoor, mut func(*provision.Record) error) (*provision.Record, error) {

+ return provision.SetRecordMove(ctx, s.cfg.Releases, d.acct.Sqid, d.app, d.host, mut)

+// offlineWriteFailed answers a transition's failure: a refusal with its

+// recorded sentence, anything else as unwritten.

+func (s *Server) offlineWriteFailed(w http.ResponseWriter, d offlineDoor, err error) {

+ if errors.Is(err, provision.ErrMoveConflict) {

+ jsonError(w, http.StatusConflict, offlineConflict(err))

+ return

+ }

+ log.Printf("console: offline move %s/%s/%s: %v", d.acct.Sqid, d.app, d.host, err)

+ jsonError(w, http.StatusBadGateway, "could not write the move. try again")

// offlineWrite applies one transition and bumps the serial, or answers the

// refusal itself. When the move it leaves is cleared (finish, or the finish

// that ends an abort) it releases the lock.

func (s *Server) offlineWrite(w http.ResponseWriter, r *http.Request, d offlineDoor, mut func(*provision.Record) error) (*provision.Record, bool) {

ctx := r.Context()

- rec, err := provision.SetRecordMove(ctx, s.cfg.Releases, d.acct.Sqid, d.app, d.host, mut)

- if errors.Is(err, provision.ErrMoveConflict) {

- jsonError(w, http.StatusConflict, offlineConflict(err))

- return nil, false

- }

+ rec, err := s.offlineApply(ctx, d, mut)

if err != nil {

- log.Printf("console: offline move %s/%s/%s: %v", d.acct.Sqid, d.app, d.host, err)

- jsonError(w, http.StatusBadGateway, "could not write the move. try again")

+ s.offlineWriteFailed(w, d, err)

return nil, false

}

s.bumpSerial(ctx)

@@ -795,7 +825,7 @@ func (s *Server) offlineSealCheck(f offlineFleet, ev offlineEvidence, mv *provis

// byte for byte. The seal is written once, so the hash holds for the whole

// move whatever the sealer's living report does each pass.

func offlineSealHolds(ev offlineEvidence, mv *provision.MoveState, host string) bool {

- return ev.seal != nil && mv.Source != nil &&

+ return ev.seal != nil && ev.seal.Box == mv.Sealer && mv.Source != nil &&

offlineEvidenceMatches(mv.Source.Evidence, moves.OfflineSealKey(host, mv.ID), ev.seal.raw)

}

diff --git a/internal/console/apiinstancemove_offline_test.go b/internal/console/apiinstancemove_offline_test.go

index 7e453142f..6a991ff82 100644

--- a/internal/console/apiinstancemove_offline_test.go

+++ b/internal/console/apiinstancemove_offline_test.go

@@ -784,6 +784,10 @@ func TestOfflineFlipDoor(t *testing.T) {

bad = w.putSeal("ie-box", func(r *moves.OfflineReport) { r.TombstoneOwner = moves.TombstoneOwner("mv_other") })

w.rehash(bad)

w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO25(provision.OfflineSealed))

+ // A seal written by a box other than the recorded sealer.

+ bad = w.putSeal("ie-x", nil)

+ w.rehash(bad)

+ w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO25(provision.OfflineSealed))

offlinePut(t, w.fx.rel, sealKey, seal)

w.rehash(seal)

// A process left, a layout journal, a stale report.

@@ -1383,3 +1387,41 @@ func TestOfflineStartIgnoresOtherRecords(t *testing.T) {

offlinePut(t, w.fx.rel, provision.RecordKey("bab", "titogo", "junk.go.tito.io"), []byte("{not a record"))

w.start()

}

+// A failed record write does not prove nothing was written, so start never

+// releases its lock on one. A write that lands and then reports an error is a

+// start that succeeded; a write that does not land leaves the lock, which

+// goes stale by itself. Either way no other host starts meanwhile.

+func TestOfflineStartKeepsTheLockWhenTheRecordWriteFails(t *testing.T) {

+ for _, landed := range []bool{true, false} {

+ t.Run(fmt.Sprintf("landed=%v", landed), func(t *testing.T) {

+ w := newOfflineWorld(t, oneEach...)

+ if c, _, out := rawReq(t, http.MethodPost, instancesURL(w.fx, "bab", "titogo"), w.token, `{"host":"beta.go.tito.io","region":"ie"}`); c != http.StatusCreated {

+ t.Fatalf("create = %d %s", c, out)

+ }

+ w.fx.srv.cfg.Releases = &recordWriteFault{Store: w.fx.rel, key: provision.RecordKey("bab", "titogo", offlineHost), land: landed}

+ code, out := w.req(http.MethodPost, "", `{"region":"uk"}`)

+ if landed {

+ // The record carries the move: the start succeeded.

+ if code != http.StatusAccepted {

+ t.Fatalf("start = %d %s", code, out)

+ }

+ if v := w.view(code, out); v.State != provision.OfflineStopping || v.ID == "" {

+ t.Fatalf("view = %+v", v)

+ }

+ } else if code != http.StatusBadGateway || errorText(out) != "could not write the move. try again" {

+ t.Fatalf("start = %d %s", code, out)

+ }

+ a, ok := w.lock()

+ if !ok || a.Host != offlineHost {

+ t.Fatalf("lock = %+v %v, want it kept", a, ok)

+ }

+ // A second host's start is refused whichever way the write went.

+ bu := instancesURL(w.fx, "bab", "titogo") + "/beta.go.tito.io/move/offline"

+ c, _, o := rawReq(t, http.MethodPost, bu, w.token, `{"region":"uk"}`)

+ if c != http.StatusConflict || errorText(o) != offlineHost+" is moving offline. move one instance at a time" {

+ t.Fatalf("beta start = %d %s", c, o)

+ }

+ })

+ }

diff --git a/internal/host/movesoffline.go b/internal/host/movesoffline.go

index 9b1832ef6..cb1019574 100644

--- a/internal/host/movesoffline.go

+++ b/internal/host/movesoffline.go

@@ -499,17 +499,12 @@ func isDir(p string) bool {

return err == nil && st.IsDir()

}

-// offlinePlatformNames are a nested instance directory's platform entries:

-// everything else in it is the app's files.

-func offlinePlatformNames(host string) map[string]bool {

- db := host + ".db"

- return map[string]bool{

- db: true, db + "-wal": true, db + "-shm": true, db + "-litestream": true,

- ".activator-" + db + "-litestream": true, db + ".corrupt": true, db + ".gen": true,

- host + ".env": true, host + ".config.env": true, host + "-sidecar.env": true,

- host + ".instance-key-adoption.json": true, ".files-gen": true,

- }

+// offlinePlatformName reports a nested instance directory's platform entry

+// (layout.IsNestedPlatformName: the database and its files, the env files,

+// the generation, seed and files markers, the adoption record, and the

+// launch descriptors, launch.gen.<n>.json included): everything else in it

+// is the app's files.

+func offlinePlatformName(host, name string) bool { return layout.IsNestedPlatformName(host, name) }

// localState reads what this box keeps for host: the DB file or either

// litestream meta dir (S8: a meta dir alone is local state, since

@@ -543,9 +538,8 @@ func (w *movesWorker) localState(host string, rt *registry.Route) (offlineLocal,

if err != nil && !errors.Is(err, os.ErrNotExist) {

return out, err

}

- platform := offlinePlatformNames(host)

for _, e := range ents {

- if !platform[e.Name()] {

+ if !offlinePlatformName(host, e.Name()) {

out.Files = true

break

}

diff --git a/internal/host/movesoffline_seal.go b/internal/host/movesoffline_seal.go

index e13489c57..cbfb60cde 100644

--- a/internal/host/movesoffline_seal.go

+++ b/internal/host/movesoffline_seal.go

@@ -2,27 +2,1172 @@ package host

import (

"context"

+ "crypto/sha256"

+ "encoding/hex"

+ "encoding/json"

+ "errors"

+ "fmt"

+ "io"

+ "log"

+ "os"

+ "path/filepath"

+ "sort"

+ "strings"

+ "sync"

+ "time"

+ "github.com/superfly/ltx"

+ "github.com/carlosframework/platform/internal/activator"

+ "github.com/carlosframework/platform/internal/alerting"

+ "github.com/carlosframework/platform/internal/appconfig"

+ "github.com/carlosframework/platform/internal/instancekey"

+ "github.com/carlosframework/platform/internal/layout"

+ "github.com/carlosframework/platform/internal/lease"

+ "github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/moves/archive"

+ "github.com/carlosframework/platform/internal/park"

"github.com/carlosframework/platform/internal/provision"

+ "github.com/carlosframework/platform/internal/registry"

"github.com/carlosframework/platform/internal/release"

)

-// offlineSealerStep is the sealer's part of an offline move

-// (docs/superpowers/plans/2026-10-20-offline-operator-move.md, Task 4, which

-// owns this file). offlineRound calls it on the box whose moveCfg.Box equals

-// rec.Move.Sealer, in every offline state, after this box's source step has

-// written its report for the pass. rec carries an offline move

-// (rec.Move.Offline()) and the box is capable (moveCfg.OfflineCapable).

-// Every step must be an ensure; an error is logged by the pass, throttled

-// per host, and the next pass runs the step again.

+// The sealer of an offline operator move (docs/superpowers/plans/2026-10-20-

+// offline-operator-move.md, Task 4): the one source box that turns "every

+// source box holds, fences and reports" into a stop at the data layer, and

+// records what the target must land.

+//

+// The tombstone is the linearization point of the whole move. Every wake on

+// every box of the source region takes the host's single-writer lease

+// (instances/<host>/lock.json in the source's own bucket) before it can

+// write anything, and from the tombstone on that lease is held by

+// moved:<id>. The order is the plan's (Review Focus 2):

-// The immutable seal record (moves.OfflineSealKey(host, id), added on

-// Task 6's branch) is written once through putOfflineCreateOnly, never

-// rewritten: on a re-run it returns the stored body with created false, and

-// the step compares instead of overwriting. The per-pass sealer report goes

-// through writeOfflineReport, which replaces.

+// 1. this box's own report for the pass says held, fenced, quiesced, no

+// process and no layout journal, and no source box keeps a sidecar;

+// 2. the local database is checkpointed unless it is behind the replica

+// (cfg.Checkpoint takes the lease, syncs and releases it);

+// 3. lease.Tombstone, which never replaces a live holder;

+// 4. only then the final position is read from the replica, and every

+// source box's local position must be at or below it;

+// 5. CarryRequest, a final MirrorPrefix round with nothing deferred, and

+// the staged listing;

+// 6. to 9. the files archive, the timers, the env and its pin, the release,

+// replication and next due;

+// 10. seal.json, written once and never rewritten (putOfflineCreateOnly),

+// then the record's CAS to offline-sealed.

-// Task 3 leaves it a no-op.

+// Every step is an ensure. Until seal.json exists the whole seal is taken

+// again on each pass (a tombstone that is already this move's is a no-op,

+// the mirror resumes from its receipts, the archive is rebuilt and its

+// objects overwritten), so a crash anywhere before it converges. Once

+// seal.json exists the sealed position is fixed, and from then on the

+// tombstone is never taken again: each pass reads lease.LockState first,

+// and a tombstone that is absent, expired or anyone else's is a lost stop,

+// as is a source report written since the seal at a position past the

+// sealed one. The pass reports failed and pages: before open the operator

+// aborts, from open on the runbook fixes it forward. Only a tombstone held

+// by this move is re-asserted (lease.Reassert, If-Match on the ETag it

+// reads, and lease.ErrTombstoneLost, never a new tombstone, when it went

+// in between). After it is lifted (finish, abort) it is never re-asserted.

+// The objects the sealer writes beside the sealed archive, under

+// moves.OfflineArchivePrefix(host, id) in the target's data bucket. The

+// landing (Task 5) reads them there.

+const (

+ // offlineArchiveSealed is the sealed files archive (archive.Seal of

+ // archive.Build's output). Its sha256 is the seal's FilesSealed.

+ offlineArchiveSealed = "files.sealed"

+ // offlineArchiveManifest is its archive.Manifest (JSON), with the

+ // sealed data key and every entry.

+ offlineArchiveManifest = "files.json"

+ // offlineTimersObject is the moves.TimersSnapshot of the row's active

+ // timers. moves.TimersDigest of its rows is the seal's TimersDigest.

+ offlineTimersObject = "timers.json"

+ // offlineEnvReportObject is the source's moves.EnvReport (names and

+ // HMACs, never a value), so the landing can name the variables that

+ // differ (moves.CompareEnv) when the env pin does not match. Its

+ // Digest is the seal's EnvDigest.

+ offlineEnvReportObject = "env-report.json"

+// offlineSealFresh is how old this box's own source report may be when the

+// sealer reads it: the source step writes it earlier in the same pass, so

+// anything older means that step failed this pass.

+const offlineSealFresh = 2 * time.Minute

+// offlinePageLostStop is the page rule of a lost stop: one page per move.

+const offlinePageLostStop = "offline-lost-stop"

+// offlineSealFaults is the sealer's test seam: a worker's entry is called

+// after each step with the step's name, and an error returned is a crash

+// there. Keyed by the worker, so tests that run side by side never share

+// one. Empty in production.

+var offlineSealFaults sync.Map // *movesWorker → func(point string) error

+func (w *movesWorker) sealFault(point string) error {

+ if f, ok := offlineSealFaults.Load(w); ok {

+ return f.(func(string) error)(point)

+ }

+ return nil

+// offlineSealerStep is the sealer's part of an offline move. offlineRound

+// calls it on the box whose label is rec.Move.Sealer, after this box's own

+// source step has written its report for the pass.

func (w *movesWorker) offlineSealerStep(ctx context.Context, s release.Store, rec provision.Record) error {

+ m := rec.Move

+ if !m.Offline() || w.cfg.Box == "" || m.Sealer != w.cfg.Box {

+ return nil

+ }

+ switch m.State {

+ case provision.OfflineSealing:

+ return w.offlineSeal(ctx, s, rec)

+ case provision.OfflineSealed, provision.OfflineLanding, provision.OfflineLanded, provision.OfflineOpen:

+ raw, err := w.offlineSealRecord(ctx, s, rec)

+ if err != nil {

+ return err

+ }

+ return w.offlineKeepStop(ctx, s, rec, raw)

+ case provision.OfflineFinishing:

+ return w.offlineFinishLift(ctx, s, rec)

+ case provision.OfflineReverting:

+ return w.offlineSealRevert(ctx, s, rec)

+ }

+ return nil

+// tombstoneAt is where the host's single-writer lease lives in this box's

+// own bucket: the activator's lease location, as ExecBackend.Acquire takes

+// it for every wake.

+func (c moveCfg) tombstoneAt(host string) (bucket, location, region string, err error) {

+ if c.OwnBucket == "" {

+ return "", "", "", errors.New("offline move: this box has no hibernate bucket")

+ }

+ return c.OwnBucket, activator.HibernatePrefix(c.OwnPrefix, host), c.OwnBucketRegion, nil

+// tombState is the lease as lease.LockState read it: an absent lock has no

+// owner and a zero expiry.

+type tombState struct {

+ owner string

+ expires time.Time

+// mine reports a tombstone this move holds now. "Expired" is the lease's

+// own reading (Acquire takes a lease the moment its expiry has passed), so

+// it is judged against the wall clock the lease is written by.

+func (t tombState) mine(id string) bool {

+ return t.owner == moves.TombstoneOwner(id) && time.Now().Before(t.expires)

+// free reports a lease nothing holds: absent, or past its expiry, whoever

+// wrote it (a lifted tombstone included: lease.IsLiftedTombstone).

+func (t tombState) free() bool {

+ return t.owner == "" || !time.Now().Before(t.expires)

+func (t tombState) String() string { return offlineLeaseText(t) }

+// offlineLeaseText is the lease as the lost stop's page and the sealer's

+// report name it (copy rows MV-O72 to MV-O75).

+func offlineLeaseText(t tombState) string {

+ switch {

+ case t.owner == "":

+ return "absent"

+ case lease.IsLiftedTombstone(t.owner, t.expires) && t.expires.Unix() == 0:

+ // LiftTombstone's expiry: the epoch.

+ return fmt.Sprintf("lifted (%s)", t.owner)

+ case t.free():

+ return fmt.Sprintf("expired (%s at %s)", t.owner, t.expires.UTC().Format(time.RFC3339))

+ }

+ return fmt.Sprintf("held by %s until %s", t.owner, t.expires.UTC().Format(time.RFC3339))

+// offlineLock reads the host's lease.

+func (w *movesWorker) offlineLock(ctx context.Context, host string) (tombState, error) {

+ bucket, loc, region, err := w.cfg.tombstoneAt(host)

+ if err != nil {

+ return tombState{}, err

+ }

+ _, owner, expires, err := lease.LockState(ctx, bucket, loc, region)

+ if err != nil {

+ return tombState{}, fmt.Errorf("read the lease of %s: %w", host, err)

+ }

+ return tombState{owner: owner, expires: expires}, nil

+// offlineTarget is the target's coordinates as its edges file publishes

+// them: the box at move.to (the lexically first, as every move reads the

+// fleet), matched exactly and then by placement under the home binding.

+func (w *movesWorker) offlineTarget(ctx context.Context, s release.Store, m *provision.MoveState) (edgesFile, error) {

+ fleet, err := loadMoveFleet(ctx, s)

+ if err != nil {

+ return edgesFile{}, fmt.Errorf("read the fleet's edges files: %w", err)

+ }

+ if t, ok := fleet.targets[m.To]; ok {

+ return t, nil

+ }

+ codes := make([]string, 0, len(fleet.targets))

+ for code := range fleet.targets {

+ codes = append(codes, code)

+ }

+ sort.Strings(codes)

+ for _, code := range codes {

+ if park.PlacementMatches(w.cfg.ParkHomeRegion, m.To, park.PlacementRegion(code)) {

+ return fleet.targets[code], nil

+ }

+ return edgesFile{}, fmt.Errorf("the target region %q publishes no data bucket yet", m.To)

+// offlineMirror is the mirror pair between this box's bucket and region.

+func (w *movesWorker) offlineMirror(ctx context.Context, region string) (lease.MirrorStore, error) {

+ if w.cfg.Mirror == nil {

+ return nil, errors.New("offline move: no S3 mirror on this box")

+ }

+ return w.mirror(ctx, region)

+// replicaPosition is the newest TXID of the replica listed in objs under

+// prefix (newestLTX), 0 when it holds no LTX file.

+func replicaPosition(objs map[string]lease.MirrorObject, prefix string) uint64 {

+ newest := newestLTX(objs, prefix, nil)

+ if newest == "" {

+ return 0

+ }

+ _, maxTX, err := ltx.ParseFilename(filepath.Base(newest))

+ if err != nil {

+ return 0

+ }

+ return uint64(maxTX)

+// sealRefusal is a seal item that failed for a reason a later pass will

+// meet again: the pass reports it (phase failed) and the operator aborts.

+// Reason is the item's reason as `carlos move offline status` shows it.

+// Each refusal is written as a literal where it is decided, so the copy

+// gate reads every sentence a Reason can carry (copy rows MV-O76 to

+// MV-O84); the two that carry an error's own text (the archive's refusal

+// and the secrets key's) are box diagnostics (movesDiagnosticWrites).

+type sealRefusal struct {

+ item, Reason string

+func (r *sealRefusal) Error() string { return r.item + ": " + r.Reason }

+// offlineSeal is the sealer's step in offline-sealing.

+func (w *movesWorker) offlineSeal(ctx context.Context, s release.Store, rec provision.Record) error {

+ m, host := rec.Move, rec.Host

+ sealKey := moves.OfflineSealKey(host, m.ID)

+ if sealKey == "" {

+ return fmt.Errorf("offline move %s: %s has no seal key", m.ID, host)

+ }

+ raw, _, err := s.Get(ctx, sealKey)

+ switch {

+ case err == nil:

+ // Sealed on an earlier pass, which crashed before the CAS: the

+ // position is fixed, so the stop is only kept, never taken again.

+ return w.offlineKeepStop(ctx, s, rec, raw)

+ case !errors.Is(err, os.ErrNotExist):

+ return fmt.Errorf("read %s: %w", sealKey, err)

+ }

+ items := map[string]moves.Check{}

+ seal, err := w.offlineTakeSeal(ctx, s, rec, items)

+ var ref *sealRefusal

+ if errors.As(err, &ref) {

+ items[ref.item] = moves.Check{OK: false, Reason: ref.Reason}

+ r := moves.OfflineReport{Op: m.ID, Host: host, Box: w.cfg.Box, Role: moves.OfflineRoleSealer, Phase: moves.OfflinePhaseFailed,

+ At: w.cfg.now().UTC(), Items: items}

+ return errors.Join(fmt.Errorf("offline move %s: %s cannot be sealed: %w", m.ID, host, err), writeOfflineReport(ctx, s, r))

+ }

+ if err != nil {

+ return fmt.Errorf("offline move %s: sealing %s: %w", m.ID, host, err)

+ }

+ body, err := json.Marshal(seal)

+ if err != nil {

+ return err

+ }

+ stored, created, err := putOfflineCreateOnly(ctx, s, sealKey, body)

+ if err != nil {

+ return err

+ }

+ if created {

+ log.Printf("moves: %s: offline move %s: sealed at TXID %d (%d staged objects, files %s)", host, m.ID, seal.FinalTXID, len(seal.Staged), seal.FilesPlain)

+ }

+ if err := w.sealFault("seal"); err != nil {

+ return err

+ }

+ return w.offlineKeepStop(ctx, s, rec, stored)

+// offlineTakeSeal runs steps 1 to 9 and returns the seal to write. items

+// collects each item's check as it is decided; a *sealRefusal is the item

+// that failed.

+func (w *movesWorker) offlineTakeSeal(ctx context.Context, s release.Store, rec provision.Record, items map[string]moves.Check) (moves.OfflineReport, error) {

+ cfg := w.cfg

+ m, host := rec.Move, rec.Host

+ var seal moves.OfflineReport

+ // 1. The stop, as the source reports say it now: this box held, fenced,

+ // quiesced, with no process and no layout journal; every box the seal

+ // named held; no sidecar anywhere.

+ if len(m.Boxes) == 0 {

+ return seal, errors.New("the record names no source boxes")

+ }

+ reports, err := readOfflineReports(ctx, s, host, m.ID, moves.OfflineRoleSource)

+ if err != nil {

+ return seal, fmt.Errorf("read the source reports: %w", err)

+ }

+ now := cfg.now().UTC()

+ own, ok := reports[cfg.Box]

+ if !ok || own.Phase != moves.OfflinePhaseStopped || !own.Held || !own.Fenced || !own.Quiesced || own.Procs != 0 || own.LayoutJournal || now.Sub(own.At) > offlineSealFresh {

+ return seal, fmt.Errorf("this box has not stopped %s this pass: report found %v, held %v, fenced %v, quiesced %v, processes %d, layout journal %v, at %s",

+ host, ok, own.Held, own.Fenced, own.Quiesced, own.Procs, own.LayoutJournal, own.At.Format(time.RFC3339))

+ }

+ boxes := append([]string(nil), m.Boxes...)

+ sort.Strings(boxes)

+ for _, b := range boxes {

+ r, ok := reports[b]

+ if !ok || !r.Held {

+ return seal, fmt.Errorf("waiting for %s to report %s held", b, host)

+ }

+ for _, b := range boxes {

+ if reports[b].Sidecar {

+ return seal, &sealRefusal{item: "sidecar", Reason: fmt.Sprintf("%s keeps a sidecar file for %s", b, host)}

+ }

+ items["sidecar"] = moves.Check{OK: true, Reason: "no source box keeps a sidecar file"}

+ rt, err := w.offlineRow(host)

+ if err != nil {

+ return seal, err

+ }

+ if rt == nil {

+ return seal, fmt.Errorf("this box has no row for %s", host)

+ }

+ tgt, err := w.offlineTarget(ctx, s, m)

+ if err != nil {

+ return seal, err

+ }

+ mirror, err := w.offlineMirror(ctx, tgt.DataBucketRegion)

+ if err != nil {

+ return seal, err

+ }

+ bucket, loc, region, err := cfg.tombstoneAt(host)

+ if err != nil {

+ return seal, err

+ }

+ srcPrefix := loc + "/"

+ dstPrefix := activator.MovesPrefix(host)

+ dbPath := rt.DBPath

+ if dbPath == "" {

+ dbPath = cfg.flatDBPath(host, rt)

+ }

+ // 2. The checkpoint, while no tombstone is down: the local database is

+ // synced under the lease, which the checkpoint releases again. That

+ // carries a tail the stop failed to checkpoint, whether litestream had

+ // already turned it into a local LTX file (the local position is past

+ // the replica's) or it is still in the WAL (the positions are equal),

+ // as the legacy cut re-checkpoints unconditionally. A local copy behind

+ // the replica is a stale warm cache: the replica is the instance, and

+ // syncing the stale copy into it is the one thing never done. A

+ // tombstone already this move's means an earlier pass checkpointed and

+ // took it, and nothing has woken since; the checkpoint could not take

+ // the lease anyway.

+ local, err := w.localState(host, rt)

+ if err != nil {

+ return seal, err

+ }

+ objs, err := mirror.List(ctx, bucket, srcPrefix)

+ if err != nil {

+ return seal, fmt.Errorf("list the replica: %w", err)

+ }

+ before, err := w.offlineLock(ctx, host)

+ if err != nil {

+ return seal, err

+ }

+ replica := replicaPosition(objs, srcPrefix)

+ switch {

+ case before.mine(m.ID):

+ case local.LocalTXID < replica:

+ log.Printf("moves: %s: offline move %s: the local copy (position %d) is behind the replica (%d); the replica is sealed as it is", host, m.ID, local.LocalTXID, replica)

+ case lexists(dbPath):

+ if cfg.Checkpoint == nil {

+ return seal, errors.New("no checkpoint on this box")

+ }

+ if err := cfg.Checkpoint(ctx, dbPath); err != nil {

+ return seal, fmt.Errorf("final checkpoint of %s: %w", dbPath, err)

+ }

+ log.Printf("moves: %s: offline move %s: checkpointed the local database (position %d, replica %d)", host, m.ID, local.LocalTXID, replica)

+ case local.LocalTXID > replica:

+ return seal, &sealRefusal{item: "db-tail", Reason: fmt.Sprintf("this box's local position %d is past the replica's %d and there is no database at %s to checkpoint", local.LocalTXID, replica, dbPath)}

+ }

+ if err := w.sealFault("checkpoint"); err != nil {

+ return seal, err

+ }

+ // 3. The tombstone: from here no wake on any box can take the lease.

+ owner := moves.TombstoneOwner(m.ID)

+ if err := lease.Tombstone(ctx, bucket, loc, region, owner, moves.TombstoneTTL); err != nil {

+ return seal, fmt.Errorf("tombstone %s/%s/lock.json: %w", bucket, loc, err)

+ }

+ if err := w.sealFault("tombstone"); err != nil {

+ return seal, err

+ }

+ // 4. The final position, read only now, and no source box past it.

+ objs, err = mirror.List(ctx, bucket, srcPrefix)

+ if err != nil {

+ return seal, fmt.Errorf("list the replica: %w", err)

+ }

+ final := replicaPosition(objs, srcPrefix)

+ if final == 0 {

+ return seal, &sealRefusal{item: "db", Reason: fmt.Sprintf("the replica at %s/%s holds no database", bucket, srcPrefix)}

+ }

+ mine, err := w.localState(host, rt) // after the checkpoint, not as reported before it

+ if err != nil {

+ return seal, err

+ }

+ for _, b := range boxes {

+ pos := reports[b].LocalTXID

+ if b == cfg.Box {

+ pos = mine.LocalTXID

+ }

+ if pos > final {

+ return seal, &sealRefusal{item: "db-tail", Reason: fmt.Sprintf("%s's local position %d is past the replica's final position %d", b, pos, final)}

+ }

+ items["db-tail"] = moves.Check{OK: true, Reason: fmt.Sprintf("every source box is at or below TXID %d", final)}

+ items["lease"] = moves.Check{OK: true, Reason: fmt.Sprintf("%s holds %s/lock.json", owner, loc)}

+ if err := w.sealFault("final"); err != nil {

+ return seal, err

+ }

+ // 5. The request record into the build state, then the byte-final

+ // round into the target's staging lane and its listing.

+ if cfg.CarryRequest != nil {

+ if err := cfg.CarryRequest(ctx, host, bucket, loc); err != nil {

+ return seal, fmt.Errorf("carry the request record into the build state: %w", err)

+ }

+ items["build-state"] = moves.Check{OK: true, Reason: "the request record is in the build state the replica prefix carries"}

+ } else {

+ items["build-state"] = moves.Check{OK: true, Reason: "this box keeps no request records; the build state travels as the replica prefix holds it"}

+ }

+ receipts := lease.Receipts{Bucket: bucket, Key: activator.MoveReceiptsKey(host)}

+ round, err := lease.MirrorPrefix(ctx, mirror, bucket, srcPrefix, tgt.DataBucket, dstPrefix, receipts)

+ if err != nil {

+ return seal, err

+ }

+ if round.Deferred > 0 {

+ return seal, fmt.Errorf("final mirror round: %d object(s) changed at the source during the copy", round.Deferred)

+ }

+ source, err := mirror.List(ctx, bucket, srcPrefix)

+ if err != nil {

+ return seal, fmt.Errorf("list the replica: %w", err)

+ }

+ listed, err := mirror.List(ctx, tgt.DataBucket, dstPrefix)

+ if err != nil {

+ return seal, fmt.Errorf("list the staged copy: %w", err)

+ }

+ staged := map[string]moves.StagedObject{}

+ for k, o := range listed {

+ staged[strings.TrimPrefix(k, dstPrefix)] = moves.StagedObject{Size: o.Size, ETag: o.ETag}

+ }

+ if err := sameListing(source, srcPrefix, staged); err != nil {

+ return seal, fmt.Errorf("the staged copy is not the replica: %w", err)

+ }

+ if replicaPosition(listed, dstPrefix) != final {

+ return seal, fmt.Errorf("the staged copy's newest TXID is %d, the replica's %d", replicaPosition(listed, dstPrefix), final)

+ }

+ items["db"] = moves.Check{OK: true, Reason: fmt.Sprintf("%d objects staged at %s/%s, newest TXID %d", len(staged), tgt.DataBucket, dstPrefix, final)}

+ if err := w.sealFault("mirror"); err != nil {

+ return seal, err

+ }

+ // 6. The files and <host>.env, sealed, beside the staging lane.

+ prefix := moves.OfflineArchivePrefix(host, m.ID)

+ files, envPath, configPath, stray, err := w.offlineFilesOf(host, rt)

+ if err != nil {

+ return seal, fmt.Errorf("read what %s keeps: %w", host, err)

+ }

+ if len(stray) > 0 {

+ sort.Strings(stray)

+ // What the app keeps there is the app's data, in a place it reads

+ // from: moving it into the files directory changes that place, and

+ // removing it loses it. So the refusal offers neither (MV-O76).

+ return seal, &sealRefusal{item: "files", Reason: fmt.Sprintf("%s keeps %s beside its files directory, and this move can carry only the files directory. see the offline move runbook", host, strings.Join(stray, ", "))}

+ }

+ a, err := w.offlineArchive(ctx, rec, mirror, tgt.DataBucket, prefix, files, envPath)

+ if err != nil {

+ return seal, err

+ }

+ items["files"] = moves.Check{OK: true, Reason: fmt.Sprintf("%d files, %d bytes, sealed at %s%s", a.entries, a.bytes, prefix, offlineArchiveSealed)}

+ if err := w.sealFault("archive"); err != nil {

+ return seal, err

+ }

+ // 7. The timers.

+ if cfg.Registry == nil {

+ return seal, errors.New("no registry on this box")

+ }

+ rows, err := cfg.Registry.TimersFor(host)

+ if err != nil {

+ return seal, fmt.Errorf("read the timers: %w", err)

+ }

+ active := moves.ActiveTimers(rows)

+ timersBody, err := moves.TimersSnapshot{V: 1, Host: host, Rows: active, TakenAt: now}.Encode()

+ if err != nil {

+ return seal, err

+ }

+ if err := mirror.Put(ctx, tgt.DataBucket, prefix+offlineTimersObject, timersBody); err != nil {

+ return seal, fmt.Errorf("put the timers: %w", err)

+ }

+ timersDigest := moves.TimersDigest(rows)

+ items["timers"] = moves.Check{OK: true, Reason: fmt.Sprintf("%d active timers at %s%s", len(active), prefix, offlineTimersObject)}

+ if err := w.sealFault("timers"); err != nil {

+ return seal, err

+ }

+ // 8. The env file, its effective instance key, and the env pin.

+ envSHA, keySHA, err := offlineEnvDigests(envPath, configPath, a.envSHA)

+ if err != nil {

+ return seal, err

+ }

+ // The digest itself is the seal's EnvSHA256, not part of the reason.

+ items["env"] = moves.Check{OK: true, Reason: "absent on the source, so none is carried"}

+ if envSHA != "" {

+ items["env"] = moves.Check{OK: true, Reason: "carried in the archive"}

+ }

+ if cfg.LaunchLayers == nil {

+ return seal, &sealRefusal{item: "env-pin", Reason: "this box has no exec backend to compose the launch environment"}

+ }

+ if cfg.SecretsKey == "" {

+ return seal, &sealRefusal{item: "env-pin", Reason: "this box holds no CARLOS_SECRETS_KEY to key the env report"}

+ }

+ env, instance, _, err := cfg.LaunchLayers(*rt)

+ if err != nil {

+ return seal, fmt.Errorf("compose the launch environment: %w", err)

+ }

+ mac, err := moves.ReportKeyFromSecrets(cfg.SecretsKey)

+ if err != nil {

+ return seal, &sealRefusal{item: "env-pin", Reason: err.Error()}

+ }

+ envReport := moves.ReportLaunch(mac, env, instance)

+ envReportBody, err := json.Marshal(envReport)

+ if err != nil {

+ return seal, err

+ }

+ if err := mirror.Put(ctx, tgt.DataBucket, prefix+offlineEnvReportObject, envReportBody); err != nil {

+ return seal, fmt.Errorf("put the env report: %w", err)

+ }

+ items["env-pin"] = moves.Check{OK: true, Reason: "launch environment digest " + envReport.Digest}

+ if err := w.sealFault("env"); err != nil {

+ return seal, err

+ }

+ // 9. The build, the replication rung and the next due time.

+ if cfg.LiveVersion == nil {

+ return seal, errors.New("no version store on this box")

+ }

+ version, err := cfg.LiveVersion(host)

+ if err != nil {

+ return seal, fmt.Errorf("read the live build: %w", err)

+ }

+ if version == "" {

+ return seal, &sealRefusal{item: "release", Reason: fmt.Sprintf("no build is live for %s on this box", host)}

+ }

+ items["release"] = moves.Check{OK: true, Reason: "live build " + version}

+ items["replication"] = moves.Check{OK: true, Reason: fmt.Sprintf("replication %q", rt.Replication)}

+ items["next-due"] = moves.Check{OK: true, Reason: fmt.Sprintf("next due %d", rt.NextDue)}

+ // The tombstone must still be this move's when the seal is written:

+ // the position it records was read under it. One lost before the seal

+ // is taken again on the next pass, which seals afresh.

+ st, err := w.offlineLock(ctx, host)

+ if err != nil {

+ return seal, err

+ }

+ if !st.mine(m.ID) {

+ return seal, fmt.Errorf("the lease is %s, not this move's tombstone; the next pass takes it again", st)

+ }

+ seal = moves.OfflineReport{

+ V: moves.OfflineReportVersion, Op: m.ID, Host: host, Box: cfg.Box, Role: moves.OfflineRoleSealer, Phase: moves.OfflinePhaseSealed,

+ At: now, Items: items, TombstoneOwner: owner,

+ FinalTXID: final, Staged: staged,

+ FilesPlain: a.plain, FilesSealed: a.sealed, FilesBytes: a.bytes, FilesEntries: a.entries,

+ EnvSHA256: envSHA, KeySHA256: keySHA, EnvDigest: envReport.Digest,

+ Release: version, TimersDigest: timersDigest, Replication: rt.Replication, NextDue: rt.NextDue,

+ }

+ if err := moves.ValidateOfflineSeal(seal, host, m.ID); err != nil {

+ return seal, err

+ }

+ return seal, nil

+// sameListing reports whether staged (keyed relative to the staging

+// prefix) holds exactly the source's objects under srcPrefix, lock.json

+// aside (the mirror never copies it), size for size. ETags may differ: a

+// multipart source's copy gets a new one (lease.Receipts).

+func sameListing(source map[string]lease.MirrorObject, srcPrefix string, staged map[string]moves.StagedObject) error {

+ want := map[string]int64{}

+ for k, o := range source {

+ rel := strings.TrimPrefix(k, srcPrefix)

+ if rel == "lock.json" {

+ continue

+ }

+ want[rel] = o.Size

+ }

+ if len(want) != len(staged) {

+ return fmt.Errorf("%d objects at the source, %d staged", len(want), len(staged))

+ }

+ for rel, size := range want {

+ got, ok := staged[rel]

+ if !ok || got.Size != size {

+ return fmt.Errorf("%s: staged %v, source size %d", rel, got, size)

+ }

return nil

}

+// offlineFilesOf is what the archive carries for the row's layout, and

+// what it cannot.

+//

+// The target lands the instance flat (the row is minted at flat paths), so

+// the archive's files root is the app's files directory as the flat layout

+// names it: <DataDir>/<host>, with <host>.env beside it. The layout pass

+// maps the flat data directory onto the nested instance directory name for

+// name (layout/extras.go: an app finds its files beside its database, and

+// nested the database is <instance dir>/<host>.db), so a nested row's

+// files directory is <instance dir>/<host>, read through the raw

+// directory as every agent read of it is, and its env file is

+// <instance dir>/<host>.env. Archiving the whole instance directory would

+// land the app's files one level down on the target (<DataDir>/<host>/

+// <host>/...), where the app would not find them.

+//

+// stray is every entry the app keeps beside its files directory, which

+// the flat files directory cannot carry: on a nested row every entry of

+// the instance directory that is neither the platform's

+// (layout.IsNestedPlatformName: the seed marker and the launch descriptors

+// included) nor <host>; on a flat row every entry of

+// the data directory the layout's naming rules give this host

+// (layout.FlatLeftovers; <host>-blobs is the live example) that is

+// neither. The seal refuses on any, rather than leave it behind.

+//

+// A host that never kept a files directory (an app that writes nothing of

+// its own) has none at the files path: the archive then carries the env

+// file alone, FilesPlain is the digest of that, and the landing extracts

+// an empty files directory.

+//

+// configPath is the rendered configuration beside the env file, which the

+// effective instance key may come from.

+func (w *movesWorker) offlineFilesOf(host string, rt *registry.Route) (files, envPath, configPath string, stray []string, err error) {

+ dd := w.cfg.DataDir

+ platform := func(name string) bool { return layout.IsNestedPlatformName(host, name) }

+ if rt != nil && provision.NestedPaths(dd, *rt) {

+ raw := layout.RawInstanceDir(dd, host)

+ ents, err := os.ReadDir(raw)

+ if err != nil && !errors.Is(err, os.ErrNotExist) {

+ return "", "", "", nil, err

+ }

+ for _, e := range ents {

+ if !platform(e.Name()) && e.Name() != host {

+ stray = append(stray, filepath.Join(raw, e.Name()))

+ }

+ return filepath.Join(raw, host), filepath.Join(raw, host+".env"), filepath.Join(raw, host+".config.env"), stray, nil

+ }

+ var hosts []string

+ if w.cfg.Registry != nil {

+ rows, err := w.cfg.Registry.List()

+ if err != nil {

+ return "", "", "", nil, err

+ }

+ for _, r := range rows {

+ hosts = append(hosts, r.Host)

+ }

+ for _, p := range layout.FlatLeftovers(dd, host, hosts) {

+ if name := filepath.Base(p); !platform(name) && name != host {

+ stray = append(stray, p)

+ }

+ return filepath.Join(dd, host), filepath.Join(dd, host+".env"), filepath.Join(dd, host+".config.env"), stray, nil

+// sealedArchive is what the seal records of the archive.

+type sealedArchive struct {

+ plain, sealed string

+ bytes int64

+ entries int

+ // envSHA is the env entry's digest as the archive recorded it

+ // ("sha256:<hex>"), "" when the archive carries no env.

+ envSHA string

+// offlineArchive builds the archive of files and envPath into a scratch

+// directory on the data disk, seals it under the deployment's config key

+// bound to the record's account and app, and puts it and its manifest

+// under prefix in the target bucket. A file the archive refuses is a

+// refusal of the files item, never a partial archive; a file that changed

+// while it was read is retried. The files directory holds nothing of the

+// platform's but the files marker, which every build leaves out, so no

+// filter is needed (archive.Build is BuildFiltered with none).

+func (w *movesWorker) offlineArchive(ctx context.Context, rec provision.Record, mirror lease.MirrorStore, bucket, prefix, files, envPath string) (sealedArchive, error) {

+ cfg := w.cfg

+ var out sealedArchive

+ if cfg.ConfigPub == nil {

+ return out, &sealRefusal{item: "files", Reason: "this box holds no CARLOS_SECRETS_KEY to seal the archive under"}

+ }

+ dir := moves.OfflineParkDir(cfg.DataDir, rec.Host, rec.Move.ID, moves.OfflineRoleSealer)

+ if dir == "" {

+ return out, fmt.Errorf("no scratch directory for %s under %q", rec.Host, cfg.DataDir)

+ }

+ if err := os.MkdirAll(dir, 0o700); err != nil {

+ return out, err

+ }

+ tmp, err := os.MkdirTemp(dir, "archive-")

+ if err != nil {

+ return out, err

+ }

+ defer os.RemoveAll(tmp)

+ built, err := os.Create(filepath.Join(tmp, "archive.tar.zst"))

+ if err != nil {

+ return out, err

+ }

+ defer built.Close()

+ plain, entries, size, err := archive.Build(files, envPath, built)

+ switch {

+ case errors.Is(err, archive.ErrChanged):

+ return out, fmt.Errorf("archive the files: %w; built again next pass", err)

+ case err != nil:

+ return out, &sealRefusal{item: "files", Reason: err.Error()}

+ }

+ if _, err := built.Seek(0, io.SeekStart); err != nil {

+ return out, err

+ }

+ sealedFile, err := os.Create(filepath.Join(tmp, "archive.sealed"))

+ if err != nil {

+ return out, err

+ }

+ defer sealedFile.Close()

+ dataKey, sealedSHA, err := archive.Seal(built, cfg.ConfigPub, appconfig.Binding{Account: rec.Account, App: rec.App}, sealedFile)

+ if err != nil {

+ return out, fmt.Errorf("seal the archive: %w", err)

+ }

+ manifest, err := json.Marshal(archive.Manifest{V: archive.ManifestVersion, Op: rec.Move.ID, Plain: plain, Sealed: sealedSHA, Bytes: size,

+ Entries: entries, SealedDataKey: dataKey, Chunk: archive.ChunkSize})

+ if err != nil {

+ return out, err

+ }

+ body, err := os.ReadFile(sealedFile.Name())

+ if err != nil {

+ return out, err

+ }

+ if err := mirror.Put(ctx, bucket, prefix+offlineArchiveSealed, body); err != nil {

+ return out, fmt.Errorf("put the sealed archive: %w", err)

+ }

+ if err := mirror.Put(ctx, bucket, prefix+offlineArchiveManifest, manifest); err != nil {

+ return out, fmt.Errorf("put the archive manifest: %w", err)

+ }

+ out = sealedArchive{plain: plain, sealed: sealedSHA, bytes: size, entries: len(entries)}

+ for _, e := range entries {

+ if e.Path == archive.EnvEntry {

+ out.envSHA = e.SHA256

+ }

+ return out, nil

+// readOwnFile reads a regular file without following a link, as the agent

+// reads every file a nested tenant can write: exists false when there is

+// none.

+func readOwnFile(path string) (body []byte, exists bool, err error) {

+ info, err := os.Lstat(path)

+ if errors.Is(err, os.ErrNotExist) {

+ return nil, false, nil

+ }

+ if err != nil {

+ return nil, false, err

+ }

+ if !info.Mode().IsRegular() {

+ return nil, true, fmt.Errorf("%s is not a regular file", path)

+ }

+ f, err := os.Open(path)

+ if err != nil {

+ return nil, true, err

+ }

+ defer f.Close()

+ got, err := f.Stat()

+ if err != nil {

+ return nil, true, err

+ }

+ if !os.SameFile(info, got) {

+ return nil, true, fmt.Errorf("%s changed while it was read", path)

+ }

+ body, err = io.ReadAll(io.LimitReader(f, instancekey.EnvReadLimit+1))

+ if err != nil {

+ return nil, true, err

+ }

+ if len(body) > instancekey.EnvReadLimit {

+ return nil, true, fmt.Errorf("%s is larger than an env file", path)

+ }

+ return body, true, nil

+// offlineEnvDigests is the env item's two values: the lowercase hex sha256

+// of <host>.env's bytes ("" when the source has none), and the lowercase

+// hex sha256 of the instance key a launch hands the app

+// (instancekey.Effective over the rendered configuration and the own file,

+// instancekey.Digest; "" when neither supplies one). archived is the env

+// entry's digest as the archive recorded it ("sha256:<hex>", or "" for no

+// env entry); the file read here must be the one archived.

+func offlineEnvDigests(envPath, configPath, archived string) (envSHA, keySHA string, err error) {

+ own, exists, err := readOwnFile(envPath)

+ if err != nil {

+ return "", "", fmt.Errorf("read the env file: %w", err)

+ }

+ if exists {

+ sum := sha256.Sum256(own)

+ envSHA = hex.EncodeToString(sum[:])

+ }

+ if want := strings.TrimPrefix(archived, "sha256:"); want != envSHA {

+ return "", "", fmt.Errorf("the env file changed while it was archived (archived %q, now %q); sealed again next pass", want, envSHA)

+ }

+ config, _, err := readOwnFile(configPath)

+ if err != nil {

+ return "", "", fmt.Errorf("read the rendered configuration: %w", err)

+ }

+ if key, _ := instancekey.Effective(instancekey.ParseEnv(config), instancekey.ParseEnv(own)); key != "" {

+ keySHA = instancekey.Digest(key)

+ }

+ return envSHA, keySHA, nil

+// offlineSealRecord reads seal.json, nil when it is not there.

+func (w *movesWorker) offlineSealRecord(ctx context.Context, s release.Store, rec provision.Record) ([]byte, error) {

+ key := moves.OfflineSealKey(rec.Host, rec.Move.ID)

+ raw, _, err := s.Get(ctx, key)

+ if errors.Is(err, os.ErrNotExist) {

+ return nil, nil

+ }

+ if err != nil {

+ return nil, fmt.Errorf("read %s: %w", key, err)

+ }

+ return raw, nil

+// decodeSeal parses and checks the seal record; nil raw is no seal.

+func decodeSeal(raw []byte, host, id string) (*moves.OfflineReport, error) {

+ if raw == nil {

+ return nil, nil

+ }

+ var seal moves.OfflineReport

+ if err := json.Unmarshal(raw, &seal); err != nil {

+ return nil, fmt.Errorf("the seal record does not parse: %w", err)

+ }

+ if err := moves.ValidateOfflineSeal(seal, host, id); err != nil {

+ return nil, err

+ }

+ return &seal, nil

+// offlineKeepStop is every pass after the seal, until the tombstone is

+// lifted: read the lease first; a tombstone that is not this move's any

+// more is a lost stop, never taken again. Then the source reports: one

+// written since the seal at a position past the sealed one is a lost stop

+// too (a write the stop did not hold off, which the staged copy lacks).

+// Only then is this move's tombstone re-asserted. In

+// offline-sealing (a seal written by a pass that crashed before the CAS)

+// it also makes the CAS. Then it rewrites the living sealer report: the

+// seal's values and items, the tombstone's owner and expiry, and At.

+func (w *movesWorker) offlineKeepStop(ctx context.Context, s release.Store, rec provision.Record, raw []byte) error {

+ cfg := w.cfg

+ m, host := rec.Move, rec.Host

+ seal, err := decodeSeal(raw, host, m.ID)

+ if err != nil {

+ return fmt.Errorf("offline move %s: %s: %w; it is never rewritten, so abort the move", m.ID, host, err)

+ }

+ if seal == nil && m.State == provision.OfflineSealing {

+ return fmt.Errorf("offline move %s: %s has no seal record", m.ID, host)

+ }

+ st, err := w.offlineLock(ctx, host)

+ if err != nil {

+ return err

+ }

+ if !st.mine(m.ID) {

+ return w.offlineLeaseLost(ctx, s, rec, seal, st)

+ }

+ if seal != nil {

+ if err := w.offlineCheckPositions(ctx, s, rec, seal); err != nil {

+ return err

+ }

+ bucket, loc, region, err := cfg.tombstoneAt(host)

+ if err != nil {

+ return err

+ }

+ if err := w.sealFault("reassert"); err != nil {

+ return err

+ }

+ if err := lease.Reassert(ctx, bucket, loc, region, moves.TombstoneOwner(m.ID), moves.TombstoneTTL); err != nil {

+ var held *lease.HeldError

+ if errors.As(err, &held) {

+ return w.offlineLeaseLost(ctx, s, rec, seal, tombState{owner: held.Owner, expires: held.ExpiresAt})

+ }

+ if errors.Is(err, lease.ErrTombstoneLost) {

+ // Gone between the read above and Reassert's own: the same

+ // lost stop, and Reassert wrote nothing. Read it again for

+ // what it is now (absent, when even that read fails).

+ now, rerr := w.offlineLock(ctx, host)

+ if rerr != nil {

+ now = tombState{}

+ }

+ return w.offlineLeaseLost(ctx, s, rec, seal, now)

+ }

+ return fmt.Errorf("offline move %s: re-assert the tombstone of %s: %w", m.ID, host, err)

+ }

+ if st, err = w.offlineLock(ctx, host); err != nil {

+ return err

+ }

+ if !st.mine(m.ID) {

+ return w.offlineLeaseLost(ctx, s, rec, seal, st)

+ }

+ if m.State == provision.OfflineSealing {

+ src := provision.MoveSource{

+ FinalTXID: seal.FinalTXID, Replication: seal.Replication, NextDue: seal.NextDue,

+ Files: &provision.MoveFilesDigest{Plain: seal.FilesPlain, Sealed: seal.FilesSealed, Bytes: seal.FilesBytes, Entries: seal.FilesEntries},

+ TimersSHA256: seal.TimersDigest, Release: seal.Release, EnvDigest: seal.EnvDigest,

+ Evidence: moves.OfflineSealEvidence(host, m.ID, raw),

+ }

+ _, err := provision.SetRecordMove(ctx, s, rec.Account, rec.App, host, provision.OfflineSealedBy(m.ID, cfg.Box, src))

+ switch {

+ case errors.Is(err, provision.ErrMoveConflict):

+ log.Printf("moves: %s: offline move %s: the record moved on before the seal's CAS: %v", host, m.ID, err)

+ case err != nil:

+ return fmt.Errorf("offline move %s: record the seal of %s: %w", m.ID, host, err)

+ default:

+ nudgeSerial(ctx, s)

+ log.Printf("moves: %s: offline move %s: sealed", host, m.ID)

+ }

+ if err := w.sealFault("cas"); err != nil {

+ return err

+ }

+ living := moves.OfflineReport{Op: m.ID, Host: host, Box: cfg.Box, Role: moves.OfflineRoleSealer}

+ if seal != nil {

+ living = *seal

+ }

+ living.Phase = moves.OfflinePhaseSealed

+ living.At = cfg.now().UTC()

+ living.TombstoneOwner = st.owner

+ living.TombstoneUntil = st.expires.UTC()

+ return writeOfflineReport(ctx, s, living)

+// offlineCheckPositions is the backstop behind the holds and the fence:

+// every source report written since the seal (at or after its At) must be

+// at or below the sealed final position. One past it is a box that wrote

+// what the staged copy does not hold, so the sealed position is no longer

+// the instance's: a lost stop, in the db-tail item.

+func (w *movesWorker) offlineCheckPositions(ctx context.Context, s release.Store, rec provision.Record, seal *moves.OfflineReport) error {

+ reports, err := readOfflineReports(ctx, s, rec.Host, rec.Move.ID, moves.OfflineRoleSource)

+ if err != nil {

+ return fmt.Errorf("read the source reports: %w", err)

+ }

+ boxes := make([]string, 0, len(reports))

+ for b := range reports {

+ boxes = append(boxes, b)

+ }

+ sort.Strings(boxes)

+ for _, b := range boxes {

+ r := reports[b]

+ if r.At.Before(seal.At) || r.LocalTXID <= seal.FinalTXID {

+ continue

+ }

+ return w.offlineLostStop(ctx, s, rec, seal, "db-tail", offlinePastSealCause(b, r.LocalTXID, seal.FinalTXID))

+ }

+ return nil

+// offlineLeaseLost is the lost stop of a tombstone that is absent,

+// expired, lifted or someone else's, in the lease item.

+func (w *movesWorker) offlineLeaseLost(ctx context.Context, s release.Store, rec provision.Record, seal *moves.OfflineReport, st tombState) error {

+ return w.offlineLostStop(ctx, s, rec, seal, "lease", offlineLeaseCause(st))

+// offlineLeaseCause and offlinePastSealCause are the two causes of a lost

+// stop as its page and the failed item say them (MV-O69, MV-O70).

+func offlineLeaseCause(st tombState) string {

+ return fmt.Sprintf("the source's lease is %s, not this move's tombstone", offlineLeaseText(st))

+func offlinePastSealCause(box string, pos, final uint64) string {

+ return fmt.Sprintf("%s reports TXID %d, past the sealed position %d", box, pos, final)

+// offlineLostStop reports a stop that no longer holds after the seal:

+// cause says why, and item is the seal item it fails. The tombstone is

+// never taken again (a wake may have written the replica in between, so

+// the sealed position is no longer the source's), the living report says

+// failed with that item not OK and no tombstone owner (so open refuses),

+// and an operator is paged once per move.

+func (w *movesWorker) offlineLostStop(ctx context.Context, s release.Store, rec provision.Record, seal *moves.OfflineReport, item, cause string) error {

+ cfg := w.cfg

+ m, host := rec.Move, rec.Host

+ r := moves.OfflineReport{Op: m.ID, Host: host, Box: cfg.Box, Role: moves.OfflineRoleSealer}

+ if seal != nil {

+ r = *seal

+ }

+ r.Phase = moves.OfflinePhaseFailed

+ r.At = cfg.now().UTC()

+ r.TombstoneOwner, r.TombstoneUntil = "", time.Time{}

+ items := map[string]moves.Check{}

+ for k, v := range r.Items {

+ items[k] = v

+ }

+ items[item] = moves.Check{OK: false, Reason: "lost stop: " + cause}

+ r.Items = items

+ werr := writeOfflineReport(ctx, s, r)

+ subject, body := offlineLostStopPage(host, m.ID, m.State, cause)

+ w.firePage(ctx, host, m.ID, offlinePageLostStop, alerting.LevelIncident, subject, body)

+ log.Printf("moves: %s: offline move %s: LOST STOP: %s", host, m.ID, cause)

+ return errors.Join(fmt.Errorf("offline move %s: %s: lost stop: %s", m.ID, host, cause), werr)

+// offlineLostStopPage is the lost stop's page (copy rows MV-O66 to

+// MV-O68; the item's reason is MV-O71): its subject, and a body for the move's state. Before open,

+// abort puts the instance back with nothing lost, so the body says to

+// abort. From open on, abort is refused (MV-O40) and the move is fixed

+// forward by hand, so the body points to the status and the runbook.

+func offlineLostStopPage(host, id, state, cause string) (subject, body string) {

+ subject = fmt.Sprintf("offline move of %s lost its stop", host)

+ if state == provision.OfflineOpen || state == provision.OfflineFinishing {

+ body = fmt.Sprintf("The offline move %s of %s is %s, and its stop no longer holds: %s. It cannot be aborted once it is open. Read carlos move offline status --host %s and follow the offline move runbook.", id, host, state, cause, host)

+ return subject, body

+ }

+ body = fmt.Sprintf("The offline move %s of %s is %s, and its stop no longer holds: %s. The move cannot go on. Read carlos move offline status --host %s and abort the move.", id, host, state, cause, host)

+ return subject, body

+// offlineFinishLift is the sealer's step in offline-finishing. Until every

+// box the seal named reports gone (it parked at open and reconcile has

+// removed its row since), the stop is kept as before. Then the tombstone

+// is lifted (only this move's, with the ETag read), the copy receipts are

+// cleared and the report says lifted. A lease that is already free (lifted

+// by an earlier pass) is left as it is; one someone else holds live is a

+// lost stop.

+func (w *movesWorker) offlineFinishLift(ctx context.Context, s release.Store, rec provision.Record) error {

+ cfg := w.cfg

+ m, host := rec.Move, rec.Host

+ reports, err := readOfflineReports(ctx, s, host, m.ID, moves.OfflineRoleSource)

+ if err != nil {

+ return fmt.Errorf("read the source reports: %w", err)

+ }

+ gone := len(m.Boxes) > 0

+ for _, b := range m.Boxes {

+ if r, ok := reports[b]; !ok || r.Phase != moves.OfflinePhaseGone {

+ gone = false

+ }

+ raw, err := w.offlineSealRecord(ctx, s, rec)

+ if err != nil {

+ return err

+ }

+ if !gone {

+ return w.offlineKeepStop(ctx, s, rec, raw)

+ }

+ st, err := w.offlineLock(ctx, host)

+ if err != nil {

+ return err

+ }

+ switch {

+ case st.mine(m.ID):

+ bucket, loc, region, err := cfg.tombstoneAt(host)

+ if err != nil {

+ return err

+ }

+ if _, err := lease.LiftTombstone(ctx, bucket, loc, region, func(owner string) bool { return owner == moves.TombstoneOwner(m.ID) }); err != nil {

+ return fmt.Errorf("offline move %s: lift the tombstone of %s: %w", m.ID, host, err)

+ }

+ log.Printf("moves: %s: offline move %s: tombstone lifted", host, m.ID)

+ case st.free():

+ default:

+ seal, _ := decodeSeal(raw, host, m.ID)

+ return w.offlineLeaseLost(ctx, s, rec, seal, st)

+ }

+ if err := w.sealFault("lift"); err != nil {

+ return err

+ }

+ own, err := w.offlineMirror(ctx, cfg.OwnBucketRegion)

+ if err != nil {

+ return err

+ }

+ clearOfflineReceipts(ctx, own, cfg.OwnBucket, host)

+ r := moves.OfflineReport{Op: m.ID, Host: host, Box: cfg.Box, Role: moves.OfflineRoleSealer, Phase: moves.OfflinePhaseLifted, At: cfg.now().UTC()}

+ return writeOfflineReport(ctx, s, r)

+// clearOfflineReceipts is clearMoveReceipts as an ensure: receipts that are

+// already gone are not deleted again, so a pass that repeats finish or

+// revert writes nothing (on a versioned bucket every DELETE of an absent

+// key leaves one more delete marker).

+func clearOfflineReceipts(ctx context.Context, m lease.MirrorStore, bucket, host string) {

+ rc := lease.Receipts{Bucket: bucket, Key: activator.MoveReceiptsKey(host)}

+ if _, err := m.Get(ctx, rc.Bucket, rc.Key); errors.Is(err, lease.ErrMirrorNotFound) {

+ return

+ }

+ clearMoveReceipts(ctx, m, rc, host)

+// offlineSealRevert is the sealer's step in offline-reverting: lift the

+// tombstone if it is this move's (a lease anyone else holds, or none, is

+// left alone), delete the two prefixes this move wrote in the target's

+// bucket (the staged replica and the archive), clear the copy receipts,

+// lift this move's fence (the source step lifts it too) and report

+// reverted. Every part is an ensure.

+func (w *movesWorker) offlineSealRevert(ctx context.Context, s release.Store, rec provision.Record) error {

+ cfg := w.cfg

+ m, host := rec.Move, rec.Host

+ bucket, loc, region, err := cfg.tombstoneAt(host)

+ if err != nil {

+ return err

+ }

+ lifted, err := lease.LiftTombstone(ctx, bucket, loc, region, func(owner string) bool { return owner == moves.TombstoneOwner(m.ID) })

+ var held *lease.HeldError

+ switch {

+ case err == nil:

+ if lifted {

+ log.Printf("moves: %s: offline move %s: tombstone lifted on abort", host, m.ID)

+ }

+ case errors.As(err, &held):

+ w.notice(host+"#offline-revert-lease", "moves: %s: offline move %s: the lease is %s's, not this move's; left as it is", host, m.ID, held.Owner)

+ default:

+ return fmt.Errorf("offline move %s: lift the tombstone of %s: %w", m.ID, host, err)

+ }

+ if err := w.sealFault("revert-lift"); err != nil {

+ return err

+ }

+ tgt, err := w.offlineTarget(ctx, s, m)

+ if err != nil {

+ return err

+ }

+ mirror, err := w.offlineMirror(ctx, tgt.DataBucketRegion)

+ if err != nil {

+ return err

+ }

+ for _, prefix := range []string{activator.MovesPrefix(host), moves.OfflineArchivePrefix(host, m.ID)} {

+ if prefix == "" || prefix == "moves/" {

+ return fmt.Errorf("offline move %s: no staging prefix for %s", m.ID, host)

+ }

+ if err := lease.DeletePrefix(ctx, mirror, tgt.DataBucket, prefix); err != nil {

+ return fmt.Errorf("offline move %s: delete %s/%s: %w", m.ID, tgt.DataBucket, prefix, err)

+ }

+ clearOfflineReceipts(ctx, mirror, bucket, host)

+ w.offlineLiftFence(host, m)

+ r := moves.OfflineReport{Op: m.ID, Host: host, Box: cfg.Box, Role: moves.OfflineRoleSealer, Phase: moves.OfflinePhaseReverted, At: cfg.now().UTC()}

+ return writeOfflineReport(ctx, s, r)

diff --git a/internal/host/movesoffline_seal_test.go b/internal/host/movesoffline_seal_test.go

new file mode 100644

index 000000000..02158838d

--- /dev/null

+++ b/internal/host/movesoffline_seal_test.go

@@ -0,0 +1,1405 @@

+//go:build linux

+package host

+import (

+ "bytes"

+ "context"

+ "crypto/ecdh"

+ "crypto/sha256"

+ "database/sql"

+ "encoding/hex"

+ "encoding/json"

+ "errors"

+ "fmt"

+ "net/http/httptest"

+ "os"

+ "path/filepath"

+ "strings"

+ "syscall"

+ "testing"

+ "time"

+ "github.com/benbjohnson/litestream"

+ "github.com/superfly/ltx"

+ "github.com/carlosframework/platform/internal/activator"

+ "github.com/carlosframework/platform/internal/appconfig"

+ "github.com/carlosframework/platform/internal/instancekey"

+ "github.com/carlosframework/platform/internal/layout"

+ "github.com/carlosframework/platform/internal/lease"

+ "github.com/carlosframework/platform/internal/lease/leasetest"

+ "github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/moves/archive"

+ "github.com/carlosframework/platform/internal/provision"

+ "github.com/carlosframework/platform/internal/registry"

+ "github.com/carlosframework/platform/internal/release"

+// The sealer (Task 4 of docs/superpowers/plans/2026-10-20-offline-operator-

+// move.md), tested through offlineRound, the moves pass's dispatch, over

+// the S3 wire protocol (leasetest.NewS3Server, with buckets data-ie and

+// data-uk behind one endpoint), real litestream replicas written by

+// lease.ReplicateWithLease, the real lease package, a real registry and

+// activator per box, and the production edges-file writer for the target's

+// coordinates. The deployment store is T3's DirStore world.

+const (

+ sealIE = "data-ie"

+ sealUK = "data-uk"

+ sealIERegion = "eu-west-1"

+ sealUKRegion = "eu-west-2"

+ sealPrefix = "instances"

+ sealVersion = "v42"

+type sealWorld struct {

+ *offWorld

+ fake *leasetest.VersionedBucket

+ keyB64 string

+ priv *ecdh.PrivateKey

+ ie, uk *offBox

+ pages []string

+ bodies []string // each page's body, beside pages

+func newSealWorld(t *testing.T) *sealWorld {

+ t.Helper()

+ fake := leasetest.NewVersionedBucket()

+ fake.AddBucket(sealIE)

+ fake.AddBucket(sealUK)

+ h, _ := leasetest.NewS3Server(fake)

+ srv := httptest.NewServer(h) // 127.0.0.1 only

+ t.Cleanup(srv.Close)

+ t.Setenv("AWS_ACCESS_KEY_ID", "test-key")

+ t.Setenv("AWS_SECRET_ACCESS_KEY", "test-secret")

+ t.Setenv("AWS_SESSION_TOKEN", "")

+ t.Setenv("AWS_EC2_METADATA_DISABLED", "true")

+ t.Setenv("AWS_CONFIG_FILE", filepath.Join(t.TempDir(), "absent"))

+ t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(t.TempDir(), "absent"))

+ t.Setenv("CARLOS_S3_ENDPOINT", srv.URL)

+ keyB64, _, err := appconfig.GenerateKey()

+ if err != nil {

+ t.Fatal(err)

+ }

+ priv, err := appconfig.ParsePrivateKey(keyB64, "test key")

+ if err != nil {

+ t.Fatal(err)

+ }

+ sw := &sealWorld{offWorld: newOffWorld(t), fake: fake, keyB64: keyB64, priv: priv}

+ sw.ie = sw.sourceBox("ie-a")

+ sw.uk = sw.box("uk-box", "uk", map[string]string{"CARLOS_HIBERNATE_BUCKET": sealUK, "CARLOS_SECRETS_KEY": keyB64})

+ var last []byte

+ if err := edgesFileWrite(context.Background(), sw.s, sw.uk.reg, edgesFile{Box: "uk-box", Placement: "uk", DataBucket: sealUK, DataBucketRegion: sealUKRegion}, "", &last, false); err != nil {

+ t.Fatal(err)

+ }

+ return sw

+// sourceBox is a home box armed as armMovesPass arms one: armOffline, then

+// the hibernate coordinates, the S3 mirror, the production checkpoint and

+// the box's exec backend for the launch environment.

+func (sw *sealWorld) sourceBox(name string) *offBox {

+ b := sw.box(name, "", map[string]string{"CARLOS_HIBERNATE_BUCKET": sealIE, "CARLOS_SECRETS_KEY": sw.keyB64})

+ sw.arm(b)

+ return b

+func (sw *sealWorld) arm(b *offBox) {

+ c := &b.w.cfg

+ c.OwnBucket, c.OwnBucketRegion, c.OwnPrefix = sealIE, sealIERegion, sealPrefix

+ c.Mirror = func(ctx context.Context, dst string) (lease.MirrorStore, error) {

+ return lease.NewS3Mirror(ctx, sealIERegion, dst)

+ }

+ // armMovesPass's Checkpoint, as it is wired there.

+ c.Checkpoint = func(ctx context.Context, dbPath string) error {

+ host := strings.TrimSuffix(filepath.Base(dbPath), ".db")

+ rep, err := lease.ReplicateWithLease(ctx, dbPath, sealIE, activator.HibernatePrefix(sealPrefix, host), sealIERegion)

+ if err != nil {

+ return err

+ }

+ defer rep.Close(ctx)

+ return rep.SyncAndWait(ctx)

+ }

+ c.CarryRequest = func(ctx context.Context, host, bucket, prefix string) error {

+ sw.fake.Put(bucket, prefix+"/build-state.json", []byte(`{"request":"carried"}`))

+ return nil

+ }

+ c.LaunchLayers = (&activator.ExecBackend{DataDir: b.dd}).LaunchLayers

+ c.Page = func(_ context.Context, level, subject, body string) error {

+ sw.pages = append(sw.pages, level+": "+subject)

+ sw.bodies = append(sw.bodies, body)

+ return nil

+ }

+// pass is a tick in the agent's order (reconcile, the edges file, the moves

+// pass) whose moves pass is movesPass's offline part: the keep view's

+// refresh from the record list, then offlineRound, returning the round's

+// error rather than logging it.

+func (sw *sealWorld) pass(b *offBox) error {

+ sw.t.Helper()

+ b.reconcile()

+ b.edges()

+ ctx := context.Background()

+ refreshOfflineView(ctx, sw.s, b.w.cfg.Offline)

+ return b.w.offlineRound(ctx, sw.s, sw.record())

+func (sw *sealWorld) mustPass(b *offBox) {

+ sw.t.Helper()

+ if err := sw.pass(b); err != nil {

+ sw.t.Fatal(err)

+ }

+func (sw *sealWorld) dbPath(b *offBox) string {

+ sw.t.Helper()

+ rt := b.row(offHost)

+ if rt == nil {

+ sw.t.Fatal("no row")

+ }

+ return rt.DBPath

+// instance gives b a live instance of offHost, as a box that has served it

+// has: the row (minted ready by reconcile), its replication rung, next due

+// and timers, a database written through lease.ReplicateWithLease into

+// data-ie, the files directory, <host>.env with an instance key, the

+// rendered configuration and a live build.

+func (sw *sealWorld) instance(b *offBox) {

+ t := sw.t

+ t.Helper()

+ b.tick()

+ rt := b.row(offHost)

+ if rt == nil || rt.HoldOp != "" {

+ t.Fatalf("fixture: row %+v", rt)

+ }

+ if err := b.reg.SetReplication(offHost, registry.ReplicationBatched); err != nil {

+ t.Fatal(err)

+ }

+ if err := b.reg.SetNextDue(offHost, 1_800_000_000); err != nil {

+ t.Fatal(err)

+ }

+ for _, tm := range []registry.Timer{

+ {Host: offHost, Name: "nightly", Kind: "cron", Spec: "0 3 * * *", Path: "/jobs/nightly", NextAt: 1_800_000_100},

+ {Host: offHost, Name: "reminder", Kind: "once", Path: "/jobs/remind", NextAt: 1_800_000_200},

+ {Host: offHost, Name: "fired", Kind: "once", Path: "/jobs/fired", LastAt: 1_700_000_000, LastStatus: "200"},

+ } {

+ if err := b.reg.UpsertTimer(tm); err != nil {

+ t.Fatal(err)

+ }

+ writeFile(t, filepath.Join(b.dd, offHost, "instance.key"), "app key file")

+ writeFile(t, filepath.Join(b.dd, offHost, "uploads", "a.png"), "png bytes")

+ writeFile(t, filepath.Join(b.dd, offHost+".env"), "CARLOS_INSTANCE_KEY=the-instance-key\nCARLOS_ADMIN_TOKEN=token\n")

+ writeFile(t, filepath.Join(b.dd, offHost+".config.env"), "APP_COLOUR=green\n")

+ sw.writeRows(b, rt.DBPath, 0, 20, true)

+ versions := filepath.Join(b.opt, "versions", sealVersion, sw.acct, "app")

+ if err := os.MkdirAll(versions, 0o755); err != nil {

+ t.Fatal(err)

+ }

+ if err := os.MkdirAll(filepath.Dir(release.LiveLink(b.opt, offHost)), 0o755); err != nil {

+ t.Fatal(err)

+ }

+ if err := os.Symlink(versions, release.LiveLink(b.opt, offHost)); err != nil {

+ t.Fatal(err)

+ }

+// writeRows inserts rows [from, to) into the database at dbPath, then, with

+// replicate, syncs it to data-ie under the lease as a wake's stop does.

+func (sw *sealWorld) writeRows(b *offBox, dbPath string, from, to int, replicate bool) {

+ t := sw.t

+ t.Helper()

+ db, err := sql.Open("sqlite", dbPath)

+ if err != nil {

+ t.Fatal(err)

+ }

+ for _, q := range []string{`PRAGMA journal_mode=WAL`, `CREATE TABLE IF NOT EXISTS t (id INTEGER PRIMARY KEY, body TEXT)`} {

+ if _, err := db.Exec(q); err != nil {

+ t.Fatal(err)

+ }

+ for i := from; i < to; i++ {

+ if _, err := db.Exec(`INSERT INTO t (id, body) VALUES (?, ?)`, i, fmt.Sprintf("row-%d", i)); err != nil {

+ t.Fatal(err)

+ }

+ if err := db.Close(); err != nil {

+ t.Fatal(err)

+ }

+ if !replicate {

+ return

+ }

+ ctx := context.Background()

+ rep, err := lease.ReplicateWithLease(ctx, dbPath, sealIE, activator.HibernatePrefix(sealPrefix, offHost), sealIERegion)

+ if err != nil {

+ t.Fatal(err)

+ }

+ if err := rep.SyncAndWait(ctx); err != nil {

+ t.Fatal(err)

+ }

+ if err := rep.Close(ctx); err != nil {

+ t.Fatal(err)

+ }

+func countRows(t *testing.T, dbPath string) int {

+ t.Helper()

+ db, err := sql.Open("sqlite", dbPath)

+ if err != nil {

+ t.Fatal(err)

+ }

+ defer db.Close()

+ var n int

+ if err := db.QueryRow(`SELECT count(*) FROM t`).Scan(&n); err != nil {

+ t.Fatal(err)

+ }

+ return n

+// start moves offHost to the record's offline-stopping, ticks every source

+// box until it holds and reports, and names the sealer and the boxes as the

+// seal door does.

+func (sw *sealWorld) startAndSeal(sealer string, boxes ...*offBox) {

+ sw.t.Helper()

+ sw.start()

+ names := []string{}

+ for _, b := range boxes {

+ b.tick()

+ names = append(names, b.name)

+ }

+ sw.move(provision.OfflineSeal(offID, sealer, names))

+func (sw *sealWorld) mirror() lease.MirrorStore {

+ sw.t.Helper()

+ m, err := lease.NewS3Mirror(context.Background(), sealIERegion, sealUKRegion)

+ if err != nil {

+ sw.t.Fatal(err)

+ }

+ return m

+func (sw *sealWorld) list(bucket, prefix string) map[string]lease.MirrorObject {

+ sw.t.Helper()

+ objs, err := sw.mirror().List(context.Background(), bucket, prefix)

+ if err != nil {

+ sw.t.Fatal(err)

+ }

+ return objs

+func (sw *sealWorld) lock() (owner string, expires time.Time, generation int64) {

+ sw.t.Helper()

+ body, ok := sw.fake.Current(sealIE, sealPrefix+"/"+offHost+"/lock.json")

+ if !ok {

+ return "", time.Time{}, 0

+ }

+ var l litestream.Lease

+ if err := json.Unmarshal(body, &l); err != nil {

+ sw.t.Fatal(err)

+ }

+ return l.Owner, l.ExpiresAt, l.Generation

+func (sw *sealWorld) sealRecord() (moves.OfflineReport, []byte, bool) {

+ sw.t.Helper()

+ raw, _, err := sw.s.Get(context.Background(), moves.OfflineSealKey(offHost, offID))

+ if errors.Is(err, os.ErrNotExist) {

+ return moves.OfflineReport{}, nil, false

+ }

+ if err != nil {

+ sw.t.Fatal(err)

+ }

+ var r moves.OfflineReport

+ if err := json.Unmarshal(raw, &r); err != nil {

+ sw.t.Fatal(err)

+ }

+ return r, raw, true

+// acquireAsAWake is what every wake does first (ExecBackend.Acquire).

+func acquireAsAWake(ctx context.Context) (*lease.Held, error) {

+ return lease.Acquire(ctx, sealIE, activator.HibernatePrefix(sealPrefix, offHost), sealIERegion)

+func wantHeldByMove(t *testing.T, err error, what string) {

+ t.Helper()

+ var he *lease.HeldError

+ if !errors.As(err, &he) || he.Owner != moves.TombstoneOwner(offID) {

+ t.Fatalf("%s: %v, want a HeldError naming %s", what, err, moves.TombstoneOwner(offID))

+ }

+// openArchive fetches the sealed archive and its manifest from data-uk,

+// verifies it and extracts it into a fresh flat files directory and env

+// path, returning them.

+func (sw *sealWorld) openArchive(seal moves.OfflineReport) (files, env string) {

+ t := sw.t

+ t.Helper()

+ prefix := moves.OfflineArchivePrefix(offHost, offID)

+ sealed, ok := sw.fake.Current(sealUK, prefix+offlineArchiveSealed)

+ if !ok {

+ t.Fatal("no sealed archive")

+ }

+ rawManifest, ok := sw.fake.Current(sealUK, prefix+offlineArchiveManifest)

+ if !ok {

+ t.Fatal("no archive manifest")

+ }

+ var m archive.Manifest

+ if err := json.Unmarshal(rawManifest, &m); err != nil {

+ t.Fatal(err)

+ }

+ bind := appconfig.Binding{Account: sw.acct, App: "app"}

+ if err := archive.Verify(bytes.NewReader(sealed), m, sw.priv, bind); err != nil {

+ t.Fatalf("Verify: %v", err)

+ }

+ sum := sha256.Sum256(sealed)

+ if m.Plain != seal.FilesPlain || m.Sealed != seal.FilesSealed || "sha256:"+hex.EncodeToString(sum[:]) != seal.FilesSealed || m.Bytes != seal.FilesBytes || len(m.Entries) != seal.FilesEntries {

+ t.Fatalf("manifest %+v against the seal %+v", m, seal)

+ }

+ var plain bytes.Buffer

+ if err := archive.Open(bytes.NewReader(sealed), m.SealedDataKey, sw.priv, bind, &plain); err != nil {

+ t.Fatalf("Open: %v", err)

+ }

+ parent := t.TempDir()

+ files, env = filepath.Join(parent, offHost), filepath.Join(parent, offHost+".env")

+ // A *bytes.Reader, not the buffer: zstd decodes a *bytes.Buffer under

+ // 128 KiB from its Bytes() without draining it, and Extract's

+ // end-of-stream check then finds the whole archive still unread.

+ if err := archive.Extract(bytes.NewReader(plain.Bytes()), files, env); err != nil {

+ t.Fatalf("Extract: %v", err)

+ }

+ if got, _, _, err := archive.PlainDigest(files, env); err != nil || got != seal.FilesPlain {

+ t.Fatalf("the extraction digests to %s (%v), the seal says %s", got, err, seal.FilesPlain)

+ }

+ return files, env

+func readString(t *testing.T, p string) string {

+ t.Helper()

+ b, err := os.ReadFile(p)

+ if err != nil {

+ t.Fatal(err)

+ }

+ return string(b)

+// (a) The happy seal, end to end, and the passes after it.

+func TestOfflineSealHappyPath(t *testing.T) {

+ sw := newSealWorld(t)

+ ie := sw.ie

+ sw.instance(ie)

+ // A tail the stop never checkpointed: rows in the local database only.

+ sw.writeRows(ie, sw.dbPath(ie), 20, 25, false)

+ sw.startAndSeal("ie-a", ie)

+ ctx := context.Background()

+ sw.mustPass(ie)

+ rec := sw.record()

+ if rec.Move.State != provision.OfflineSealed {

+ t.Fatalf("state %s, want sealed", rec.Move.State)

+ }

+ seal, raw, ok := sw.sealRecord()

+ if !ok {

+ t.Fatal("no seal record")

+ }

+ if err := moves.ValidateOfflineSeal(seal, offHost, offID); err != nil {

+ t.Fatal(err)

+ }

+ src := rec.Move.Source

+ if src == nil || src.Evidence != moves.OfflineSealEvidence(offHost, offID, raw) || src.FinalTXID != seal.FinalTXID ||

+ src.Files == nil || src.Files.Plain != seal.FilesPlain || src.Files.Sealed != seal.FilesSealed || src.TimersSHA256 != seal.TimersDigest ||

+ src.Release != sealVersion || src.EnvDigest != seal.EnvDigest || src.Replication != registry.ReplicationBatched || src.NextDue != 1_800_000_000 {

+ t.Fatalf("record source %+v, seal %+v", src, seal)

+ }

+ // The tombstone, and a wake meets it.

+ owner, expires, _ := sw.lock()

+ if owner != moves.TombstoneOwner(offID) || time.Until(expires) < moves.TombstoneTTL-time.Minute {

+ t.Fatalf("lock %s until %s", owner, expires)

+ }

+ if h, err := acquireAsAWake(ctx); err == nil {

+ h.Release(ctx)

+ t.Fatal("a wake took the lease after the seal")

+ } else {

+ wantHeldByMove(t, err, "a wake after the seal")

+ }

+ // The staged copy is the replica, object for object, and its listing is

+ // the seal's.

+ srcObjs := sw.list(sealIE, sealPrefix+"/"+offHost+"/")

+ staged := sw.list(sealUK, activator.MovesPrefix(offHost))

+ if len(staged) != len(seal.Staged) || len(staged) != len(srcObjs)-1 {

+ t.Fatalf("staged %d objects, seal %d, source %d (lock.json included)", len(staged), len(seal.Staged), len(srcObjs))

+ }

+ for k, o := range staged {

+ rel := strings.TrimPrefix(k, activator.MovesPrefix(offHost))

+ if s := seal.Staged[rel]; s.Size != o.Size || s.ETag != o.ETag {

+ t.Fatalf("%s staged %+v, sealed %+v", rel, o, s)

+ }

+ if so := srcObjs[sealPrefix+"/"+offHost+"/"+rel]; so.Size != o.Size {

+ t.Fatalf("%s staged %+v, source %+v", rel, o, so)

+ }

+ if _, ok := seal.Staged["build-state.json"]; !ok {

+ t.Fatal("the carried build state is not staged")

+ }

+ if replicaPosition(staged, activator.MovesPrefix(offHost)) != seal.FinalTXID {

+ t.Fatal("the staged newest TXID is not the seal's")

+ }

+ // The staged replica restores to every row, the tail included: the

+ // checkpoint ran before the tombstone.

+ restored := filepath.Join(t.TempDir(), "restored.db")

+ if ok, err := lease.Restore(ctx, restored, sealUK, strings.TrimSuffix(activator.MovesPrefix(offHost), "/"), sealUKRegion); err != nil || !ok {

+ t.Fatalf("restore the staged copy: %v %v", ok, err)

+ }

+ if n := countRows(t, restored); n != 25 {

+ t.Fatalf("the staged copy holds %d rows, want 25", n)

+ }

+ // The archive opens, verifies and extracts to the same plaintext, with

+ // the files and the env byte for byte.

+ files, env := sw.openArchive(seal)

+ for rel, want := range map[string]string{"instance.key": "app key file", "uploads/a.png": "png bytes"} {

+ if got := readString(t, filepath.Join(files, rel)); got != want {

+ t.Fatalf("%s = %q", rel, got)

+ }

+ envBody := readString(t, filepath.Join(ie.dd, offHost+".env"))

+ if readString(t, env) != envBody {

+ t.Fatal("the env file did not travel byte for byte")

+ }

+ sum := sha256.Sum256([]byte(envBody))

+ if seal.EnvSHA256 != hex.EncodeToString(sum[:]) || seal.KeySHA256 != instancekey.Digest("the-instance-key") {

+ t.Fatalf("env %s key %s", seal.EnvSHA256, seal.KeySHA256)

+ }

+ // The timers snapshot (active rows only) and the env report.

+ prefix := moves.OfflineArchivePrefix(offHost, offID)

+ timersBody, _ := sw.fake.Current(sealUK, prefix+offlineTimersObject)

+ snap, err := moves.DecodeTimers(timersBody)

+ if err != nil || len(snap.Rows) != 2 || moves.TimersDigest(snap.Rows) != seal.TimersDigest {

+ t.Fatalf("timers snapshot %+v %v", snap, err)

+ }

+ reportBody, _ := sw.fake.Current(sealUK, prefix+offlineEnvReportObject)

+ var envReport moves.EnvReport

+ if err := json.Unmarshal(reportBody, &envReport); err != nil || envReport.Digest != seal.EnvDigest || envReport.Vars["APP_COLOUR"] == "" || strings.Contains(string(reportBody), "green") {

+ t.Fatalf("env report %s %v", reportBody, err)

+ }

+ if seal.Release != sealVersion || seal.Replication != registry.ReplicationBatched || seal.NextDue != 1_800_000_000 {

+ t.Fatalf("seal %+v", seal)

+ }

+ // The living report: the tombstone and the pass's time.

+ living := sw.report(moves.OfflineRoleSealer, "ie-a")

+ if living.Phase != moves.OfflinePhaseSealed || living.TombstoneOwner != moves.TombstoneOwner(offID) || !living.TombstoneUntil.Equal(expires) || !living.At.Equal(sw.clock()) {

+ t.Fatalf("living report %+v", living)

+ }

+ if ok, bad := living.SourceItemsOK(); !ok {

+ t.Fatalf("living report items %v", bad)

+ }

+ // Later passes re-assert the expiry, keep the tombstone's generation,

+ // never touch the seal record, and rewrite the living report.

+ _, _, gen := sw.lock()

+ for _, state := range []func(){func() {}, sw.flip, sw.landed} {

+ state()

+ time.Sleep(5 * time.Millisecond)

+ sw.advance(time.Second)

+ if err := ie.w.offlineSealerStep(ctx, sw.s, sw.record()); err != nil {

+ t.Fatal(err)

+ }

+ owner2, expires2, gen2 := sw.lock()

+ if owner2 != owner || !expires2.After(expires) || gen2 != gen {

+ t.Fatalf("after a later pass: %s until %s (generation %d), before %s until %s (generation %d)", owner2, expires2, gen2, owner, expires, gen)

+ }

+ expires = expires2

+ living = sw.report(moves.OfflineRoleSealer, "ie-a")

+ if !living.TombstoneUntil.Equal(expires2) || !living.At.Equal(sw.clock()) || living.Phase != moves.OfflinePhaseSealed {

+ t.Fatalf("living report after a later pass: %+v", living)

+ }

+ if _, again, _ := sw.sealRecord(); !bytes.Equal(again, raw) {

+ t.Fatal("the seal record was rewritten")

+ }

+ if len(sw.pages) != 0 {

+ t.Fatalf("paged: %v", sw.pages)

+ }

+// (b) A live writer's lease: with a local database the checkpoint cannot

+// take it, without one the tombstone cannot; either way the state stays

+// sealing and nothing is staged or sealed. A wake that races the tombstone

+// meets it.

+func TestOfflineSealWaitsForALiveWriter(t *testing.T) {

+ ctx := context.Background()

+ for _, local := range []bool{true, false} {

+ t.Run(fmt.Sprintf("local database %v", local), func(t *testing.T) {

+ sw := newSealWorld(t)

+ ie := sw.ie

+ sw.instance(ie)

+ if !local {

+ db := sw.dbPath(ie)

+ for _, p := range []string{db, db + "-wal", db + "-shm", activatorMetaDir(db)} {

+ if err := os.RemoveAll(p); err != nil {

+ t.Fatal(err)

+ }

+ sw.startAndSeal("ie-a", ie)

+ wake, err := acquireAsAWake(ctx)

+ if err != nil {

+ t.Fatal(err)

+ }

+ err = sw.pass(ie)

+ var he *lease.HeldError

+ if !errors.As(err, &he) || he.Owner == moves.TombstoneOwner(offID) {

+ t.Fatalf("a live writer: %v", err)

+ }

+ if !local && !strings.Contains(err.Error(), "tombstone") {

+ t.Fatalf("not refused at the tombstone: %v", err)

+ }

+ if sw.record().Move.State != provision.OfflineSealing {

+ t.Fatal("the state moved")

+ }

+ if _, _, ok := sw.sealRecord(); ok {

+ t.Fatal("sealed under a live writer")

+ }

+ if n := len(sw.list(sealUK, "moves/")); n != 0 {

+ t.Fatalf("%d objects staged under a live writer", n)

+ }

+ if err := wake.Err(); err != nil {

+ t.Fatalf("the writer lost its lease: %v", err)

+ }

+ if err := wake.Release(ctx); err != nil {

+ t.Fatal(err)

+ }

+ // The race: a wake that comes for the lease the moment after

+ // the tombstone is written is refused by it.

+ raced := false

+ offlineSealFaults.Store(ie.w, func(point string) error {

+ if point == "tombstone" {

+ raced = true

+ h, err := acquireAsAWake(ctx)

+ if err == nil {

+ h.Release(ctx)

+ return errors.New("a wake took the lease after the tombstone")

+ }

+ wantHeldByMove(t, err, "a wake racing the tombstone")

+ }

+ return nil

+ })

+ t.Cleanup(func() { offlineSealFaults.Delete(ie.w) })

+ sw.mustPass(ie)

+ if !raced || sw.record().Move.State != provision.OfflineSealed {

+ t.Fatalf("raced %v, state %s", raced, sw.record().Move.State)

+ }

+ })

+ }

+// (c) A source box whose local position is past the replica's final

+// position refuses the seal: the db-tail item fails, the report says

+// failed, nothing is sealed and the state stays sealing.

+func TestOfflineSealRefusesASourceAheadOfTheFinalPosition(t *testing.T) {

+ sw := newSealWorld(t)

+ ie := sw.ie

+ sw.instance(ie)

+ ieB := sw.sourceBox("ie-b")

+ ieB.tick()

+ final := replicaPosition(sw.list(sealIE, sealPrefix+"/"+offHost+"/"), sealPrefix+"/"+offHost+"/")

+ if final == 0 {

+ t.Fatal("fixture: no replica")

+ }

+ // ie-b woke the host once and kept a position past the replica's.

+ meta := filepath.Join(activatorMetaDir(provision.DefaultDBPath(ieB.dd, offHost)), "ltx", "0")

+ writeFile(t, filepath.Join(meta, ltxName(final+1, final+5)), "ltx")

+ sw.startAndSeal("ie-a", ie, ieB)

+ if r := sw.report(moves.OfflineRoleSource, "ie-b"); r.LocalTXID != final+5 {

+ t.Fatalf("fixture: ie-b reports %d", r.LocalTXID)

+ }

+ err := sw.pass(ie)

+ var ref *sealRefusal

+ if !errors.As(err, &ref) || ref.item != "db-tail" || !strings.Contains(ref.Reason, "ie-b") {

+ t.Fatalf("pass: %v", err)

+ }

+ r := sw.report(moves.OfflineRoleSealer, "ie-a")

+ if r.Phase != moves.OfflinePhaseFailed || r.Items["db-tail"].OK || !r.Items["sidecar"].OK {

+ t.Fatalf("report %+v", r)

+ }

+ if _, _, ok := sw.sealRecord(); ok || sw.record().Move.State != provision.OfflineSealing {

+ t.Fatal("sealed past a source ahead of the replica")

+ }

+func ltxName(min, max uint64) string {

+ return ltx.FormatFilename(ltx.TXID(min), ltx.TXID(max))

+// (d) After the seal the stop is kept, never taken again: a tombstone

+// stolen by another move, expired, lifted or deleted is a lost stop. The

+// pass reports failed with the lease item down, pages once for the move,

+// and leaves the lease exactly as it found it.

+func TestOfflineSealLostStopIsNeverRetaken(t *testing.T) {

+ sw := newSealWorld(t)

+ ie := sw.ie

+ sw.instance(ie)

+ sw.startAndSeal("ie-a", ie)

+ sw.mustPass(ie)

+ ctx := context.Background()

+ key := sealPrefix + "/" + offHost + "/lock.json"

+ ours, _ := sw.fake.Current(sealIE, key)

+ lockBody := func(owner string, expires time.Time) []byte {

+ b, err := json.Marshal(litestream.Lease{Generation: 1, Owner: owner, ExpiresAt: expires})

+ if err != nil {

+ t.Fatal(err)

+ }

+ return b

+ }

+ cases := []struct {

+ name string

+ damage func()

+ }{

+ {"stolen by another move", func() {

+ sw.fake.Put(sealIE, key, lockBody(moves.TombstoneOwner(offID2), time.Now().Add(time.Hour)))

+ }},

+ {"stolen by a wake", func() {

+ sw.fake.Put(sealIE, key, lockBody("box-x", time.Now().Add(time.Hour)))

+ }},

+ {"expired", func() {

+ sw.fake.Put(sealIE, key, lockBody(moves.TombstoneOwner(offID), time.Now().Add(-time.Second)))

+ }},

+ {"lifted", func() {

+ if _, err := lease.LiftTombstone(ctx, sealIE, sealPrefix+"/"+offHost, sealIERegion, func(o string) bool { return o == moves.TombstoneOwner(offID) }); err != nil {

+ t.Fatal(err)

+ }

+ }},

+ {"deleted", func() { sw.fake.BareDelete(sealIE, key) }},

+ }

+ for _, c := range cases {

+ // The stop holds: a pass is clean.

+ sw.fake.Put(sealIE, key, ours)

+ if err := ie.w.offlineSealerStep(ctx, sw.s, sw.record()); err != nil {

+ t.Fatalf("%s: before: %v", c.name, err)

+ }

+ c.damage()

+ before := sw.fake.Snapshot()

+ for i := range 2 {

+ err := ie.w.offlineSealerStep(ctx, sw.s, sw.record())

+ if err == nil || !strings.Contains(err.Error(), "lost stop") {

+ t.Fatalf("%s, pass %d: %v", c.name, i, err)

+ }

+ if sw.fake.Snapshot() != before {

+ t.Fatalf("%s, pass %d: the lease was written after the stop was lost", c.name, i)

+ }

+ r := sw.report(moves.OfflineRoleSealer, "ie-a")

+ if r.Phase != moves.OfflinePhaseFailed || r.Items["lease"].OK || !strings.Contains(r.Items["lease"].Reason, "lost stop") || r.TombstoneOwner != "" || !r.Items["files"].OK {

+ t.Fatalf("%s: report %+v", c.name, r)

+ }

+ if len(sw.pages) != 1 || !strings.HasPrefix(sw.pages[0], "incident: ") {

+ t.Fatalf("pages %v, want one incident", sw.pages)

+ }

+ if sw.record().Move.State != provision.OfflineSealed {

+ t.Fatal("the record moved")

+ }

+// (e) A crash after each step, then the next pass: one tombstone (one

+// generation, created once), staged objects copied once and identical, the

+// archive the seal names, and a sealed record.

+func TestOfflineSealReplaysAfterACrashAtEachStep(t *testing.T) {

+ sw := newSealWorld(t)

+ ie := sw.ie

+ sw.instance(ie)

+ sw.writeRows(ie, sw.dbPath(ie), 20, 23, false)

+ sw.startAndSeal("ie-a", ie)

+ ctx := context.Background()

+ points := []string{"checkpoint", "tombstone", "final", "mirror", "archive", "timers", "env", "seal", "cas"}

+ t.Cleanup(func() { offlineSealFaults.Delete(ie.w) })

+ var gen int64

+ var stagedAfter map[string]lease.MirrorObject

+ for i, crash := range points {

+ hit := false

+ offlineSealFaults.Store(ie.w, func(point string) error {

+ if point == crash {

+ hit = true

+ return errors.New("crash at " + point)

+ }

+ return nil

+ })

+ err := sw.pass(ie)

+ if !hit || err == nil || !strings.Contains(err.Error(), "crash at "+crash) {

+ t.Fatalf("crash at %s: hit %v, %v", crash, hit, err)

+ }

+ if i >= 1 { // the tombstone is down from the crash after it on

+ owner, _, g := sw.lock()

+ if owner != moves.TombstoneOwner(offID) || (gen != 0 && g != gen) {

+ t.Fatalf("after the crash at %s: lock %s generation %d (was %d)", crash, owner, g, gen)

+ }

+ gen = g

+ }

+ if crash == "mirror" {

+ stagedAfter = sw.list(sealUK, activator.MovesPrefix(offHost))

+ }

+ wantState := provision.OfflineSealing

+ if crash == "cas" {

+ wantState = provision.OfflineSealed

+ }

+ if s := sw.record().Move.State; s != wantState {

+ t.Fatalf("after the crash at %s: state %s", crash, s)

+ }

+ offlineSealFaults.Delete(ie.w)

+ sw.mustPass(ie)

+ if sw.record().Move.State != provision.OfflineSealed {

+ t.Fatal("not sealed after the replays")

+ }

+ if r := sw.report(moves.OfflineRoleSealer, "ie-a"); r.Phase != moves.OfflinePhaseSealed || r.TombstoneOwner != moves.TombstoneOwner(offID) {

+ t.Fatalf("living report %+v", r)

+ }

+ // One tombstone: created once (one version with the move's owner and

+ // no expiry change before the seal), one generation throughout.

+ if _, _, g := sw.lock(); g != gen {

+ t.Fatalf("generation %d, was %d", g, gen)

+ }

+ versions, err := sw.fake.ListVersions(ctx, sealIE, sealPrefix+"/"+offHost+"/lock.json")

+ if err != nil {

+ t.Fatal(err)

+ }

+ created := 0

+ for _, v := range versions {

+ if v.DeleteMarker {

+ continue

+ }

+ body, err := sw.fake.GetVersion(ctx, sealIE, v.Key, v.VersionID)

+ if err != nil {

+ t.Fatal(err)

+ }

+ var l litestream.Lease

+ if err := json.Unmarshal(body, &l); err == nil && l.Owner == moves.TombstoneOwner(offID) && l.Generation != gen {

+ t.Fatalf("a tombstone of another generation %d", l.Generation)

+ }

+ if l.Owner == moves.TombstoneOwner(offID) {

+ created++

+ }

+ // The tombstone is written once before the seal (the replays after it

+ // find it this move's and write nothing); after the seal the re-asserts

+ // of the last two passes rewrite it, same generation.

+ if created != 3 {

+ t.Fatalf("%d versions of the tombstone", created)

+ }

+ // The staged objects: identical to those after the first mirror round,

+ // and each copied once.

+ final := sw.list(sealUK, activator.MovesPrefix(offHost))

+ if len(final) != len(stagedAfter) {

+ t.Fatalf("staged %d objects, %d after the first round", len(final), len(stagedAfter))

+ }

+ for k, o := range final {

+ if stagedAfter[k] != o {

+ t.Fatalf("%s changed: %+v, was %+v", k, o, stagedAfter[k])

+ }

+ vs, err := sw.fake.ListVersions(ctx, sealUK, k)

+ if err != nil || len(vs) != 1 {

+ t.Fatalf("%s has %d versions (%v)", k, len(vs), err)

+ }

+ seal, _, ok := sw.sealRecord()

+ if !ok {

+ t.Fatal("no seal record")

+ }

+ sw.openArchive(seal)

+ if n := len(seal.Staged); n != len(final) {

+ t.Fatalf("seal names %d staged objects, %d staged", n, len(final))

+ }

+// (f) The revert: the sealer lifts this move's tombstone, deletes exactly

+// the staging lane and the archive prefix in the target's bucket, clears

+// the receipts and lifts the fence; a wake can take the lease again. A

+// lease that is not this move's is left alone. Every part is an ensure.

+func TestOfflineSealRevertLiftsAndUnfences(t *testing.T) {

+ sw := newSealWorld(t)

+ ie := sw.ie

+ sw.instance(ie)

+ sw.startAndSeal("ie-a", ie)

+ sw.mustPass(ie)

+ ctx := context.Background()

+ // Neighbours the revert must not touch.

+ sw.fake.Put(sealUK, "moves/other.example.com/0000/x.ltx", []byte("x"))

+ sw.fake.Put(sealUK, "moves/@offline/other.example.com/"+offID2+"/files.sealed", []byte("x"))

+ sw.fake.Put(sealUK, sealPrefix+"/"+offHost+"/0000/y.ltx", []byte("y"))

+ // Copy receipts, as a round over a multipart source leaves them.

+ sw.fake.Put(sealIE, activator.MoveReceiptsKey(offHost), []byte(`{"v":1}`))

+ sw.flip()

+ sw.abort()

+ if sw.record().Move.State != provision.OfflineReverting {

+ t.Fatal("fixture: not reverting")

+ }

+ if !ie.svc.Fenced(offHost) {

+ t.Fatal("fixture: not fenced")

+ }

+ // The sealer's own step lifts the fence too (the source step does it

+ // as well, earlier in the same round).

+ if err := ie.w.offlineSealerStep(ctx, sw.s, sw.record()); err != nil {

+ t.Fatal(err)

+ }

+ if ie.svc.Fenced(offHost) {

+ t.Fatal("still fenced after the revert")

+ }

+ owner, expires, _ := sw.lock()

+ if !lease.IsLiftedTombstone(owner, expires) || owner != moves.TombstoneOwner(offID) {

+ t.Fatalf("lock %s %s, want this move's lifted tombstone", owner, expires)

+ }

+ for _, prefix := range []string{activator.MovesPrefix(offHost), moves.OfflineArchivePrefix(offHost, offID)} {

+ if n := len(sw.list(sealUK, prefix)); n != 0 {

+ t.Fatalf("%s holds %d objects", prefix, n)

+ }

+ for _, k := range []string{"moves/other.example.com/0000/x.ltx", "moves/@offline/other.example.com/" + offID2 + "/files.sealed", sealPrefix + "/" + offHost + "/0000/y.ltx"} {

+ if _, ok := sw.fake.Current(sealUK, k); !ok {

+ t.Fatalf("the revert deleted %s", k)

+ }

+ if _, ok := sw.fake.Current(sealIE, activator.MoveReceiptsKey(offHost)); ok {

+ t.Fatal("the receipts are still there")

+ }

+ if r := sw.report(moves.OfflineRoleSealer, "ie-a"); r.Phase != moves.OfflinePhaseReverted {

+ t.Fatalf("report %+v", r)

+ }

+ // Again: nothing changes, and the lifted tombstone is not taken again.

+ before := sw.fake.Snapshot()

+ sw.mustPass(ie)

+ if sw.fake.Snapshot() != before {

+ t.Fatal("a second revert pass wrote to the buckets")

+ }

+ wake, err := acquireAsAWake(ctx)

+ if err != nil {

+ t.Fatalf("a wake after the revert: %v", err)

+ }

+ // A lease that is not this move's (a live wake now) is left alone.

+ if err := ie.w.offlineSealerStep(ctx, sw.s, sw.record()); err != nil {

+ t.Fatal(err)

+ }

+ if err := wake.Err(); err != nil {

+ t.Fatalf("the revert disturbed a wake's lease: %v", err)

+ }

+ wake.Release(ctx)

+ // Another move's tombstone is never lifted by this one.

+ other, err := json.Marshal(litestream.Lease{Generation: 9, Owner: moves.TombstoneOwner(offID2), ExpiresAt: time.Now().Add(time.Hour)})

+ if err != nil {

+ t.Fatal(err)

+ }

+ sw.fake.Put(sealIE, sealPrefix+"/"+offHost+"/lock.json", other)

+ if err := ie.w.offlineSealerStep(ctx, sw.s, sw.record()); err != nil {

+ t.Fatal(err)

+ }

+ if owner, expires, _ := sw.lock(); owner != moves.TombstoneOwner(offID2) || !time.Now().Before(expires) {

+ t.Fatalf("another move's tombstone was lifted: %s %s", owner, expires)

+ }

+// The revert before anything was sealed (abort while sealing failed on a

+// refused item) lifts the tombstone the failed seal took, and a revert

+// with no tombstone at all is a clean pass.

+func TestOfflineSealRevertBeforeTheSeal(t *testing.T) {

+ sw := newSealWorld(t)

+ ie := sw.ie

+ sw.instance(ie)

+ writeFile(t, filepath.Join(ie.dd, offHost, "big"), "")

+ if err := os.Truncate(filepath.Join(ie.dd, offHost, "big"), archive.Cap+1); err != nil {

+ t.Fatal(err)

+ }

+ sw.startAndSeal("ie-a", ie)

+ if err := sw.pass(ie); err == nil {

+ t.Fatal("sealed over the cap")

+ }

+ if owner, _, _ := sw.lock(); owner != moves.TombstoneOwner(offID) {

+ t.Fatal("fixture: no tombstone")

+ }

+ sw.abort()

+ sw.mustPass(ie)

+ if owner, expires, _ := sw.lock(); !lease.IsLiftedTombstone(owner, expires) {

+ t.Fatalf("lock %s %s after the revert", owner, expires)

+ }

+ if n := len(sw.list(sealUK, "moves/")); n != 0 {

+ t.Fatalf("%d objects left in the staging prefixes", n)

+ }

+ sw.mustPass(ie)

+// The files item: a file over the cap, a FIFO, and an entry of the host's

+// beside its files directory are failed items (the report says which),

+// never a partial archive and never a seal; once the file is gone the next

+// pass seals.

+func TestOfflineSealFailsAFileOverTheCapAndAFIFO(t *testing.T) {

+ sw := newSealWorld(t)

+ ie := sw.ie

+ sw.instance(ie)

+ sw.startAndSeal("ie-a", ie)

+ files := filepath.Join(ie.dd, offHost)

+ big := filepath.Join(files, "big.bin")

+ writeFile(t, big, "")

+ if err := os.Truncate(big, archive.Cap+1); err != nil {

+ t.Fatal(err)

+ }

+ fifo := filepath.Join(files, "pipe")

+ blobs := filepath.Join(ie.dd, offHost+"-blobs")

+ cases := []struct {

+ name string

+ setup func()

+ want string

+ }{

+ {"over the cap", func() {}, strings.TrimPrefix(archive.ErrOverCap.Error(), "archive: ")},

+ {"a FIFO", func() {

+ if err := os.Remove(big); err != nil {

+ t.Fatal(err)

+ }

+ if err := syscall.Mkfifo(fifo, 0o644); err != nil {

+ t.Fatal(err)

+ }

+ }, strings.TrimPrefix(archive.ErrSpecialFile.Error(), "archive: ") + ": pipe"},

+ // The flat layout's rules give <host>-blobs to the host, and the

+ // archive carries the files directory alone.

+ {"an entry beside the files directory", func() {

+ if err := os.Remove(fifo); err != nil {

+ t.Fatal(err)

+ }

+ writeFile(t, filepath.Join(blobs, "b"), "blob")

+ }, blobs},

+ }

+ for _, c := range cases {

+ c.setup()

+ err := sw.pass(ie)

+ var ref *sealRefusal

+ if !errors.As(err, &ref) || ref.item != "files" || !strings.Contains(ref.Reason, c.want) {

+ t.Fatalf("%s: %v", c.name, err)

+ }

+ r := sw.report(moves.OfflineRoleSealer, "ie-a")

+ if r.Phase != moves.OfflinePhaseFailed || r.Items["files"].OK || !r.Items["db"].OK {

+ t.Fatalf("%s: report %+v", c.name, r)

+ }

+ if _, _, ok := sw.sealRecord(); ok || sw.record().Move.State != provision.OfflineSealing {

+ t.Fatalf("%s: sealed", c.name)

+ }

+ if n := len(sw.list(sealUK, moves.OfflineArchivePrefix(offHost, offID))); n != 0 {

+ t.Fatalf("%s: %d archive objects written", c.name, n)

+ }

+ if err := os.RemoveAll(blobs); err != nil {

+ t.Fatal(err)

+ }

+ sw.mustPass(ie)

+ if sw.record().Move.State != provision.OfflineSealed {

+ t.Fatal("not sealed once the files were fixed")

+ }

+ if r := sw.report(moves.OfflineRoleSealer, "ie-a"); r.Phase != moves.OfflinePhaseSealed || !r.Items["files"].OK {

+ t.Fatalf("report %+v", r)

+ }

+// A nested instance, made nested by the real layout pass, which takes the

+// flat files directory <data>/<host> into the instance directory as

+// <instance dir>/<host>, where the app finds it beside its database. The

+// archive is read through the raw instance directory and carries that

+// files directory and the env file, never the database, its litestream

+// state, the rendered configuration or any other platform entry; it

+// extracts flat with the app's files at the flat files directory's root,

+// where the flat app looks. An entry beside the files directory that the

+// flat layout cannot carry refuses the files item until it is gone.

+func TestOfflineSealArchivesANestedInstanceFlat(t *testing.T) {

+ sw := newSealWorld(t)

+ ie := sw.ie

+ sw.instance(ie)

+ raw := sw.nest(ie)

+ // More platform entries, which stay behind: the generation and files

+ // markers, the adoption record, the seed marker of a seeded database,

+ // and the launch descriptors (fixed and per generation) as a real

+ // nested host keeps them.

+ for _, name := range []string{offHost + ".db.gen", ".files-gen", offHost + instancekey.AdoptionSuffix, offHost + ".db" + activator.SeedMarkerSuffix,

+ "launch.run.json", "launch.seed.json", "launch.gen.7.json"} {

+ writeFile(t, filepath.Join(raw, name), "platform")

+ }

+ // An entry of the app's beside its files directory, which no flat

+ // files directory can hold.

+ stray := filepath.Join(raw, offHost+"-blobs", "b")

+ writeFile(t, stray, "blob")

+ sw.startAndSeal("ie-a", ie)

+ err := sw.pass(ie)

+ var ref *sealRefusal

+ if !errors.As(err, &ref) || ref.item != "files" || !strings.Contains(ref.Reason, offHost+"-blobs") {

+ t.Fatalf("a stray entry: %v", err)

+ }

+ // The refusal says the move cannot carry it, and offers no fix that

+ // moves or removes the app's data (review I3).

+ if !strings.Contains(ref.Reason, "this move can carry only the files directory. see the offline move runbook") || strings.Contains(ref.Reason, "move it") || strings.Contains(ref.Reason, "remove") {

+ t.Fatalf("the refusal: %q", ref.Reason)

+ }

+ for _, name := range []string{"launch.run.json", "launch.gen.7.json", ".db.seed"} {

+ if strings.Contains(ref.Reason, name) {

+ t.Fatalf("the refusal names the platform's %s: %q", name, ref.Reason)

+ }

+ if r := sw.report(moves.OfflineRoleSealer, "ie-a"); r.Phase != moves.OfflinePhaseFailed || r.Items["files"].OK {

+ t.Fatalf("report %+v", r)

+ }

+ if err := os.RemoveAll(filepath.Dir(stray)); err != nil {

+ t.Fatal(err)

+ }

+ sw.mustPass(ie)

+ if sw.record().Move.State != provision.OfflineSealed {

+ t.Fatal("not sealed")

+ }

+ seal, _, _ := sw.sealRecord()

+ files, env := sw.openArchive(seal)

+ got := map[string]bool{}

+ if err := filepath.Walk(files, func(p string, info os.FileInfo, err error) error {

+ if err != nil {

+ return err

+ }

+ if !info.IsDir() {

+ rel, _ := filepath.Rel(files, p)

+ got[rel] = true

+ }

+ return nil

+ }); err != nil {

+ t.Fatal(err)

+ }

+ if len(got) != 2 || !got["instance.key"] || !got[filepath.Join("uploads", "a.png")] {

+ t.Fatalf("the archive carried %v, want the app's two files", got)

+ }

+ if readString(t, env) != readString(t, filepath.Join(raw, offHost+".env")) {

+ t.Fatal("the env file did not travel byte for byte")

+ }

+ if seal.KeySHA256 != instancekey.Digest("the-instance-key") {

+ t.Fatal("the key digest")

+ }

+// nest makes b's instance of offHost nested with the real layout pass,

+// which takes the flat files directory <data>/<host> into the instance

+// directory as <instance dir>/<host>, and returns the raw instance

+// directory.

+func (sw *sealWorld) nest(ie *offBox) string {

+ t := sw.t

+ t.Helper()

+ ctx := context.Background()

+ // The bind mount at <dd>/instances, as a link in the fixture.

+ if err := os.MkdirAll(filepath.Join(ie.dd, ".instances"), 0o755); err != nil {

+ t.Fatal(err)

+ }

+ if err := os.Symlink(".instances", filepath.Join(ie.dd, "instances")); err != nil {

+ t.Fatal(err)

+ }

+ // The layout pass works on the row's own paths; the sockets live in a

+ // short directory of the test's, as the pass's tests keep them.

+ sd, err := os.MkdirTemp("", "ly")

+ if err != nil {

+ t.Fatal(err)

+ }

+ t.Cleanup(func() { os.RemoveAll(sd) })

+ rt := ie.row(offHost)

+ if err := ie.reg.SetInstancePaths(offHost, rt.DBPath, rt.Socket, rt.DBPath, filepath.Join(sd, offHost+".sock")); err != nil {

+ t.Fatal(err)

+ }

+ pass := &layout.Pass{Registry: ie.reg, DataDir: ie.dd, SocketDir: sd, SocketsGID: -1, Now: time.Now, SweptAtStart: true, Logf: t.Logf}

+ res := pass.Converge(ctx, layout.ModeNested)

+ migrated := false

+ for _, r := range res {

+ if r.Host == offHost && r.Outcome == "migrated" {

+ migrated = true

+ }

+ if !migrated {

+ t.Fatalf("the layout pass: %+v", res)

+ }

+ ie.nested = true

+ raw := layout.RawInstanceDir(ie.dd, offHost)

+ if rt := ie.row(offHost); !provision.NestedPaths(ie.dd, *rt) {

+ t.Fatalf("fixture: row %+v is not nested", rt)

+ }

+ if lexists(filepath.Join(ie.dd, offHost)) {

+ t.Fatal("fixture: the layout pass left the flat files directory")

+ }

+ for _, name := range []string{offHost + ".db", offHost + ".env", offHost + ".config.env", ".activator-" + offHost + ".db-litestream", filepath.Join(offHost, "instance.key")} {

+ if !lexists(filepath.Join(raw, name)) {

+ t.Fatalf("fixture: the layout pass did not move %s into the instance directory", name)

+ }

+ return raw

+// Finishing lifts the tombstone only once every box the seal named reports

+// gone: until then each pass keeps the stop. After the lift a wake takes

+// the lease, the report says lifted, and a later pass changes nothing.

+func TestOfflineSealFinishingLiftsOnlyAfterEverySourceIsGone(t *testing.T) {

+ sw := newSealWorld(t)

+ ie := sw.ie

+ sw.instance(ie)

+ ieB := sw.sourceBox("ie-b")

+ sw.startAndSeal("ie-a", ie, ieB)

+ sw.mustPass(ie)

+ ieB.tick()

+ if sw.record().Move.State != provision.OfflineSealed {

+ t.Fatal("not sealed")

+ }

+ sw.flip()

+ sw.landed()

+ sw.open()

+ sw.mustPass(ie)

+ sw.mustPass(ieB)

+ for _, b := range []string{"ie-a", "ie-b"} {

+ if r := sw.report(moves.OfflineRoleSource, b); r.Phase != moves.OfflinePhaseParked {

+ t.Fatalf("%s: %+v", b, r)

+ }

+ sw.finishing()

+ ctx := context.Background()

+ // ie-a goes; ie-b has not run a pass since.

+ for range 3 {

+ sw.mustPass(ie)

+ }

+ if r := sw.report(moves.OfflineRoleSource, "ie-a"); r.Phase != moves.OfflinePhaseGone {

+ t.Fatalf("ie-a: %+v", r)

+ }

+ owner, _, _ := sw.lock()

+ if owner != moves.TombstoneOwner(offID) {

+ t.Fatal("fixture: no tombstone")

+ }

+ if _, err := acquireAsAWake(ctx); err == nil {

+ t.Fatal("the tombstone was lifted before ie-b was gone")

+ }

+ if r := sw.report(moves.OfflineRoleSealer, "ie-a"); r.Phase != moves.OfflinePhaseSealed || r.TombstoneOwner != moves.TombstoneOwner(offID) {

+ t.Fatalf("sealer report while waiting: %+v", r)

+ }

+ for range 3 {

+ sw.mustPass(ieB)

+ }

+ if r := sw.report(moves.OfflineRoleSource, "ie-b"); r.Phase != moves.OfflinePhaseGone {

+ t.Fatalf("ie-b: %+v", r)

+ }

+ sw.mustPass(ie)

+ if r := sw.report(moves.OfflineRoleSealer, "ie-a"); r.Phase != moves.OfflinePhaseLifted {

+ t.Fatalf("sealer report: %+v", r)

+ }

+ owner, expires, _ := sw.lock()

+ if !lease.IsLiftedTombstone(owner, expires) || owner != moves.TombstoneOwner(offID) {

+ t.Fatalf("lock %s %s", owner, expires)

+ }

+ before := sw.fake.Snapshot()

+ sw.mustPass(ie)

+ if sw.fake.Snapshot() != before {

+ t.Fatal("a pass after the lift wrote to the buckets")

+ }

+ wake, err := acquireAsAWake(ctx)

+ if err != nil {

+ t.Fatalf("a wake after the lift: %v", err)

+ }

+ wake.Release(ctx)

+ if len(sw.pages) != 0 {

+ t.Fatalf("paged: %v", sw.pages)

+ }

+// A host that never kept a files directory (seen on real nested hosts: the

+// instance directory holds the database, the env files and the launch

+// descriptors, and no <host>) seals with an empty files item: the archive

+// carries the env file alone, FilesPlain is archive.PlainDigest of no files

+// directory and that env file (FilesEntries 1), and it extracts to an

+// empty files directory beside the env file. Nested and flat alike.

+func TestOfflineSealAHostWithNoFilesDirectory(t *testing.T) {

+ for _, nested := range []bool{true, false} {

+ t.Run(map[bool]string{true: "nested", false: "flat"}[nested], func(t *testing.T) {

+ sw := newSealWorld(t)

+ ie := sw.ie

+ sw.instance(ie)

+ files := filepath.Join(ie.dd, offHost)

+ envPath := filepath.Join(ie.dd, offHost+".env")

+ if nested {

+ raw := sw.nest(ie)

+ files, envPath = filepath.Join(raw, offHost), filepath.Join(raw, offHost+".env")

+ writeFile(t, filepath.Join(raw, "launch.run.json"), "{}")

+ }

+ if err := os.RemoveAll(files); err != nil {

+ t.Fatal(err)

+ }

+ sw.startAndSeal("ie-a", ie)

+ sw.mustPass(ie)

+ if sw.record().Move.State != provision.OfflineSealed {

+ t.Fatal("not sealed")

+ }

+ seal, _, _ := sw.sealRecord()

+ want, wantBytes, wantEntries, err := archive.PlainDigest(files, envPath)

+ if err != nil {

+ t.Fatal(err)

+ }

+ if seal.FilesPlain != want || seal.FilesBytes != wantBytes || seal.FilesEntries != wantEntries || wantEntries != 1 {

+ t.Fatalf("seal files %s %d bytes %d entries, want %s %d bytes 1 entry (env only)", seal.FilesPlain, seal.FilesBytes, seal.FilesEntries, want, wantBytes)

+ }

+ if r := sw.report(moves.OfflineRoleSealer, "ie-a"); r.Phase != moves.OfflinePhaseSealed || !r.Items["files"].OK {

+ t.Fatalf("report %+v", r)

+ }

+ got, env := sw.openArchive(seal)

+ ents, err := os.ReadDir(got)

+ if err != nil || len(ents) != 0 {

+ t.Fatalf("the landing's files directory: %v entries, %v", len(ents), err)

+ }

+ if readString(t, env) != readString(t, envPath) {

+ t.Fatal("the env file did not travel byte for byte")

+ }

+ if lexists(files) {

+ t.Fatal("the seal made a files directory on the source")

+ }

+ })

+ }

+// The tombstone goes between the sealer's LockState and Reassert's own

+// read: Reassert reports it lost (lease.ErrTombstoneLost) and writes

+// nothing, so the sealer pages a lost stop once and never takes it again.

+func TestOfflineSealLostBetweenTheReadAndTheReassert(t *testing.T) {

+ sw := newSealWorld(t)

+ ie := sw.ie

+ sw.instance(ie)

+ sw.startAndSeal("ie-a", ie)

+ sw.mustPass(ie)

+ ctx := context.Background()

+ key := sealPrefix + "/" + offHost + "/lock.json"

+ var after string

+ t.Cleanup(func() { offlineSealFaults.Delete(ie.w) })

+ offlineSealFaults.Store(ie.w, func(point string) error {

+ if point == "reassert" {

+ sw.fake.BareDelete(sealIE, key)

+ after = sw.fake.Snapshot()

+ }

+ return nil

+ })

+ err := ie.w.offlineSealerStep(ctx, sw.s, sw.record())

+ if err == nil || !strings.Contains(err.Error(), "lost stop") {

+ t.Fatalf("pass: %v", err)

+ }

+ if after == "" || sw.fake.Snapshot() != after {

+ t.Fatal("the bucket was written after the tombstone went")

+ }

+ if _, ok := sw.fake.Current(sealIE, key); ok {

+ t.Fatal("the tombstone was taken again")

+ }

+ r := sw.report(moves.OfflineRoleSealer, "ie-a")

+ if r.Phase != moves.OfflinePhaseFailed || r.Items["lease"].OK || r.Items["lease"].Reason != "lost stop: "+offlineLeaseCause(tombState{}) {

+ t.Fatalf("report %+v", r)

+ }

+ // The next pass finds it gone at LockState: still nothing written,

+ // still one page.

+ offlineSealFaults.Delete(ie.w)

+ if err := ie.w.offlineSealerStep(ctx, sw.s, sw.record()); err == nil || !strings.Contains(err.Error(), "lost stop") {

+ t.Fatalf("second pass: %v", err)

+ }

+ if sw.fake.Snapshot() != after {

+ t.Fatal("the second pass wrote to the bucket")

+ }

+ if len(sw.pages) != 1 {

+ t.Fatalf("pages %v, want one", sw.pages)

+ }

+// The lost stop's page says abort only while abort can still put the

+// instance back. From open on, abort is refused, so it points to the

+// status and the runbook instead. Each state in its own world: the page

+// goes out once per move.

+func TestOfflineSealLostStopPageByState(t *testing.T) {

+ for _, state := range []string{provision.OfflineSealed, provision.OfflineOpen, provision.OfflineFinishing} {

+ t.Run(state, func(t *testing.T) {

+ sw := newSealWorld(t)

+ ie := sw.ie

+ sw.instance(ie)

+ sw.startAndSeal("ie-a", ie)

+ sw.mustPass(ie)

+ ctx := context.Background()

+ if state != provision.OfflineSealed {

+ sw.flip()

+ sw.landed()

+ sw.open()

+ }

+ if state == provision.OfflineFinishing {

+ sw.finishing()

+ }

+ if err := ie.w.offlineSealerStep(ctx, sw.s, sw.record()); err != nil {

+ t.Fatalf("the stop held: %v", err)

+ }

+ if got := sw.record().Move.State; got != state {

+ t.Fatalf("fixture: state %s", got)

+ }

+ sw.fake.BareDelete(sealIE, sealPrefix+"/"+offHost+"/lock.json")

+ if err := ie.w.offlineSealerStep(ctx, sw.s, sw.record()); err == nil || !strings.Contains(err.Error(), "lost stop") {

+ t.Fatalf("lost: %v", err)

+ }

+ if len(sw.pages) != 1 || sw.pages[0] != "incident: offline move of "+offHost+" lost its stop" {

+ t.Fatalf("pages %v", sw.pages)

+ }

+ body := sw.bodies[0]

+ if !strings.HasPrefix(body, "The offline move "+offID+" of "+offHost+" is "+state+", and its stop no longer holds: the source's lease is absent, not this move's tombstone. ") {

+ t.Fatalf("body: %s", body)

+ }

+ abort := strings.HasSuffix(body, "The move cannot go on. Read carlos move offline status --host "+offHost+" and abort the move.")

+ runbook := strings.HasSuffix(body, "It cannot be aborted once it is open. Read carlos move offline status --host "+offHost+" and follow the offline move runbook.")

+ if wantAbort := state == provision.OfflineSealed; abort != wantAbort || runbook == wantAbort {

+ t.Fatalf("body in %s: %s", state, body)

+ }

+ })

+ }

+// The backstop behind the holds and the fence (review M2): a source report

+// written since the seal at a position past the sealed one is a lost stop

+// in the db-tail item, even with the tombstone held. Nothing is written to

+// the lease, and the page names the box.

+func TestOfflineSealLostStopOnASourcePastTheSealedPosition(t *testing.T) {

+ sw := newSealWorld(t)

+ ie := sw.ie

+ sw.instance(ie)

+ ieB := sw.sourceBox("ie-b")

+ sw.startAndSeal("ie-a", ie, ieB)

+ sw.mustPass(ie)

+ if sw.record().Move.State != provision.OfflineSealed {

+ t.Fatal("not sealed")

+ }

+ seal, _, _ := sw.sealRecord()

+ ctx := context.Background()

+ // A pass with every report at or below the seal keeps the stop.

+ ieB.tick()

+ if err := ie.w.offlineSealerStep(ctx, sw.s, sw.record()); err != nil {

+ t.Fatalf("at the seal: %v", err)

+ }

+ // ie-b writes past the sealed position, and reports it.

+ meta := filepath.Join(activatorMetaDir(provision.DefaultDBPath(ieB.dd, offHost)), "ltx", "0")

+ writeFile(t, filepath.Join(meta, ltxName(seal.FinalTXID+1, seal.FinalTXID+3)), "ltx")

+ ieB.tick()

+ if r := sw.report(moves.OfflineRoleSource, "ie-b"); r.LocalTXID != seal.FinalTXID+3 || r.At.Before(seal.At) {

+ t.Fatalf("fixture: ie-b reports %d at %s", r.LocalTXID, r.At)

+ }

+ before := sw.fake.Snapshot()

+ err := ie.w.offlineSealerStep(ctx, sw.s, sw.record())

+ if err == nil || !strings.Contains(err.Error(), "lost stop") {

+ t.Fatalf("past the seal: %v", err)

+ }

+ if sw.fake.Snapshot() != before {

+ t.Fatal("the lease was written after the stop was lost")

+ }

+ cause := offlinePastSealCause("ie-b", seal.FinalTXID+3, seal.FinalTXID)

+ r := sw.report(moves.OfflineRoleSealer, "ie-a")

+ if r.Phase != moves.OfflinePhaseFailed || r.Items["db-tail"].OK || r.Items["db-tail"].Reason != "lost stop: "+cause || r.TombstoneOwner != "" || !r.Items["lease"].OK {

+ t.Fatalf("report %+v", r)

+ }

+ if len(sw.bodies) != 1 || !strings.Contains(sw.bodies[0], "its stop no longer holds: "+cause+".") {

+ t.Fatalf("pages %v %v", sw.pages, sw.bodies)

+ }

+ if owner, _, _ := sw.lock(); owner != moves.TombstoneOwner(offID) {

+ t.Fatalf("the tombstone is %s", owner)

+ }

diff --git a/internal/layout/plan.go b/internal/layout/plan.go

index ed1b17194..2fc6a9120 100644

--- a/internal/layout/plan.go

+++ b/internal/layout/plan.go

@@ -10,8 +10,13 @@ package layout

import (

"fmt"

"path/filepath"

+ "strconv"

+ "strings"

+ "github.com/carlosframework/platform/internal/activator"

"github.com/carlosframework/platform/internal/controlname"

+ "github.com/carlosframework/platform/internal/instancectl"

+ "github.com/carlosframework/platform/internal/instancekey"

"github.com/carlosframework/platform/internal/provision"

"github.com/carlosframework/platform/internal/release"

)

@@ -154,6 +159,78 @@ func nestedFiles(dataDir, socketDir, host string) []string {

}

+// filesMarkerName is the files directory's generation marker

+// (archive.MarkerName, hostfiles' marker), named here rather than imported:

+// the archive package depends on hostfiles and appconfig, which this

+// package has no other reason to.

+const filesMarkerName = ".files-gen"

+// NestedPlatformNames is every name in a nested host's instance directory

+// that is the platform's, not the app's: the names nestedFiles puts in the

+// raw instance directory (the database, its WAL, SHM and quarantine, the

+// own and sidecar env files, the rendered configuration, the activator's

+// litestream shadow; the socket is in the run directory and is not here),

+// the database's generation marker (<db>.gen), litestream's stock meta

+// directory (<db>-litestream), the files generation marker (.files-gen) and

+// the instance-key adoption record, the seed marker a seeded database

+// keeps beside it for its life (<db>.seed, activator.SeedMarkerSuffix,

+// written by ExecBackend.seededRestore in activator/seedplace.go), and the

+// unit launch descriptors written into the instance directory:

+// launch.<role>.json for every instancectl role

+// (instancectl.LaunchFileName; written by ExecBackend.writeLaunchFor in

+// activator/prepare.go, and launch.run.json also by root's role launch in

+// activator/rolelaunch_root_linux.go). A

+// generation's descriptor, launch.gen.<n>.json (instancectl.

+// GenLaunchFileName, written by writeLaunchFor), has no fixed name:

+// IsNestedPlatformName answers for it. Everything else in the directory is

+// the app's: its files directory <host> and anything it keeps beside it.

+//

+// The offline move's sealer uses these to find what the app keeps beside

+// its files directory, which the move cannot carry. It archives the files

+// directory alone (archive.Build over <instance dir>/<host>), so none of

+// these is ever carried; the source's retire parks every entry of the

+// instance directory, these included.

+func NestedPlatformNames(host string) map[string]bool {

+ const dataDir, socketDir = "/data", "/run/carlos"

+ raw := rawInstanceDir(dataDir, host)

+ out := map[string]bool{}

+ for _, p := range nestedFiles(dataDir, socketDir, host) {

+ if filepath.Dir(p) == raw {

+ out[filepath.Base(p)] = true

+ }

+ db := host + ".db"

+ out[db+".gen"] = true

+ out[db+"-litestream"] = true

+ out[db+activator.SeedMarkerSuffix] = true

+ out[filesMarkerName] = true

+ out[host+instancekey.AdoptionSuffix] = true

+ for _, role := range []instancectl.Role{instancectl.RoleRun, instancectl.RoleSeed, instancectl.RoleSidecar, instancectl.RoleCmd} {

+ out[instancectl.LaunchFileName(role)] = true

+ }

+ return out

+// IsNestedPlatformName reports whether name, an entry of host's instance

+// directory, is the platform's: one of NestedPlatformNames, or a

+// generation's launch descriptor, exactly launch.gen.<n>.json with n a

+// decimal number as instancectl.GenLaunchFileName writes it.

+func IsNestedPlatformName(host, name string) bool {

+ if NestedPlatformNames(host)[name] {

+ return true

+ }

+ n, ok := strings.CutPrefix(name, "launch.gen.")

+ if !ok {

+ return false

+ }

+ n, ok = strings.CutSuffix(n, ".json")

+ if !ok || n == "" {

+ return false

+ }

+ gen, err := strconv.ParseUint(n, 10, 64)

+ return err == nil && instancectl.GenLaunchFileName(gen) == name

// PlanFor derives the plan for host in the given direction from the host

// name alone (design §8.2 step 3). The endpoints are the row values the

// pass will compare against and swap; the artifacts are the files (and one

diff --git a/internal/layout/plan_test.go b/internal/layout/plan_test.go

index 27381c460..eaebf3593 100644

--- a/internal/layout/plan_test.go

+++ b/internal/layout/plan_test.go

@@ -150,6 +150,69 @@ func TestRawRunDirIsTheDottedRawSocketTree(t *testing.T) {

}

+// NestedPlatformNames is exactly the plan's raw-directory names (every

+// nested artifact the pass moves into the instance directory, the socket

+// aside) plus the generation markers, litestream's stock meta directory and

+// the adoption record; and it covers every name a nested plan's artifacts

+// land at in the raw directory, so no platform file of the layout pass can

+// reach an app archive.

+func TestNestedPlatformNamesAreThePlansRawNames(t *testing.T) {

+ const h = "h.example.com"

+ got := NestedPlatformNames(h)

+ want := map[string]bool{

+ h + ".db": true, h + ".db-wal": true, h + ".db-shm": true, h + ".db.corrupt": true,

+ h + ".env": true, h + "-sidecar.env": true, h + ".config.env": true,

+ ".activator-" + h + ".db-litestream": true, ".files-gen": true,

+ h + ".db.gen": true, h + ".db-litestream": true, h + ".instance-key-adoption.json": true,

+ h + ".db.seed": true,

+ "launch.run.json": true, "launch.seed.json": true, "launch.sidecar.json": true, "launch.cmd.json": true,

+ }

+ if len(got) != len(want) {

+ t.Fatalf("NestedPlatformNames = %v, want %v", got, want)

+ }

+ for n := range want {

+ if !got[n] {

+ t.Errorf("%s missing", n)

+ }

+ p, err := PlanFor("/data", "/run/carlos", h, ModeNested)

+ if err != nil {

+ t.Fatal(err)

+ }

+ raw := rawInstanceDir("/data", h)

+ for _, a := range p.Artifacts {

+ if filepath.Dir(a.Dst) == raw && !got[filepath.Base(a.Dst)] {

+ t.Errorf("artifact %s lands at %s, which NestedPlatformNames does not name", a.Name, a.Dst)

+ }

+ if a.Name == "socket" && got[filepath.Base(a.Dst)] {

+ t.Errorf("the socket is in the run directory, not the instance directory")

+ }

+// IsNestedPlatformName is NestedPlatformNames plus a generation's launch

+// descriptor exactly as instancectl.GenLaunchFileName writes it; anything

+// shaped almost like one is the app's.

+func TestIsNestedPlatformNameMatchesGenerationLaunchFilesExactly(t *testing.T) {

+ const h = "h.example.com"

+ for name := range NestedPlatformNames(h) {

+ if !IsNestedPlatformName(h, name) {

+ t.Errorf("%s: not a platform name", name)

+ }

+ for _, name := range []string{"launch.gen.0.json", "launch.gen.7.json", "launch.gen.18446744073709551615.json"} {

+ if !IsNestedPlatformName(h, name) {

+ t.Errorf("%s: not a platform name", name)

+ }

+ for _, name := range []string{h, "launch.gen.json", "launch.gen..json", "launch.gen.07.json", "launch.gen.-1.json", "launch.gen.7.json.bak",

+ "launch.gen.7", "xlaunch.gen.7.json", "launch.gen.x.json", "launch.gen.18446744073709551616.json", "launch.other.json", h + "-blobs", "uploads"} {

+ if IsNestedPlatformName(h, name) {

+ t.Errorf("%s: taken for a platform name", name)

+ }

func TestPlanForRefusesBadInput(t *testing.T) {

if _, err := PlanFor("/data", "/run/carlos", "h.example.com", Mode("sideways")); err == nil {

t.Error("unknown mode accepted")

diff --git a/internal/lease/leasetest/s3server.go b/internal/lease/leasetest/s3server.go

index 051b9f391..ee7819d3b 100644

--- a/internal/lease/leasetest/s3server.go

+++ b/internal/lease/leasetest/s3server.go

@@ -2,6 +2,7 @@ package leasetest

import (

"crypto/sha256"

+ "encoding/base64"

"encoding/hex"

"encoding/xml"

"errors"

@@ -9,6 +10,7 @@ import (

"io"

"net/http"

"net/url"

+ "sort"

"strconv"

"strings"

"sync/atomic"

@@ -105,7 +107,7 @@ func writeErr(w http.ResponseWriter, err error) {

}

// NewS3Server speaks the subset of the S3 wire protocol VersionedStore uses,

-// path-style, over f, holding f.mu for each request. The second result

+// and ListObjectsV2 (serveListV2), path-style, over f, holding f.mu for each request. The second result

// counts requests served. With serves non-empty, the endpoint answers only

// for those request buckets (a regional endpoint); any other bucket gets

// 301 PermanentRedirect, as S3 answers a wrong-region request. A copy's

@@ -145,6 +147,8 @@ func NewS3Server(f *VersionedBucket, serves ...string) (http.Handler, func() int

switch {

case r.Method == http.MethodGet && key == "" && q.Has("versions"):

f.serveList(w, bucket, q)

+ case r.Method == http.MethodGet && key == "" && q.Get("list-type") == "2":

+ f.serveListV2(w, bucket, q)

case r.Method == http.MethodGet && key != "":

versionID := q.Get("versionId")

if versionID == "" { // no version named: the current one

@@ -289,6 +293,127 @@ func (f *VersionedBucket) serveList(w http.ResponseWriter, bucket string, q url.

xml.NewEncoder(w).Encode(res)

}

+type xmlObject struct {

+ Key string

+ LastModified string

+ ETag string

+ Size int64

+ StorageClass string

+type xmlCommonPrefix struct {

+ Prefix string

+type listBucketResult struct {

+ XMLName xml.Name `xml:"ListBucketResult"`

+ Name string

+ Prefix string

+ Delimiter string `xml:",omitempty"`

+ MaxKeys int

+ KeyCount int

+ IsTruncated bool

+ ContinuationToken string `xml:",omitempty"`

+ NextContinuationToken string `xml:",omitempty"`

+ StartAfter string `xml:",omitempty"`

+ Contents []xmlObject

+ CommonPrefixes []xmlCommonPrefix

+// serveListV2 is ListObjectsV2, the subset litestream's replica client

+// (its restore lists with a delimiter) and lease.NewS3Mirror page through:

+// the current objects under prefix in key order (an object whose newest

+// version is a delete marker is not listed), a delimiter rolling every key

+// that holds it after the prefix into one common prefix, max-keys (at most

+// 1000, and at most PageSize when that is set lower; keys and common

+// prefixes both count) and continuation-token pagination. The token is

+// opaque to the client; here it is the last key or common prefix of the

+// previous page, and a page resumes after it (after every key under it,

+// for a common prefix). start-after is honoured on the first page.

+func (f *VersionedBucket) serveListV2(w http.ResponseWriter, bucket string, q url.Values) {

+ prefix, delim := q.Get("prefix"), q.Get("delimiter")

+ if err := f.fail("list", prefix); err != nil {

+ writeErr(w, err)

+ return

+ }

+ size := 1000

+ if f.PageSize > 0 && f.PageSize < size {

+ size = f.PageSize

+ }

+ if raw := q.Get("max-keys"); raw != "" {

+ n, err := strconv.Atoi(raw)

+ if err != nil || n < 0 {

+ s3err(w, http.StatusBadRequest, "InvalidArgument")

+ return

+ }

+ size = min(size, n)

+ }

+ after := q.Get("start-after")

+ token := q.Get("continuation-token")

+ if token != "" {

+ last, err := base64.StdEncoding.DecodeString(token)

+ if err != nil || len(last) == 0 {

+ s3errMsg(w, http.StatusBadRequest, "InvalidArgument", "The continuation token provided is incorrect")

+ return

+ }

+ after = string(last)

+ }

+ // entries are the page candidates in key order: a key, or a common

+ // prefix (isPrefix) standing for every key under it.

+ type entry struct {

+ name string

+ isPrefix bool

+ }

+ var keys []string

+ for k, vs := range f.buckets[bucket] {

+ if strings.HasPrefix(k, prefix) && len(vs) > 0 && !vs[0].deleteMarker {

+ keys = append(keys, k)

+ }

+ sort.Strings(keys)

+ var entries []entry

+ for _, k := range keys {

+ e := entry{name: k}

+ if delim != "" {

+ if i := strings.Index(k[len(prefix):], delim); i >= 0 {

+ e = entry{name: k[:len(prefix)+i+len(delim)], isPrefix: true}

+ }

+ if n := len(entries); n > 0 && entries[n-1] == e {

+ continue

+ }

+ // A common prefix that ended the last page is that page's token

+ // itself, and so is every key rolled into it.

+ if e.name <= after {

+ continue

+ }

+ entries = append(entries, e)

+ }

+ res := listBucketResult{Name: bucket, Prefix: prefix, Delimiter: delim, MaxKeys: size, ContinuationToken: token}

+ if token == "" {

+ res.StartAfter = q.Get("start-after")

+ }

+ page := entries

+ if len(page) > size {

+ page = page[:size]

+ res.IsTruncated = size > 0

+ }

+ for _, e := range page {

+ if e.isPrefix {

+ res.CommonPrefixes = append(res.CommonPrefixes, xmlCommonPrefix{Prefix: e.name})

+ continue

+ }

+ v := f.buckets[bucket][e.name][0]

+ res.Contents = append(res.Contents, xmlObject{Key: e.name, LastModified: v.modified.UTC().Format(time.RFC3339), ETag: `"` + v.etag + `"`, Size: int64(len(v.body)), StorageClass: "STANDARD"})

+ }

+ res.KeyCount = len(page)

+ if res.IsTruncated && len(page) > 0 {

+ res.NextContinuationToken = base64.StdEncoding.EncodeToString([]byte(page[len(page)-1].name))

+ }

+ w.Header().Set("Content-Type", "application/xml")

+ xml.NewEncoder(w).Encode(res)

func (f *VersionedBucket) serveCopy(w http.ResponseWriter, r *http.Request, bucket, key string) {

src := strings.TrimPrefix(r.Header.Get("x-amz-copy-source"), "/")

srcPath, rawQ, _ := strings.Cut(src, "?")

diff --git a/internal/lease/leasetest/s3server_test.go b/internal/lease/leasetest/s3server_test.go

index 3b3581c2b..93e5efee8 100644

--- a/internal/lease/leasetest/s3server_test.go

+++ b/internal/lease/leasetest/s3server_test.go

@@ -4,12 +4,17 @@ import (

"context"

"crypto/sha256"

"encoding/hex"

+ "encoding/xml"

"errors"

+ "fmt"

"io"

"net/http"

"net/http/httptest"

+ "net/url"

"strings"

"testing"

+ "github.com/carlosframework/platform/internal/lease"

)

// failingReader yields some bytes and then an error, like a dropped upload.

@@ -287,3 +292,130 @@ func TestS3ServerConditionalDeleteUsesReadETag(t *testing.T) {

t.Fatal("plain delete changed", got.Code)

}

+// listV2 decodes one ListObjectsV2 page from the raw wire.

+func listV2(t *testing.T, h http.Handler, query string) (listBucketResult, int) {

+ t.Helper()

+ rec := do(t, h, http.MethodGet, "/data?list-type=2"+query, "", nil)

+ var res listBucketResult

+ if rec.Code == http.StatusOK {

+ if err := xml.Unmarshal(rec.Body.Bytes(), &res); err != nil {

+ t.Fatalf("decode %q: %v", rec.Body.String(), err)

+ }

+ return res, rec.Code

+// ListObjectsV2 on the wire: the current objects under a prefix in key

+// order, each with its size and quoted ETag; a key whose newest version is

+// a delete marker, an older version and a sibling prefix are not listed;

+// max-keys pages, and the continuation token resumes after the page;

+// start-after skips; a bad token and a negative max-keys are 400s; a

+// delimiter rolls keys into common prefixes, which page like keys.

+func TestS3ServerListObjectsV2(t *testing.T) {

+ m := NewVersionedBucket()

+ m.AddBucket("data")

+ h, _ := NewS3Server(m)

+ for _, k := range []string{"p/c", "p/a", "p/b/x", "p/d", "q/a", "p2/a"} {

+ m.Put("data", k, []byte("body of "+k))

+ }

+ m.Put("data", "p/a", []byte("newer"))

+ m.BareDelete("data", "p/d")

+ res, code := listV2(t, h, "&prefix=p/")

+ if code != http.StatusOK {

+ t.Fatal(code)

+ }

+ var keys []string

+ for _, o := range res.Contents {

+ keys = append(keys, o.Key)

+ }

+ if strings.Join(keys, ",") != "p/a,p/b/x,p/c" || res.IsTruncated || res.KeyCount != 3 || res.NextContinuationToken != "" {

+ t.Fatalf("listing %+v", res)

+ }

+ if o := res.Contents[0]; o.Size != int64(len("newer")) || o.ETag != `"`+md5hex([]byte("newer"))+`"` {

+ t.Fatalf("p/a listed as %+v", o)

+ }

+ first, _ := listV2(t, h, "&prefix=p/&max-keys=2")

+ if len(first.Contents) != 2 || !first.IsTruncated || first.NextContinuationToken == "" || first.Contents[1].Key != "p/b/x" {

+ t.Fatalf("first page %+v", first)

+ }

+ second, _ := listV2(t, h, "&prefix=p/&max-keys=2&continuation-token="+url.QueryEscape(first.NextContinuationToken))

+ if len(second.Contents) != 1 || second.Contents[0].Key != "p/c" || second.IsTruncated || second.ContinuationToken != first.NextContinuationToken {

+ t.Fatalf("second page %+v", second)

+ }

+ after, _ := listV2(t, h, "&prefix=p/&start-after=p/a")

+ if len(after.Contents) != 2 || after.Contents[0].Key != "p/b/x" {

+ t.Fatalf("start-after %+v", after)

+ }

+ if _, code := listV2(t, h, "&continuation-token=%21%21"); code != http.StatusBadRequest {

+ t.Fatal("bad token", code)

+ }

+ if _, code := listV2(t, h, "&max-keys=-1"); code != http.StatusBadRequest {

+ t.Fatal("negative max-keys", code)

+ }

+ // A delimiter rolls p/b/x into the common prefix p/b/, which counts

+ // against max-keys and can end a page.

+ m.Put("data", "p/b/y", []byte("y"))

+ d1, _ := listV2(t, h, "&prefix=p/&delimiter=/&max-keys=2")

+ if len(d1.Contents) != 1 || d1.Contents[0].Key != "p/a" || len(d1.CommonPrefixes) != 1 || d1.CommonPrefixes[0].Prefix != "p/b/" ||

+ d1.KeyCount != 2 || !d1.IsTruncated || d1.Delimiter != "/" {

+ t.Fatalf("delimited first page %+v", d1)

+ }

+ d2, _ := listV2(t, h, "&prefix=p/&delimiter=/&max-keys=2&continuation-token="+url.QueryEscape(d1.NextContinuationToken))

+ if len(d2.Contents) != 1 || d2.Contents[0].Key != "p/c" || len(d2.CommonPrefixes) != 0 || d2.IsTruncated {

+ t.Fatalf("delimited second page %+v", d2)

+ }

+ top, _ := listV2(t, h, "&delimiter=/")

+ var tops []string

+ for _, p := range top.CommonPrefixes {

+ tops = append(tops, p.Prefix)

+ }

+ if len(top.Contents) != 0 || strings.Join(tops, ",") != "p/,p2/,q/" {

+ t.Fatalf("top level %+v", top)

+ }

+// The SDK's own paginator, through lease.NewS3Mirror, reads every page:

+// 2,500 keys at the default 1,000 a page, and again at a PageSize of 7.

+func TestS3ServerListObjectsV2PaginatesTheSDK(t *testing.T) {

+ m := NewVersionedBucket()

+ m.AddBucket("data")

+ h, _ := NewS3Server(m)

+ srv := httptest.NewServer(h)

+ defer srv.Close()

+ t.Setenv("AWS_ACCESS_KEY_ID", "test-key")

+ t.Setenv("AWS_SECRET_ACCESS_KEY", "test-secret")

+ t.Setenv("AWS_SESSION_TOKEN", "")

+ t.Setenv("AWS_EC2_METADATA_DISABLED", "true")

+ t.Setenv("AWS_CONFIG_FILE", t.TempDir()+"/absent")

+ t.Setenv("AWS_SHARED_CREDENTIALS_FILE", t.TempDir()+"/absent")

+ t.Setenv("CARLOS_S3_ENDPOINT", srv.URL)

+ want := map[string]bool{}

+ for i := range 2500 {

+ k := fmt.Sprintf("big/%05d", i)

+ m.Put("data", k, []byte(k))

+ want[k] = true

+ }

+ m.Put("data", "bigger/x", []byte("x")) // a sibling prefix, never listed

+ ctx := context.Background()

+ mirror, err := lease.NewS3Mirror(ctx, "eu-west-1", "eu-west-1")

+ if err != nil {

+ t.Fatal(err)

+ }

+ for _, page := range []int{1000, 7} {

+ m.PageSize = page

+ got, err := mirror.List(ctx, "data", "big/")

+ if err != nil {

+ t.Fatalf("page size %d: %v", page, err)

+ }

+ if len(got) != len(want) {

+ t.Fatalf("page size %d: listed %d keys, want %d", page, len(got), len(want))

+ }

+ for k, o := range got {

+ if !want[k] || o.Size != int64(len(k)) || o.ETag != md5hex([]byte(k)) {

+ t.Fatalf("page size %d: %s listed as %+v", page, k, o)

+ }

diff --git a/internal/lease/tombstone.go b/internal/lease/tombstone.go

index c0054208f..7f9969a21 100644

--- a/internal/lease/tombstone.go

+++ b/internal/lease/tombstone.go

@@ -60,6 +60,9 @@ var (

// between the read and the conditional write. Nothing was overwritten;

// the caller's next pass reads it afresh.

ErrTombstoneChanged = errors.New("lease: the lock changed under the tombstone write")

+ // ErrTombstoneLost reports that the tombstone Reassert was asked to

+ // keep is absent or expired: the stop was lost. Nothing was written.

+ ErrTombstoneLost = errors.New("lease: the tombstone is absent or expired; the stop was lost")

)

// leaseFree is the predicate litestream's AcquireLease applies to the lease

@@ -209,14 +212,15 @@ func Tombstone(ctx context.Context, bucket, location, region, owner string, ttl

// - live and held by owner: rewritten with IfMatch on the ETag just read.

// If the lock changed since, ErrTombstoneChanged and nothing is written;

-// - absent, or expired (including lifted): Tombstone, which writes a new

-// one;

+// - absent, or expired (including lifted), whoever wrote it:

+// ErrTombstoneLost, and nothing is written. The stop was lost: in

+// between a wake may have acquired, written and released the lease, so

+// a tombstone is never taken again here;

// - live and held by anyone else: a *HeldError, nothing written.

-// Callers must read LockState first and treat absent or expired as a lost

-// stop; never Reassert after Lift. Reassert cannot tell a tombstone that was

-// kept from one that lapsed and was re-created, and in between a wake may

-// have acquired, written and released the lease.

+// Callers read LockState first and treat absent or expired as a lost stop;

+// ErrTombstoneLost is the same verdict for a lock that went between that

+// read and this one. Never Reassert after Lift.

func Reassert(ctx context.Context, bucket, location, region, owner string, ttl time.Duration) error {

if !tombstoneOwnerOK(owner) {

return ErrTombstoneOwner

@@ -232,8 +236,11 @@ func Reassert(ctx context.Context, bucket, location, region, owner string, ttl t

if err != nil {

return err

}

- if absent || leaseFree(cur.ExpiresAt, time.Now()) {

- return Tombstone(ctx, bucket, location, region, owner, ttl)

+ if absent {

+ return fmt.Errorf("reassert tombstone: %w", ErrTombstoneLost)

+ }

+ if leaseFree(cur.ExpiresAt, time.Now()) {

+ return fmt.Errorf("reassert tombstone: %w (%s, expired %s)", ErrTombstoneLost, cur.Owner, cur.ExpiresAt.UTC().Format(time.RFC3339))

}

if cur.Owner != owner {

return heldErr(cur)

diff --git a/internal/lease/tombstone_test.go b/internal/lease/tombstone_test.go

index 9dc522871..abdc5e70b 100644

--- a/internal/lease/tombstone_test.go

+++ b/internal/lease/tombstone_test.go

@@ -1,6 +1,7 @@

package lease_test

import (

+ "bytes"

"context"

"encoding/json"

"errors"

@@ -387,32 +388,38 @@ func TestReassertMovesTheExpiry(t *testing.T) {

wantHeld(t, err, tombMove)

}

-func TestReassertWithNothingToReassertTombstones(t *testing.T) {

+// With nothing live to keep, Reassert reports the stop lost and writes

+// nothing: absent, its own tombstone expired or lifted, and someone else's

+// expired lease. It never takes a tombstone again (a wake may have used the

+// lease in between).

+func TestReassertWithNothingToReassertReportsTheStopLost(t *testing.T) {

s := tombFixture(t)

ctx := context.Background()

- if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {

- t.Fatalf("absent: %v", err)

- }

- if l, _, ok := s.lock(t, tombLoc); !ok || l.Owner != tombMove {

- t.Fatalf("absent: lock = %+v present %v", l, ok)

- }

- // An expired tombstone of its own is replaced and live again.

- s.putLock(t, tombLoc, tombMove, 5, time.Now().Add(-time.Hour))

- if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {

- t.Fatalf("expired: %v", err)

- }

- l, _, _ := s.lock(t, tombLoc)

- if l.Owner != tombMove || l.Generation != 6 {

- t.Fatalf("expired: lock = %+v, want owner %s at generation 6", l, tombMove)

+ if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); !errors.Is(err, lease.ErrTombstoneLost) {

+ t.Fatalf("absent: %v, want ErrTombstoneLost", err)

}

- near(t, l.ExpiresAt, time.Hour)

- // An expired lease of someone else is replaced the same way.

- s.putLock(t, tombLoc, tombOther, 2, time.Now().Add(-time.Hour))

- if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {

- t.Fatalf("expired other: %v", err)

+ if _, _, ok := s.lock(t, tombLoc); ok || s.writeCount() != 0 {

+ t.Fatalf("absent: a lock was written (%d writes)", s.writeCount())

}

- if l, _, _ := s.lock(t, tombLoc); l.Owner != tombMove {

- t.Fatalf("expired other: lock = %+v", l)

+ for _, c := range []struct {

+ name string

+ owner string

+ expires time.Time

+ }{

+ {"its own, expired", tombMove, time.Now().Add(-time.Hour)},

+ {"its own, lifted", tombMove, time.Unix(0, 0)},

+ {"someone else's, expired", tombOther, time.Now().Add(-time.Hour)},

+ {"an ordinary lease, expired", "box-a:1", time.Now().Add(-time.Second)},

+ } {

+ s.putLock(t, tombLoc, c.owner, 5, c.expires)

+ _, before, _ := s.lock(t, tombLoc)

+ writes := s.writeCount()

+ if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); !errors.Is(err, lease.ErrTombstoneLost) {

+ t.Fatalf("%s: %v, want ErrTombstoneLost", c.name, err)

+ }

+ if _, after, _ := s.lock(t, tombLoc); !bytes.Equal(before, after) || s.writeCount() != writes {

+ t.Fatalf("%s: the lock was rewritten", c.name)

+ }

}

diff --git a/internal/moves/archive/archive.go b/internal/moves/archive/archive.go

index 1c3287872..a017e20ad 100644

--- a/internal/moves/archive/archive.go

+++ b/internal/moves/archive/archive.go

@@ -143,7 +143,14 @@ type item struct {

// marker. The files directory is read through an os.Root, so nothing a

// tenant does inside it (a directory swapped for a symlink mid-walk) can

// make the walk or a later open leave it.

-func walk(root *os.Root, envPath string) ([]item, error) {

+//

+// skip, when non-nil, is asked each path relative to the files directory

+// (slash-separated, as the archive names it) before anything else is read

+// about it: a path it accepts is left out as the marker is, a directory

+// with everything under it, and is neither typed nor counted, so a

+// platform artifact of any kind (a socket, a FIFO) cannot fail the walk.

+// envPath is never asked.

+func walk(root *os.Root, envPath string, skip func(rel string) bool) ([]item, error) {

var items []item

add := func(it item) error {

if len(items) >= entryLimit {

@@ -160,9 +167,10 @@ func walk(root *os.Root, envPath string) ([]item, error) {

if p == "." {

return nil

}

- if p == MarkerName {

+ if p == MarkerName || (skip != nil && skip(p)) {

// The marker is never archived, nor anything under it if

- // it is (wrongly) a directory.

+ // it is (wrongly) a directory; nor is a path the caller

+ // skips.

if d.IsDir() {

return fs.SkipDir

}

@@ -323,6 +331,11 @@ func openFilesRoot(filesDir string) (*os.Root, error) {

// build writes the tar of filesDir and envPath to out and returns what

// Build and PlainDigest report.

func build(filesDir, envPath string, out io.Writer) (built, error) {

+ return buildFiltered(filesDir, envPath, nil, out)

+// buildFiltered is build over the paths skip leaves in (walk).

+func buildFiltered(filesDir, envPath string, skip func(rel string) bool, out io.Writer) (built, error) {

root, err := openFilesRoot(filesDir)

if err != nil {

return built{}, err

@@ -330,7 +343,7 @@ func build(filesDir, envPath string, out io.Writer) (built, error) {

if root != nil {

defer root.Close()

}

- items, err := walk(root, envPath)

+ items, err := walk(root, envPath, skip)

if err != nil {

return built{}, err

}

@@ -410,11 +423,24 @@ func copyItem(tw io.Writer, it item, size int64) (Entry, error) {

// can never collide with the overlay. A missing envPath (or "") means the

// archive has no env entry; a missing filesDir is an empty one.

func Build(filesDir, envPath string, w io.Writer) (plain string, entries []Entry, bytes int64, err error) {

+ return BuildFiltered(filesDir, envPath, nil, w)

+// BuildFiltered is Build over the files directory without the paths skip

+// accepts. skip is asked each path relative to filesDir, slash-separated

+// (a top-level entry is its bare name); a skipped directory is skipped

+// whole, and a skipped entry of any type is never read, typed or counted.

+// The env entry is never asked. A nil skip is Build. Nothing calls it with

+// a filter today: the offline move's sealer archives a nested instance's

+// files directory (<instance dir>/<host>, which holds nothing of the

+// platform's but the marker) with Build, and refuses the seal over any

+// entry of the app's beside it.

+func BuildFiltered(filesDir, envPath string, skip func(rel string) bool, w io.Writer) (plain string, entries []Entry, bytes int64, err error) {

zw, err := zstd.NewWriter(w, zstd.WithEncoderConcurrency(1), zstd.WithWindowSize(zstdWindow))

if err != nil {

return "", nil, 0, err

}

- b, err := build(filesDir, envPath, zw)

+ b, err := buildFiltered(filesDir, envPath, skip, zw)

if err != nil {

zw.Close()

return "", nil, 0, err

diff --git a/internal/moves/archive/archive_test.go b/internal/moves/archive/archive_test.go

index 9cf54e7d7..833507a49 100644

--- a/internal/moves/archive/archive_test.go

+++ b/internal/moves/archive/archive_test.go

@@ -104,6 +104,98 @@ func TestBuildIsDeterministicAndExcludesTheMarker(t *testing.T) {

}

+// BuildFiltered over a nested-shaped directory leaves out exactly what

+// skip accepts (a top-level file, a top-level directory with everything

+// under it, and a symlink, which unskipped would refuse the walk), never

+// asks about env, and writes byte for byte the archive Build writes over a

+// flat directory holding only the app's files; that archive extracts flat

+// and its extraction digests to the same plaintext. Build is BuildFiltered

+// with a nil skip.

+func TestBuildFilteredLeavesOutWhatSkipAccepts(t *testing.T) {

+ flat, env := filesFixture(t)

+ nested := filepath.Join(t.TempDir(), "nested")

+ if err := os.CopyFS(nested, os.DirFS(flat)); err != nil {

+ t.Fatal(err)

+ }

+ for _, rel := range []string{"a.txt", "sub", "sub/b.bin", "env", "sub/env", "sub/env/x", ".files-gen"} {

+ info, err := os.Lstat(filepath.Join(flat, rel))

+ if err != nil {

+ t.Fatal(err)

+ }

+ if err := os.Chmod(filepath.Join(nested, rel), info.Mode().Perm()); err != nil {

+ t.Fatal(err)

+ }

+ mustWrite(t, filepath.Join(nested, "h.db"), []byte("database"), 0o600)

+ if err := os.MkdirAll(filepath.Join(nested, ".activator-h.db-litestream", "ltx", "0"), 0o700); err != nil {

+ t.Fatal(err)

+ }

+ mustWrite(t, filepath.Join(nested, ".activator-h.db-litestream", "ltx", "0", "f.ltx"), []byte("ltx"), 0o600)

+ if err := os.Symlink("/etc/passwd", filepath.Join(nested, "h.config.env")); err != nil {

+ t.Fatal(err)

+ }

+ // A nested file that shares a skipped name below the top level is kept.

+ mustWrite(t, filepath.Join(nested, "sub", "h.db"), []byte("an app file"), 0o644)

+ mustWrite(t, filepath.Join(flat, "sub", "h.db"), []byte("an app file"), 0o644)

+ platform := map[string]bool{"h.db": true, ".activator-h.db-litestream": true, "h.config.env": true}

+ var asked []string

+ skip := func(rel string) bool {

+ asked = append(asked, rel)

+ return !strings.Contains(rel, "/") && platform[rel]

+ }

+ if _, _, _, err := Build(nested, env, io.Discard); !errors.Is(err, ErrSpecialFile) {

+ t.Fatalf("unfiltered, the symlink should refuse the walk: %v", err)

+ }

+ var got, want bytes.Buffer

+ plain, entries, n, err := BuildFiltered(nested, env, skip, &got)

+ if err != nil {

+ t.Fatal(err)

+ }

+ wplain, wentries, wn, err := Build(flat, env, &want)

+ if err != nil {

+ t.Fatal(err)

+ }

+ if plain != wplain || n != wn || len(entries) != len(wentries) || !bytes.Equal(got.Bytes(), want.Bytes()) {

+ t.Fatalf("filtered %s %d %v, flat %s %d %v", plain, n, entries, wplain, wn, wentries)

+ }

+ for _, rel := range asked {

+ if strings.HasPrefix(rel, ".activator-h.db-litestream/") {

+ t.Fatalf("asked about %q inside a skipped directory", rel)

+ }

+ if !slicesContains(asked, "sub/h.db") || !slicesContains(asked, "h.db") {

+ t.Fatalf("asked %v", asked)

+ }

+ // Extract flat, then digest the extraction as Build would.

+ parent := t.TempDir()

+ files, outEnv := filepath.Join(parent, "h"), filepath.Join(parent, "h.env")

+ if err := Extract(bytes.NewReader(got.Bytes()), files, outEnv); err != nil {

+ t.Fatal(err)

+ }

+ if p, _, _, err := PlainDigest(files, outEnv); err != nil || p != plain {

+ t.Fatalf("extraction digests to %s (%v), want %s", p, err, plain)

+ }

+ for _, name := range []string{"h.db", ".activator-h.db-litestream", "h.config.env"} {

+ if _, err := os.Lstat(filepath.Join(files, name)); !errors.Is(err, fs.ErrNotExist) {

+ t.Fatalf("%s extracted: %v", name, err)

+ }

+ // A nil skip is Build.

+ var nilSkip bytes.Buffer

+ if p, _, _, err := BuildFiltered(flat, env, nil, &nilSkip); err != nil || p != wplain || !bytes.Equal(nilSkip.Bytes(), want.Bytes()) {

+ t.Fatalf("nil skip: %s %v", p, err)

+ }

+func slicesContains(s []string, v string) bool {

+ for _, x := range s {

+ if x == v {

+ return true

+ }

+ return false

func mustDecode(t *testing.T, b []byte) io.Reader {

t.Helper()

zr, err := zstd.NewReader(bytes.NewReader(b))

@@ -831,6 +923,45 @@ func TestExtractWithoutEnvLeavesNoEnv(t *testing.T) {

}

+// A small archive handed over in a *bytes.Buffer (under the decoder's

+// 128 KiB shortcut for a reader with Bytes() and Len()) is read as a

+// stream like any other: Extract, the decode Verify runs, and Verify itself

+// accept it, rather than finding the whole archive still unread after it.

+func TestASmallArchiveInABytesBufferExtractsAndVerifies(t *testing.T) {

+ dir, env := filesFixture(t)

+ var tarz bytes.Buffer

+ plain, entries, n, err := Build(dir, env, &tarz)

+ if err != nil {

+ t.Fatal(err)

+ }

+ if tarz.Len() >= 128<<10 {

+ t.Fatalf("fixture: %d bytes is not under 128 KiB", tarz.Len())

+ }

+ raw := append([]byte(nil), tarz.Bytes()...)

+ parent := t.TempDir()

+ files, envOut := filepath.Join(parent, "h"), filepath.Join(parent, "h.env")

+ if err := Extract(bytes.NewBuffer(append([]byte(nil), raw...)), files, envOut); err != nil {

+ t.Fatalf("Extract from a *bytes.Buffer: %v", err)

+ }

+ if got, _, _, err := PlainDigest(files, envOut); err != nil || got != plain {

+ t.Fatalf("the extraction digests to %s (%v), want %s", got, err, plain)

+ }

+ m := Manifest{V: ManifestVersion, Op: "mv_x", Plain: plain, Bytes: n, Entries: entries, Chunk: ChunkSize}

+ if got, err := verifyStream(bytes.NewBuffer(append([]byte(nil), raw...)), m); err != nil || got != plain {

+ t.Fatalf("verifyStream from a *bytes.Buffer: %s, %v", got, err)

+ }

+ pub, priv, b := keys(t)

+ var sealed bytes.Buffer

+ key, sealedSHA, err := Seal(bytes.NewReader(raw), pub, b, &sealed)

+ if err != nil {

+ t.Fatal(err)

+ }

+ m.Sealed, m.SealedDataKey = sealedSHA, key

+ if err := Verify(&sealed, m, priv, b); err != nil {

+ t.Fatalf("Verify from a *bytes.Buffer: %v", err)

+ }

// sealedArchive is a good archive with its manifest, over a tree large

// enough to seal into three chunks.

type sealedArchive struct {

diff --git a/internal/moves/archive/archive_unix_test.go b/internal/moves/archive/archive_unix_test.go

index caab62dde..ef987b669 100644

--- a/internal/moves/archive/archive_unix_test.go

+++ b/internal/moves/archive/archive_unix_test.go

@@ -135,3 +135,24 @@ func TestBuildRefusalsCarryTheirPath(t *testing.T) {

})

}

+// A FIFO (a platform socket or pipe in a nested instance directory) that

+// skip accepts is never opened or typed: BuildFiltered archives around it.

+// The same FIFO unskipped is refused, naming it.

+func TestBuildFilteredNeverTypesASkippedFIFO(t *testing.T) {

+ files := t.TempDir()

+ if err := os.WriteFile(filepath.Join(files, "a"), []byte("a"), 0o644); err != nil {

+ t.Fatal(err)

+ }

+ if err := syscall.Mkfifo(filepath.Join(files, "app.fifo"), 0o644); err != nil {

+ t.Fatal(err)

+ }

+ var pe *PathError

+ if _, _, _, err := BuildFiltered(files, "", nil, io.Discard); !errors.Is(err, ErrSpecialFile) || !errors.As(err, &pe) || pe.Path != "app.fifo" {

+ t.Fatalf("unskipped FIFO: %v", err)

+ }

+ _, entries, _, err := BuildFiltered(files, "", func(rel string) bool { return rel == "app.fifo" }, io.Discard)

+ if err != nil || len(entries) != 1 || entries[0].Path != "files/a" {

+ t.Fatalf("skipped FIFO: %v %v", entries, err)

+ }

diff --git a/internal/moves/archive/limits.go b/internal/moves/archive/limits.go

index 44f6bf90c..0a4fcd66c 100644

--- a/internal/moves/archive/limits.go

+++ b/internal/moves/archive/limits.go

@@ -209,8 +209,13 @@ func (b *boundedReader) Read(p []byte) (int, error) {

return n, err

}

+// newDecoder always streams from r. WithDecodeBuffersBelow(0) turns off

+// the decoder's shortcut for a reader with Bytes() and Len() (a

+// *bytes.Buffer) under 128 KiB, which decodes the whole buffer at once and

+// never drains it: requireEOF would then read the archive again and refuse

+// it as data after the archive.

func newDecoder(r io.Reader) (*zstd.Decoder, error) {

- return zstd.NewReader(r, zstd.WithDecoderConcurrency(1), zstd.WithDecoderMaxWindow(zstdWindow))

+ return zstd.NewReader(r, zstd.WithDecoderConcurrency(1), zstd.WithDecoderMaxWindow(zstdWindow), zstd.WithDecodeBuffersBelow(0))

}

// requireEOF reads the raw stream once more after the decoder finished. A

diff --git a/move.go b/move.go

index 6f4c67a7d..3ebb08307 100644

--- a/move.go

+++ b/move.go

@@ -9,6 +9,7 @@ import (

"flag"

"fmt"

"io"

+ "maps"

"math"

"net/http"

"net/url"

@@ -21,6 +22,7 @@ import (

"github.com/carlosframework/platform/internal/hostfence"

"github.com/carlosframework/platform/internal/hostresolve"

"github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/provision"

)

// `carlos move` (spec §12, moves Phase 1 Plan 6 Task 1a): request a move of

@@ -81,6 +83,8 @@ const (

moveNoFlipTime = "The move's flip time could not be read, so its target cannot be declared lost from here."

moveOtherLoss = "The console asks to confirm a loss carlos move does not know. Use carlos ops host resolve."

moveLossChanged = "The move changed since it was read. Nothing was declared. Run the command again."

+ // The offline subcommand named no verb, or one it does not have (MV-O56).

+ moveOfflineUsage = "carlos move offline needs start, seal, flip, open, finish, abort or status."

)

func moveSay(out io.Writer, line string) {

@@ -117,7 +121,8 @@ type moveDeps struct {

}

type moveArgs struct {

- verb string // request | status | cut | abort | resolve

host, to, app string

account, accountID string

console string

@@ -135,9 +140,19 @@ func parseMoveArgs(args []string, out io.Writer) (moveArgs, error) {

switch args[0] {

case "status", "cut", "abort", "resolve":

a.verb, args = args[0], args[1:]

+ case "offline":

+ if len(args) < 2 || !slices.Contains(offlineVerbs, args[1]) {

+ moveSay(out, moveOfflineUsage)

+ return moveArgs{}, errors.New("no offline verb")

+ }

+ a.verb, a.offline, args = "offline", args[1], args[2:]

}

- fs := flag.NewFlagSet("move "+a.verb, flag.ContinueOnError)

+ name := "move " + a.verb

+ if a.offline != "" {

+ name += " " + a.offline

+ }

+ fs := flag.NewFlagSet(name, flag.ContinueOnError)

var parsed strings.Builder

fs.SetOutput(&parsed)

fs.Usage = func() {} // the help is said by moveFlagHelp

@@ -156,6 +171,10 @@ func parseMoveArgs(args []string, out io.Writer) (moveArgs, error) {

fs.BoolVar(&a.wait, "wait", false, "wait for the move in flight, or the last one, to finish")

case "cut", "abort":

fs.BoolVar(&a.noWait, "no-wait", false, "return once the console has accepted it")

+ case "offline":

+ if a.offline == "start" {

+ fs.StringVar(&a.to, "to", "", "the region to move the instance to")

+ }

case "resolve":

fs.BoolVar(&a.retry, "retry", false, "retry a stuck move once its cause is fixed")

fs.StringVar(&a.lostBox, "declare-lost", "", "declare the move's target box lost, once AWS shows it stopped or terminated")

@@ -173,7 +192,7 @@ func parseMoveArgs(args []string, out io.Writer) (moveArgs, error) {

case a.host == "":

moveSay(out, moveNeedsHost)

return moveArgs{}, errors.New("no --host")

- case a.verb == "request" && a.to == "":

+ case (a.verb == "request" || a.offline == "start") && a.to == "":

moveSay(out, moveNeedsTo)

return moveArgs{}, errors.New("no --to")

case a.verb == "resolve" && a.retry && a.lostBox != "":

@@ -183,7 +202,7 @@ func parseMoveArgs(args []string, out io.Writer) (moveArgs, error) {

moveSay(out, moveNeedsRetry)

return moveArgs{}, errors.New("no --retry or --declare-lost")

}

- if a.verb != "status" {

+ if a.verb != "status" && a.verb != "offline" {

a.wait = !a.noWait

}

return a, nil

@@ -231,6 +250,13 @@ func moveMain(ctx context.Context, args []string, out io.Writer, d moveDeps) int

m := &mover{sess: sess, out: out, host: a.host, poll: d.poll,

path: apiPath(acct, app, "/instances/"+url.PathEscape(a.host)+"/move")}

+ // The offline move is the operator's tool for a console with Phase 1 off.

+ // It has no Phase 1 marker to check: its doors refuse for themselves.

+ if a.verb == "offline" {

+ m.path += "/offline"

+ return m.offline(ctx, a)

+ }

// L4: read the status first, and send nothing to a console that does

// not answer as Phase 1.

view, code := m.readFlat(ctx)

@@ -293,6 +319,11 @@ type moveAnswer struct {

// (hasStuck when there is one).

stuck serverText

hasStuck bool

+ // waiting is the offline status view's "waiting": the sentence the next

+ // verb would be refused with, as the console wrote it (set by roundTrip).

+ WireWaiting string `json:"waiting"` // read only by roundTrip

+ waiting serverText

+ hasWaiting bool

}

type moveDetailView struct {

@@ -419,6 +450,9 @@ func (m *mover) roundTrip(ctx context.Context, method, path string, in any) move

if d := r.answer.Detail; d != nil && d.Move != nil && d.Move.WireStuck != "" {

r.answer.stuck, r.answer.hasStuck = serverText{spoken: d.Move.WireStuck}, true

}

+ if r.answer.WireWaiting != "" {

+ r.answer.waiting, r.answer.hasWaiting = serverText{spoken: r.answer.WireWaiting}, true

+ }

return r

}

var e struct {

@@ -468,31 +502,37 @@ func (m *mover) limit() {

// get reads one answer, riding out 429s (backing off, said once) and a few

// transient failures. code is -1 on success, else the exit code to return.

func (m *mover) get(ctx context.Context, path string) (moveAnswer, int) {

+ r, code := m.getReply(ctx, path)

+ return r.answer, code

+// getReply is get with the whole reply, for a reader of the raw body.

+func (m *mover) getReply(ctx context.Context, path string) (moveReply, int) {

failures := 0

for {

r := m.roundTrip(ctx, http.MethodGet, path, nil)

ok := !r.failed && r.moved == ""

switch {

case ok && r.status == http.StatusOK:

- return r.answer, -1

+ return r, -1

case r.moved != "" || r.stopped:

m.sayReply(r)

- return moveAnswer{}, moveExitError

+ return moveReply{}, moveExitError

case ok && r.status == http.StatusTooManyRequests:

m.limit()

case ok && r.status >= 400 && r.status < 500:

m.sayReply(r)

- return moveAnswer{}, moveExitRefused

+ return moveReply{}, moveExitRefused

default:

failures++

if failures >= moveTransient {

m.sayReply(r)

- return moveAnswer{}, moveExitError

+ return moveReply{}, moveExitError

}

if !m.sleep(ctx) {

moveSay(m.out, moveStopped)

- return moveAnswer{}, moveExitError

+ return moveReply{}, moveExitError

}

@@ -937,3 +977,680 @@ func (m *mover) refusal(r moveReply) int {

}

return -1

}

+// ---- carlos move offline ----

+//

+// An operator moves one instance with planned downtime, one step at a time:

+// start, seal, flip, open, finish, abort, and status (offline-move plan,

+// Task 7). Each verb posts to its door, then polls the status door every

+// interval until the step it started is done or has failed, and says which

+// verb comes next. It never runs the next verb by itself. The exit codes are

+// the family's: 0 the step is done, 3 the move was aborted, 4 the step failed

+// or the move waits for the operator, 5 the console refused, 1 an error.

+// Refusals are the console's own sentences (serverText); the CLI invents none.

+// offlineVerbs are the seven verbs after `carlos move offline`.

+var offlineVerbs = []string{"start", "seal", "flip", "open", "finish", "abort", "status"}

+// offlineView is the offline status view (GET) and the 202 every mutating

+// door answers with.

+type offlineView struct {

+ Host string `json:"host"`

+ State string `json:"state"`

+ ID string `json:"id"`

+ From string `json:"from"`

+ To string `json:"to"`

+ Sealer string `json:"sealer"`

+ Boxes []string `json:"boxes"`

+ Next string `json:"next"`

+ CanAbort bool `json:"can_abort"`

+ // Failed: a box's report says the step cannot finish (the status view's

+ // own flag; the CLI says MV-O6 or MV-O9 for it).

+ Failed bool `json:"failed"`

+ Left []offlineLeftView `json:"left"`

+ Reports []offlineReportView `json:"reports"`

+ Items []offlineItemView `json:"items"`

+ Last *moveOutcome `json:"last"`

+type offlineReportView struct {

+ Box string `json:"box"`

+ Role string `json:"role"`

+ Phase string `json:"phase"`

+ Fresh bool `json:"fresh"`

+ Verdicts map[string]offlineVerdict `json:"items"`

+// offlineLeftView is a path a source box could not park and left in place,

+// with the reason it gave (a box diagnostic, not copy).

+type offlineLeftView struct {

+ Holder string `json:"box"`

+ Where string `json:"path"`

+ Because string `json:"reason"`

+// offlineVerdict is one check of a report; its names are apart from the

+// tree's for the copy gate (see offlineItemView).

+type offlineVerdict struct {

+ Passed bool `json:"ok"`

+ Why string `json:"reason"`

+// The fields are named apart from the rest of the tree's (Detail, ID): the

+// copy gate reads a field by its name across the scanned files.

+type offlineItemView struct {

+ Slug string `json:"id"`

+ Owner string `json:"owner"`

+ Check string `json:"status"`

+ Diagnostic string `json:"detail"`

+// offlineReply is one answer of the offline doors: the view, and the sentence

+// the console says the next verb waits for (server text).

+type offlineReply struct {

+ v offlineView

+ waiting serverText

+ hasWaiting bool

+// report is the box's report of a role, if it wrote one.

+func (v offlineView) report(role, box string) *offlineReportView {

+ for i := range v.Reports {

+ if v.Reports[i].Role == role && v.Reports[i].Box == box {

+ return &v.Reports[i]

+ }

+ return nil

+// failed names the items an owner's report failed, by the box's own

+// diagnostic (or the item's name when it gave none), joined. Empty when none

+// failed.

+func (v offlineView) failed(owner string) string {

+ var parts []string

+ for _, it := range v.Items {

+ if it.Owner != owner || it.Check != "failed" {

+ continue

+ }

+ if it.Diagnostic == "" {

+ parts = append(parts, it.Slug)

+ continue

+ }

+ parts = append(parts, it.Diagnostic)

+ }

+ return strings.Join(parts, "; ")

+// failedReport is the role's report when it says it failed.

+func (v offlineView) failedReport(role string) *offlineReportView {

+ for i := range v.Reports {

+ if v.Reports[i].Role == role && v.Reports[i].Phase == moves.OfflinePhaseFailed {

+ return &v.Reports[i]

+ }

+ return nil

+// reason is what a failed report says: its failing checks, else its phase.

+func (r offlineReportView) reason() string {

+ var parts []string

+ for _, id := range slices.Sorted(maps.Keys(r.Verdicts)) {

+ c := r.Verdicts[id]

+ switch {

+ case c.Passed:

+ case c.Why != "":

+ parts = append(parts, c.Why)

+ default:

+ parts = append(parts, id)

+ }

+ if len(parts) > 0 {

+ return strings.Join(parts, "; ")

+ }

+ return r.Phase

+// The roles a failed step can be: the sealer sealing, the target landing,

+// or a source box cleaning up after the open.

+const (

+ failSealer = "sealer"

+ failTarget = "target"

+ failSource = "source"

+// failure says whether a box has failed its step, which role it was and why:

+// the items the role's report failed, else what its failed report says, else

+// the word failed. Before the open it is the target landing, else the sealer

+// sealing (both at once go by the state the move is in). Once the move is

+// open or finishing a failed report of any role counts, source boxes included;

+// while it reverts only a failed source box does, since a target or sealer

+// that failed earlier is being put back.

+func (v offlineView) failure() (kind, reason string, ok bool) {

+ switch v.State {

+ case provision.OfflineOpen, provision.OfflineFinishing, provision.OfflineReverting:

+ roles := []string{moves.OfflineRoleSealer, moves.OfflineRoleTarget, moves.OfflineRoleSource}

+ if v.State == provision.OfflineReverting {

+ roles = roles[2:]

+ }

+ for _, role := range roles {

+ if r := v.failedReport(role); r != nil {

+ return role, r.reason(), true

+ }

+ return "", "", false

+ }

+ sealing := v.State == provision.OfflineStopping || v.State == provision.OfflineSealing || v.State == provision.OfflineSealed

+ sealReason, landReason := v.failed(moves.OfflineOwnerSeal), v.failed(moves.OfflineOwnerLanding)

+ sealRep, landRep := v.failedReport(moves.OfflineRoleSealer), v.failedReport(moves.OfflineRoleTarget)

+ sealFailed := sealRep != nil || (sealing && sealReason != "")

+ landFailed := landRep != nil || (!sealing && landReason != "")

+ switch {

+ case sealFailed && (sealing || !landFailed):

+ if sealReason == "" && sealRep != nil {

+ sealReason = sealRep.reason()

+ }

+ return failSealer, sealReason, true

+ case landFailed:

+ if landReason == "" && landRep != nil {

+ landReason = landRep.reason()

+ }

+ return failTarget, landReason, true

+ case v.Failed:

+ if sealing {

+ return failSealer, moves.OfflinePhaseFailed, true

+ }

+ return failTarget, moves.OfflinePhaseFailed, true

+ }

+ return "", "", false

+// sayFailure says a failed step: MV-O6 for the target before the open,

+// MV-O9 for the sealer or a target once open (abort refuses there), MV-O65

+// for a source box.

+func (w *offlineRun) sayFailure(v offlineView) bool {

+ kind, reason, ok := v.failure()

+ if !ok {

+ return false

+ }

+ host := w.m.host

+ if kind == failTarget && (v.State == provision.OfflineOpen || v.State == provision.OfflineFinishing) {

+ // Once the move is open, abort refuses (MV-O27), so MV-O6's advice

+ // would send the operator to a verb that says no; MV-O9 says it.

+ kind = failSealer

+ }

+ switch kind {

+ case failTarget:

+ moveSay(w.m.out, fmt.Sprintf("%s could not take %s: %s. Run carlos move offline abort --host %s to move it back.", v.To, host, reason, host))

+ case failSource:

+ for _, r := range v.Reports {

+ if r.Role == moves.OfflineRoleSource && r.Phase == moves.OfflinePhaseFailed {

+ moveSay(w.m.out, fmt.Sprintf("%s could not clean up %s: %s. Run carlos move offline status --host %s for each item.", r.Box, host, reason, host))

+ return true

+ }

+ default:

+ moveSay(w.m.out, fmt.Sprintf("The move could not finish a step: %s. Run carlos move offline status --host %s for each item.", reason, host))

+ }

+ return true

+// offlineRun is one `carlos move offline` verb's conversation: the move it

+// follows, as the console's answers named it, and what has been said.

+type offlineRun struct {

+ m *mover

+ id, from, to string

+ said map[string]bool

+ lastWaiting serverText

+func (w *offlineRun) learn(v offlineView) {

+ if v.ID != "" && (w.id == "" || w.id == v.ID) {

+ w.id, w.from, w.to = v.ID, v.From, v.To

+ }

+func (w *offlineRun) once(key string) bool {

+ if w.said[key] {

+ return false

+ }

+ w.said[key] = true

+ return true

+// read decodes a 2xx reply from an offline door.

+func (w *offlineRun) read(r moveReply) (offlineReply, int) {

+ var v offlineView

+ if json.Unmarshal(r.body, &v) != nil {

+ moveSay(w.m.out, moveUnreadable)

+ return offlineReply{}, moveExitError

+ }

+ w.learn(v)

+ return offlineReply{v: v, waiting: r.answer.waiting, hasWaiting: r.answer.hasWaiting}, -1

+// poll reads the status door.

+func (w *offlineRun) poll(ctx context.Context) (offlineReply, int) {

+ r, code := w.m.getReply(ctx, w.m.path)

+ if code >= 0 {

+ return offlineReply{}, code

+ }

+ return w.read(r)

+// post sends a verb's door. A refusal is said as the console wrote it.

+func (w *offlineRun) post(ctx context.Context, tail string, body any) (offlineReply, int) {

+ r := w.m.send(ctx, http.MethodPost, tail, body)

+ if code := w.m.refusal(r); code >= 0 {

+ return offlineReply{}, code

+ }

+ return w.read(r)

+// follow polls until step says the verb's step has ended, and returns the

+// exit code it gave.

+func (w *offlineRun) follow(ctx context.Context, cur offlineReply, step func(offlineReply) (int, bool)) int {

+ for {

+ if code, done := step(cur); done {

+ return code

+ }

+ if !w.m.sleep(ctx) {

+ moveSay(w.m.out, moveStopped)

+ return moveExitError

+ }

+ var code int

+ if cur, code = w.poll(ctx); code >= 0 {

+ return code

+ }

+// sayWaiting says the console's sentence for what the next verb waits for,

+// each time it changes.

+func (w *offlineRun) sayWaiting(r offlineReply) {

+ if r.hasWaiting && r.waiting != w.lastWaiting {

+ saySpoken(w.m.out, r.waiting)

+ w.lastWaiting = r.waiting

+ }

+// sayCleaning names, once each, the boxes the finish waits for.

+func (w *offlineRun) sayCleaning(v offlineView) {

+ var source, sealer string

+ switch v.State {

+ case provision.OfflineOpen:

+ source = moves.OfflinePhaseParked

+ case provision.OfflineFinishing:

+ source, sealer = moves.OfflinePhaseGone, moves.OfflinePhaseLifted

+ case provision.OfflineReverting:

+ source, sealer = moves.OfflinePhaseReverted, moves.OfflinePhaseReverted

+ default:

+ return

+ }

+ var boxes []string

+ for _, b := range v.Boxes {

+ if r := v.report(moves.OfflineRoleSource, b); r == nil || r.Phase != source {

+ boxes = append(boxes, b)

+ }

+ if sealer != "" && v.Sealer != "" {

+ if r := v.report(moves.OfflineRoleSealer, v.Sealer); r == nil || r.Phase != sealer {

+ boxes = append(boxes, v.Sealer)

+ }

+ for _, b := range boxes {

+ if w.once("clean:" + v.State + ":" + b) {

+ moveSay(w.m.out, fmt.Sprintf("Waiting for %s to clean up %s.", b, w.m.host))

+ }

+// ended says the end of the move this run followed, from the console's

+// `last`, which counts only when it is that move's own.

+func (w *offlineRun) ended(v offlineView) int {

+ host := w.m.host

+ l := v.Last

+ if w.id == "" || l == nil || l.ID != w.id {

+ moveSay(w.m.out, moveNotRecorded)

+ return moveExitWaiting

+ }

+ switch l.Outcome {

+ case "done":

+ if w.to == "" {

+ offlineOutcomeLine(w.m, l)

+ return moveExitDone

+ }

+ moveSay(w.m.out, moves.MovedLine(host, w.to, int(math.Round(l.OfflineS))))

+ return moveExitDone

+ case "aborted":

+ if w.from == "" {

+ offlineOutcomeLine(w.m, l)

+ return moveExitRolled

+ }

+ moveSay(w.m.out, moves.MovedBackLine(host, w.from))

+ return moveExitRolled

+ }

+ moveSay(w.m.out, moveNotRecorded)

+ return moveExitWaiting

+// offlineOutcomeLine says an outcome without the regions: the move, the host

+// and how it ended.

+func offlineOutcomeLine(m *mover, l *moveOutcome) {

+ moveSay(m.out, fmt.Sprintf("Move %s of %s is %s.", l.ID, m.host, l.Outcome))

+// served reports that the target serves the instance: its report says

+// opened, the move has gone on to finishing, or the next step is finish.

+func (w *offlineRun) served(v offlineView) bool {

+ if v.State == provision.OfflineFinishing || v.Next == "finish" {

+ return true

+ }

+ for _, r := range v.Reports {

+ if r.Role == moves.OfflineRoleTarget && r.Phase == moves.OfflinePhaseOpened {

+ return true

+ }

+ return false

+// sayServed is MV-O7, said once.

+func (w *offlineRun) sayServed(v offlineView) {

+ if w.once("served") {

+ host := w.m.host

+ moveSay(w.m.out, fmt.Sprintf("%s is served from %s. Run carlos move offline finish --host %s once %s has cleaned up.", host, v.To, host, v.From))

+ }

+// headline says where a move in flight stands, in the verbs' own sentences.

+func (w *offlineRun) headline(r offlineReply) {

+ v, host := r.v, w.m.host

+ moveSay(w.m.out, moves.MoveStartingLine(host, v.From, v.To))

+ if w.sayFailure(v) {

+ return

+ }

+ switch v.State {

+ case provision.OfflineStopping:

+ if v.Next == "seal" {

+ moveSay(w.m.out, fmt.Sprintf("Every box in %s has stopped %s. Run carlos move offline seal --host %s to copy it.", v.From, host, host))

+ return

+ }

+ w.sayWaiting(r)

+ case provision.OfflineSealing:

+ moveSay(w.m.out, fmt.Sprintf("Copying %s to %s.", host, v.To))

+ case provision.OfflineSealed:

+ if v.Next == "flip" {

+ moveSay(w.m.out, fmt.Sprintf("Copied and checked. Run carlos move offline flip --host %s to start it in %s.", host, v.To))

+ return

+ }

+ w.sayWaiting(r)

+ case provision.OfflineLanding:

+ moveSay(w.m.out, moves.FlippedLine(v.From, v.To))

+ case provision.OfflineLanded:

+ if v.Next == "open" {

+ moveSay(w.m.out, fmt.Sprintf("%s holds everything %s needs. Nothing is served yet. Run carlos move offline open --host %s to serve it from %s.", v.To, host, host, v.To))

+ return

+ }

+ w.sayWaiting(r)

+ case provision.OfflineOpen, provision.OfflineFinishing:

+ if w.served(v) {

+ w.sayServed(v)

+ } else {

+ w.sayWaiting(r)

+ }

+ w.sayCleaning(v)

+ case provision.OfflineReverting:

+ moveSay(w.m.out, fmt.Sprintf("Moving %s back to %s.", host, v.From))

+ w.sayCleaning(v)

+ if v.Next == "finish" {

+ moveSay(w.m.out, fmt.Sprintf("Every box has put %s back in %s. Run carlos move offline finish --host %s to end the move.", host, v.From, host))

+ }

+// offline runs one `carlos move offline` verb.

+func (m *mover) offline(ctx context.Context, a moveArgs) int {

+ w := &offlineRun{m: m, said: map[string]bool{}}

+ switch a.offline {

+ case "status":

+ return w.runStatus(ctx)

+ case "start":

+ return w.runStart(ctx, a.to)

+ case "seal":

+ return w.runSeal(ctx)

+ case "flip":

+ return w.runFlip(ctx)

+ case "open":

+ return w.runOpen(ctx)

+ case "finish":

+ return w.runFinish(ctx)

+ default: // abort

+ return w.runAbort(ctx)

+ }

+// stray is a view the verb's step did not expect: a move that is no longer

+// in flight says how it ended; any other state is said where it stands, and

+// the operator decides.

+func (w *offlineRun) stray(r offlineReply) int {

+ if r.v.State == "none" {

+ return w.ended(r.v)

+ }

+ w.headline(r)

+ return moveExitWaiting

+func (w *offlineRun) runStatus(ctx context.Context) int {

+ r, code := w.poll(ctx)

+ if code >= 0 {

+ return code

+ }

+ v, host := r.v, w.m.host

+ if v.State == "none" {

+ if v.Last != nil {

+ offlineOutcomeLine(w.m, v.Last)

+ } else {

+ moveSay(w.m.out, fmt.Sprintf("No move is in flight for %s.", host))

+ }

+ return moveExitDone

+ }

+ w.headline(r)

+ failed := false

+ for _, it := range v.Items {

+ if it.Diagnostic == "" {

+ moveSay(w.m.out, fmt.Sprintf("Item %s is %s.", it.Slug, it.Check))

+ } else {

+ moveSay(w.m.out, fmt.Sprintf("Item %s is %s: %s.", it.Slug, it.Check, it.Diagnostic))

+ }

+ failed = failed || it.Check == "failed"

+ }

+ for _, rep := range v.Reports {

+ if rep.Fresh {

+ moveSay(w.m.out, fmt.Sprintf("%s is %s as the %s.", rep.Box, rep.Phase, rep.Role))

+ } else {

+ moveSay(w.m.out, fmt.Sprintf("%s is %s as the %s. Its report is not current.", rep.Box, rep.Phase, rep.Role))

+ }

+ for _, l := range v.Left {

+ moveSay(w.m.out, fmt.Sprintf("%s left %s in place: %s.", l.Holder, l.Where, l.Because))

+ }

+ if _, _, stepFailed := v.failure(); failed || v.Failed || stepFailed {

+ return moveExitWaiting

+ }

+ return moveExitDone

+func (w *offlineRun) runStart(ctx context.Context, to string) int {

+ r, code := w.post(ctx, "", map[string]string{"region": to})

+ if code >= 0 {

+ return code

+ }

+ host := w.m.host

+ moveSay(w.m.out, moves.MoveStartingLine(host, r.v.From, r.v.To))

+ moveSay(w.m.out, fmt.Sprintf("%s is offline from now until the move opens or is aborted.", host))

+ return w.follow(ctx, r, func(r offlineReply) (int, bool) {

+ switch r.v.State {

+ case provision.OfflineStopping:

+ if r.v.Next == "seal" {

+ moveSay(w.m.out, fmt.Sprintf("Every box in %s has stopped %s. Run carlos move offline seal --host %s to copy it.", r.v.From, host, host))

+ return moveExitDone, true

+ }

+ w.sayWaiting(r)

+ return 0, false

+ }

+ return w.stray(r), true

+ })

+func (w *offlineRun) runSeal(ctx context.Context) int {

+ r, code := w.post(ctx, "/seal", struct{}{})

+ if code >= 0 {

+ return code

+ }

+ host := w.m.host

+ moveSay(w.m.out, fmt.Sprintf("Copying %s to %s.", host, r.v.To))

+ return w.follow(ctx, r, func(r offlineReply) (int, bool) {

+ switch r.v.State {

+ case provision.OfflineSealing, provision.OfflineSealed:

+ if w.sayFailure(r.v) {

+ return moveExitWaiting, true

+ }

+ if r.v.State == provision.OfflineSealed {

+ if r.v.Next == "flip" {

+ moveSay(w.m.out, fmt.Sprintf("Copied and checked. Run carlos move offline flip --host %s to start it in %s.", host, r.v.To))

+ return moveExitDone, true

+ }

+ w.sayWaiting(r)

+ }

+ return 0, false

+ }

+ return w.stray(r), true

+ })

+func (w *offlineRun) runFlip(ctx context.Context) int {

+ r, code := w.post(ctx, "/flip", struct{}{})

+ if code >= 0 {

+ return code

+ }

+ host := w.m.host

+ moveSay(w.m.out, moves.FlippedLine(r.v.From, r.v.To))

+ return w.follow(ctx, r, func(r offlineReply) (int, bool) {

+ switch r.v.State {

+ case provision.OfflineLanding, provision.OfflineLanded:

+ if w.sayFailure(r.v) {

+ return moveExitWaiting, true

+ }

+ if r.v.State == provision.OfflineLanded {

+ if r.v.Next == "open" {

+ moveSay(w.m.out, fmt.Sprintf("%s holds everything %s needs. Nothing is served yet. Run carlos move offline open --host %s to serve it from %s.", r.v.To, host, host, r.v.To))

+ return moveExitDone, true

+ }

+ w.sayWaiting(r)

+ }

+ return 0, false

+ }

+ return w.stray(r), true

+ })

+func (w *offlineRun) runOpen(ctx context.Context) int {

+ r, code := w.post(ctx, "/open", struct{}{})

+ if code >= 0 {

+ return code

+ }

+ // The line is said once the target serves (its report says opened, or the

+ // next step is finish), not when the door only accepted the step.

+ return w.follow(ctx, r, func(r offlineReply) (int, bool) {

+ if r.v.State == provision.OfflineOpen {

+ if w.sayFailure(r.v) {

+ return moveExitWaiting, true

+ }

+ if w.served(r.v) {

+ w.sayServed(r.v)

+ } else {

+ w.sayWaiting(r)

+ }

+ if r.v.Next == "finish" {

+ return moveExitDone, true

+ }

+ if w.sayFailure(r.v) {

+ return moveExitWaiting, true

+ }

+ w.sayCleaning(r.v)

+ return 0, false

+ }

+ return w.stray(r), true

+ })

+// finish posts until the move is cleared. The first post (from open) starts

+// the clean-up and the console answers offline-finishing; once the boxes have

+// cleaned up and the tombstone is lifted, the status shows the next verb is

+// finish again and the second post ends the move. A move reverting from an

+// abort needs only the one.

+func (w *offlineRun) runFinish(ctx context.Context) int {

+ cur, code := w.poll(ctx)

+ if code >= 0 {

+ return code

+ }

+ for {

+ // Wait until the door would accept: the boxes' clean-up shows in the

+ // status before the post, so the door is never asked early.

+ switch cur.v.State {

+ case provision.OfflineOpen, provision.OfflineFinishing, provision.OfflineReverting:

+ if cur.v.Next != "finish" {

+ code := w.follow(ctx, cur, func(r offlineReply) (int, bool) {

+ switch r.v.State {

+ case provision.OfflineOpen, provision.OfflineFinishing, provision.OfflineReverting:

+ if r.v.Next == "finish" {

+ cur = r

+ return -1, true

+ }

+ if w.sayFailure(r.v) {

+ return moveExitWaiting, true

+ }

+ w.sayCleaning(r.v)

+ return 0, false

+ }

+ cur = r

+ return -1, true

+ })

+ if code >= 0 {

+ return code

+ }

+ prev := cur.v.State

+ r, code := w.post(ctx, "/finish", struct{}{})

+ if code >= 0 {

+ return code

+ }

+ if r.v.State == "none" {

+ return w.ended(r.v)

+ }

+ if r.v.State == prev {

+ return w.stray(r)

+ }

+ cur = r

+ }

+func (w *offlineRun) runAbort(ctx context.Context) int {

+ r, code := w.post(ctx, "/abort", struct{}{})

+ if code >= 0 {

+ return code

+ }

+ host := w.m.host

+ moveSay(w.m.out, fmt.Sprintf("Moving %s back to %s.", host, r.v.From))

+ return w.follow(ctx, r, func(r offlineReply) (int, bool) {

+ if r.v.State == provision.OfflineReverting {

+ if r.v.Next == "finish" {

+ moveSay(w.m.out, fmt.Sprintf("Every box has put %s back in %s. Run carlos move offline finish --host %s to end the move.", host, r.v.From, host))

+ return moveExitRolled, true

+ }

+ w.sayCleaning(r.v)

+ return 0, false

+ }

+ return w.stray(r), true

+ })

diff --git a/move_test.go b/move_test.go

index e2f3a7390..3ae90a02d 100644

--- a/move_test.go

+++ b/move_test.go

@@ -1094,3 +1094,715 @@ func TestMoveStatusSaysWhyTheCutDidNotStart(t *testing.T) {

}

wantLines(t, out, mc12)

}

+// ---- carlos move offline ----

+const tOfflinePth = tMovePth + "/offline"

+// offPost is one scripted answer of a mutating door; the last one repeats.

+type offPost struct {

+ code int

+ body any

+// fakeOfflineConsole answers the offline doors in T6's shapes. Each GET of the

+// status door returns the next scripted view (the last repeats); each POST

+// returns the next scripted answer of its door. Anything outside the offline

+// doors is a 404, so a verb that read the Phase 1 door would fail.

+type fakeOfflineConsole struct {

+ mu sync.Mutex

+ gets []frame

+ i int

+ posts map[string][]offPost // keyed by the door's tail: "", "/seal", ...

+ n map[string]int

+ log []string // method path body

+func (c *fakeOfflineConsole) ServeHTTP(w http.ResponseWriter, r *http.Request) {

+ c.mu.Lock()

+ defer c.mu.Unlock()

+ write := func(code int, v any) {

+ w.Header().Set("Content-Type", "application/json")

+ w.WriteHeader(code)

+ _ = json.NewEncoder(w).Encode(v)

+ }

+ body, _ := io.ReadAll(r.Body)

+ c.log = append(c.log, r.Method+" "+strings.TrimPrefix(r.URL.Path, tMovePth)+" "+strings.TrimSpace(string(body)))

+ if !strings.HasPrefix(r.URL.Path, tOfflinePth) {

+ http.NotFound(w, r)

+ return

+ }

+ if r.Method == http.MethodGet && r.URL.Path == tOfflinePth {

+ n := c.i

+ if n >= len(c.gets) {

+ n = len(c.gets) - 1

+ }

+ c.i++

+ write(http.StatusOK, c.gets[n])

+ return

+ }

+ tail := strings.TrimPrefix(r.URL.Path, tOfflinePth)

+ list := c.posts[tail]

+ if r.Method != http.MethodPost || len(list) == 0 {

+ http.NotFound(w, r)

+ return

+ }

+ if c.n == nil {

+ c.n = map[string]int{}

+ }

+ k := c.n[tail]

+ if k >= len(list) {

+ k = len(list) - 1

+ }

+ c.n[tail]++

+ write(list[k].code, list[k].body)

+func (c *fakeOfflineConsole) requests() []string {

+ c.mu.Lock()

+ defer c.mu.Unlock()

+ return append([]string(nil), c.log...)

+// posted counts the POSTs a door got.

+func (c *fakeOfflineConsole) posted(tail string) int {

+ c.mu.Lock()

+ defer c.mu.Unlock()

+ return c.n[tail]

+func runOfflineAgainst(t *testing.T, c *fakeOfflineConsole, args ...string) (int, string) {

+ t.Helper()

+ ts := httptest.NewServer(c)

+ t.Cleanup(ts.Close)

+ var out bytes.Buffer

+ deps := moveDeps{

+ session: func(string) (*apiSession, error) {

+ return &apiSession{c: newConsoleClient(ts.URL, "cli-token"), console: ts.URL, address: "op@keymail.dev"}, nil

+ },

+ euid: func() int { return 1000 },

+ poll: time.Millisecond,

+ stdin: strings.NewReader(""),

+ }

+ args = append([]string{"offline"}, args...)

+ args = append(args, "--host", tHost, "--account", "bab", "--app", "titogo")

+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)

+ defer cancel()

+ code := moveMain(ctx, args, &out, deps)

+ return code, out.String()

+// offView is the offline status view of a move in state, as T6 builds it.

+func offView(state string, extra frame) frame {

+ f := frame{"host": tHost, "state": state, "id": tMoveID, "from": "ie", "to": "uk",

+ "sealer": "ie-box", "boxes": []string{"ie-box"}}

+ for k, v := range extra {

+ f[k] = v

+ }

+ return f

+func offDone(outcome string, offlineS float64) frame {

+ return frame{"host": tHost, "state": "none",

+ "last": frame{"id": tMoveID, "outcome": outcome, "at": "2026-10-20T10:00:00Z", "offline_s": offlineS}}

+func offReport(box, role, phase string, fresh bool) frame {

+ return frame{"box": box, "role": role, "phase": phase, "fresh": fresh}

+const (

+ oHost = "acme.go.tito.io"

+ oStarting = "Moving acme.go.tito.io from ie to uk."

+ oOffline = "acme.go.tito.io is offline from now until the move opens or is aborted."

+ oStopped = "Every box in ie has stopped acme.go.tito.io. Run carlos move offline seal --host acme.go.tito.io to copy it."

+ oCopying = "Copying acme.go.tito.io to uk."

+ oCopied = "Copied and checked. Run carlos move offline flip --host acme.go.tito.io to start it in uk."

+ oFlipped = "Stopped in ie. Starting in uk."

+ oLanded = "uk holds everything acme.go.tito.io needs. Nothing is served yet. Run carlos move offline open --host acme.go.tito.io to serve it from uk."

+ oServed = "acme.go.tito.io is served from uk. Run carlos move offline finish --host acme.go.tito.io once ie has cleaned up."

+ oBack = "Moving acme.go.tito.io back to ie."

+ oPutBack = "Every box has put acme.go.tito.io back in ie. Run carlos move offline finish --host acme.go.tito.io to end the move."

+ oWaitBox = "Waiting for ie-box in ie to confirm it has stopped acme.go.tito.io. acme.go.tito.io is offline."

+func TestParseMoveArgsOffline(t *testing.T) {

+ for _, c := range []struct {

+ args []string

+ ok bool

+ sub string

+ }{

+ {[]string{"offline", "start", "--host", tHost, "--to", "uk"}, true, "start"},

+ {[]string{"offline", "seal", "--host", tHost}, true, "seal"},

+ {[]string{"offline", "flip", "--host", tHost}, true, "flip"},

+ {[]string{"offline", "open", "--host", tHost}, true, "open"},

+ {[]string{"offline", "finish", "--host", tHost}, true, "finish"},

+ {[]string{"offline", "abort", "--host", tHost}, true, "abort"},

+ {[]string{"offline", "status", "--host", tHost, "--app", "x", "--account", "y"}, true, "status"},

+ {[]string{"offline", "start", "--host", tHost}, false, ""},

+ {[]string{"offline", "start", "--to", "uk"}, false, ""},

+ {[]string{"offline", "seal", "--host", tHost, "--to", "uk"}, false, ""},

+ {[]string{"offline", "seal", "--host", tHost, "--no-wait"}, false, ""},

+ {[]string{"offline", "seal", "--host", tHost, "extra"}, false, ""},

+ {[]string{"offline"}, false, ""},

+ {[]string{"offline", "cut", "--host", tHost}, false, ""},

+ {[]string{"offline", "--host", tHost}, false, ""},

+ } {

+ got, err := parseMoveArgs(c.args, io.Discard)

+ if (err == nil) != c.ok || (c.ok && (got.verb != "offline" || got.offline != c.sub)) {

+ t.Errorf("parseMoveArgs(%q) = %+v, %v; want ok %v offline %q", c.args, got, err, c.ok, c.sub)

+ }

+ var out bytes.Buffer

+ if _, err := parseMoveArgs([]string{"offline", "cut", "--host", tHost}, &out); err == nil {

+ t.Fatal("an unknown offline verb parsed")

+ }

+ wantLines(t, out.String(), "carlos move offline needs start, seal, flip, open, finish, abort or status.")

+ out.Reset()

+ _, _ = parseMoveArgs([]string{"offline", "start", "--host", tHost}, &out)

+ wantLines(t, out.String(), "carlos move needs --to <region>.")

+ // The Phase 1 verbs are untouched.

+ if got, err := parseMoveArgs([]string{"abort", "--host", tHost}, io.Discard); err != nil || got.verb != "abort" || got.offline != "" {

+ t.Errorf("carlos move abort = %+v, %v", got, err)

+ }

+func TestMoveOfflineStart(t *testing.T) {

+ c := &fakeOfflineConsole{

+ posts: map[string][]offPost{"": {{202, offView("offline-stopping", frame{"waiting": oWaitBox})}}},

+ gets: []frame{

+ offView("offline-stopping", frame{"waiting": oWaitBox}),

+ offView("offline-stopping", frame{"next": "seal"}),

+ },

+ }

+ code, out := runOfflineAgainst(t, c, "start", "--to", "uk")

+ if code != 0 {

+ t.Fatalf("exit %d:\n%s", code, out)

+ }

+ want := oStarting + "\n" + oOffline + "\n" + oWaitBox + "\n" + oStopped + "\n"

+ if out != want {

+ t.Fatalf("output:\n%s\nwant:\n%s", out, want)

+ }

+ reqs := c.requests()

+ if reqs[0] != `POST {"region":"uk"}` && reqs[0] != `POST /offline {"region":"uk"}` {

+ t.Fatalf("first request %q", reqs[0])

+ }

+ for _, r := range reqs {

+ if !strings.Contains(r, "/offline") {

+ t.Errorf("a request left the offline doors: %q", r)

+ }

+func TestMoveOfflineRefusals(t *testing.T) {

+ for _, c := range []struct {

+ name string

+ code int

+ body any

+ exit int

+ say string

+ }{

+ {"claim console", 409, frame{"error": "offline moves are not available while host claims or Phase 1 moves are on"}, 5, "offline moves are not available while host claims or Phase 1 moves are on"},

+ {"not found", 404, "404 page not found", 5, "Not found. Check --account and --app, and that you are a member of the instance's account."},

+ {"console error", 502, frame{"error": "could not write the move. try again"}, 1, "could not write the move. try again"},

+ {"no reason", 500, frame{}, 1, "The console answered 500 and gave no reason."},

+ } {

+ t.Run(c.name, func(t *testing.T) {

+ f := &fakeOfflineConsole{posts: map[string][]offPost{"": {{c.code, c.body}}, "/seal": {{c.code, c.body}}}, gets: []frame{offView("offline-stopping", nil)}}

+ for _, args := range [][]string{{"start", "--to", "uk"}, {"seal"}} {

+ code, out := runOfflineAgainst(t, f, args...)

+ if code != c.exit || out != c.say+"\n" {

+ t.Errorf("%v: exit %d, output %q; want %d, %q", args, code, out, c.exit, c.say)

+ }

+ for _, r := range f.requests() {

+ if strings.HasPrefix(r, "GET") {

+ t.Errorf("a refused verb polled: %q", r)

+ }

+ })

+ }

+func TestMoveOfflineSeal(t *testing.T) {

+ proc := "ie-box still runs a process for acme.go.tito.io. wait for it to stop, then flip"

+ c := &fakeOfflineConsole{

+ posts: map[string][]offPost{"/seal": {{202, offView("offline-sealing", nil)}}},

+ gets: []frame{

+ offView("offline-sealing", nil),

+ offView("offline-sealed", frame{"waiting": proc}),

+ offView("offline-sealed", frame{"next": "flip"}),

+ },

+ }

+ code, out := runOfflineAgainst(t, c, "seal")

+ if want := oCopying + "\n" + proc + "\n" + oCopied + "\n"; code != 0 || out != want {

+ t.Fatalf("exit %d, output:\n%s\nwant:\n%s", code, out, want)

+ }

+ if got := c.requests()[0]; !strings.HasPrefix(got, "POST /offline/seal") {

+ t.Errorf("first request %q", got)

+ }

+func TestMoveOfflineSealFails(t *testing.T) {

+ item := func(status, detail string) frame {

+ return frame{"id": "files", "owner": "seal", "status": status, "detail": detail}

+ }

+ for _, c := range []struct {

+ name string

+ extra frame

+ say string

+ }{

+ {"item", frame{"items": []frame{item("ok", ""), item("failed", "a file over 256 MiB")}}, "The move could not finish a step: a file over 256 MiB. Run carlos move offline status --host acme.go.tito.io for each item."},

+ {"item without detail", frame{"items": []frame{item("failed", "")}}, "The move could not finish a step: files. Run carlos move offline status --host acme.go.tito.io for each item."},

+ {"failed flag and report", frame{"failed": true, "reports": []frame{{"box": "ie-box", "role": "sealer", "phase": "failed", "fresh": true,

+ "items": frame{"db": frame{"ok": true, "reason": ""}, "files": frame{"ok": false, "reason": "special file in the archive"}}}}},

+ "The move could not finish a step: special file in the archive. Run carlos move offline status --host acme.go.tito.io for each item."},

+ } {

+ t.Run(c.name, func(t *testing.T) {

+ f := &fakeOfflineConsole{

+ posts: map[string][]offPost{"/seal": {{202, offView("offline-sealing", nil)}}},

+ gets: []frame{offView("offline-sealing", c.extra)},

+ }

+ code, out := runOfflineAgainst(t, f, "seal")

+ if want := oCopying + "\n" + c.say + "\n"; code != 4 || out != want {

+ t.Fatalf("exit %d, output:\n%s\nwant 4:\n%s", code, out, want)

+ }

+ })

+ }

+func TestMoveOfflineFlip(t *testing.T) {

+ c := &fakeOfflineConsole{

+ posts: map[string][]offPost{"/flip": {{202, offView("offline-landing", nil)}}},

+ gets: []frame{

+ offView("offline-landing", nil),

+ offView("offline-landed", frame{"waiting": "this move is offline-landed. run carlos move offline status --host acme.go.tito.io to see what it waits for"}),

+ offView("offline-landed", frame{"next": "open"}),

+ },

+ }

+ code, out := runOfflineAgainst(t, c, "flip")

+ want := oFlipped + "\nthis move is offline-landed. run carlos move offline status --host acme.go.tito.io to see what it waits for\n" + oLanded + "\n"

+ if code != 0 || out != want {

+ t.Fatalf("exit %d, output:\n%s\nwant:\n%s", code, out, want)

+ }

+func TestMoveOfflineFlipFails(t *testing.T) {

+ failedItem := frame{"id": "db", "owner": "landing", "status": "failed", "detail": "the listing differs"}

+ for _, c := range []struct {

+ name string

+ extra frame

+ say string

+ }{

+ {"item", frame{"items": []frame{failedItem}}, "uk could not take acme.go.tito.io: the listing differs. Run carlos move offline abort --host acme.go.tito.io to move it back."},

+ {"failed flag, check in the report", frame{"failed": true, "reports": []frame{{"box": "uk-box", "role": "target", "phase": "failed", "fresh": true,

+ "items": frame{"env-pin": frame{"ok": false, "reason": "the app settings differ"}}}}},

+ "uk could not take acme.go.tito.io: the app settings differ. Run carlos move offline abort --host acme.go.tito.io to move it back."},

+ {"failed flag only", frame{"failed": true}, "uk could not take acme.go.tito.io: failed. Run carlos move offline abort --host acme.go.tito.io to move it back."},

+ } {

+ t.Run(c.name, func(t *testing.T) {

+ f := &fakeOfflineConsole{

+ posts: map[string][]offPost{"/flip": {{202, offView("offline-landing", nil)}}},

+ gets: []frame{offView("offline-landing", nil), offView("offline-landing", c.extra)},

+ }

+ code, out := runOfflineAgainst(t, f, "flip")

+ if want := oFlipped + "\n" + c.say + "\n"; code != 4 || out != want {

+ t.Fatalf("exit %d, output:\n%s\nwant 4:\n%s", code, out, want)

+ }

+ if f.posted("/flip") != 1 {

+ t.Error("flip posted more than once")

+ }

+ })

+ }

+func TestMoveOfflineOpen(t *testing.T) {

+ src := func(phase string) frame { return offReport("ie-box", "source", phase, true) }

+ tgt := func(phase string) frame { return offReport("uk-box", "target", phase, true) }

+ c := &fakeOfflineConsole{

+ posts: map[string][]offPost{"/open": {{202, offView("offline-open", nil)}}},

+ gets: []frame{

+ offView("offline-open", frame{"reports": []frame{src("stopped"), tgt("landed")}}),

+ offView("offline-open", frame{"reports": []frame{src("stopped"), tgt("opened")}}),

+ offView("offline-open", frame{"next": "finish", "reports": []frame{src("parked"), tgt("opened")}}),

+ },

+ }

+ code, out := runOfflineAgainst(t, c, "open")

+ // The served line waits for the target's opened report.

+ want := "Waiting for ie-box to clean up acme.go.tito.io.\n" + oServed + "\n"

+ if code != 0 || out != want {

+ t.Fatalf("exit %d, output:\n%s\nwant:\n%s", code, out, want)

+ }

+ // Or for finish to be next, when the target's report is not in the view.

+ c = &fakeOfflineConsole{

+ posts: map[string][]offPost{"/open": {{202, offView("offline-open", nil)}}},

+ gets: []frame{offView("offline-open", frame{"next": "finish"})},

+ }

+ if code, out := runOfflineAgainst(t, c, "open"); code != 0 || out != "Waiting for ie-box to clean up acme.go.tito.io.\n"+oServed+"\n" {

+ t.Fatalf("exit %d, output:\n%s", code, out)

+ }

+// finish posts twice for a move that was open: the first starts the clean-up

+// and answers offline-finishing; once the boxes are gone and the tombstone is

+// lifted, the status shows finish again and the second post ends the move.

+func TestMoveOfflineFinishPostsTwice(t *testing.T) {

+ c := &fakeOfflineConsole{

+ posts: map[string][]offPost{"/finish": {

+ {202, offView("offline-finishing", frame{"reports": []frame{offReport("ie-box", "source", "parked", true)}})},

+ {202, offDone("done", 1800)},

+ }},

+ gets: []frame{

+ offView("offline-open", frame{"next": "finish"}),

+ offView("offline-finishing", frame{"reports": []frame{offReport("ie-box", "source", "parked", true)}}),

+ offView("offline-finishing", frame{"reports": []frame{offReport("ie-box", "source", "gone", true), offReport("ie-box", "sealer", "sealed", true)}}),

+ offView("offline-finishing", frame{"next": "finish", "reports": []frame{offReport("ie-box", "source", "gone", true), offReport("ie-box", "sealer", "lifted", true)}}),

+ },

+ }

+ code, out := runOfflineAgainst(t, c, "finish")

+ want := "Waiting for ie-box to clean up acme.go.tito.io.\nMoved. acme.go.tito.io is served from uk. It was offline for 1800 seconds.\n"

+ if code != 0 || out != want {

+ t.Fatalf("exit %d, output:\n%s\nwant:\n%s", code, out, want)

+ }

+ if n := c.posted("/finish"); n != 2 {

+ t.Fatalf("finish posted %d times, want 2:\n%v", n, c.requests())

+ }

+// A finish before the boxes have parked does not ask the door early.

+func TestMoveOfflineFinishWaitsBeforeThePost(t *testing.T) {

+ c := &fakeOfflineConsole{

+ posts: map[string][]offPost{"/finish": {{202, offDone("done", 12)}}},

+ gets: []frame{

+ offView("offline-open", frame{"reports": []frame{offReport("ie-box", "source", "stopped", true)}}),

+ offView("offline-open", frame{"next": "finish"}),

+ },

+ }

+ code, out := runOfflineAgainst(t, c, "finish")

+ if code != 0 || !strings.HasSuffix(out, "It was offline for 12 seconds.\n") || !strings.HasPrefix(out, "Waiting for ie-box to clean up acme.go.tito.io.\n") {

+ t.Fatalf("exit %d:\n%s", code, out)

+ }

+ reqs := c.requests()

+ post := -1

+ for i, r := range reqs {

+ if strings.HasPrefix(r, "POST") {

+ post = i

+ }

+ if post < 2 {

+ t.Fatalf("the door was posted before the status showed finish:\n%v", reqs)

+ }

+func TestMoveOfflineFinishAfterAbort(t *testing.T) {

+ c := &fakeOfflineConsole{

+ posts: map[string][]offPost{"/finish": {{202, offDone("aborted", 0)}}},

+ gets: []frame{offView("offline-reverting", frame{"next": "finish"})},

+ }

+ code, out := runOfflineAgainst(t, c, "finish")

+ if want := "acme.go.tito.io is back in ie and serving. Nothing was lost.\n"; code != 3 || out != want {

+ t.Fatalf("exit %d, output:\n%s\nwant 3:\n%s", code, out, want)

+ }

+ if c.posted("/finish") != 1 {

+ t.Error("finish after an abort posted more than once")

+ }

+func TestMoveOfflineFinishRefusedAndOutcomeNotRecorded(t *testing.T) {

+ // Nothing in flight: the door's own sentence, exit 5.

+ c := &fakeOfflineConsole{

+ posts: map[string][]offPost{"/finish": {{409, frame{"error": "acme.go.tito.io has no offline move in flight"}}}},

+ gets: []frame{{"host": tHost, "state": "none"}},

+ }

+ if code, out := runOfflineAgainst(t, c, "finish"); code != 5 || out != "acme.go.tito.io has no offline move in flight\n" {

+ t.Fatalf("exit %d:\n%s", code, out)

+ }

+ // An outcome that is another move's is not this move's.

+ c = &fakeOfflineConsole{

+ posts: map[string][]offPost{"/finish": {{202, frame{"host": tHost, "state": "none", "last": frame{"id": "mv_other", "outcome": "done"}}}}},

+ gets: []frame{offView("offline-reverting", frame{"next": "finish"})},

+ }

+ if code, out := runOfflineAgainst(t, c, "finish"); code != 4 || out != "This move's outcome is not recorded.\n" {

+ t.Fatalf("exit %d:\n%s", code, out)

+ }

+func TestMoveOfflineAbort(t *testing.T) {

+ c := &fakeOfflineConsole{

+ posts: map[string][]offPost{"/abort": {{202, offView("offline-reverting", nil)}}},

+ gets: []frame{

+ offView("offline-reverting", frame{"reports": []frame{offReport("ie-box", "source", "stopped", true)}}),

+ offView("offline-reverting", frame{"next": "finish", "reports": []frame{offReport("ie-box", "source", "reverted", true)}}),

+ },

+ }

+ code, out := runOfflineAgainst(t, c, "abort")

+ want := oBack + "\nWaiting for ie-box to clean up acme.go.tito.io.\n" + oPutBack + "\n"

+ if code != 3 || out != want {

+ t.Fatalf("exit %d, output:\n%s\nwant 3:\n%s", code, out, want)

+ }

+ if c.posted("/finish") != 0 {

+ t.Error("abort ran finish by itself")

+ }

+ // Once open, the door refuses and the CLI says so as sent.

+ c = &fakeOfflineConsole{posts: map[string][]offPost{"/abort": {{409, frame{"error": "acme.go.tito.io is already served from uk. move it back with a new move"}}}}}

+ if code, out := runOfflineAgainst(t, c, "abort"); code != 5 || out != "acme.go.tito.io is already served from uk. move it back with a new move\n" {

+ t.Fatalf("exit %d:\n%s", code, out)

+ }

+// A verb whose step is overtaken (another terminal aborted) says where the

+// move stands and leaves the decision to the operator.

+func TestMoveOfflineOvertaken(t *testing.T) {

+ c := &fakeOfflineConsole{

+ posts: map[string][]offPost{"/seal": {{202, offView("offline-sealing", nil)}}},

+ gets: []frame{offView("offline-reverting", nil)},

+ }

+ code, out := runOfflineAgainst(t, c, "seal")

+ if want := oCopying + "\n" + oStarting + "\n" + oBack + "\nWaiting for ie-box to clean up acme.go.tito.io.\n"; code != 4 || out != want {

+ t.Fatalf("exit %d, output:\n%s\nwant 4:\n%s", code, out, want)

+ }

+func TestMoveOfflineStatus(t *testing.T) {

+ c := &fakeOfflineConsole{gets: []frame{offView("offline-sealed", frame{

+ "next": "flip",

+ "items": []frame{

+ {"id": "db", "owner": "landing", "status": "waiting"},

+ {"id": "files", "owner": "seal", "status": "ok", "detail": "12 files"},

+ },

+ "reports": []frame{offReport("ie-box", "source", "stopped", true), offReport("ie-box", "sealer", "sealed", false)},

+ })}}

+ code, out := runOfflineAgainst(t, c, "status")

+ want := oStarting + "\n" + oCopied + "\n" +

+ "Item db is waiting.\nItem files is ok: 12 files.\n" +

+ "ie-box is stopped as the source.\nie-box is sealed as the sealer. Its report is not current.\n"

+ if code != 0 || out != want {

+ t.Fatalf("exit %d, output:\n%s\nwant:\n%s", code, out, want)

+ }

+ if reqs := c.requests(); len(reqs) != 1 || !strings.HasPrefix(reqs[0], "GET /offline") {

+ t.Fatalf("status sent %v", reqs)

+ }

+ // A failed item is the operator's: exit 4.

+ c = &fakeOfflineConsole{gets: []frame{offView("offline-landing", frame{"items": []frame{{"id": "db", "owner": "landing", "status": "failed", "detail": "digest differs"}}})}}

+ code, out = runOfflineAgainst(t, c, "status")

+ wantF := oStarting + "\nuk could not take acme.go.tito.io: digest differs. Run carlos move offline abort --host acme.go.tito.io to move it back.\nItem db is failed: digest differs.\n"

+ if code != 4 || out != wantF {

+ t.Fatalf("exit %d, output:\n%s\nwant 4:\n%s", code, out, wantF)

+ }

+ // Waiting states say the console's sentence.

+ c = &fakeOfflineConsole{gets: []frame{offView("offline-stopping", frame{"waiting": oWaitBox})}}

+ if code, out := runOfflineAgainst(t, c, "status"); code != 0 || out != oStarting+"\n"+oWaitBox+"\n" {

+ t.Fatalf("exit %d:\n%s", code, out)

+ }

+ // Nothing in flight.

+ c = &fakeOfflineConsole{gets: []frame{{"host": tHost, "state": "none"}}}

+ if code, out := runOfflineAgainst(t, c, "status"); code != 0 || out != "No move is in flight for acme.go.tito.io.\n" {

+ t.Fatalf("exit %d:\n%s", code, out)

+ }

+ c = &fakeOfflineConsole{gets: []frame{offDone("done", 5)}}

+ if code, out := runOfflineAgainst(t, c, "status"); code != 0 || out != "Move "+tMoveID+" of acme.go.tito.io is done.\n" {

+ t.Fatalf("exit %d:\n%s", code, out)

+ }

+func TestMoveOfflineWaitingAfterOpenAndAbort(t *testing.T) {

+ // Status of an open move names the boxes still cleaning up.

+ c := &fakeOfflineConsole{gets: []frame{offView("offline-finishing", frame{

+ "reports": []frame{offReport("ie-box", "source", "gone", true), offReport("ie-box", "sealer", "sealed", true)}})}}

+ code, out := runOfflineAgainst(t, c, "status")

+ want := oStarting + "\n" + oServed + "\nWaiting for ie-box to clean up acme.go.tito.io.\n" +

+ "ie-box is gone as the source.\nie-box is sealed as the sealer.\n"

+ if code != 0 || out != want {

+ t.Fatalf("exit %d, output:\n%s\nwant:\n%s", code, out, want)

+ }

+func TestMoveOfflineNeedsNoPhase1AndRefusesRoot(t *testing.T) {

+ // The Phase 1 door is never read: the fake 404s everything outside

+ // /move/offline, and the verb still works (the other tests). Root is

+ // refused as for every carlos move.

+ c := &fakeOfflineConsole{gets: []frame{{"host": tHost, "state": "none"}}}

+ ts := httptest.NewServer(c)

+ defer ts.Close()

+ var out bytes.Buffer

+ deps := moveDeps{session: func(string) (*apiSession, error) { t.Error("signed in as root"); return nil, nil }, euid: func() int { return 0 }, poll: time.Millisecond}

+ if code := moveMain(context.Background(), []string{"offline", "status", "--host", tHost, "--app", "titogo"}, &out, deps); code != 5 {

+ t.Fatalf("exit %d:\n%s", code, out.String())

+ }

+ wantLines(t, out.String(), "Run carlos move as yourself, not as root. It uses your own credentials.")

+ if len(c.requests()) != 0 {

+ t.Error("a root run reached the console")

+ }

+func TestMoveOfflineRidesOutARateLimit(t *testing.T) {

+ c := &fakeOfflineConsole{

+ posts: map[string][]offPost{"/seal": {{429, frame{"error": "too many instance requests. wait a minute and try again"}}, {202, offView("offline-sealing", nil)}}},

+ gets: []frame{offView("offline-sealed", frame{"next": "flip"})},

+ }

+ code, out := runOfflineAgainst(t, c, "seal")

+ if want := "The console is limiting requests. Checking every 10 seconds.\n" + oCopying + "\n" + oCopied + "\n"; code != 0 || out != want {

+ t.Fatalf("exit %d, output:\n%s\nwant:\n%s", code, out, want)

+ }

+ if c.posted("/seal") != 2 {

+ t.Errorf("seal posted %d times, want 2", c.posted("/seal"))

+ }

+// The page's paragraph is the copy record's rows MV-O32 and MV-O33, word for

+// word, and its examples parse as the seven offline verbs.

+func TestMoveDocsOfflineParagraph(t *testing.T) {

+ raw, err := os.ReadFile("docs/site/cli.md")

+ if err != nil {

+ t.Fatal(err)

+ }

+ page := string(raw)

+ rec, err := os.ReadFile("docs/superpowers/plans/2026-10-06-moves-phase1-copy.md")

+ if err != nil {

+ t.Fatal(err)

+ }

+ for _, id := range []string{"MV-O32", "MV-O33"} {

+ text := ""

+ for _, line := range strings.Split(string(rec), "\n") {

+ if cells := strings.Split(line, "|"); len(cells) >= 5 && strings.TrimSpace(cells[1]) == id {

+ text = strings.TrimSpace(cells[3])

+ }

+ if text == "" {

+ t.Fatalf("the copy record has no row %s", id)

+ }

+ if !strings.Contains(page, "\n"+text+"\n") {

+ t.Errorf("the page does not carry %s word for word: %q", id, text)

+ }

+ seen := map[string]bool{}

+ for _, line := range strings.Split(page, "\n") {

+ fields := strings.Fields(line)

+ if len(fields) < 4 || fields[0] != "carlos" || fields[1] != "move" || fields[2] != "offline" {

+ continue

+ }

+ args := fields[2:]

+ for i, a := range args {

+ if strings.HasPrefix(a, "<") {

+ args[i] = "x"

+ }

+ got, err := parseMoveArgs(args, io.Discard)

+ if err != nil {

+ t.Errorf("the page's example %q does not parse: %v", line, err)

+ continue

+ }

+ seen[got.offline] = true

+ }

+ for _, v := range offlineVerbs {

+ if !seen[v] {

+ t.Errorf("the page has no example of carlos move offline %s", v)

+ }

+// status of a failed move: exit 4 on the flag alone, a failed seal is not

+// headlined as progress, and the role the view marks decides MV-O6 or MV-O9.

+func TestMoveOfflineStatusFailed(t *testing.T) {

+ sealerFailed := frame{"box": "ie-box", "role": "sealer", "phase": "failed", "fresh": true,

+ "items": frame{"files": frame{"ok": false, "reason": "special file in the archive"}}}

+ targetFailed := frame{"box": "uk-box", "role": "target", "phase": "failed", "fresh": true}

+ for _, c := range []struct {

+ name string

+ state string

+ extra frame

+ say string

+ }{

+ {"failed seal, flag only", "offline-sealing", frame{"failed": true},

+ "The move could not finish a step: failed. Run carlos move offline status --host acme.go.tito.io for each item."},

+ {"failed seal, sealer report", "offline-sealing", frame{"failed": true, "reports": []frame{sealerFailed}},

+ "The move could not finish a step: special file in the archive. Run carlos move offline status --host acme.go.tito.io for each item."},

+ {"failed landing, target report", "offline-landed", frame{"failed": true, "reports": []frame{targetFailed}},

+ "uk could not take acme.go.tito.io: failed. Run carlos move offline abort --host acme.go.tito.io to move it back."},

+ {"failed target before the flip", "offline-stopping", frame{"failed": true, "reports": []frame{targetFailed}},

+ "uk could not take acme.go.tito.io: failed. Run carlos move offline abort --host acme.go.tito.io to move it back."},

+ } {

+ t.Run(c.name, func(t *testing.T) {

+ f := &fakeOfflineConsole{gets: []frame{offView(c.state, c.extra)}}

+ code, out := runOfflineAgainst(t, f, "status")

+ if code != 4 {

+ t.Errorf("exit %d, want 4:\n%s", code, out)

+ }

+ if !strings.HasPrefix(out, oStarting+"\n"+c.say+"\n") {

+ t.Errorf("output:\n%s\nwant the failure line after the headline: %s", out, c.say)

+ }

+ wantNoLine(t, out, "Copying")

+ })

+ }

+func TestMoveOfflineStatusShowsPathsLeft(t *testing.T) {

+ c := &fakeOfflineConsole{gets: []frame{offView("offline-finishing", frame{

+ "next": "finish",

+ "left": []frame{{"box": "ie-box", "path": "/opt/carlos/live/acme.go.tito.io", "reason": "a root-owned directory"}},

+ })}}

+ code, out := runOfflineAgainst(t, c, "status")

+ if code != 0 {

+ t.Fatalf("exit %d:\n%s", code, out)

+ }

+ wantLines(t, out, "ie-box left /opt/carlos/live/acme.go.tito.io in place: a root-owned directory.")

+// A failed report seen after the open is not silent: status says it and exits

+// 4, and open stops polling with the same line.

+func TestMoveOfflineFailedReportAfterTheOpen(t *testing.T) {

+ failedRep := func(box, role, check string) frame {

+ f := offReport(box, role, "failed", true)

+ f["items"] = frame{check: frame{"ok": false, "reason": "the " + check + " check failed"}}

+ return f

+ }

+ for _, c := range []struct {

+ name string

+ state string

+ rep frame

+ say string

+ }{

+ // Abort refuses once open, so a target failure there says MV-O9.

+ {"target, open", "offline-open", failedRep("uk-box", "target", "config"),

+ "The move could not finish a step: the config check failed. Run carlos move offline status --host acme.go.tito.io for each item."},

+ {"sealer, finishing", "offline-finishing", failedRep("ie-box", "sealer", "lease"),

+ "The move could not finish a step: the lease check failed. Run carlos move offline status --host acme.go.tito.io for each item."},

+ {"source, open", "offline-open", failedRep("ie-box", "source", "warm-cache"),

+ "ie-box could not clean up acme.go.tito.io: the warm-cache check failed. Run carlos move offline status --host acme.go.tito.io for each item."},

+ {"source, reverting", "offline-reverting", failedRep("ie-box", "source", "warm-cache"),

+ "ie-box could not clean up acme.go.tito.io: the warm-cache check failed. Run carlos move offline status --host acme.go.tito.io for each item."},

+ } {

+ t.Run(c.name, func(t *testing.T) {

+ f := &fakeOfflineConsole{gets: []frame{offView(c.state, frame{"reports": []frame{c.rep}})}}

+ code, out := runOfflineAgainst(t, f, "status")

+ if code != 4 || !strings.HasPrefix(out, oStarting+"\n"+c.say+"\n") {

+ t.Errorf("status exit %d, output:\n%s\nwant 4 and %q after the headline", code, out, c.say)

+ }

+ if c.state == "offline-open" {

+ f := &fakeOfflineConsole{

+ posts: map[string][]offPost{"/open": {{202, offView("offline-open", nil)}}},

+ gets: []frame{offView(c.state, frame{"reports": []frame{c.rep}})},

+ }

+ code, out := runOfflineAgainst(t, f, "open")

+ if code != 4 || !strings.HasSuffix(out, c.say+"\n") {

+ t.Errorf("open exit %d, output:\n%s\nwant 4 ending in %q", code, out, c.say)

+ }

+ })

+ }

+ // A target or sealer that failed before an abort is being put back: a

+ // reverting move shows only a failed source box.

+ stale := offReport("uk-box", "target", "failed", true)

+ f := &fakeOfflineConsole{gets: []frame{offView("offline-reverting", frame{"next": "finish", "reports": []frame{stale}})}}

+ if code, out := runOfflineAgainst(t, f, "status"); code != 0 || strings.Contains(out, "could not") {

+ t.Errorf("a reverting move with an old failed target: exit %d:\n%s", code, out)

+ }

diff --git a/offline_move_e2e_test.go b/offline_move_e2e_test.go

new file mode 100644

index 000000000..a924edb49

--- /dev/null

+++ b/offline_move_e2e_test.go

@@ -0,0 +1,218 @@

+package main

+import (

+ "bytes"

+ "context"

+ "crypto/ecdh"

+ "encoding/base64"

+ "encoding/json"

+ "net/http"

+ "net/http/httptest"

+ "path/filepath"

+ "strings"

+ "testing"

+ "time"

+ "github.com/carlosframework/platform/internal/appconfig"

+ "github.com/carlosframework/platform/internal/console"

+ "github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/provision"

+ "github.com/carlosframework/platform/internal/release"

+ "github.com/carlosframework/platform/internal/status"

+// TestMoveOfflineAgainstTheRealDoors drives `carlos move offline start` and

+// `status` through moveMain against a real console.Server: the stub tests

+// type the status view by hand, and this one ties the CLI's view structs to

+// the JSON the console writes (next, waiting, failed, items, reports, left).

+// The boxes are played by reports written where the real boxes write them.

+func TestMoveOfflineAgainstTheRealDoors(t *testing.T) {

+ const (

+ host = "acme.go.tito.io"

+ addr = "paul@keymail.dev"

+ )

+ tmp := t.TempDir()

+ rel := release.NewDirStore(filepath.Join(tmp, "store"))

+ seed, pubB64, err := release.GenerateKey()

+ if err != nil {

+ t.Fatal(err)

+ }

+ priv, err := release.ParseSigningKey(seed, "e2e")

+ if err != nil {

+ t.Fatal(err)

+ }

+ pub, err := release.ParsePublicKey(pubB64, "e2e")

+ if err != nil {

+ t.Fatal(err)

+ }

+ cst, err := console.Open(filepath.Join(tmp, "console.db"))

+ if err != nil {

+ t.Fatal(err)

+ }

+ t.Cleanup(func() { cst.Close() })

+ if err := cst.SetOperator(addr, true); err != nil {

+ t.Fatal(err)

+ }

+ secrets, err := ecdh.P256().NewPrivateKey(bytes.Repeat([]byte{1}, 32))

+ if err != nil {

+ t.Fatal(err)

+ }

+ fp := appconfig.KeyFingerprint(secrets.PublicKey())

+ var srv *console.Server

+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv.ServeHTTP(w, r) }))

+ t.Cleanup(ts.Close)

+ srv, err = console.NewServer(console.Config{

+ Origin: ts.URL,

+ Keymail: "http://keymail.invalid",

+ Registry: filepath.Join(tmp, "registry.db"),

+ Releases: rel,

+ ReleasePub: pub,

+ Key: priv,

+ AppScheme: "http",

+ SecretsPub: base64.StdEncoding.EncodeToString(secrets.PublicKey().Bytes()),

+ }, cst)

+ if err != nil {

+ t.Fatal(err)

+ }

+ if err := cst.AddMember(console.OpsAccountID, addr, console.RoleOwner, time.Now()); err != nil {

+ t.Fatal(err)

+ }

+ if _, err := cst.CreateAppIn(console.OpsAccountID, "titogo", addr, time.Now()); err != nil {

+ t.Fatal(err)

+ }

+ token, _ := e2eDeviceLogin(t, ts.URL, cst, addr, "offline@laptop")

+ ctx := context.Background()

+ // A hibernating instance in ie, and the two regions' boxes published.

+ if err := provision.PutManifest(ctx, rel, release.OpsAccount, "titogo",

+ provision.Manifest{Schema: 1, App: "titogo", Domain: "go.tito.io", Channel: "stable"}); err != nil {

+ t.Fatal(err)

+ }

+ req, _ := http.NewRequest(http.MethodPost, ts.URL+"/api/cli/apps/"+release.OpsAccount+"/titogo/instances",

+ strings.NewReader(`{"host":"`+host+`","region":"ie"}`))

+ req.Header.Set("Authorization", "Bearer "+token)

+ req.Header.Set("Content-Type", "application/json")

+ res, err := http.DefaultClient.Do(req)

+ if err != nil {

+ t.Fatal(err)

+ }

+ res.Body.Close()

+ if res.StatusCode != http.StatusCreated {

+ t.Fatalf("create the instance = %d", res.StatusCode)

+ }

+ for _, b := range []struct{ box, placement, aws, bucket string }{

+ {"ie-box", "ie", "eu-west-1", "data-ie"},

+ {"uk-box", "uk", "eu-west-2", "data-uk"},

+ } {

+ body, _ := json.Marshal(map[string]any{

+ "v": 1, "box": b.box, "box_id": b.box, "eip": "1.2.3.4", "region": b.aws, "placement": b.placement,

+ "data_bucket": b.bucket, "hibernation_configured": true, "serves_shared_pool": true,

+ "accepts_inbound": true, "secrets_key_fp": fp,

+ "capabilities": []string{"region-ha-1", "offline-move-1"}, "hosts": map[string]any{},

+ })

+ if _, err := rel.Put(ctx, "control/edges/"+b.box+".json", body, ""); err != nil {

+ t.Fatal(err)

+ }

+ if err := status.WriteBox(ctx, rel, status.Box{Label: b.box, GeneratedAt: time.Now(), Metrics: &status.BoxMetrics{}}); err != nil {

+ t.Fatal(err)

+ }

+ run := func(c context.Context, args ...string) (int, string) {

+ var out bytes.Buffer

+ deps := moveDeps{

+ session: func(string) (*apiSession, error) {

+ return &apiSession{c: newConsoleClient(ts.URL, token), console: ts.URL, address: addr}, nil

+ },

+ euid: func() int { return 1000 },

+ poll: 20 * time.Millisecond,

+ }

+ args = append(append([]string{"offline"}, args...), "--host", host, "--account", release.OpsAccount, "--app", "titogo")

+ code := moveMain(c, args, &out, deps)

+ return code, out.String()

+ }

+ put := func(id, box, role, phase string, edit func(*moves.OfflineReport)) {

+ t.Helper()

+ r := moves.OfflineReport{V: moves.OfflineReportVersion, Op: id, Host: host, Box: box, Role: role, Phase: phase,

+ At: time.Now().UTC(), Held: true, Fenced: true, Quiesced: true}

+ if edit != nil {

+ edit(&r)

+ }

+ body, err := json.Marshal(r)

+ if err != nil {

+ t.Fatal(err)

+ }

+ key := moves.OfflineReportKey(host, id, role, box)

+ _, etag, _ := rel.Get(ctx, key)

+ if _, err := rel.Put(ctx, key, body, etag); err != nil {

+ t.Fatal(err)

+ }

+ // start waits for the source box; it runs on while the test plays it.

+ ctxStart, cancel := context.WithTimeout(ctx, 30*time.Second)

+ defer cancel()

+ type result struct {

+ code int

+ out string

+ }

+ started := make(chan result, 1)

+ go func() {

+ code, out := run(ctxStart, "start", "--to", "uk")

+ started <- result{code, out}

+ }()

+ var id string

+ for deadline := time.Now().Add(20 * time.Second); id == "" && time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) {

+ if rec, err := provision.GetRecord(ctx, rel, release.OpsAccount, "titogo", host); err == nil && rec.Move != nil {

+ id = rec.Move.ID

+ }

+ if id == "" {

+ t.Fatal("start never moved the record")

+ }

+ // status, before the box has reported: the console's own waiting sentence.

+ code, out := run(ctx, "status")

+ wantWait := "Waiting for ie-box in ie to confirm it has stopped " + host + ". " + host + " is offline."

+ wantLines(t, out, "Moving "+host+" from ie to uk.", wantWait, "Item class is ok.", "Item db is waiting.")

+ if code != 0 {

+ t.Fatalf("status exit %d:\n%s", code, out)

+ }

+ // The box reports; start's next poll sees next == seal.

+ put(id, "ie-box", moves.OfflineRoleSource, moves.OfflinePhaseStopped, nil)

+ select {

+ case r := <-started:

+ if r.code != 0 {

+ t.Fatalf("start exit %d:\n%s", r.code, r.out)

+ }

+ wantLines(t, r.out, "Moving "+host+" from ie to uk.", host+" is offline from now until the move opens or is aborted.",

+ "Every box in ie has stopped "+host+". Run carlos move offline seal --host "+host+" to copy it.")

+ case <-time.After(20 * time.Second):

+ t.Fatal("start never finished")

+ }

+ code, out = run(ctx, "status")

+ wantLines(t, out, "Every box in ie has stopped "+host+". Run carlos move offline seal --host "+host+" to copy it.",

+ "ie-box is stopped as the source.")

+ if code != 0 {

+ t.Fatalf("status exit %d:\n%s", code, out)

+ }

+ // A target that says it failed sets the view's failed flag and its check.

+ put(id, "uk-box", moves.OfflineRoleTarget, moves.OfflinePhaseFailed, func(r *moves.OfflineReport) {

+ r.Items = map[string]moves.Check{"config": {OK: false, Reason: "the app settings differ"}}

+ })

+ code, out = run(ctx, "status")

+ if code != 4 {

+ t.Fatalf("status of a failed move exit %d, want 4:\n%s", code, out)

+ }

+ wantLines(t, out, "uk could not take "+host+": the app settings differ. Run carlos move offline abort --host "+host+" to move it back.")

+ // A source box that left a path in place says so, with its reason.

+ put(id, "ie-box", moves.OfflineRoleSource, moves.OfflinePhaseParked, func(r *moves.OfflineReport) {

+ r.Retired = []moves.RetiredPath{{Path: "/opt/carlos/live/" + host, Action: moves.RetiredLeft, Reason: "a root-owned directory"}}

+ })

+ _, out = run(ctx, "status")

+ wantLines(t, out, "ie-box left /opt/carlos/live/"+host+" in place: a root-owned directory.")