carlos / platform Public

host: preparing and ready rounds on the per-operation staging layout, under the staging lease (§7.3)

Behind CARLOS_MOVES_PHASE1. Off, the worker is main's (runMoveRound is
legacyMoveRound, unchanged), an id'd record is skipped with one log line
a minute, and ExecBackend.Strict stays nil. On, the source box takes the
staging-work lease, revalidates the claim under it, re-checks the box-step
refusals (S1), touches the target region and box, writes move.json
create-only, mirrors the claim's generation into replica/, stages the
sealed files archive when its plain digest changes, records the
per-operation ledger, publishes its readiness report, and promotes to
ready (pinning the target's inputs) and, for an automatic cut, to cutting,
every CAS guarded by the move's identity. Every staging write goes
through guardedMirror: one guard check, one signature, one request. The
interim abort releases the claim with RequireReleasedMove at the crash
boundary.

The named target box writes T1 to T12 every pass (Task 11's host side):
T3 composes the flat held row on a scratch directory, T5 verifies the
archive by exact version, T7 stages the pinned release or relays it to
the root stage pass, T11 prices the cut from measured rates (O_DIRECT on
linux) and the archive's entry count. S1 probes through
edge.ServeUncached. TestT4StorageProbeMinio runs in 30-minio-contract.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

claude 2 weeks ago 05fdae5e1f77e9d4d1a97c83c943c8e31bee6bde parent 81184b9

16 files changed, +4081 −12

.amadan/ci.d/30-minio-contract +29 −0
docs/superpowers/plans/2026-10-06-moves-phase1-copy.md +34 −1
internal/host/agent.go +4 −1
internal/host/movesguard.go +99 −0
internal/host/movesmeasure.go +41 −0
internal/host/movesmeasure_linux.go +60 −0
internal/host/movesmeasure_other.go +18 −0
internal/host/movespass.go +256 −8
internal/host/movespass_test.go +11 −2
internal/host/movesphase1.go +755 −0
internal/host/movesphase1_harness_test.go +513 −0
internal/host/movesphase1_test.go +607 −0
internal/host/movesreport.go +893 −0
internal/host/movesreport_minio_test.go +61 −0
internal/host/movesreport_test.go +527 −0
internal/host/movesstage.go +173 −0

diff --git a/.amadan/ci.d/30-minio-contract b/.amadan/ci.d/30-minio-contract

index a93b0bad9..a0834406b 100755

--- a/.amadan/ci.d/30-minio-contract

+++ b/.amadan/ci.d/30-minio-contract

@@ -359,4 +359,33 @@ if ! grep -q '^--- PASS: TestDeliveredContractMinio (' "$out"; then

exit 1

+# T4's store probe (moves spec §7.3, §14.1): the same server, the same

+# scoped user and bucket, and the same rule that its PASS line must appear,

+# because TestT4StorageProbeMinio skips itself without CARLOS_MINIO_TEST too.

+out2=$rundir/test-t4.log

+set +e

+CARLOS_MINIO_TEST=1 \

+ MINIO_ENDPOINT="http://127.0.0.1:$port" \

+ MINIO_SCOPED_KEY=scopeduser \

+ MINIO_SCOPED_SECRET=scopedsecret1 \

+ MINIO_BUCKET=contract-bucket \

+ MINIO_DENIED_BUCKET=denied-bucket \

+ go test ./internal/host/ -run TestT4StorageProbeMinio -count=1 -v >"$out2" 2>&1

+test_status=$?

+set -e

+cat "$out2"

+if [ "$test_status" -ne 0 ]; then

+ exit "$test_status"

+fi

+if ! grep -q '^--- PASS: TestT4StorageProbeMinio (' "$out2"; then

+ echo >&2

+ echo "the T4 probe test did not PASS: it was skipped, filtered out, or renamed." >&2

+ echo "It skips unless CARLOS_MINIO_TEST=1. Check the env block above and that" >&2

+ echo "TestT4StorageProbeMinio still exists in internal/host." >&2

+ exit 1

+fi

echo "minio contract: PASS"

diff --git a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md

index 1196e17bf..3729f0c81 100644

--- a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md

+++ b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md

@@ -48,7 +48,7 @@ Columns: id, file, text, status.

@@ -153,6 +153,39 @@ Columns: id, file, text, status.

## Plan 2

diff --git a/internal/host/agent.go b/internal/host/agent.go

index 744364817..49aef548c 100644

--- a/internal/host/agent.go

+++ b/internal/host/agent.go

@@ -398,7 +398,7 @@ func runAgent(args []string, buildClaims ClaimFactory) {

} else {

eb.Mirror = m

}

- armMovesPass(*cfg, dd, reg, svc)

+ armMovesPass(*cfg, dd, reg, svc, eb)

// The startup sweep (design §6.7): a leftover instance unit from a

// previous agent process, one this box's registry no longer holds

// a live wake for, is stopped before this process starts spawning

@@ -612,6 +612,9 @@ func runAgent(args []string, buildClaims ClaimFactory) {

edgeDoorOptions(os.Getenv, readPublicIP)...)

edge.RunWithActivatorReady(args, act, cache, func(e *edge.Edge) {

go tunnelLoop(e)

+ // A Phase 1 move's S1 probe asks through this edge, uncached

+ // (moves design §7.3); before this it answers "edge not ready".

+ setMovesReadyProbe(e)

// The edge owns the health probers, so its state table is only

// reachable once it exists — statusLoop (already running above) reads

// it through this published source on its next tick. Method value, so

diff --git a/internal/host/movesguard.go b/internal/host/movesguard.go

new file mode 100644

index 000000000..60bedba55

--- /dev/null

+++ b/internal/host/movesguard.go

@@ -0,0 +1,99 @@

+package host

+import (

+ "context"

+ "time"

+ "github.com/carlosframework/platform/internal/lease"

+ "github.com/carlosframework/platform/internal/moves"

+// guardedMirror is the one path every staging write takes (moves spec §7.3,

+// model 14 and 18): each mutation of the target bucket calls Guarded.Before

+// and is signed at the instant the guard checked the staging lease, through

+// a lease.StagingWriter whose At methods sign once and never retry. A

+// caller that wants another attempt calls again and gets a new check and a

+// new signature.

+//

+// It is a lease.MirrorStore, a VersionedCopier and a VersionedPutter, so

+// lease.MirrorPrefixKeep runs over it in its versioned mode. Writes to any

+// other bucket are the source's own receipts and ledger in its own bucket:

+// they are not staging, need no guard, and go through own, the ordinary

+// mirror client. Reads go through the staging writer's ordinary reads.

+// Nothing here ever calls a lease.VersionedStore against the target bucket.

+type guardedMirror struct {

+ w lease.StagingWriter

+ g moves.Guarded

+ dst string // the target bucket: the only bucket the guard covers

+ own lease.MirrorStore // the source's own bucket: receipts and the ledger

+var (

+ _ lease.MirrorStore = (*guardedMirror)(nil)

+ _ lease.VersionedCopier = (*guardedMirror)(nil)

+ _ lease.VersionedPutter = (*guardedMirror)(nil)

+func (m *guardedMirror) List(ctx context.Context, bucket, prefix string) (map[string]lease.MirrorObject, error) {

+ return m.w.List(ctx, bucket, prefix)

+func (m *guardedMirror) Get(ctx context.Context, bucket, key string) ([]byte, error) {

+ if bucket != m.dst {

+ return m.own.Get(ctx, bucket, key)

+ }

+ return m.w.Get(ctx, bucket, key)

+func (m *guardedMirror) CopyVersioned(ctx context.Context, srcBucket, srcKey, srcETag, dstBucket, dstKey string) (etag, versionID string, err error) {

+ err = m.g.Before(func(signAt time.Time) error {

+ var cerr error

+ etag, versionID, cerr = m.w.CopyVersionedAt(ctx, signAt, srcBucket, srcKey, srcETag, dstBucket, dstKey)

+ return cerr

+ })

+ return etag, versionID, err

+// Copy is CopyVersioned without the version. MirrorPrefixKeep never calls

+// it on a VersionedCopier; it exists so the type is a MirrorStore.

+func (m *guardedMirror) Copy(ctx context.Context, srcBucket, srcKey, srcETag, dstBucket, dstKey string) (string, error) {

+ etag, _, err := m.CopyVersioned(ctx, srcBucket, srcKey, srcETag, dstBucket, dstKey)

+ return etag, err

+func (m *guardedMirror) PutVersioned(ctx context.Context, bucket, key string, body []byte) (versionID string, err error) {

+ err = m.g.Before(func(signAt time.Time) error {

+ var perr error

+ versionID, perr = m.w.PutVersionedAt(ctx, signAt, bucket, key, body)

+ return perr

+ })

+ return versionID, err

+// PutIfAbsentVersioned is move.json's create-only write; an existing object

+// is lease.ErrExists.

+func (m *guardedMirror) PutIfAbsentVersioned(ctx context.Context, bucket, key string, body []byte) (versionID string, err error) {

+ err = m.g.Before(func(signAt time.Time) error {

+ var perr error

+ versionID, perr = m.w.PutIfAbsentVersionedAt(ctx, signAt, bucket, key, body)

+ return perr

+ })

+ return versionID, err

+func (m *guardedMirror) Put(ctx context.Context, bucket, key string, body []byte) error {

+ if bucket != m.dst {

+ return m.own.Put(ctx, bucket, key, body)

+ }

+ _, err := m.PutVersioned(ctx, bucket, key, body)

+ return err

+func (m *guardedMirror) Delete(ctx context.Context, bucket, key string) error {

+ if bucket != m.dst {

+ return m.own.Delete(ctx, bucket, key)

+ }

+ return m.g.Before(func(signAt time.Time) error {

+ return m.w.DeleteAt(ctx, signAt, bucket, key)

+ })

diff --git a/internal/host/movesmeasure.go b/internal/host/movesmeasure.go

new file mode 100644

index 000000000..57ad5160b

--- /dev/null

+++ b/internal/host/movesmeasure.go

@@ -0,0 +1,41 @@

+package host

+import (

+ "os"

+ "path/filepath"

+ "time"

+ "github.com/carlosframework/platform/internal/moves"

+// measureDiskSynced is the disk measurement without O_DIRECT: write, fsync,

+// then read measureDiskSize. The read may be answered from the page cache,

+// so it overstates the read rate; it is the fallback where O_DIRECT is

+// unavailable (movesmeasure_other.go, and tmpfs on linux).

+func measureDiskSynced(dir string, now func() time.Time) (moves.MeasuredRate, error) {

+ path := filepath.Join(dir, "disk-synced")

+ f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o600)

+ if err != nil {

+ return moves.MeasuredRate{}, err

+ }

+ defer f.Close()

+ buf := make([]byte, 1<<20)

+ start := now()

+ for n := 0; n < measureDiskSize; n += len(buf) {

+ if _, err := f.Write(buf); err != nil {

+ return moves.MeasuredRate{}, err

+ }

+ if err := f.Sync(); err != nil {

+ return moves.MeasuredRate{}, err

+ }

+ if _, err := f.Seek(0, 0); err != nil {

+ return moves.MeasuredRate{}, err

+ }

+ for n := 0; n < measureDiskSize; n += len(buf) {

+ if _, err := f.Read(buf); err != nil {

+ return moves.MeasuredRate{}, err

+ }

+ return moves.RateOf(2*measureDiskSize, now().Sub(start), now().UTC()), nil

diff --git a/internal/host/movesmeasure_linux.go b/internal/host/movesmeasure_linux.go

new file mode 100644

index 000000000..255c5b48d

--- /dev/null

+++ b/internal/host/movesmeasure_linux.go

@@ -0,0 +1,60 @@

+//go:build linux

+package host

+import (

+ "os"

+ "path/filepath"

+ "syscall"

+ "time"

+ "unsafe"

+ "github.com/carlosframework/platform/internal/moves"

+// measureDiskSize is how much T11's disk measurement writes and reads back.

+const measureDiskSize = 64 << 20

+// measureDiskRate times writing and reading measureDiskSize under dir with

+// O_DIRECT, so the page cache neither absorbs the write nor answers the

+// read (F12). A filesystem that refuses O_DIRECT (tmpfs) is measured with a

+// plain write, fsync and read instead.

+func measureDiskRate(dir string, now func() time.Time) (moves.MeasuredRate, error) {

+ path := filepath.Join(dir, "disk")

+ f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR|syscall.O_DIRECT, 0o600)

+ if err != nil {

+ return measureDiskSynced(dir, now)

+ }

+ defer f.Close()

+ buf := alignedBlock(1 << 20)

+ start := now()

+ for n := 0; n < measureDiskSize; n += len(buf) {

+ if _, err := f.Write(buf); err != nil {

+ return moves.MeasuredRate{}, err

+ }

+ if err := f.Sync(); err != nil {

+ return moves.MeasuredRate{}, err

+ }

+ if _, err := f.Seek(0, 0); err != nil {

+ return moves.MeasuredRate{}, err

+ }

+ for n := 0; n < measureDiskSize; n += len(buf) {

+ if _, err := f.Read(buf); err != nil {

+ return moves.MeasuredRate{}, err

+ }

+ return moves.RateOf(2*measureDiskSize, now().Sub(start), now().UTC()), nil

+// alignedBlock is a size-byte buffer aligned to 4096 bytes, as O_DIRECT

+// requires.

+func alignedBlock(size int) []byte {

+ const align = 4096

+ b := make([]byte, size+align)

+ off := 0

+ if r := int(uintptr(unsafe.Pointer(&b[0])) & (align - 1)); r != 0 {

+ off = align - r

+ }

+ return b[off : off+size]

diff --git a/internal/host/movesmeasure_other.go b/internal/host/movesmeasure_other.go

new file mode 100644

index 000000000..115fded9a

--- /dev/null

+++ b/internal/host/movesmeasure_other.go

@@ -0,0 +1,18 @@

+//go:build !linux

+package host

+import (

+ "time"

+ "github.com/carlosframework/platform/internal/moves"

+// measureDiskSize is how much T11's disk measurement writes and reads back.

+const measureDiskSize = 64 << 20

+// measureDiskRate has no O_DIRECT off linux (F12): it writes, fsyncs and

+// reads back.

+func measureDiskRate(dir string, now func() time.Time) (moves.MeasuredRate, error) {

+ return measureDiskSynced(dir, now)

diff --git a/internal/host/movespass.go b/internal/host/movespass.go

index 92ef00c34..6f51188a4 100644

--- a/internal/host/movespass.go

+++ b/internal/host/movespass.go

@@ -2,6 +2,7 @@ package host

import (

"context"

+ "crypto/ecdh"

"encoding/json"

"errors"

"fmt"

@@ -15,6 +16,7 @@ import (

"time"

"github.com/carlosframework/platform/internal/activator"

+ "github.com/carlosframework/platform/internal/appconfig"

"github.com/carlosframework/platform/internal/layout"

"github.com/carlosframework/platform/internal/lease"

"github.com/carlosframework/platform/internal/moves"

@@ -71,6 +73,69 @@ type movesWorker struct {

nudged sync.Map // host → true: status→quiesce hop nudged

trouble sync.Map // host → time.Time of last logged error

fenced sync.Map // host → uint64: the cut's wake-fence generation

+ // Phase 1 (CARLOS_MOVES_PHASE1). stagers memoizes the staging writer per

+ // target bucket region; archives keeps T5's verdict per move and exact

+ // archive version; measured is this box's last T11 measurements;

+ // notices rate-limits the worker's standing log lines; strict is the

+ // set of hosts with a Phase 1 move in flight here (ExecBackend.Strict).

+ stagers map[string]lease.StagingWriter

+ archives map[string]verifiedArchive

+ measured moves.Measurements

+ notices sync.Map // key → time.Time of the last line

+ strict atomic.Pointer[map[string]bool]

+// staging is the memoized staging writer for a target bucket region.

+func (w *movesWorker) staging(ctx context.Context, dstRegion string) (lease.StagingWriter, error) {

+ if w.cfg.Staging == nil {

+ return nil, errors.New("no staging writer on this box")

+ }

+ w.mu.Lock()

+ m := w.stagers[dstRegion]

+ w.mu.Unlock()

+ if m != nil {

+ return m, nil

+ }

+ m, err := w.cfg.Staging(ctx, dstRegion)

+ if err != nil {

+ return nil, err

+ }

+ w.mu.Lock()

+ if w.stagers == nil {

+ w.stagers = map[string]lease.StagingWriter{}

+ }

+ w.stagers[dstRegion] = m

+ w.mu.Unlock()

+ return m, nil

+// notice logs a standing condition at most once a minute per key.

+func (w *movesWorker) notice(key, format string, args ...any) {

+ now := time.Now()

+ if last, ok := w.notices.Load(key); ok && now.Sub(last.(time.Time)) < time.Minute {

+ return

+ }

+ w.notices.Store(key, now)

+ log.Printf(format, args...)

+// strictHost is ExecBackend.Strict with the switch on (ruling F9): a host

+// is strict while its record has a Phase 1 move in flight that names this

+// box's region, as the last pass read it.

+func (w *movesWorker) strictHost(host string) bool {

+ m := w.strict.Load()

+ return m != nil && (*m)[host]

+func (w *movesWorker) setStrict(recs []provision.Record) {

+ m := map[string]bool{}

+ for _, rec := range recs {

+ if rec.Move != nil && !rec.Move.Legacy() && (rec.Region == w.cfg.Region || rec.Move.To == w.cfg.Region) {

+ m[rec.Host] = true

+ }

+ w.strict.Store(&m)

}

var armedMovesWorker atomic.Pointer[movesWorker]

@@ -94,10 +159,12 @@ func (w *movesWorker) mirror(ctx context.Context, dstRegion string) (lease.Mirro

// armMovesPass wires the worker with what only RunAgent has in hand: the

// activator's hibernate coordinates (the own-bucket side of every mirror),

-// the data dir the cut purges, the registry the cut quiesces through, and

-// the service that answers whether a child is down. Called once, before

-// svc.Start.

-func armMovesPass(hib activator.Config, dataDir string, reg *registry.Registry, svc *activator.Service) {

+// the data dir the cut purges, the registry the cut quiesces through, the

+// service that answers whether a child is down, and the exec backend (the

+// Phase 1 launch environment and S2). Called once, before svc.Start: with

+// the Phase 1 switch on it sets eb.Strict, which the spawn path reads; with

+// it off eb.Strict stays nil, so every spawn is main's.

+func armMovesPass(hib activator.Config, dataDir string, reg *registry.Registry, svc *activator.Service, eb *activator.ExecBackend) {

readyDelta, ok := moveReadyDeltaFromEnv(os.Getenv)

if !ok {

log.Printf("agent: %s=%q ignored: want a whole number of bytes, 0 or more — keeping the built-in default of %d", moveReadyDeltaEnv, os.Getenv(moveReadyDeltaEnv), defaultMoveReadyDelta)

@@ -144,7 +211,73 @@ func armMovesPass(hib activator.Config, dataDir string, reg *registry.Registry,

ReadyDelta: readyDelta,

Enabled: moves.Enabled(os.Getenv),

}

- armedMovesWorker.Store(&movesWorker{cfg: cfg, mirrors: map[string]lease.MirrorStore{}})

+ w := &movesWorker{mirrors: map[string]lease.MirrorStore{}}

+ if cfg.Enabled {

+ armPhase1(&cfg, w, hib, dataDir, reg, eb, os.Getenv)

+ }

+ w.cfg = cfg

+ armedMovesWorker.Store(w)

+// armPhase1 wires the Phase 1 hooks (moves design §7.3). Only with the

+// switch on: off, none is set and eb.Strict stays nil.

+func armPhase1(cfg *moveCfg, w *movesWorker, hib activator.Config, dataDir string, reg *registry.Registry, eb *activator.ExecBackend, getenv func(string) string) {

+ cfg.Box = metricsBoxLabel(getenv, readMachineID())

+ cfg.OptDir = release.OptDir()

+ cfg.DeployBucket = strings.TrimSpace(getenv("CARLOS_DEPLOYMENT_BUCKET"))

+ if cfg.DeployBucket != "" {

+ cfg.DeployBucketRegion = edgesFileRegion(getenv, readPlacementRegion())

+ bucket, region := cfg.DeployBucket, cfg.DeployBucketRegion

+ cfg.StagingLease = func(ctx context.Context, targetRegion, host string) (moves.SafeUntilGuard, func(context.Context) error, error) {

+ h, err := lease.AcquireTTL(ctx, bucket, moves.StagingLeasePrefix(targetRegion, host), region, moves.StagingLeaseTTL)

+ if err != nil {

+ return nil, nil, err

+ }

+ return h, h.Release, nil

+ }

+ cfg.Staging = func(ctx context.Context, dstRegion string) (lease.StagingWriter, error) {

+ return lease.NewStagingMirror(ctx, hib.Region, dstRegion)

+ }

+ cfg.Versioned = func(ctx context.Context, bucket, region string) (lease.VersionedStore, error) {

+ return lease.NewVersionedS3(ctx, map[string]string{bucket: region})

+ }

+ if raw := strings.TrimSpace(getenv("CARLOS_SECRETS_KEY")); raw != "" {

+ if priv, err := appconfig.ParsePrivateKey(raw, "CARLOS_SECRETS_KEY"); err != nil {

+ log.Printf("agent: moves: CARLOS_SECRETS_KEY does not parse; Phase 1 rounds refuse: %v", err)

+ } else {

+ cfg.SecretsKey, cfg.SecretsPriv, cfg.SecretsPub = raw, priv, priv.PublicKey()

+ }

+ cfg.FilesDir = func(host string) (string, string) {

+ return filepath.Join(dataDir, host), filepath.Join(dataDir, host+".env")

+ }

+ cfg.ReadyProbe = probeThroughEdge

+ cfg.DiskFree = diskBytes

+ cfg.ProbeStore = probeStores

+ cfg.LaneProbe = laneProbe(cfg.Versioned)

+ cfg.MeasureDisk = measureDiskIn(dataDir, time.Now)

+ cfg.MeasureS3 = measureS3(func(ctx context.Context) (lease.MirrorStore, error) { return w.mirror(ctx, hib.Region) }, hib.Bucket, time.Now)

+ if reg != nil {

+ cfg.ReleaseReady = heldReleaseReady(reg, cfg.OptDir, dataDir, cfg.Box, stageRequester(dataDir))

+ }

+ if eb != nil {

+ cfg.StrictNext = eb.NextSpawnStrict

+ cfg.ComposeHeldEnv = composeHeldEnv(cfg.SecretsPriv, cfg.Region, eb.Cache != nil)

+ if reg != nil {

+ cfg.LaunchEnv = func(host string) (map[string]string, []string, error) {

+ rt, err := reg.Get(host)

+ if err != nil {

+ return nil, nil, err

+ }

+ if rt == nil {

+ return nil, nil, fmt.Errorf("%s has no registry row on this box", host)

+ }

+ return eb.LaunchEnvironment(*rt)

+ }

+ eb.Strict = w.strictHost

+ }

}

// runMovesPass is the loops' call: inert until armed (no activator = no

@@ -207,6 +340,55 @@ type moveCfg struct {

// Enabled is the Phase 1 moves switch (moves.Enabled, set in

// armMovesPass). Off, the worker runs the legacy protocol only.

Enabled bool

+ // Phase 1 (moves design §7.3), wired by armMovesPass only with the

+ // switch on. A nil hook refuses its step, never falls back.

+ //

+ // Box is this box's label (metricsBoxLabel), the name the claim's

+ // participants and op.intent.target use. OptDir is the version store

+ // (the source report's Release is LiveVersion there).

+ Box string

+ OptDir string

+ // DeployBucket and DeployBucketRegion are where the staging-work lease

+ // lives (CARLOS_DEPLOYMENT_BUCKET, CARLOS_AWS_REGION or the IMDS

+ // region). StagingLease is nil without a deployment bucket, and then a

+ // Phase 1 round refuses (round-5 finding 11).

+ DeployBucket, DeployBucketRegion string

+ StagingLease func(ctx context.Context, targetRegion, host string) (moves.SafeUntilGuard, func(context.Context) error, error)

+ // Staging is the staging writer for a target bucket region (Task 6,

+ // F17): every staging write goes through it behind the guard.

+ Staging func(ctx context.Context, dstRegion string) (lease.StagingWriter, error)

+ // Versioned reads a bucket by exact version: the target's T5 in its own

+ // bucket and T10's lane in the source's. Never the source against the

+ // target's lane.

+ Versioned func(ctx context.Context, bucket, region string) (lease.VersionedStore, error)

+ // SecretsKey is CARLOS_SECRETS_KEY (the launch report's HMAC key is

+ // derived from it); SecretsPub seals the archive and SecretsPriv opens

+ // it on the target.

+ SecretsKey string

+ SecretsPub *ecdh.PublicKey

+ SecretsPriv *ecdh.PrivateKey

+ // LaunchEnv is the source's composed launch environment

+ // (ExecBackend.LaunchEnvironment over the host's row).

+ LaunchEnv func(host string) (env map[string]string, shadowed []string, err error)

+ // FilesDir is the files directory and <host>.env the archive carries

+ // (flat: <DataDir>/<host>/ and <DataDir>/<host>.env).

+ FilesDir func(host string) (dir, envPath string)

+ // ReadyProbe is S1: GET path on host through the edge, uncached and

+ // waking (setMovesReadyProbe). StrictNext is S2

+ // (ExecBackend.NextSpawnStrict).

+ ReadyProbe func(ctx context.Context, host, path string) moves.Check

+ StrictNext func(host string) moves.Check

+ // The target's hooks: T3's composition of the held row, T4's store

+ // probe, T6's disk reading, T7's release, T10's lane probe and T11's

+ // regional S3 and disk measurements.

+ ComposeHeldEnv func(ctx context.Context, s release.Store, rec provision.Record, carriedEnv []byte) (env map[string]string, shadowed []string, in provision.MoveInputs, err error)

+ ProbeStore func(ctx context.Context, env map[string]string, moveID string) (prefix string, err error)

+ DiskFree func(dir string) (total, avail uint64, err error)

+ ReleaseReady func(ctx context.Context, s release.Store, rec provision.Record, version string) moves.Check

+ LaneProbe func(ctx context.Context, bucket, region, moveID string) error

+ MeasureS3 func(ctx context.Context, key string) (moves.MeasuredRate, error)

+ MeasureDisk func(ctx context.Context) (moves.MeasuredRate, moves.FilesRate, error)

}

// defaultMoveReadyDelta is the readiness bar when

@@ -249,7 +431,52 @@ func movesPass(ctx context.Context, s release.Store, w *movesWorker) error {

return fmt.Errorf("moves: list records: %w", err)

}

var targets map[string]edgesFile // lazy: most passes carry no move

+ var fleet *moveFleet // lazy, Phase 1 only

+ loadFleet := func() bool {

+ if fleet == nil {

+ f, err := loadMoveFleet(ctx, s)

+ if err != nil {

+ log.Printf("moves: %v. every Phase 1 move waits for the next pass", err)

+ return false

+ }

+ fleet = &f

+ }

+ return true

+ }

+ if cfg.Enabled {

+ w.setStrict(recs)

+ }

for _, rec := range recs {

+ if rec.Move != nil && !rec.Move.Legacy() {

+ // A Phase 1 record. With the switch off it is never touched

+ // (none exists unless the switch was on somewhere).

+ if !cfg.Enabled {

+ if rec.Region == cfg.Region {

+ w.notice(rec.Host+"#switch", "moves: %s: move %s waits: %s is off on this box", rec.Host, rec.Move.ID, moves.SwitchEnv)

+ }

+ continue

+ }

+ if rec.Region != cfg.Region || (cfg.OwnsRoute != nil && !cfg.OwnsRoute(rec.Host)) {

+ continue

+ }

+ if !loadFleet() {

+ return nil

+ }

+ if err := phase1MoveRound(ctx, s, w, rec, *fleet); err != nil {

+ w.logTrouble(rec.Host, err)

+ continue

+ }

+ w.trouble.Delete(rec.Host)

+ continue

+ }

+ // A legacy record with the switch on: only its abort completes, by

+ // the legacy path (F7(a)); anything else is refused (MV-N7).

+ if cfg.Enabled && rec.Move != nil && rec.Move.State != provision.MoveAborting {

+ if rec.Region == cfg.Region {

+ w.notice(rec.Host+"#legacy", "moves: %s: %s", rec.Host, legacyMoveSentence)

+ }

+ continue

+ }

if rec.Move == nil || rec.Region != cfg.Region {

continue

}

@@ -271,7 +498,7 @@ func movesPass(ctx context.Context, s release.Store, w *movesWorker) error {

return nil

}

- if err := runMoveRound(ctx, s, w, rec, targets); err != nil {

+ if err := legacyMoveRound(ctx, s, w, rec, targets); err != nil {

// One move's trouble must not starve its neighbours — and a

// PERSISTENT one (an unapplied staging grant's AccessDenied,

// review round 1) must not flood the journal at the fast

@@ -281,10 +508,27 @@ func movesPass(ctx context.Context, s release.Store, w *movesWorker) error {

}

w.trouble.Delete(rec.Host)

}

+ if cfg.Enabled && hasPhase1Into(recs, cfg.Region) && loadFleet() {

+ w.targetReportPass(ctx, s, recs, *fleet)

+ }

w.liftFences(recs)

return nil

}

+// legacyMoveSentence is MV-N7, logged (never shown to a user) when the

+// switch is on and a record carries a move with no id.

+const legacyMoveSentence = "this move was started by an older console and has no id. abort it and request it again"

+// hasPhase1Into reports whether any Phase 1 move targets region.

+func hasPhase1Into(recs []provision.Record, region string) bool {

+ for _, rec := range recs {

+ if rec.Move != nil && !rec.Move.Legacy() && rec.Move.To == region {

+ return true

+ }

+ return false

// moveFenceGeneration is the cut fence's generation: the move's request

// time, so every pass of one cut installs the same fence (FenceAndStop is

// idempotent at an equal generation) and a later move of the same host

@@ -336,7 +580,7 @@ func (w *movesWorker) liftFences(recs []provision.Record) {

// moveTargets folds the fleet's edges files into placement → coordinates.

// A duplicate placement code keeps the lexically-first box (deterministic

// everywhere, same rule as foldHomes) — and a target with no data bucket is

-// simply absent, which runMoveRound reports per-move.

+// simply absent, which legacyMoveRound reports per-move.

func moveTargets(ctx context.Context, s release.Store) (map[string]edgesFile, error) {

bodies, err := loadEdgesFileBodies(ctx, s)

if err != nil {

@@ -356,7 +600,11 @@ func moveTargets(ctx context.Context, s release.Store) (map[string]edgesFile, er

return out, nil

}

-func runMoveRound(ctx context.Context, s release.Store, w *movesWorker, rec provision.Record, targets map[string]edgesFile) error {

+// legacyMoveRound is the worker's round for a legacy record (no move id),

+// main's runMoveRound unchanged. It is the only round with the Phase 1

+// switch off; with it on, it completes a legacy aborting record and nothing

+// else (F7(a)).

+func legacyMoveRound(ctx context.Context, s release.Store, w *movesWorker, rec provision.Record, targets map[string]edgesFile) error {

cfg := w.cfg

tgt, ok := targets[rec.Move.To]

if !ok {

diff --git a/internal/host/movespass_test.go b/internal/host/movespass_test.go

index 6d00fce88..d6362c070 100644

--- a/internal/host/movespass_test.go

+++ b/internal/host/movespass_test.go

@@ -7,6 +7,7 @@ import (

"os"

"path/filepath"

"strings"

+ "sync"

"testing"

"time"

@@ -26,6 +27,14 @@ type fakeMoveMirror struct {

buckets map[string]map[string]fakeMoveObject

ops []string

beforeCopy func(srcKey string)

+ // The Phase 1 staging writer's state (movesphase1_harness_test.go):

+ // the guarded call log, a failure hook, and object versions.

+ mu sync.Mutex

+ at []stagingCall

+ failAt func(op, key string) error

+ versions int

+ vers map[string]string

}

type fakeMoveObject struct {

@@ -910,7 +919,7 @@ func TestArmMovesPassReadsTheReadyDelta(t *testing.T) {

{"lots", defaultMoveReadyDelta},

} {

t.Setenv("CARLOS_MOVE_READY_DELTA_BYTES", tc.raw)

- armMovesPass(activator.Config{}, t.TempDir(), nil, nil)

+ armMovesPass(activator.Config{}, t.TempDir(), nil, nil, nil)

if got := armedMovesWorker.Load().cfg.ReadyDelta; got != tc.want {

t.Errorf("CARLOS_MOVE_READY_DELTA_BYTES=%q armed a bar of %d, want %d", tc.raw, got, tc.want)

}

@@ -929,7 +938,7 @@ func TestArmMovesPassReadsTheSwitch(t *testing.T) {

{"", false}, {"on", true}, {"1", true}, {"true", true}, {"off", false}, {"yes", false},

} {

t.Setenv(moves.SwitchEnv, tc.raw)

- armMovesPass(activator.Config{}, t.TempDir(), nil, nil)

+ armMovesPass(activator.Config{}, t.TempDir(), nil, nil, nil)

if got := armedMovesWorker.Load().cfg.Enabled; got != tc.want {

t.Errorf("%s=%q armed Enabled=%v, want %v", moves.SwitchEnv, tc.raw, got, tc.want)

}

diff --git a/internal/host/movesphase1.go b/internal/host/movesphase1.go

new file mode 100644

index 000000000..2876874bf

--- /dev/null

+++ b/internal/host/movesphase1.go

@@ -0,0 +1,755 @@

+package host

+import (

+ "context"

+ "encoding/hex"

+ "encoding/json"

+ "errors"

+ "fmt"

+ "io"

+ "log"

+ "os"

+ "path/filepath"

+ "sort"

+ "strings"

+ "time"

+ "github.com/carlosframework/platform/internal/appconfig"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/lease"

+ "github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/moves/archive"

+ "github.com/carlosframework/platform/internal/objstore"

+ "github.com/carlosframework/platform/internal/provision"

+ "github.com/carlosframework/platform/internal/registry"

+ "github.com/carlosframework/platform/internal/release"

+// The Phase 1 source round (moves spec §7.3, §7.11, K5, K12), behind

+// CARLOS_MOVES_PHASE1. It drives a move whose record carries an id: the

+// staging lane under moves/<host>/<id>/ in the target bucket, written only

+// through the guarded staging mirror while this box holds the staging-work

+// lease; the per-operation receipts and ledger in this box's own bucket; the

+// source's readiness report in the deployment bucket; and the promotion to

+// ready (and, for an automatic cut, to cutting) by guarded CAS. The legacy

+// worker (legacyMoveRound) is untouched and runs only with the switch off.

+//

+// The operator-visible sentences below are rows of the copy record

+// (docs/superpowers/plans/2026-10-06-moves-phase1-copy.md), pending review;

+// they reach `carlos move status` through the source report's S1.

+const (

+ s1NoDeployBucket = "this deployment has no deployment bucket for the staging lease"

+ s1Nested = "nested instances cannot move yet"

+ s1NoSecretsKey = "this box has no secrets key, so it cannot report the launch environment or seal the archive"

+ s1EdgeNotReady = "this box's edge is not ready to probe the instance yet"

+func s1NotExec(host string) string {

+ return host + " is not an exec-backed instance, so it cannot move"

+func s1NoHibernate(host string) string { return host + " does not hibernate, so it cannot move" }

+func s1Sidecar(host string) string {

+ return host + " has a sidecar environment file, so it cannot move"

+func s1Column(host, column string) string {

+ return host + " has " + column + " set in the registry, so it cannot move"

+// s1PrivateStorage is M-C17, the door's sentence repeated every round.

+func s1PrivateStorage(host string) string {

+ return host + " uses private storage. Instances with private storage cannot move yet."

+// s1SpecialFile and s1OverCap are M-C14 and M-C15.

+func s1SpecialFile(host, path string) string {

+ return host + " keeps a file this move cannot copy: " + path + ". Only regular files and folders move."

+func s1OverCap(host string) string {

+ return host + " keeps more than 256 MiB of files beside its database. Keep large files in object storage."

+// s1NoAnswer is M-C16.

+func s1NoAnswer(host string) string {

+ return host + " does not answer its readiness path. Declare moves.ready_path for this app."

+// crossRefreshAfter is how old the source's cross-region rate may get before

+// a round that copied nothing measures it again by recopying one staged

+// object. Half of moves.MeasurementFreshness, so a quiet (hibernating)

+// source keeps a fresh rate in its report while it waits for ready.

+const crossRefreshAfter = moves.MeasurementFreshness / 2

+// crossMinBytes is the least a round must copy for its rate to be recorded:

+// a round of a few small files measures latency, not throughput.

+const crossMinBytes = 1 << 20

+// moveLedger is the per-operation ledger beside the copy receipts, at

+// moves.LedgerKey in this box's own bucket (§6.3). It is what the rounds

+// record about the lane: move.json's version (or "present" when a replay

+// found it already there), the published archive by digest and by the

+// version ids its PUTs returned, and the most recent copying round's

+// cross-region rate.

+type moveLedger struct {

+ V int `json:"v"`

+ MoveJSONVersion string `json:"move_json_version,omitempty"`

+ ArchivePlain string `json:"archive_plain,omitempty"`

+ ArchiveSealed string `json:"archive_sealed,omitempty"`

+ ArchiveVersions *moves.ArchiveRef `json:"archive_versions,omitempty"`

+ LastRoundBytes int64 `json:"last_round_bytes,omitempty"`

+ // LastRoundSeconds and LastRoundAt time the most recent round that

+ // copied at least crossMinBytes, or the recopy that refreshed it.

+ LastRoundSeconds float64 `json:"last_round_seconds,omitempty"`

+ LastRoundAt time.Time `json:"last_round_at,omitempty"`

+// moveJSONPresent is the ledger's note that move.json already existed when

+// this source wrote it: its own lost-response PUT, or a foreign object. The

+// source cannot tell which (M17 forbids the read), so the target's T9

+// validates the object's fields.

+const moveJSONPresent = "present"

+func (l moveLedger) crossRate() (float64, time.Time) {

+ if l.LastRoundBytes <= 0 || l.LastRoundSeconds <= 0 {

+ return 0, time.Time{}

+ }

+ return float64(l.LastRoundBytes) / (1 << 20) / l.LastRoundSeconds, l.LastRoundAt

+func readMoveLedger(ctx context.Context, m lease.MirrorStore, bucket, key string) (moveLedger, error) {

+ body, err := m.Get(ctx, bucket, key)

+ if errors.Is(err, lease.ErrMirrorNotFound) {

+ return moveLedger{V: 1}, nil

+ }

+ if err != nil {

+ return moveLedger{}, fmt.Errorf("read the move ledger %s/%s: %w", bucket, key, err)

+ }

+ var l moveLedger

+ if err := json.Unmarshal(body, &l); err != nil || l.V != 1 {

+ // An unreadable ledger is no evidence: start over. Every fact in it

+ // is recomputed (move.json answers present, the archive is rebuilt).

+ return moveLedger{V: 1}, nil

+ }

+ return l, nil

+func writeMoveLedger(ctx context.Context, m lease.MirrorStore, bucket, key string, l moveLedger) error {

+ body, err := json.Marshal(l)

+ if err != nil {

+ return err

+ }

+ if err := m.Put(ctx, bucket, key, body); err != nil {

+ return fmt.Errorf("write the move ledger %s/%s: %w", bucket, key, err)

+ }

+ return nil

+// phase1MoveRound is the Phase 1 worker's step for one id'd record this box

+// is the source of.

+func phase1MoveRound(ctx context.Context, s release.Store, w *movesWorker, rec provision.Record, fleet moveFleet) error {

+ switch rec.Move.State {

+ case provision.MovePreparing, provision.MoveReady:

+ return w.preparingRound(ctx, s, rec, fleet)

+ case provision.MoveAborting:

+ return w.interimAbort(ctx, s, rec)

+ }

+ return nil // cutting and later: Plan 2

+// srcRound is what one preparing round learned for the source report.

+type srcRound struct {

+ s1 *moves.Check // a refusal; nil means probe the instance

+ ledger moveLedger

+ deltaBytes int64

+// preparingRound is §7.3's source round, in order: no deployment bucket

+// refuses; the staging lease is taken and the claim revalidated under it;

+// the box-step refusals; the store binding; Touch; move.json; the replica

+// lane; the archive; the source report; Ready; the promotion; the

+// automatic cut.

+func (w *movesWorker) preparingRound(ctx context.Context, s release.Store, rec provision.Record, fleet moveFleet) error {

+ cfg := w.cfg

+ id := rec.Move.ID

+ g := provision.MoveGuard{ID: id, Incarnation: rec.Move.Incarnation}

+ // Round-5 finding 11: a lease in a per-participant bucket, or a

+ // 15-minute one, cannot fence staging writers. Without the deployment

+ // bucket there is no shared place for it, so the round refuses.

+ if cfg.StagingLease == nil {

+ return w.writeSourceReport(ctx, s, rec, &srcRound{s1: &moves.Check{Reason: s1NoDeployBucket}})

+ }

+ tgt, ok := fleet.targets[rec.Move.To]

+ if !ok {

+ return fmt.Errorf("target region %s publishes no data bucket yet. waiting", rec.Move.To)

+ }

+ held, releaseLease, err := cfg.StagingLease(ctx, rec.Move.To, rec.Host)

+ if err != nil {

+ var he *lease.HeldError

+ if errors.As(err, &he) {

+ log.Printf("moves: %s: another worker holds the staging lease; this round waits", rec.Host)

+ return nil

+ }

+ return fmt.Errorf("staging lease: %w", err)

+ }

+ defer func() {

+ rctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)

+ defer cancel()

+ if err := releaseLease(rctx); err != nil {

+ log.Printf("moves: %s: release the staging lease: %v", rec.Host, err)

+ }

+ }()

+ // K5: revalidate the claim under the lease.

+ obs, err := hostclaim.Read(ctx, s, rec.Host)

+ if err != nil {

+ return fmt.Errorf("read the host claim: %w", err)

+ }

+ if obs.Absent {

+ return fmt.Errorf("move %s has no host claim", id)

+ }

+ if err := hostclaim.RequireMoveOp(id, hostclaim.KindMoving, hostclaim.PhaseMoving)(obs.Claim); err != nil {

+ return fmt.Errorf("move %s: %w", id, err)

+ }

+ if h := obs.Claim.Holder; h == nil || h.Incarnation != rec.Move.Incarnation {

+ return fmt.Errorf("move %s: the host claim names another incarnation", id)

+ }

+ if cfg.Box != "" && obs.Claim.Op.Exec != cfg.Box {

+ return fmt.Errorf("move %s: the claim's executor is %q, not this box", id, obs.Claim.Op.Exec)

+ }

+ // The box-step refusals (§7.1: every box step re-checks).

+ if why, err := w.sourceRefusal(ctx, s, rec); err != nil {

+ return err

+ } else if why != "" {

+ return w.writeSourceReport(ctx, s, rec, &srcRound{s1: &moves.Check{Reason: why}})

+ }

+ // K12 and T9: the target region and box are in touched before the

+ // first staging write.

+ targetBox := obs.Claim.Op.Intent["target"]

+ if !containsString(obs.Claim.Touched.Regions, rec.Move.To) || (targetBox != "" && !containsString(obs.Claim.Touched.Boxes, targetBox)) {

+ var boxes []string

+ if targetBox != "" {

+ boxes = []string{targetBox}

+ }

+ if _, err := hostclaim.Update(ctx, s, obs, hostclaim.Touch(id, []string{rec.Move.To}, boxes)); err != nil {

+ return fmt.Errorf("touch %s in the host claim: %w", rec.Move.To, err)

+ }

+ stg, err := w.staging(ctx, tgt.DataBucketRegion)

+ if err != nil {

+ return err

+ }

+ own, err := w.mirror(ctx, tgt.DataBucketRegion)

+ if err != nil {

+ return err

+ }

+ gm := &guardedMirror{w: stg, g: moves.Guarded{Lease: held, Now: cfg.Now}, dst: tgt.DataBucket, own: own}

+ ledgerKey := moves.LedgerKey(rec.Host, id)

+ ledger, err := readMoveLedger(ctx, own, cfg.OwnBucket, ledgerKey)

+ if err != nil {

+ return err

+ }

+ before := ledger

+ saveLedger := func() error {

+ if ledger == before {

+ return nil

+ }

+ if err := writeMoveLedger(ctx, own, cfg.OwnBucket, ledgerKey, ledger); err != nil {

+ return err

+ }

+ before = ledger

+ return nil

+ }

+ // move.json, create-only, before the first round. Once the ledger

+ // knows it, it is not written again.

+ if ledger.MoveJSONVersion == "" {

+ body, err := json.Marshal(moves.MoveJSON{MoveID: id, Incarnation: rec.Move.Incarnation, Host: rec.Host, From: rec.Move.From, To: rec.Move.To})

+ if err != nil {

+ return err

+ }

+ ver, err := gm.PutIfAbsentVersioned(ctx, tgt.DataBucket, moves.MoveJSONKey(rec.Host, id), body)

+ switch {

+ case errors.Is(err, lease.ErrExists):

+ ledger.MoveJSONVersion = moveJSONPresent

+ case err != nil:

+ return fmt.Errorf("write move.json: %w", err)

+ default:

+ ledger.MoveJSONVersion = ver

+ }

+ if err := saveLedger(); err != nil {

+ return err

+ }

+ // The replica lane: the claim's generation, flat under replica/.

+ srcPrefix, keep := moves.ReplicaSource(*obs.Claim.Holder)

+ dstPrefix := moves.ReplicaPrefix(rec.Host, id)

+ receipts := lease.Receipts{Bucket: cfg.OwnBucket, Key: moves.ReceiptsKey(rec.Host, id)}

+ var round lease.MirrorRound

+ start := cfg.Now()

+ for attempt := 0; ; attempt++ {

+ round, err = lease.MirrorPrefixKeep(ctx, gm, cfg.OwnBucket, srcPrefix, tgt.DataBucket, dstPrefix, receipts, keep)

+ // One retry, and only by calling again: a new check, a new

+ // signature. A guard refusal is never retried.

+ if err == nil || attempt == 1 || errors.Is(err, moves.ErrLeaseUnsafe) || errors.Is(err, moves.ErrInvalidSkew) {

+ break

+ }

+ log.Printf("moves: %s: replica round failed, retrying once: %v", rec.Host, err)

+ }

+ if err != nil {

+ return fmt.Errorf("replica lane: %w", err)

+ }

+ if round.BytesCopied >= crossMinBytes {

+ ledger.LastRoundBytes, ledger.LastRoundSeconds, ledger.LastRoundAt = round.BytesCopied, cfg.Now().Sub(start).Seconds(), cfg.Now().UTC()

+ } else if _, at := ledger.crossRate(); at.IsZero() || cfg.Now().Sub(at) > crossRefreshAfter {

+ if err := w.refreshCrossRate(ctx, gm, srcPrefix, keep, tgt.DataBucket, dstPrefix, &ledger); err != nil {

+ return err

+ }

+ sr := &srcRound{deltaBytes: round.DeltaBytes()}

+ // The files archive, rebuilt only when its plaintext digest changes.

+ if why, err := w.publishArchive(ctx, gm, rec, tgt.DataBucket, &ledger); err != nil {

+ return err

+ } else if why != "" {

+ sr.s1 = &moves.Check{Reason: why}

+ }

+ if err := saveLedger(); err != nil {

+ return err

+ }

+ sr.ledger = ledger

+ if err := w.writeSourceReport(ctx, s, rec, sr); err != nil {

+ return err

+ }

+ return w.decideReady(ctx, s, rec, g, sr)

+// refreshCrossRate measures the cross-region path when no recent round did:

+// it recopies the largest object of the lane (guarded, as every staging

+// write) and times it. The recopy lands a new version of an object the lane

+// already holds with the same bytes; the receipts keep naming the earlier

+// version, which holds the same bytes too.

+func (w *movesWorker) refreshCrossRate(ctx context.Context, gm *guardedMirror, srcPrefix string, keep func(string) bool, dstBucket, dstPrefix string, ledger *moveLedger) error {

+ src, err := gm.List(ctx, w.cfg.OwnBucket, srcPrefix)

+ if err != nil {

+ return fmt.Errorf("list the replica source: %w", err)

+ }

+ var key string

+ var obj lease.MirrorObject

+ for k, o := range src {

+ rel := strings.TrimPrefix(k, srcPrefix)

+ if keep(rel) && (o.Size > obj.Size || (o.Size == obj.Size && k < key) || key == "") {

+ key, obj = k, o

+ }

+ if key == "" || obj.Size <= 0 {

+ return nil // nothing to measure with yet

+ }

+ start := w.cfg.Now()

+ if _, _, err := gm.CopyVersioned(ctx, w.cfg.OwnBucket, key, obj.ETag, dstBucket, dstPrefix+strings.TrimPrefix(key, srcPrefix)); err != nil {

+ if errors.Is(err, lease.ErrCopySourceChanged) {

+ return nil // the next round copies it anyway, and times that

+ }

+ return fmt.Errorf("measure the cross-region copy: %w", err)

+ }

+ if d := w.cfg.Now().Sub(start); d > 0 {

+ ledger.LastRoundBytes, ledger.LastRoundSeconds, ledger.LastRoundAt = obj.Size, d.Seconds(), w.cfg.Now().UTC()

+ }

+ return nil

+// sourceRefusal is the box-step re-check (§6.2, §7.1, F14): the reason this

+// source cannot move, or "". A registry or bucket read that fails is an

+// error and ends the round.

+func (w *movesWorker) sourceRefusal(ctx context.Context, s release.Store, rec provision.Record) (string, error) {

+ cfg := w.cfg

+ if cfg.Route == nil {

+ return "", errors.New("no registry to re-check the source against")

+ }

+ rt, err := cfg.Route(rec.Host)

+ if err != nil {

+ return "", fmt.Errorf("read the registry row: %w", err)

+ }

+ if rt == nil {

+ return "", fmt.Errorf("%s has no registry row on this box", rec.Host)

+ }

+ if why := routeRefusal(*rt, cfg.DataDir); why != "" {

+ return why, nil

+ }

+ if fileExists(filepath.Join(cfg.DataDir, rec.Host+"-sidecar.env")) {

+ return s1Sidecar(rec.Host), nil

+ }

+ // The door's private-storage check (Task 4), repeated before the first

+ // staging write of every round.

+ if env := provision.InstanceStoreEnv(&rec); env == "" || appconfig.ValidEnvironmentName(env) {

+ store, _, err := objstore.GetIn(ctx, s, rec.Account, rec.App, env)

+ if err != nil {

+ return "", fmt.Errorf("read the instance's storage binding: %w", err)

+ }

+ if store != nil {

+ return s1PrivateStorage(rec.Host), nil

+ }

+ if cfg.SecretsPub == nil || cfg.SecretsKey == "" {

+ return s1NoSecretsKey, nil

+ }

+ return "", nil

+// routeRefusal is the registry half of the box-step re-check. Backing is

+// refused whenever it is set: §6.2 names only Backing (unit), and the plan

+// refuses any non-empty value, which is the stricter reading (F14).

+func routeRefusal(rt registry.Route, dataDir string) string {

+ switch {

+ case rt.Kind != "instance" || rt.Platform:

+ return s1NotExec(rt.Host)

+ case !rt.Hibernate:

+ return s1NoHibernate(rt.Host)

+ case rt.DBPath != provision.DefaultDBPath(dataDir, rt.Host):

+ return s1Nested

+ }

+ for _, c := range []struct {

+ name string

+ set bool

+ }{

+ {"caps", rt.Caps != ""}, {"unit", rt.Unit != ""}, {"backing", rt.Backing != ""},

+ {"tunnel", rt.Tunnel != ""}, {"fleet_box", rt.FleetBox != ""}, {"pool", rt.Pool != ""},

+ {"canary", rt.Canary}, {"addr", rt.Addr != ""}, {"addrs", rt.Addrs != ""}, {"health", rt.Health != ""},

+ {"replica", rt.Replica}, {"replicas", rt.Replicas != ""},

+ } {

+ if c.set {

+ return s1Column(rt.Host, c.name)

+ }

+ return ""

+func fileExists(path string) bool {

+ _, err := os.Lstat(path)

+ return err == nil

+func containsString(xs []string, x string) bool {

+ for _, v := range xs {

+ if v == x {

+ return true

+ }

+ return false

+// publishArchive builds, seals and stages the files archive when its

+// plaintext digest differs from the one the ledger published, and records

+// the new one. It returns an S1 reason for an archive this move can never

+// carry; a file that changed while it was read is a retry next round, and

+// the archive published before stays.

+func (w *movesWorker) publishArchive(ctx context.Context, gm *guardedMirror, rec provision.Record, dstBucket string, ledger *moveLedger) (string, error) {

+ cfg := w.cfg

+ dir, envPath := cfg.FilesDir(rec.Host)

+ plain, _, _, err := archive.PlainDigest(dir, envPath)

+ if why, retry, ferr := archiveFailure(rec.Host, err); ferr != nil || why != "" || retry {

+ return why, ferr

+ }

+ if plain == ledger.ArchivePlain && ledger.ArchiveVersions != nil {

+ return "", nil

+ }

+ tmp, err := os.MkdirTemp("", "carlos-move-archive-")

+ if err != nil {

+ return "", err

+ }

+ defer os.RemoveAll(tmp)

+ built, err := os.Create(filepath.Join(tmp, "archive.tar.zst"))

+ if err != nil {

+ return "", err

+ }

+ defer built.Close()

+ plain, entries, bytes, err := archive.Build(dir, envPath, built)

+ if why, retry, ferr := archiveFailure(rec.Host, err); ferr != nil || why != "" || retry {

+ return why, ferr

+ }

+ if _, err := built.Seek(0, io.SeekStart); err != nil {

+ return "", err

+ }

+ sealedPath := filepath.Join(tmp, "archive.sealed")

+ sealed, err := os.Create(sealedPath)

+ if err != nil {

+ return "", err

+ }

+ defer sealed.Close()

+ dataKey, sealedSHA, err := archive.Seal(built, cfg.SecretsPub, appconfig.Binding{Account: rec.Account, App: rec.App}, sealed)

+ if err != nil {

+ return "", fmt.Errorf("seal the archive: %w", err)

+ }

+ m := archive.Manifest{V: archive.ManifestVersion, Op: rec.Move.ID, Incarnation: rec.Move.Incarnation, Plain: plain,

+ Sealed: sealedSHA, Bytes: bytes, Entries: entries, SealedDataKey: dataKey, Chunk: archive.ChunkSize}

+ manifest, err := json.Marshal(m)

+ if err != nil {

+ return "", err

+ }

+ sealedBody, err := os.ReadFile(sealedPath)

+ if err != nil {

+ return "", err

+ }

+ name := "files-" + digestHex(sealedSHA)

+ prefix := moves.FilesPrefix(rec.Host, rec.Move.ID)

+ ref := &moves.ArchiveRef{ManifestKey: prefix + name + ".json", SealedObjectKey: prefix + name + ".sealed", Plain: plain}

+ if ref.SealedVersion, err = gm.PutVersioned(ctx, dstBucket, ref.SealedObjectKey, sealedBody); err != nil {

+ return "", fmt.Errorf("stage the sealed archive: %w", err)

+ }

+ if ref.ManifestVersion, err = gm.PutVersioned(ctx, dstBucket, ref.ManifestKey, manifest); err != nil {

+ return "", fmt.Errorf("stage the archive manifest: %w", err)

+ }

+ ledger.ArchivePlain, ledger.ArchiveSealed, ledger.ArchiveVersions = plain, sealedSHA, ref

+ log.Printf("moves: %s: staged the files archive %s (%d files, %d bytes)", rec.Host, plain, len(entries), bytes)

+ return "", nil

+// archiveFailure maps an archive error to an S1 reason (a standing refusal

+// the operator must act on), a retry (a file changed under the walk), or an

+// error that ends the round.

+func archiveFailure(host string, err error) (why string, retry bool, ferr error) {

+ switch {

+ case err == nil:

+ return "", false, nil

+ case errors.Is(err, archive.ErrChanged):

+ log.Printf("moves: %s: %v; the archive is rebuilt next round", host, err)

+ return "", true, nil

+ case errors.Is(err, archive.ErrSpecialFile):

+ path := strings.TrimPrefix(err.Error(), archive.ErrSpecialFile.Error()+": ")

+ return s1SpecialFile(host, strings.TrimSuffix(path, " is a hard link")), false, nil

+ case errors.Is(err, archive.ErrOverCap):

+ return s1OverCap(host), false, nil

+ case errors.Is(err, archive.ErrTooManyEntries), errors.Is(err, archive.ErrBadPath):

+ return err.Error(), false, nil // MV-N79, MV-N81

+ }

+ return "", false, fmt.Errorf("archive the files: %w", err)

+// digestHex is "sha256:<hex>" without its algorithm, for an object name.

+func digestHex(d string) string {

+ _, h, ok := strings.Cut(d, ":")

+ if !ok {

+ return d

+ }

+ if _, err := hex.DecodeString(h); err != nil {

+ return d

+ }

+ return h

+// writeSourceReport publishes the source's readiness report (§7.3): the

+// launch environment as HMACs, S1 (a refusal, or the instance's own answer

+// to its readiness path), S2, the published archive, the live release and

+// the cross-region rate. It is replaced every round.

+func (w *movesWorker) writeSourceReport(ctx context.Context, s release.Store, rec provision.Record, r *srcRound) error {

+ cfg := w.cfg

+ now := cfg.Now().UTC()

+ sr := moves.SourceReport{V: 1, Box: cfg.Box, MoveID: rec.Move.ID, At: now,

+ FilesPlain: r.ledger.ArchivePlain, Archive: r.ledger.ArchiveVersions}

+ sr.CrossRate, sr.CrossRateAt = r.ledger.crossRate()

+ if r.s1 != nil {

+ sr.S1 = *r.s1

+ } else {

+ sr.S1 = w.probeSource(ctx, s, rec)

+ }

+ sr.S2 = moves.Check{Reason: "the next spawn of this host is not strict"}

+ if cfg.StrictNext != nil {

+ sr.S2 = cfg.StrictNext(rec.Host)

+ // NextSpawnStrict prefixes its reasons with "S2: ", which Ready adds.

+ sr.S2.Reason = strings.TrimPrefix(sr.S2.Reason, "S2: ")

+ }

+ if cfg.OptDir != "" {

+ if v, err := release.LiveVersion(cfg.OptDir, rec.Host); err == nil {

+ sr.Release = v

+ } else {

+ log.Printf("moves: %s: live version: %v", rec.Host, err)

+ }

+ if r.s1 == nil && cfg.LaunchEnv != nil {

+ key, err := moves.ReportKeyFromSecrets(cfg.SecretsKey)

+ if err != nil {

+ sr.S1 = moves.Check{Reason: s1NoSecretsKey}

+ } else if env, _, err := cfg.LaunchEnv(rec.Host); err != nil {

+ return fmt.Errorf("compose the launch environment: %w", err)

+ } else {

+ sr.Env = moves.ReportEnv(key, env)

+ }

+ body, err := json.Marshal(sr)

+ if err != nil {

+ return err

+ }

+ return moves.PutReplace(ctx, s, moves.SourceReportKey(moves.ReportPrefix(rec.Account, rec.App, rec.Host, rec.Move.ID)), body)

+// probeSource is S1: the instance's own answer to its readiness path,

+// woken if it sleeps, never from the response cache.

+func (w *movesWorker) probeSource(ctx context.Context, s release.Store, rec provision.Record) moves.Check {

+ if w.cfg.ReadyProbe == nil {

+ return moves.Check{Reason: s1EdgeNotReady}

+ }

+ path := "/"

+ if m, err := provision.GetManifest(ctx, s, rec.Account, rec.App); err == nil && m != nil {

+ path = m.ReadyPathOrDefault()

+ }

+ return w.cfg.ReadyProbe(ctx, rec.Host, path)

+// decideReady reads the target's report and applies §7.3's rule; on ready

+// from preparing it pins the inputs and promotes, and an automatic cut then

+// marks the cut requested (§7.4). Every CAS is guarded by the op identity

+// the round read: a replacement move is ErrMoveConflict and the round ends.

+func (w *movesWorker) decideReady(ctx context.Context, s release.Store, rec provision.Record, g provision.MoveGuard, r *srcRound) error {

+ cfg := w.cfg

+ state := rec.Move.State

+ if state == provision.MovePreparing {

+ target, err := readTargetReport(ctx, s, rec)

+ if err != nil {

+ return err

+ }

+ source, err := readSourceReport(ctx, s, rec)

+ if err != nil {

+ return err

+ }

+ ok, why := moves.Ready(r.deltaBytes, cfg.ReadyDelta, target, source, cfg.Now())

+ if !ok {

+ w.notice(rec.Host+"#ready", "moves: %s: not ready: %s", rec.Host, why)

+ return nil

+ }

+ in := target.Inputs

+ in.Release = source.Release

+ if _, err := provision.SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, provision.MoveGuardedPromoteReady(g, cfg.Now(), in)); err != nil {

+ if errors.Is(err, provision.ErrMoveConflict) {

+ log.Printf("moves: %s: the record no longer holds move %s as preparing (%v); this round ends", rec.Host, g.ID, err)

+ return nil

+ }

+ return err

+ }

+ state = provision.MoveReady

+ nudgeSerial(ctx, s)

+ log.Printf("moves: %s: move %s is ready (last round moved %d bytes; bar %d)", rec.Host, g.ID, r.deltaBytes, cfg.ReadyDelta)

+ }

+ if state == provision.MoveReady && rec.Move.Cut == "auto" {

+ if _, err := provision.SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, provision.MoveGuardedCut(g)); err != nil {

+ if errors.Is(err, provision.ErrMoveConflict) {

+ log.Printf("moves: %s: the automatic cut of %s was refused (%v)", rec.Host, g.ID, err)

+ return nil

+ }

+ return err

+ }

+ nudgeSerial(ctx, s)

+ log.Printf("moves: %s: move %s cut requested automatically", rec.Host, g.ID)

+ }

+ return nil

+func readSourceReport(ctx context.Context, s release.Store, rec provision.Record) (*moves.SourceReport, error) {

+ var r moves.SourceReport

+ ok, err := readReport(ctx, s, moves.SourceReportKey(moves.ReportPrefix(rec.Account, rec.App, rec.Host, rec.Move.ID)), &r)

+ if !ok {

+ return nil, err

+ }

+ return &r, nil

+func readTargetReport(ctx context.Context, s release.Store, rec provision.Record) (*moves.TargetReport, error) {

+ var r moves.TargetReport

+ ok, err := readReport(ctx, s, moves.TargetReportKey(moves.ReportPrefix(rec.Account, rec.App, rec.Host, rec.Move.ID)), &r)

+ if !ok {

+ return nil, err

+ }

+ return &r, nil

+// readReport reads a report: (false, nil) when absent or unreadable as a

+// report (Ready then says no report), an error when the read failed.

+func readReport(ctx context.Context, s release.Store, key string, into any) (bool, error) {

+ body, _, err := s.Get(ctx, key)

+ if errors.Is(err, os.ErrNotExist) {

+ return false, nil

+ }

+ if err != nil {

+ return false, fmt.Errorf("read %s: %w", key, err)

+ }

+ if err := json.Unmarshal(body, into); err != nil {

+ log.Printf("moves: %s does not parse: %v", key, err)

+ return false, nil

+ }

+ return true, nil

+// interimAbort is the abort's box half until Plan 5 replaces it (§7.11):

+// the source deletes its own snapshots, receipts and ledger for the move,

+// releases the claim back to serving (K8), and then completes the record.

+// The staging lane in the target bucket is left for the target's park.

+//

+// The crash boundary between the claim release and the record write: a

+// round that finds the record aborting and the claim already serving

+// completes the record only when the serving op records this move as the

+// one it released (RequireReleasedMove). A serving claim without that

+// evidence was released by someone else (the console pass, Plan 5), and

+// the record is left to it.

+func (w *movesWorker) interimAbort(ctx context.Context, s release.Store, rec provision.Record) error {

+ cfg := w.cfg

+ id := rec.Move.ID

+ g := provision.MoveGuard{ID: id, Incarnation: rec.Move.Incarnation}

+ obs, err := hostclaim.Read(ctx, s, rec.Host)

+ if err != nil {

+ return fmt.Errorf("read the host claim: %w", err)

+ }

+ if obs.Absent {

+ w.notice(rec.Host+"#abort", "moves: %s: aborting move %s has no host claim; left for the console pass", rec.Host, id)

+ return nil

+ }

+ c := obs.Claim

+ switch {

+ case c.Op.Kind == hostclaim.KindMoving && c.Op.ID == id:

+ if err := w.deleteOwnMoveState(ctx, rec); err != nil {

+ return err

+ }

+ box := cfg.Box

+ if box == "" {

+ box = c.Op.Exec

+ }

+ if _, err := hostclaim.Update(ctx, s, obs, hostclaim.ReleaseMove(id, box)); err != nil {

+ return fmt.Errorf("release the host claim: %w", err)

+ }

+ case hostclaim.RequireReleasedMove(id)(c) == nil:

+ // The claim was released by this abort before a crash: finish.

+ if err := w.deleteOwnMoveState(ctx, rec); err != nil {

+ return err

+ }

+ default:

+ w.notice(rec.Host+"#abort", "moves: %s: aborting move %s: the claim is %s/%s and records no release of this move; left for the console pass", rec.Host, id, c.Op.Kind, c.Op.ID)

+ return nil

+ }

+ if _, err := provision.SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, provision.MoveGuardedAbortComplete(g, cfg.Now())); err != nil {

+ if errors.Is(err, provision.ErrMoveConflict) {

+ log.Printf("moves: %s: abort of %s: the record moved on (%v)", rec.Host, id, err)

+ return nil

+ }

+ return err

+ }

+ nudgeSerial(ctx, s)

+ log.Printf("moves: %s: move %s aborted; the instance stays in %q", rec.Host, id, rec.Region)

+ return nil

+// deleteOwnMoveState removes the move's snapshots, receipts and ledger from

+// this box's own bucket. Idempotent.

+func (w *movesWorker) deleteOwnMoveState(ctx context.Context, rec provision.Record) error {

+ cfg := w.cfg

+ own, err := w.mirror(ctx, cfg.OwnBucketRegion)

+ if err != nil {

+ return err

+ }

+ if err := lease.DeletePrefix(ctx, own, cfg.OwnBucket, moves.SnapshotsPrefix(rec.Host, rec.Move.ID)); err != nil {

+ return fmt.Errorf("delete the move's snapshots: %w", err)

+ }

+ keys := []string{moves.ReceiptsKey(rec.Host, rec.Move.ID), moves.LedgerKey(rec.Host, rec.Move.ID)}

+ sort.Strings(keys)

+ for _, k := range keys {

+ if err := own.Delete(ctx, cfg.OwnBucket, k); err != nil && !errors.Is(err, lease.ErrMirrorNotFound) {

+ return fmt.Errorf("delete %s: %w", k, err)

+ }

+ return nil

diff --git a/internal/host/movesphase1_harness_test.go b/internal/host/movesphase1_harness_test.go

new file mode 100644

index 000000000..35486bd7c

--- /dev/null

+++ b/internal/host/movesphase1_harness_test.go

@@ -0,0 +1,513 @@

+package host

+import (

+ "bytes"

+ "context"

+ "errors"

+ "fmt"

+ "io"

+ "os"

+ "path/filepath"

+ "strings"

+ "sync"

+ "testing"

+ "time"

+ "github.com/carlosframework/platform/internal/appconfig"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/lease"

+ "github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/provision"

+ "github.com/carlosframework/platform/internal/registry"

+ "github.com/carlosframework/platform/internal/release"

+// The Phase 1 harness: the legacy moveHarness, its fake mirror extended

+// with the staging writer's guarded At methods (F17) and a call log, a

+// lease.VersionedStore over the same buckets, a staging-lease fake, a

+// stepping clock, a MoveStart'ed record, and the host claim the request

+// door would have written. fakeMoveMirror deliberately does NOT gain

+// CopyVersioned: MirrorPrefix would then run the legacy tests in its

+// versioned mode.

+// stagingCall is one guarded staging request the fake received.

+type stagingCall struct {

+ Op, Bucket, Key string

+ SignAt time.Time

+func (f *fakeMoveMirror) logAt(op string, signAt time.Time, bucket, key string) error {

+ f.mu.Lock()

+ defer f.mu.Unlock()

+ f.at = append(f.at, stagingCall{op, bucket, key, signAt})

+ if f.failAt != nil {

+ return f.failAt(op, key)

+ }

+ return nil

+func (f *fakeMoveMirror) nextVersion() string {

+ f.mu.Lock()

+ defer f.mu.Unlock()

+ f.versions++

+ return fmt.Sprintf("ver-%04d", f.versions)

+func (f *fakeMoveMirror) CopyVersionedAt(ctx context.Context, signAt time.Time, srcBucket, srcKey, srcETag, dstBucket, dstKey string) (string, string, error) {

+ if err := f.logAt("copy", signAt, dstBucket, dstKey); err != nil {

+ return "", "", err

+ }

+ etag, err := f.Copy(ctx, srcBucket, srcKey, srcETag, dstBucket, dstKey)

+ if err != nil {

+ return "", "", err

+ }

+ ver := f.nextVersion()

+ f.setVersion(dstBucket, dstKey, ver)

+ return etag, ver, nil

+func (f *fakeMoveMirror) putAt(op string, signAt time.Time, bucket, key string, body []byte, createOnly bool) (string, error) {

+ if err := f.logAt(op, signAt, bucket, key); err != nil {

+ return "", err

+ }

+ if _, ok := f.buckets[bucket][key]; ok && createOnly {

+ return "", lease.ErrExists

+ }

+ ver := f.nextVersion()

+ f.store(bucket, key, fakeMoveObject{lease.MirrorObject{ETag: "etag-" + ver, Size: int64(len(body))}, string(body)})

+ f.setVersion(bucket, key, ver)

+ return ver, nil

+// setVersion records the current version of bucket/key. The legacy

+// fakeMoveObject has no version field (its positional literals stay as

+// they are), so versions live beside it.

+func (f *fakeMoveMirror) setVersion(bucket, key, ver string) {

+ f.mu.Lock()

+ defer f.mu.Unlock()

+ if f.vers == nil {

+ f.vers = map[string]string{}

+ }

+ f.vers[bucket+"/"+key] = ver

+func (f *fakeMoveMirror) versionOf(bucket, key string) string {

+ f.mu.Lock()

+ defer f.mu.Unlock()

+ return f.vers[bucket+"/"+key]

+func (f *fakeMoveMirror) PutVersionedAt(_ context.Context, signAt time.Time, bucket, key string, body []byte) (string, error) {

+ return f.putAt("put", signAt, bucket, key, body, false)

+func (f *fakeMoveMirror) PutIfAbsentVersionedAt(_ context.Context, signAt time.Time, bucket, key string, body []byte) (string, error) {

+ return f.putAt("put-if-absent", signAt, bucket, key, body, true)

+func (f *fakeMoveMirror) DeleteAt(ctx context.Context, signAt time.Time, bucket, key string) error {

+ if err := f.logAt("delete", signAt, bucket, key); err != nil {

+ return err

+ }

+ return f.Delete(ctx, bucket, key)

+var _ lease.StagingWriter = (*fakeMoveMirror)(nil)

+// stagingCalls is the guarded call log, by op and key.

+func (f *fakeMoveMirror) stagingCalls() []stagingCall {

+ f.mu.Lock()

+ defer f.mu.Unlock()

+ return append([]stagingCall(nil), f.at...)

+// fakeVersioned is a lease.VersionedStore over the fake mirror's buckets,

+// reading by exact version. deny refuses a call the way an IAM policy does.

+type fakeVersioned struct {

+ f *fakeMoveMirror

+ mu sync.Mutex

+ calls []string

+ deny func(op, bucket, key string) error

+func (v *fakeVersioned) call(op, bucket, key string) error {

+ v.mu.Lock()

+ v.calls = append(v.calls, op+" "+bucket+"/"+key)

+ v.mu.Unlock()

+ if v.deny != nil {

+ return v.deny(op, bucket, key)

+ }

+ return nil

+func (v *fakeVersioned) ListVersions(_ context.Context, bucket, prefix string) ([]lease.ObjectVersion, error) {

+ if err := v.call("list-versions", bucket, prefix); err != nil {

+ return nil, err

+ }

+ var out []lease.ObjectVersion

+ for k, o := range v.f.buckets[bucket] {

+ if strings.HasPrefix(k, prefix) {

+ out = append(out, lease.ObjectVersion{Key: k, VersionID: v.f.versionOf(bucket, k), ETag: o.ETag, Size: o.Size, IsLatest: true})

+ }

+ return out, nil

+func (v *fakeVersioned) GetVersion(_ context.Context, bucket, key, versionID string) ([]byte, error) {

+ if err := v.call("get-version", bucket, key); err != nil {

+ return nil, err

+ }

+ o, ok := v.f.buckets[bucket][key]

+ if !ok || versionID == "" || v.f.versionOf(bucket, key) != versionID {

+ return nil, lease.ErrMirrorNotFound

+ }

+ return []byte(o.body), nil

+func (v *fakeVersioned) OpenVersion(ctx context.Context, bucket, key, versionID string) (io.ReadCloser, error) {

+ b, err := v.GetVersion(ctx, bucket, key, versionID)

+ if err != nil {

+ return nil, err

+ }

+ return io.NopCloser(bytes.NewReader(b)), nil

+func (v *fakeVersioned) CopyVersion(context.Context, string, string, string, string, string, bool) (string, string, error) {

+ return "", "", errors.New("fakeVersioned: CopyVersion is not used by Phase 1 Plan 1")

+func (v *fakeVersioned) DeleteVersion(_ context.Context, bucket, key, versionID string) error {

+ if err := v.call("delete-version", bucket, key); err != nil {

+ return err

+ }

+ if _, ok := v.f.buckets[bucket][key]; ok && v.f.versionOf(bucket, key) == versionID {

+ delete(v.f.buckets[bucket], key)

+ }

+ return nil

+func (v *fakeVersioned) PutIfAbsent(_ context.Context, bucket, key string, body []byte) (string, error) {

+ if err := v.call("put-if-absent", bucket, key); err != nil {

+ return "", err

+ }

+ if _, ok := v.f.buckets[bucket][key]; ok {

+ return "", lease.ErrExists

+ }

+ ver := v.f.nextVersion()

+ v.f.store(bucket, key, fakeMoveObject{lease.MirrorObject{ETag: "etag-" + ver, Size: int64(len(body))}, string(body)})

+ v.f.setVersion(bucket, key, ver)

+ return ver, nil

+func (v *fakeVersioned) callsOn(prefix string) []string {

+ v.mu.Lock()

+ defer v.mu.Unlock()

+ var out []string

+ for _, c := range v.calls {

+ if strings.Contains(c, prefix) {

+ out = append(out, c)

+ }

+ return out

+// fakeStagingLease is the staging-work lease: safe until `until`.

+type fakeStagingLease struct {

+ mu sync.Mutex

+ until time.Time

+ acquired int

+ released int

+ // unsafeAfter, when positive, makes every SafeUntil read after that

+ // many answer the lease's start: too close to its end to sign.

+ reads int

+ unsafeAfter int

+func (l *fakeStagingLease) SafeUntil() (time.Time, error) {

+ l.mu.Lock()

+ defer l.mu.Unlock()

+ l.reads++

+ if l.unsafeAfter > 0 && l.reads > l.unsafeAfter {

+ return l.until.Add(-moves.StagingLeaseTTL), nil

+ }

+ return l.until, nil

+// stepClock advances a millisecond each read, so every timed span is

+// positive and the order of events is the order of reads.

+type stepClock struct {

+ mu sync.Mutex

+ now time.Time

+func (c *stepClock) Now() time.Time {

+ c.mu.Lock()

+ defer c.mu.Unlock()

+ c.now = c.now.Add(time.Millisecond)

+ return c.now

+func (c *stepClock) Advance(d time.Duration) {

+ c.mu.Lock()

+ c.now = c.now.Add(d)

+ c.mu.Unlock()

+const (

+ p1Host = "a.go.example.com"

+ p1ID = "mv_abcdefghijklmnopqrstuvwxyz"

+ p1ID2 = "mv_bcdefghijklmnopqrstuvwxyza"

+var p1Created = time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)

+type phase1Harness struct {

+ *moveHarness

+ vs *fakeVersioned

+ lease *fakeStagingLease

+ clock *stepClock

+ inc string

+ route registry.Route

+ env map[string]string // the source's launch environment

+ optDir string

+ probe moves.Check

+ strict moves.Check

+ // worker1 is the source box's worker and worker2 the target's, each

+ // kept across passes as one agent keeps its own.

+ worker1, worker2 *movesWorker

+func newPhase1Harness(t *testing.T) *phase1Harness {

+ t.Helper()

+ h := &phase1Harness{moveHarness: newMoveHarness(t), lease: &fakeStagingLease{}, clock: &stepClock{now: time.Date(2026, 10, 7, 9, 0, 0, 0, time.UTC)}}

+ h.vs = &fakeVersioned{f: h.f}

+ h.lease.until = h.clock.now.Add(moves.StagingLeaseTTL)

+ inc, err := hostclaim.Incarnation("bab", "app", p1Host, p1Created)

+ if err != nil {

+ t.Fatal(err)

+ }

+ h.inc = inc

+ privB64, _, err := appconfig.GenerateKey()

+ if err != nil {

+ t.Fatal(err)

+ }

+ priv, err := appconfig.ParsePrivateKey(privB64, "test")

+ if err != nil {

+ t.Fatal(err)

+ }

+ h.optDir = t.TempDir()

+ h.route = registry.Route{Host: p1Host, Kind: "instance", App: "app", Account: "bab", Hibernate: true, Status: "ready",

+ Socket: provision.DefaultSocket(p1Host), DBPath: provision.DefaultDBPath(h.cfg.DataDir, p1Host), Provisioned: true}

+ h.env = map[string]string{"CARLOS_REGION": "ie", "CARLOS_CONTROL_SOCKET": "/run/carlos/" + p1Host + ".sock.control", "APP_MODE": "prod"}

+ h.probe, h.strict = moves.Check{OK: true}, moves.Check{OK: true}

+ ctx := context.Background()

+ h.cfg.Enabled = true

+ h.cfg.Box = "ie-box"

+ h.cfg.Now = h.clock.Now

+ h.cfg.OptDir = h.optDir

+ h.cfg.DeployBucket = "deploy"

+ h.cfg.StagingLease = func(context.Context, string, string) (moves.SafeUntilGuard, func(context.Context) error, error) {

+ h.lease.mu.Lock()

+ h.lease.acquired++

+ h.lease.mu.Unlock()

+ return h.lease, func(context.Context) error {

+ h.lease.mu.Lock()

+ h.lease.released++

+ h.lease.mu.Unlock()

+ return nil

+ }, nil

+ }

+ h.cfg.Staging = func(context.Context, string) (lease.StagingWriter, error) { return h.f, nil }

+ h.cfg.Versioned = func(context.Context, string, string) (lease.VersionedStore, error) { return h.vs, nil }

+ h.cfg.SecretsKey, h.cfg.SecretsPriv, h.cfg.SecretsPub = privB64, priv, priv.PublicKey()

+ h.cfg.LaunchEnv = func(string) (map[string]string, []string, error) { return h.env, nil, nil }

+ h.cfg.FilesDir = func(host string) (string, string) {

+ return filepath.Join(h.cfg.DataDir, host), filepath.Join(h.cfg.DataDir, host+".env")

+ }

+ h.cfg.ReadyProbe = func(context.Context, string, string) moves.Check { return h.probe }

+ h.cfg.StrictNext = func(string) moves.Check { return h.strict }

+ h.cfg.Route = func(host string) (*registry.Route, error) {

+ if host != p1Host {

+ return nil, nil

+ }

+ rt := h.route

+ return &rt, nil

+ }

+ // The source's files and carried .env, its live release v7, and its

+ // replica: two generations under the scoped location (g-old and the

+ // live g-cur), plus a generation.json and a lock.json beside them.

+ files := filepath.Join(h.cfg.DataDir, p1Host)

+ if err := os.MkdirAll(files, 0o755); err != nil {

+ t.Fatal(err)

+ }

+ if err := os.WriteFile(filepath.Join(files, "upload.txt"), []byte("hello"), 0o644); err != nil {

+ t.Fatal(err)

+ }

+ if err := os.WriteFile(filepath.Join(h.cfg.DataDir, p1Host+".env"), []byte("APP_MODE=prod\n"), 0o600); err != nil {

+ t.Fatal(err)

+ }

+ pointLiveForTest(t, h.optDir, p1Host, "v7")

+ loc := "instances/" + p1Host + "/" + inc + "/"

+ h.f.putSized("data-ie", loc+"g-cur/0000/0000000000000001-0000000000000002.ltx", "cur-a", 2<<20)

+ h.f.putSized("data-ie", loc+"g-cur/0001/0000000000000001-0000000000000002.ltx", "cur-b", 4096)

+ h.f.putSized("data-ie", loc+"g-old/0000/0000000000000001-0000000000000001.ltx", "old-a", 4096)

+ h.f.put("data-ie", loc+"generation.json", "gen")

+ h.f.put("data-ie", loc+"g-cur/lock.json", "lock")

+ // The source's edges file: ie publishes its data bucket too (T10).

+ if _, err := h.s.Put(ctx, "control/edges/ie-box.json",

+ []byte(`{"v":1,"box":"ie-box","eip":"1.2.3.5","placement":"ie","data_bucket":"data-ie","data_bucket_region":"eu-west-1","hosts":{}}`), ""); err != nil {

+ t.Fatal(err)

+ }

+ // The console's record (T2).

+ if _, err := h.s.Put(ctx, moves.ConsoleKey, []byte(`{"v":1,"protocol":1,"secrets_key_fp":"`+appconfig.KeyFingerprint(priv.PublicKey())+`"}`), ""); err != nil {

+ t.Fatal(err)

+ }

+ return h

+// pointLiveForTest makes optDir/live/<host> point at version v, the shape

+// release.LiveVersion reads.

+func pointLiveForTest(t *testing.T, optDir, host, v string) {

+ t.Helper()

+ target := filepath.Join(optDir, "versions", v, "bab", "app")

+ if err := os.MkdirAll(target, 0o755); err != nil {

+ t.Fatal(err)

+ }

+ if err := os.MkdirAll(filepath.Join(optDir, "live"), 0o755); err != nil {

+ t.Fatal(err)

+ }

+ link := filepath.Join(optDir, "live", host)

+ os.Remove(link)

+ if err := os.Symlink(target, link); err != nil {

+ t.Fatal(err)

+ }

+// start writes the record (MoveStart'ed, id p1ID, cut mode cut) and the

+// claim the door's CAS leaves: serving under exec ie-box, then moving with

+// target uk-box. The claim has touched ie and ie-box only.

+func (h *phase1Harness) start(t *testing.T, cut string) provision.Record {

+ t.Helper()

+ ctx := context.Background()

+ rec := provision.Record{Schema: 1, Host: p1Host, Slug: "s", App: "app", Account: "bab", Region: "ie", Channel: "stable", CreatedAt: p1Created}

+ if err := provision.CreateRecord(ctx, h.s, rec); err != nil {

+ t.Fatal(err)

+ }

+ if _, err := provision.SetRecordMove(ctx, h.s, "bab", "app", p1Host, provision.MoveStartTransition(provision.MoveStart{

+ ID: p1ID, Incarnation: h.inc, From: "ie", To: "uk", Cut: cut, By: "op", Now: h.clock.Now()})); err != nil {

+ t.Fatal(err)

+ }

+ obs, err := hostclaim.Update(ctx, h.s, hostclaim.Observed{Host: p1Host, Absent: true}, func(c *hostclaim.Claim) error {

+ c.Holder = &hostclaim.Holder{Family: "instance", Key: provision.RecordKey("bab", "app", p1Host), Account: "bab", App: "app",

+ Incarnation: h.inc, CreatedAt: p1Created, Location: "instances/" + p1Host + "/" + h.inc, Generation: "cur", Stamp: 1}

+ c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "ie-box"}

+ c.Touched = hostclaim.Touched{Complete: true, Regions: []string{"ie"}, Boxes: []string{"ie-box"}}

+ return nil

+ })

+ if err != nil {

+ t.Fatal(err)

+ }

+ if _, err := hostclaim.Update(ctx, h.s, obs, hostclaim.BeginMove(p1ID, "op", "ie-box", "uk-box", "uk", h.clock.Now())); err != nil {

+ t.Fatal(err)

+ }

+ return h.read(t, p1Host)

+func (h *phase1Harness) claim(t *testing.T) *hostclaim.Claim {

+ t.Helper()

+ obs, err := hostclaim.Read(context.Background(), h.s, p1Host)

+ if err != nil || obs.Absent {

+ t.Fatalf("claim: %v absent=%v", err, obs.Absent)

+ }

+ return obs.Claim

+// pass runs one source pass.

+func (h *phase1Harness) pass(t *testing.T) {

+ t.Helper()

+ if err := movesPass(context.Background(), h.s, h.w()); err != nil {

+ t.Fatal(err)

+ }

+// w is one worker kept across the harness's passes, as one agent keeps it.

+func (h *phase1Harness) w() *movesWorker {

+ if h.worker1 == nil {

+ h.worker1 = &movesWorker{cfg: h.cfg, mirrors: map[string]lease.MirrorStore{}}

+ }

+ h.worker1.cfg = h.cfg

+ return h.worker1

+// targetCfg is the uk box's configuration over the same store and buckets,

+// its hooks stubbed to pass (each test turns one off).

+func (h *phase1Harness) targetCfg() moveCfg {

+ c := h.cfg

+ c.Region, c.Box, c.OwnBucket, c.OwnBucketRegion = "uk", "uk-box", "data-uk", "eu-west-2"

+ c.OwnsRoute = func(string) bool { return false }

+ c.Route = func(string) (*registry.Route, error) { return nil, nil }

+ c.ComposeHeldEnv = func(context.Context, release.Store, provision.Record, []byte) (map[string]string, []string, provision.MoveInputs, error) {

+ env := map[string]string{}

+ for k, v := range h.env {

+ env[k] = v

+ }

+ env["CARLOS_REGION"] = "uk"

+ return env, nil, provision.MoveInputs{BundleETags: map[string]string{"default": "e1", "environment": ""}}, nil

+ }

+ c.ProbeStore = func(context.Context, map[string]string, string) (string, error) { return "", nil }

+ c.DiskFree = func(string) (uint64, uint64, error) { return 100 << 30, 90 << 30, nil }

+ c.ReleaseReady = func(context.Context, release.Store, provision.Record, string) moves.Check {

+ return moves.Check{OK: true}

+ }

+ c.LaneProbe = func(context.Context, string, string, string) error { return nil }

+ c.MeasureS3 = func(context.Context, string) (moves.MeasuredRate, error) {

+ return moves.MeasuredRate{MiBPerS: 250, At: h.clock.Now(), Bytes: 32 << 20}, nil

+ }

+ c.MeasureDisk = func(context.Context) (moves.MeasuredRate, moves.FilesRate, error) {

+ now := h.clock.Now()

+ return moves.MeasuredRate{MiBPerS: 500, At: now, Bytes: 128 << 20}, moves.FilesRate{PerS: 1600, At: now, Files: 512}, nil

+ }

+ return c

+// targetPass runs one pass as the target box.

+func (h *phase1Harness) targetPass(t *testing.T, c moveCfg) {

+ t.Helper()

+ if h.worker2 == nil {

+ h.worker2 = &movesWorker{mirrors: map[string]lease.MirrorStore{}}

+ }

+ h.worker2.cfg = c

+ if err := movesPass(context.Background(), h.s, h.worker2); err != nil {

+ t.Fatal(err)

+ }

+func (h *phase1Harness) targetReport(t *testing.T) *moves.TargetReport {

+ t.Helper()

+ r, err := readTargetReport(context.Background(), h.s, h.read(t, p1Host))

+ if err != nil {

+ t.Fatal(err)

+ }

+ return r

+func (h *phase1Harness) sourceReport(t *testing.T) *moves.SourceReport {

+ t.Helper()

+ r, err := readSourceReport(context.Background(), h.s, h.read(t, p1Host))

+ if err != nil {

+ t.Fatal(err)

+ }

+ return r

+func (h *phase1Harness) ledger(t *testing.T) moveLedger {

+ t.Helper()

+ l, err := readMoveLedger(context.Background(), h.f, "data-ie", moves.LedgerKey(p1Host, p1ID))

+ if err != nil {

+ t.Fatal(err)

+ }

+ return l

diff --git a/internal/host/movesphase1_test.go b/internal/host/movesphase1_test.go

new file mode 100644

index 000000000..98b606248

--- /dev/null

+++ b/internal/host/movesphase1_test.go

@@ -0,0 +1,607 @@

+package host

+import (

+ "context"

+ "errors"

+ "os"

+ "path/filepath"

+ "sort"

+ "strings"

+ "testing"

+ "time"

+ "github.com/carlosframework/platform/internal/activator"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/lease"

+ "github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/objstore"

+ "github.com/carlosframework/platform/internal/provision"

+ "github.com/carlosframework/platform/internal/release"

+func stagedKeys(h *phase1Harness, prefix string) []string {

+ var keys []string

+ for k := range h.f.buckets["data-uk"] {

+ if strings.HasPrefix(k, prefix) {

+ keys = append(keys, strings.TrimPrefix(k, prefix))

+ }

+ sort.Strings(keys)

+ return keys

+// The whole preparing round on the new layout: the replica lane holds the

+// claim's generation flat under replica/<level>/, move.json is there, the

+// archive is staged, and the move is not promoted until a passing, fresh

+// target report exists; the target's own pass writes one, and the next

+// source round promotes, pinning the inputs the target checked.

+func TestPhase1PrepareAndPromote(t *testing.T) {

+ h := newPhase1Harness(t)

+ h.start(t, "manual")

+ h.pass(t)

+ if got := h.read(t, p1Host).Move.State; got != provision.MovePreparing {

+ t.Fatalf("after the first round: %s, want preparing", got)

+ }

+ lane := moves.StagingPrefix(p1Host, p1ID)

+ keys := stagedKeys(h, lane)

+ want := []string{"move.json", "replica/0000/0000000000000001-0000000000000002.ltx", "replica/0001/0000000000000001-0000000000000002.ltx"}

+ for _, k := range want {

+ if !containsString(keys, k) {

+ t.Fatalf("the lane lacks %s: %v", k, keys)

+ }

+ for _, k := range keys {

+ if strings.HasPrefix(k, "replica/") && !containsString(want, k) {

+ t.Fatalf("the lane holds %s, outside the claim's generation", k)

+ }

+ sr := h.sourceReport(t)

+ if sr == nil || sr.Archive == nil || sr.FilesPlain == "" || sr.Release != "v7" || !sr.S1.OK || !sr.S2.OK || sr.Env.Digest == "" || sr.CrossRate <= 0 {

+ t.Fatalf("source report: %+v", sr)

+ }

+ // No target report: still preparing, however many rounds.

+ h.pass(t)

+ if got := h.read(t, p1Host).Move.State; got != provision.MovePreparing {

+ t.Fatalf("promoted without a target report: %s", got)

+ }

+ h.targetPass(t, h.targetCfg())

+ tr := h.targetReport(t)

+ if ok, why := tr.Passing(); !ok {

+ t.Fatalf("the target's report does not pass: %s (%+v)", why, tr.Checks)

+ }

+ h.pass(t)

+ got := h.read(t, p1Host)

+ if got.Move.State != provision.MoveReady || got.Move.Inputs == nil || got.Move.Inputs.FilesPlain != sr.FilesPlain || got.Move.Inputs.Release != "v7" || got.Move.Inputs.EnvReport != sr.Env.Digest {

+ t.Fatalf("after a passing target report: %+v inputs %+v", got.Move, got.Move.Inputs)

+ }

+// A stale target report after ready demotes nothing (Review Focus 5): the

+// worker keeps mirroring a ready move and never writes it back.

+func TestPhase1StaleReportAfterReadyDemotesNothing(t *testing.T) {

+ h := newPhase1Harness(t)

+ h.start(t, "manual")

+ h.pass(t)

+ h.targetPass(t, h.targetCfg())

+ h.pass(t)

+ if got := h.read(t, p1Host).Move.State; got != provision.MoveReady {

+ t.Fatalf("not ready: %s", got)

+ }

+ h.clock.Advance(2 * moves.ReportFreshness)

+ h.pass(t)

+ if got := h.read(t, p1Host).Move.State; got != provision.MoveReady {

+ t.Fatalf("a stale report demoted a ready move to %s", got)

+ }

+// With the switch off, an id'd record is inert: no lease, no staging write,

+// no claim write, no CAS, and one log line however many passes run. The

+// exec backend's Strict stays nil, so every spawn is main's.

+func TestPhase1IsInertWithTheSwitchOff(t *testing.T) {

+ h := newPhase1Harness(t)

+ before := h.start(t, "auto")

+ claimBefore := h.claim(t)

+ h.cfg.Enabled = false

+ logs := captureLog(t)

+ h.pass(t)

+ if n := len(h.f.stagingCalls()); n != 0 || h.lease.acquired != 0 {

+ t.Fatalf("switch off: %d staging requests, %d leases", n, h.lease.acquired)

+ }

+ for _, op := range h.f.ops {

+ if strings.HasPrefix(op, "copy") || strings.Contains(op, "moves/") {

+ t.Fatalf("switch off: %s", op)

+ }

+ if got := h.read(t, p1Host); got.Move.State != before.Move.State || !got.Move.UpdatedAt.Equal(before.Move.UpdatedAt) {

+ t.Fatalf("switch off: the record changed: %+v", got.Move)

+ }

+ if c := h.claim(t); c.Gen != claimBefore.Gen {

+ t.Fatalf("switch off: the claim was written (gen %d → %d)", claimBefore.Gen, c.Gen)

+ }

+ if r, _ := readSourceReport(context.Background(), h.s, before); r != nil {

+ t.Fatal("switch off: a source report was written")

+ }

+ if n := strings.Count(logs(), "is off on this box"); n != 1 {

+ t.Fatalf("switch off: %d log lines, want 1:\n%s", n, logs())

+ }

+ // eb.Strict: nil with the switch off, set with it on.

+ prev := armedMovesWorker.Load()

+ t.Cleanup(func() { armedMovesWorker.Store(prev) })

+ for _, on := range []bool{false, true} {

+ v := ""

+ if on {

+ v = "on"

+ }

+ t.Setenv(moves.SwitchEnv, v)

+ eb := &activator.ExecBackend{}

+ armMovesPass(activator.Config{}, t.TempDir(), nil, nil, eb)

+ if (eb.Strict != nil) != on {

+ t.Fatalf("switch %v: eb.Strict set = %v", on, eb.Strict != nil)

+ }

+// Without a deployment bucket there is no shared place for the staging

+// lease (round-5 finding 11): the round writes S1 and does nothing else.

+func TestPhase1WithoutADeploymentBucketRefuses(t *testing.T) {

+ h := newPhase1Harness(t)

+ h.start(t, "manual")

+ h.cfg.StagingLease, h.cfg.DeployBucket = nil, ""

+ gen := h.claim(t).Gen

+ h.pass(t)

+ if n := len(h.f.stagingCalls()); n != 0 {

+ t.Fatalf("%d staging requests", n)

+ }

+ if h.claim(t).Gen != gen {

+ t.Fatal("the claim was written")

+ }

+ if sr := h.sourceReport(t); sr == nil || sr.S1.OK || sr.S1.Reason != s1NoDeployBucket {

+ t.Fatalf("source report: %+v", sr)

+ }

+// Review Focus 2: the source's own lost-response PUT of move.json and a

+// foreign object look alike, and the source may not read the lane. With

+// the ledger lost and move.json present, the put answers ErrExists, the

+// round records "present" and goes on, and nothing reads the target lane.

+func TestStagingMoveJSONReplayAfterLostLedger(t *testing.T) {

+ h := newPhase1Harness(t)

+ h.start(t, "manual")

+ h.f.store("data-uk", moves.MoveJSONKey(p1Host, p1ID), fakeMoveObject{lease.MirrorObject{ETag: "x", Size: 2}, "{}"})

+ h.vs.deny = func(op, bucket, key string) error {

+ if bucket == "data-uk" && strings.HasPrefix(key, "moves/") {

+ return errors.New("AccessDenied")

+ }

+ return nil

+ }

+ h.pass(t)

+ if got := h.ledger(t).MoveJSONVersion; got != moveJSONPresent {

+ t.Fatalf("ledger move_json_version = %q, want %q", got, moveJSONPresent)

+ }

+ if keys := stagedKeys(h, moves.ReplicaPrefix(p1Host, p1ID)); len(keys) != 2 {

+ t.Fatalf("the round did not go on: %v", keys)

+ }

+ if calls := h.vs.callsOn(moves.StagingPrefix(p1Host, p1ID)); len(calls) != 0 {

+ t.Fatalf("the source read the target lane: %v", calls)

+ }

+// The interim abort (§7.11): the source deletes its own snapshots, receipts

+// and ledger, releases the claim to serving, then completes the record;

+// the staging lane is left for the target's park. A crash between the two

+// writes completes on the next round only with the release's evidence, and

+// a serving claim without it is left to the console pass.

+func TestInterimAbortReleasesTheClaim(t *testing.T) {

+ ctx := context.Background()

+ h := newPhase1Harness(t)

+ h.start(t, "manual")

+ h.pass(t)

+ h.f.put("data-ie", moves.SnapshotsPrefix(p1Host, p1ID)+"files-x.sealed", "s")

+ g := provision.MoveGuard{ID: p1ID, Incarnation: h.inc}

+ if _, err := provision.SetRecordMove(ctx, h.s, "bab", "app", p1Host, provision.MoveGuardedAbort(g)); err != nil {

+ t.Fatal(err)

+ }

+ lane := len(stagedKeys(h, moves.StagingPrefix(p1Host, p1ID)))

+ h.pass(t)

+ got := h.read(t, p1Host)

+ if got.Move != nil || got.LastMove == nil || got.LastMove.Outcome != "aborted" {

+ t.Fatalf("abort did not complete: %+v %+v", got.Move, got.LastMove)

+ }

+ if c := h.claim(t); c.Op.Kind != hostclaim.KindServing || c.Op.Exec != "ie-box" {

+ t.Fatalf("claim after the abort: %+v", c.Op)

+ }

+ for _, k := range []string{moves.ReceiptsKey(p1Host, p1ID), moves.LedgerKey(p1Host, p1ID), moves.SnapshotsPrefix(p1Host, p1ID) + "files-x.sealed"} {

+ if _, ok := h.f.buckets["data-ie"][k]; ok {

+ t.Fatalf("%s survived the abort", k)

+ }

+ if n := len(stagedKeys(h, moves.StagingPrefix(p1Host, p1ID))); n != lane || n == 0 {

+ t.Fatalf("the source touched the lane: %d keys, had %d", n, lane)

+ }

+ // The crash case: the claim released, the record still aborting.

+ h2 := newPhase1Harness(t)

+ h2.start(t, "manual")

+ if _, err := provision.SetRecordMove(ctx, h2.s, "bab", "app", p1Host, provision.MoveGuardedAbort(g)); err != nil {

+ t.Fatal(err)

+ }

+ obs, _ := hostclaim.Read(ctx, h2.s, p1Host)

+ if _, err := hostclaim.Update(ctx, h2.s, obs, hostclaim.ReleaseMove(p1ID, "ie-box")); err != nil {

+ t.Fatal(err)

+ }

+ h2.pass(t)

+ if got := h2.read(t, p1Host); got.Move != nil {

+ t.Fatalf("the replay did not complete the record: %+v", got.Move)

+ }

+ // A serving claim with no evidence of this release: left alone.

+ h3 := newPhase1Harness(t)

+ h3.start(t, "manual")

+ if _, err := provision.SetRecordMove(ctx, h3.s, "bab", "app", p1Host, provision.MoveGuardedAbort(g)); err != nil {

+ t.Fatal(err)

+ }

+ rewriteClaimOp(t, h3.s, func(op *hostclaim.Op) { *op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "ie-box"} })

+ h3.pass(t)

+ if got := h3.read(t, p1Host); got.Move == nil || got.Move.State != provision.MoveAborting {

+ t.Fatalf("a serving claim without evidence completed the record: %+v", got.Move)

+ }

+// rewriteClaimOp edits p1Host's claim op in place, as another writer would.

+func rewriteClaimOp(t *testing.T, s release.Store, edit func(*hostclaim.Op)) {

+ t.Helper()

+ obs, err := hostclaim.Read(context.Background(), s, p1Host)

+ if err != nil {

+ t.Fatal(err)

+ }

+ if _, err := hostclaim.Update(context.Background(), s, obs, func(c *hostclaim.Claim) error { edit(&c.Op); return nil }); err != nil {

+ t.Fatal(err)

+ }

+// Every transition is guarded by op identity: a replacement move written

+// between the round's read and its CAS is ErrMoveConflict, and the round

+// ends without advancing it.

+func TestRoundNeverAdvancesAReplacementMove(t *testing.T) {

+ ctx := context.Background()

+ h := newPhase1Harness(t)

+ h.start(t, "auto")

+ h.pass(t)

+ h.targetPass(t, h.targetCfg())

+ // Between the next round's read and its promotion, the record's move is

+ // replaced by another in the same state.

+ h.f.failAt = func(op, key string) error {

+ h.f.failAt = nil

+ _, err := provision.SetRecordMove(ctx, h.s, "bab", "app", p1Host, func(rec *provision.Record) error {

+ rec.Move.ID = p1ID2

+ return nil

+ })

+ return err

+ }

+ h.f.putSized("data-ie", "instances/"+p1Host+"/"+h.inc+"/g-cur/0000/0000000000000003-0000000000000003.ltx", "cur-c", 10)

+ h.pass(t)

+ got := h.read(t, p1Host)

+ if got.Move.ID != p1ID2 || got.Move.State != provision.MovePreparing || got.Move.Inputs != nil {

+ t.Fatalf("the replacement move was advanced: %+v", got.Move)

+ }

+// The source copy is bound to the claim's location and generation (round-3

+// finding 5): from a scoped location with two generations only g-cur's

+// level files reach the lane; from a legacy root location with g-x/ beside

+// its level directories, only the level directories do.

+func TestReplicaSourceSelectsTheClaimsGeneration(t *testing.T) {

+ h := newPhase1Harness(t)

+ h.start(t, "manual")

+ c := h.claim(t)

+ prefix, _ := moves.ReplicaSource(*c.Holder)

+ if want := "instances/" + p1Host + "/" + h.inc + "/g-cur/"; prefix != want {

+ t.Fatalf("prefix %q, want %q", prefix, want)

+ }

+ h.pass(t)

+ got := stagedKeys(h, moves.ReplicaPrefix(p1Host, p1ID))

+ if want := []string{"0000/0000000000000001-0000000000000002.ltx", "0001/0000000000000001-0000000000000002.ltx"}; strings.Join(got, ",") != strings.Join(want, ",") {

+ t.Fatalf("scoped location staged %v, want %v", got, want)

+ }

+ // The legacy root location.

+ h2 := newPhase1Harness(t)

+ h2.start(t, "manual")

+ rewriteP1Claim(t, h2.s, func(c *hostclaim.Claim) {

+ c.Holder.Location, c.Holder.Generation = "instances/"+p1Host, hostclaim.RootGeneration

+ })

+ root := "instances/" + p1Host + "/"

+ h2.f.putSized("data-ie", root+"0000/0000000000000001-0000000000000005.ltx", "root-a", 10)

+ h2.f.putSized("data-ie", root+"0009/0000000000000001-0000000000000005.ltx", "root-snap", 10)

+ h2.f.putSized("data-ie", root+"g-x/0000/0000000000000001-0000000000000001.ltx", "gx", 10)

+ h2.pass(t)

+ got = stagedKeys(h2, moves.ReplicaPrefix(p1Host, p1ID))

+ if want := []string{"0000/0000000000000001-0000000000000005.ltx", "0009/0000000000000001-0000000000000005.ltx"}; strings.Join(got, ",") != strings.Join(want, ",") {

+ t.Fatalf("root location staged %v, want %v", got, want)

+ }

+func rewriteP1Claim(t *testing.T, s release.Store, edit func(*hostclaim.Claim)) {

+ t.Helper()

+ obs, err := hostclaim.Read(context.Background(), s, p1Host)

+ if err != nil {

+ t.Fatal(err)

+ }

+ if _, err := hostclaim.Update(context.Background(), s, obs, func(c *hostclaim.Claim) error { edit(c); return nil }); err != nil {

+ t.Fatal(err)

+ }

+// Every staging write goes through the guard: with the lease too close to

+// its end, the round issues no staging request of any class (move.json,

+// the replica copy, the archive) and nothing reaches the target bucket.

+// And for every point at which the lease turns unsafe, the requests issued

+// are exactly the checks that passed: no class signs without one.

+func TestEveryStagingWriteIsGuarded(t *testing.T) {

+ h := newPhase1Harness(t)

+ h.start(t, "manual")

+ h.lease.until = h.clock.now.Add(moves.SignatureLife) // inside SignatureLife + skew

+ h.pass(t)

+ if calls := h.f.stagingCalls(); len(calls) != 0 {

+ t.Fatalf("guarded requests issued past the bound: %+v", calls)

+ }

+ if keys := stagedKeys(h, "moves/"); len(keys) != 0 {

+ t.Fatalf("the target bucket was written: %v", keys)

+ }

+ // A full first round with a safe lease: every class, each signed at

+ // the guard's check time, every write to the target bucket guarded.

+ full := newPhase1Harness(t)

+ full.start(t, "manual")

+ full.pass(t)

+ calls := full.f.stagingCalls()

+ ops := map[string]bool{}

+ for _, c := range calls {

+ ops[c.Op] = true

+ if c.SignAt.IsZero() || c.Bucket != "data-uk" {

+ t.Fatalf("unguarded or misdirected request: %+v", c)

+ }

+ if !ops["put-if-absent"] || !ops["copy"] || !ops["put"] {

+ t.Fatalf("not every class went through the guard: %v", ops)

+ }

+ if full.lease.reads != len(calls) {

+ t.Fatalf("%d guard checks for %d requests", full.lease.reads, len(calls))

+ }

+ for _, op := range full.f.ops {

+ if strings.Contains(op, "data-uk/") && !strings.HasPrefix(op, "copy ") {

+ t.Fatalf("an unguarded write reached the target bucket: %s", op)

+ }

+ for k := 1; k < len(calls); k++ {

+ hk := newPhase1Harness(t)

+ hk.start(t, "manual")

+ hk.lease.unsafeAfter = k

+ hk.pass(t)

+ if n := len(hk.f.stagingCalls()); n != k {

+ t.Fatalf("lease unsafe after %d checks: %d requests issued", k, n)

+ }

+// The box-step refusals, written into S1 with nothing staged: a legacy

+// record with the switch on (logged, never touched), a nested row, a

+// sidecar environment file, every refused registry column, a non-instance

+// or non-hibernating row (F14), and a bound private store.

+func TestPreparingRefusesLegacyNestedAndRefusedColumns(t *testing.T) {

+ ctx := context.Background()

+ t.Run("legacy record", func(t *testing.T) {

+ h := newPhase1Harness(t)

+ h.record(t, p1Host, provision.MoveRequest("uk", time.Now()))

+ logs := captureLog(t)

+ h.pass(t)

+ if len(h.f.ops) != 0 || len(h.f.stagingCalls()) != 0 || h.read(t, p1Host).Move.State != provision.MovePreparing {

+ t.Fatalf("a legacy record was driven: %v", h.f.ops)

+ }

+ if !strings.Contains(logs(), legacyMoveSentence) {

+ t.Fatalf("no MV-N7 log line:\n%s", logs())

+ }

+ })

+ for name, tc := range map[string]struct {

+ edit func(*phase1Harness)

+ want string

+ }{

+ "nested": {func(h *phase1Harness) { h.route.DBPath = filepath.Join(h.cfg.DataDir, "instances", "x", p1Host+".db") }, s1Nested},

+ "sidecar": {func(h *phase1Harness) { writeFileForTest(filepath.Join(h.cfg.DataDir, p1Host+"-sidecar.env"), "X=1") }, s1Sidecar(p1Host)},

+ "not instance": {func(h *phase1Harness) { h.route.Kind = "service" }, s1NotExec(p1Host)},

+ "platform": {func(h *phase1Harness) { h.route.Platform = true }, s1NotExec(p1Host)},

+ "no hibernate": {func(h *phase1Harness) { h.route.Hibernate = false }, s1NoHibernate(p1Host)},

+ "caps": {func(h *phase1Harness) { h.route.Caps = "x" }, s1Column(p1Host, "caps")},

+ "unit": {func(h *phase1Harness) { h.route.Unit = "x" }, s1Column(p1Host, "unit")},

+ "backing": {func(h *phase1Harness) { h.route.Backing = "unit" }, s1Column(p1Host, "backing")},

+ "tunnel": {func(h *phase1Harness) { h.route.Tunnel = "x" }, s1Column(p1Host, "tunnel")},

+ "fleet box": {func(h *phase1Harness) { h.route.FleetBox = "x" }, s1Column(p1Host, "fleet_box")},

+ "pool": {func(h *phase1Harness) { h.route.Pool = "x" }, s1Column(p1Host, "pool")},

+ "canary": {func(h *phase1Harness) { h.route.Canary = true }, s1Column(p1Host, "canary")},

+ "addr": {func(h *phase1Harness) { h.route.Addr = "x:1" }, s1Column(p1Host, "addr")},

+ "addrs": {func(h *phase1Harness) { h.route.Addrs = "x:1" }, s1Column(p1Host, "addrs")},

+ "health": {func(h *phase1Harness) { h.route.Health = "/h" }, s1Column(p1Host, "health")},

+ "replica": {func(h *phase1Harness) { h.route.Replica = true }, s1Column(p1Host, "replica")},

+ "replicas": {func(h *phase1Harness) { h.route.Replicas = "uk" }, s1Column(p1Host, "replicas")},

+ "private storage": {func(h *phase1Harness) {

+ // The record has no Environment, so its store hangs off its

+ // slug (provision.InstanceStoreEnv).

+ if _, err := h.s.Put(ctx, objstore.KeyIn("bab", "app", "s"), []byte(`{"environment":"s"}`), ""); err != nil {

+ panic(err)

+ }

+ }, s1PrivateStorage(p1Host)},

+ } {

+ t.Run(name, func(t *testing.T) {

+ h := newPhase1Harness(t)

+ h.start(t, "manual")

+ tc.edit(h)

+ h.pass(t)

+ if calls := h.f.stagingCalls(); len(calls) != 0 {

+ t.Fatalf("staged despite the refusal: %+v", calls)

+ }

+ sr := h.sourceReport(t)

+ if sr == nil || sr.S1.OK || sr.S1.Reason != tc.want {

+ t.Fatalf("S1 = %+v, want %q", sr, tc.want)

+ }

+ })

+ }

+// K12 and T9: the claim's touched gains the target region and box before

+// the first staging write, from a claim that touched neither.

+func TestPreparingTouchesTheTargetRegionAndBoxFirst(t *testing.T) {

+ h := newPhase1Harness(t)

+ h.start(t, "manual")

+ if c := h.claim(t); containsString(c.Touched.Regions, "uk") || containsString(c.Touched.Boxes, "uk-box") {

+ t.Fatalf("the fixture already touched uk: %+v", c.Touched)

+ }

+ var touched *hostclaim.Touched

+ h.f.failAt = func(op, key string) error {

+ if touched == nil {

+ c := h.claim(t)

+ touched = &c.Touched

+ }

+ return nil

+ }

+ h.pass(t)

+ if touched == nil {

+ t.Fatal("no staging write happened")

+ }

+ if !containsString(touched.Regions, "uk") || !containsString(touched.Boxes, "uk-box") {

+ t.Fatalf("at the first staging write touched = %+v", *touched)

+ }

+// A failed copy is retried only by calling again: the lane asks the guard

+// again before the retry, and a lease past the bound by then ends the round

+// with no second request.

+func TestStagingWriterRetriesWithANewCheck(t *testing.T) {

+ h := newPhase1Harness(t)

+ h.start(t, "manual")

+ // The first copy fails, and by its retry the lease is too close to

+ // its end.

+ copies := 0

+ h.f.failAt = func(op, key string) error {

+ if op != "copy" {

+ return nil

+ }

+ copies++

+ if copies == 1 {

+ h.lease.until = h.clock.now.Add(time.Minute)

+ return errors.New("injected 503")

+ }

+ return nil

+ }

+ logs := captureLog(t)

+ h.pass(t)

+ if copies != 1 {

+ t.Fatalf("%d copy requests reached the store, want 1", copies)

+ }

+ if !strings.Contains(logs(), "retrying once") || !strings.Contains(logs(), moves.ErrLeaseUnsafe.Error()) {

+ t.Fatalf("the retry did not go back to the guard:\n%s", logs())

+ }

+ // With the lease safe, the retry's request is signed at a new check.

+ h2 := newPhase1Harness(t)

+ h2.start(t, "manual")

+ var signs []time.Time

+ failed := false

+ h2.f.failAt = func(op, key string) error {

+ if op != "copy" {

+ return nil

+ }

+ calls := h2.f.at

+ signs = append(signs, calls[len(calls)-1].SignAt)

+ if !failed {

+ failed = true

+ return errors.New("injected 503")

+ }

+ return nil

+ }

+ h2.pass(t)

+ if len(signs) < 2 || !signs[1].After(signs[0]) {

+ t.Fatalf("the retry reused the first signature time: %v", signs)

+ }

+// The source report is replaced each round (PutReplace), never refused as

+// an existing object.

+func TestSourceReportIsWrittenEachRound(t *testing.T) {

+ h := newPhase1Harness(t)

+ h.start(t, "manual")

+ h.pass(t)

+ first := h.sourceReport(t)

+ h.probe = moves.Check{Reason: s1NoAnswer(p1Host)}

+ h.pass(t)

+ second := h.sourceReport(t)

+ if first == nil || second == nil || !second.At.After(first.At) || second.S1.OK || second.S1.Reason != s1NoAnswer(p1Host) {

+ t.Fatalf("first %+v second %+v", first, second)

+ }

+// The archive is rebuilt only when the plaintext digest changes.

+func TestArchiveIsRebuiltOnlyWhenPlainChanges(t *testing.T) {

+ h := newPhase1Harness(t)

+ h.start(t, "manual")

+ sealedPuts := func() int {

+ n := 0

+ for _, c := range h.f.stagingCalls() {

+ if c.Op == "put" && strings.HasSuffix(c.Key, ".sealed") {

+ n++

+ }

+ return n

+ }

+ h.pass(t)

+ if n := sealedPuts(); n != 1 {

+ t.Fatalf("%d archives staged over two unchanged rounds, want 1", n)

+ }

+ first := h.ledger(t).ArchivePlain

+ if err := os.WriteFile(filepath.Join(h.cfg.DataDir, p1Host, "upload.txt"), []byte("changed"), 0o644); err != nil {

+ t.Fatal(err)

+ }

+ h.pass(t)

+ if n := sealedPuts(); n != 2 {

+ t.Fatalf("%d archives staged after a change, want 2", n)

+ }

+ if l := h.ledger(t); l.ArchivePlain == first || l.ArchiveVersions == nil || l.ArchiveVersions.SealedVersion == "" || l.ArchiveVersions.ManifestVersion == "" {

+ t.Fatalf("ledger after the change: %+v", l)

+ }

+ // An archive this move can never carry is a standing S1 refusal.

+ if err := os.Symlink("/etc/passwd", filepath.Join(h.cfg.DataDir, p1Host, "link")); err != nil {

+ t.Fatal(err)

+ }

+ h.pass(t)

+ if sr := h.sourceReport(t); sr.S1.OK || !strings.Contains(sr.S1.Reason, "keeps a file this move cannot copy") {

+ t.Fatalf("S1 = %+v", sr.S1)

+ }

+// cut == "auto": from ready the source itself marks the cut requested.

+func TestAutoCutMarksCuttingFromReady(t *testing.T) {

+ h := newPhase1Harness(t)

+ h.start(t, "auto")

+ h.pass(t)

+ h.targetPass(t, h.targetCfg())

+ h.pass(t)

+ got := h.read(t, p1Host)

+ if got.Move.State != provision.MoveCutting || got.Move.CutPhase != provision.CutRequested || got.Move.Inputs == nil {

+ t.Fatalf("auto cut: %+v", got.Move)

+ }

+ // A manual move stays ready.

+ h2 := newPhase1Harness(t)

+ h2.start(t, "manual")

+ h2.pass(t)

+ h2.targetPass(t, h2.targetCfg())

+ h2.pass(t)

+ if got := h2.read(t, p1Host).Move; got.State != provision.MoveReady {

+ t.Fatalf("manual cut: %+v", got)

+ }

diff --git a/internal/host/movesreport.go b/internal/host/movesreport.go

new file mode 100644

index 000000000..9e79b6fe1

--- /dev/null

+++ b/internal/host/movesreport.go

@@ -0,0 +1,893 @@

+package host

+import (

+ "context"

+ "crypto/ecdh"

+ "encoding/json"

+ "errors"

+ "fmt"

+ "io"

+ "log"

+ "net/http"

+ "os"

+ "path/filepath"

+ "sort"

+ "strconv"

+ "strings"

+ "sync/atomic"

+ "time"

+ "github.com/carlosframework/platform/internal/activator"

+ "github.com/carlosframework/platform/internal/appconfig"

+ "github.com/carlosframework/platform/internal/edge"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/lease"

+ "github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/moves/archive"

+ "github.com/carlosframework/platform/internal/objstore"

+ "github.com/carlosframework/platform/internal/provision"

+ "github.com/carlosframework/platform/internal/registry"

+ "github.com/carlosframework/platform/internal/release"

+// The target's readiness report (moves spec §7.3, T1 to T12), behind

+// CARLOS_MOVES_PHASE1. The named target box (the claim's op.intent.target)

+// writes it every pass for every move into its placement that is preparing

+// or ready; no other box writes one. Each check is its own small function

+// returning a moves.Check. The reasons are rows of the copy record, pending

+// review; they reach `carlos move status` through the report.

+const (

+ t1FloorUnreadable = "the moves protocol floor could not be read"

+ t1LocalUnreadable = "this box cannot read its own copy of the moves protocol floor"

+ t1NotArmed = "this box has not armed the moves protocol floor yet"

+ t2NoConsole = "the console has not published its secrets key fingerprint yet"

+ t2NoKey = "this box has no secrets key"

+ t2Differs = "this box's secrets key is not the console's"

+ t3NoSource = "the source has not published a readiness report yet" // MV-N62's text

+ t3NoCompose = "this box could not compose the launch environment of the held row"

+ t5NoArchive = "the source has published no archive yet" // MV-N73

+ t5Unreadable = "this box could not read the archive the source published"

+ t5Refused = "the archive does not open with this box's key or does not match its manifest"

+ t6Unmeasured = "this box could not measure its free disk space"

+ t7NoRelease = "the source has not reported its release yet" // MV-N76

+ t7Waiting = "waiting for the root stage pass" // MV-N75

+ t7Failed = "this box could not stage the source's release"

+ t9NotThisMove = "the host claim is not held by this move"

+ t9Incarnation = "the host claim names another incarnation of this instance"

+ t9NoMoveJSON = "the staging lane has no move.json yet"

+ t9OtherMove = "the staging lane holds another move's move.json" // MV-N9, aligned

+ t10NoBucket = "the source region publishes no data bucket"

+ t10Failed = "this box cannot write the return lane in the source's bucket"

+ t11NoReplica = "no replica has been staged yet"

+func t1Below(min int) string {

+ return fmt.Sprintf("this box speaks moves protocol %d, below the floor's %d", moves.ProtocolVersion, min)

+func t3Differs(names []string) string {

+ return "the launch environment differs from the source's: " + strings.Join(names, ", ")

+func t3Shadowed(name string) string {

+ return name + " in the carried .env overrides the rendered configuration"

+func t4Failed(prefix string) string {

+ return "the store bound as " + prefix + " did not answer the probe from this box"

+func t6NoRoom(needGiB float64) string {

+ return fmt.Sprintf("this box needs %.1f GiB free for the move without filling its disk past 85%%", needGiB)

+func t8Count(n int, to string) string {

+ if n == 0 {

+ return "no box publishes " + to

+ }

+ return fmt.Sprintf("%d boxes publish %s, and a move needs exactly one", n, to)

+func t9Untouched(what string) string { return "the host claim has not touched " + what + " yet" } // MV-N74

+func t12NotQuiet(region string, until time.Time) string {

+ return "the instance left " + region + " recently and can move back after " + until.UTC().Format(time.RFC3339)

+// moveFleet is the fleet as the edges files publish it: per placement, the

+// box whose coordinates a move uses (the lexically first, as moveTargets

+// picks) and every box that publishes it (T8). The home publishes no

+// placement code, so it is kept under "", where a move out of the home

+// finds its source bucket (T10).

+type moveFleet struct {

+ targets map[string]edgesFile

+ boxes map[string][]string

+func loadMoveFleet(ctx context.Context, s release.Store) (moveFleet, error) {

+ bodies, err := loadEdgesFileBodies(ctx, s)

+ if err != nil {

+ return moveFleet{}, err

+ }

+ f := moveFleet{targets: map[string]edgesFile{}, boxes: map[string][]string{}}

+ for _, body := range bodies {

+ var t edgesFile

+ if err := json.Unmarshal(body, &t); err != nil || t.V < 1 {

+ continue

+ }

+ if !containsString(f.boxes[t.Placement], t.Box) {

+ f.boxes[t.Placement] = append(f.boxes[t.Placement], t.Box)

+ }

+ if t.DataBucket == "" {

+ continue

+ }

+ if prev, dup := f.targets[t.Placement]; dup && prev.Box <= t.Box {

+ continue

+ }

+ f.targets[t.Placement] = t

+ }

+ return f, nil

+// targetReportPass writes the target report for every move into this box's

+// placement whose claim names this box as the target.

+func (w *movesWorker) targetReportPass(ctx context.Context, s release.Store, recs []provision.Record, fleet moveFleet) {

+ for _, rec := range recs {

+ m := rec.Move

+ if m == nil || m.Legacy() || m.To != w.cfg.Region || (m.State != provision.MovePreparing && m.State != provision.MoveReady) {

+ continue

+ }

+ if err := w.targetReport(ctx, s, rec, fleet); err != nil {

+ w.logTrouble(rec.Host+" (target)", err)

+ continue

+ }

+ w.trouble.Delete(rec.Host + " (target)")

+ }

+// verifiedArchive is what T5 learned from one published archive, kept per

+// exact version so a pass does not open the same archive again.

+type verifiedArchive struct {

+ ref moves.ArchiveRef

+ check moves.Check

+ plain string

+ entries int

+ bytes int64

+ env []byte // the carried <host>.env, nil when the archive has none

+// heldEnv is T3's composition: the environment, the names a carried .env

+// shadows, and the input versions rendered.

+type heldEnv struct {

+ env map[string]string

+ shadowed []string

+ inputs provision.MoveInputs

+func (w *movesWorker) targetReport(ctx context.Context, s release.Store, rec provision.Record, fleet moveFleet) error {

+ cfg := w.cfg

+ id := rec.Move.ID

+ obs, err := hostclaim.Read(ctx, s, rec.Host)

+ if err != nil {

+ return fmt.Errorf("read the host claim: %w", err)

+ }

+ if obs.Absent || obs.Claim.Op.ID != id {

+ w.notice(rec.Host+"#target", "moves: %s: the host claim does not name move %s; no target report", rec.Host, id)

+ return nil

+ }

+ // T8 and T9 bind to the named target: only it writes the report.

+ if target := obs.Claim.Op.Intent["target"]; cfg.Box == "" || target != cfg.Box {

+ w.notice(rec.Host+"#target", "moves: %s: this box is not the move's target (%q); no target report", rec.Host, target)

+ return nil

+ }

+ source, err := readSourceReport(ctx, s, rec)

+ if err != nil {

+ return err

+ }

+ now := cfg.Now().UTC()

+ rep := moves.TargetReport{V: 1, Box: cfg.Box, Region: cfg.Region, MoveID: id, At: now,

+ Checks: map[string]moves.Check{}, CutBudgetS: int(moves.CutWindowDefault)}

+ rep.Checks[moves.CheckProtocol] = w.checkProtocol(ctx, s)

+ rep.Checks[moves.CheckSecrets] = w.checkSecrets(ctx, s)

+ art := w.checkArchive(ctx, rec, source)

+ rep.Checks[moves.CheckArchive] = art.check

+ held, t3, notes := w.checkEnv(ctx, s, rec, source, art)

+ rep.Checks[moves.CheckEnv] = t3

+ rep.Notes = notes

+ rep.Checks[moves.CheckStorage] = w.checkStorage(ctx, held, id)

+ replicaBytes, replicaErr := w.stagedReplicaBytes(ctx, rec)

+ rep.Checks[moves.CheckDisk] = w.checkDisk(replicaBytes, replicaErr, art)

+ rep.Checks[moves.CheckRelease] = w.checkRelease(ctx, s, rec, source)

+ rep.Checks[moves.CheckOneOwner] = checkOneOwner(fleet, rec.Move.To)

+ rep.Checks[moves.CheckClaim] = w.checkClaim(ctx, obs, rec)

+ rep.Checks[moves.CheckLane] = w.checkLane(ctx, rec, fleet)

+ rep.Checks[moves.CheckQuiet] = checkQuiet(obs, rec.Move.To, now)

+ // T11 from measured rates: this box's own, and the source's cross rate.

+ m := w.measure(ctx, rec)

+ m.Cross = source.CrossMeasurement()

+ dbMiB := float64(replicaBytes) / (1 << 20)

+ if replicaErr != nil || replicaBytes == 0 {

+ rep.Checks[moves.CheckBudget] = moves.Check{Reason: t11NoReplica}

+ } else {

+ res := moves.BudgetCheck(dbMiB, art.entries, m, cfg.ReadyDelta, moves.CutWindowDefault, now)

+ rep.Checks[moves.CheckBudget] = res.Check

+ rep.RecordBudget(dbMiB, art.entries, m, res)

+ }

+ rep.Inputs = held.inputs

+ if source != nil {

+ rep.Inputs.Release = source.Release

+ if t3.OK {

+ rep.Inputs.EnvReport = source.Env.Digest

+ }

+ if art.check.OK {

+ rep.Inputs.FilesPlain = art.plain

+ }

+ body, err := json.Marshal(rep)

+ if err != nil {

+ return err

+ }

+ return moves.PutReplace(ctx, s, moves.TargetReportKey(moves.ReportPrefix(rec.Account, rec.App, rec.Host, id)), body)

+// checkProtocol is T1 (§11): this box speaks at least the published floor

+// and has armed it locally, so its executable selection enforces it. No

+// published floor is a pass: the floor is opt-in until the first

+// production move (§11).

+func (w *movesWorker) checkProtocol(ctx context.Context, s release.Store) moves.Check {

+ fl, err := moves.ReadFloor(ctx, s)

+ if err != nil {

+ log.Printf("moves: T1: %v", err)

+ return moves.Check{Reason: t1FloorUnreadable}

+ }

+ if fl == nil {

+ return moves.Check{OK: true}

+ }

+ if moves.ProtocolVersion < fl.Min {

+ return moves.Check{Reason: t1Below(fl.Min)}

+ }

+ local, err := readLocalFloor(w.cfg.DataDir)

+ if err != nil {

+ log.Printf("moves: T1: local floor: %v", err)

+ return moves.Check{Reason: t1LocalUnreadable}

+ }

+ if local == nil || local.MinRelease != fl.MinRelease || local.Min < fl.Min {

+ return moves.Check{Reason: t1NotArmed}

+ }

+ return moves.Check{OK: true}

+// checkSecrets is T2 (§10): this box's secrets key fingerprint equals the

+// one the console publishes.

+func (w *movesWorker) checkSecrets(ctx context.Context, s release.Store) moves.Check {

+ if w.cfg.SecretsPub == nil {

+ return moves.Check{Reason: t2NoKey}

+ }

+ var rec moves.ConsoleRecord

+ if ok, err := readReport(ctx, s, moves.ConsoleKey, &rec); err != nil || !ok || rec.SecretsKeyFP == "" {

+ if err != nil {

+ log.Printf("moves: T2: %v", err)

+ }

+ return moves.Check{Reason: t2NoConsole}

+ }

+ if appconfig.KeyFingerprint(w.cfg.SecretsPub) != rec.SecretsKeyFP {

+ return moves.Check{Reason: t2Differs}

+ }

+ return moves.Check{OK: true}

+// checkArchive is T5: the archive the source report names, read by exact

+// version from this box's bucket, opens with this box's key and matches its

+// manifest (archive.Verify: digests, every entry, the entry types, the

+// limits), and is this move's. Its carried <host>.env is kept for T3. A

+// verdict is kept per exact version; a read that fails is not.

+func (w *movesWorker) checkArchive(ctx context.Context, rec provision.Record, source *moves.SourceReport) verifiedArchive {

+ if source == nil || source.Archive == nil {

+ return verifiedArchive{check: moves.Check{Reason: t5NoArchive}}

+ }

+ ref := *source.Archive

+ w.mu.Lock()

+ cached, ok := w.archives[rec.Move.ID]

+ w.mu.Unlock()

+ if ok && cached.ref == ref {

+ return cached

+ }

+ v, keep := w.verifyArchive(ctx, rec, ref)

+ if keep {

+ w.mu.Lock()

+ if w.archives == nil {

+ w.archives = map[string]verifiedArchive{}

+ }

+ w.archives[rec.Move.ID] = v

+ w.mu.Unlock()

+ }

+ return v

+func (w *movesWorker) verifyArchive(ctx context.Context, rec provision.Record, ref moves.ArchiveRef) (verifiedArchive, bool) {

+ cfg := w.cfg

+ out := verifiedArchive{ref: ref}

+ fail := func(reason string, keep bool, err error) (verifiedArchive, bool) {

+ log.Printf("moves: %s: T5: %v", rec.Host, err)

+ out.check = moves.Check{Reason: reason}

+ return out, keep

+ }

+ if cfg.SecretsPriv == nil {

+ out.check = moves.Check{Reason: t2NoKey}

+ return out, false

+ }

+ if cfg.Versioned == nil {

+ return fail(t5Unreadable, false, errors.New("no versioned store"))

+ }

+ vs, err := cfg.Versioned(ctx, cfg.OwnBucket, cfg.OwnBucketRegion)

+ if err != nil {

+ return fail(t5Unreadable, false, err)

+ }

+ raw, err := vs.GetVersion(ctx, cfg.OwnBucket, ref.ManifestKey, ref.ManifestVersion)

+ if err != nil {

+ return fail(t5Unreadable, false, err)

+ }

+ var m archive.Manifest

+ if err := json.Unmarshal(raw, &m); err != nil {

+ return fail(t5Refused, true, err)

+ }

+ if m.Op != rec.Move.ID || m.Incarnation != rec.Move.Incarnation || m.Plain != ref.Plain {

+ return fail(t5Refused, true, fmt.Errorf("the manifest is for %s/%s/%s", m.Op, m.Incarnation, m.Plain))

+ }

+ tmp, err := os.MkdirTemp("", "carlos-move-t5-")

+ if err != nil {

+ return fail(t5Unreadable, false, err)

+ }

+ defer os.RemoveAll(tmp)

+ sealedPath := filepath.Join(tmp, "archive.sealed")

+ if err := downloadVersion(ctx, vs, cfg.OwnBucket, ref.SealedObjectKey, ref.SealedVersion, sealedPath); err != nil {

+ return fail(t5Unreadable, false, err)

+ }

+ bind := appconfig.Binding{Account: rec.Account, App: rec.App}

+ f, err := os.Open(sealedPath)

+ if err != nil {

+ return fail(t5Unreadable, false, err)

+ }

+ defer f.Close()

+ if err := archive.Verify(f, m, cfg.SecretsPriv, bind); err != nil {

+ return fail(t5Refused, true, err)

+ }

+ env, err := carriedEnv(f, m, cfg.SecretsPriv, bind, tmp, rec.Host)

+ if err != nil {

+ return fail(t5Refused, true, err)

+ }

+ out.check, out.plain, out.entries, out.bytes, out.env = moves.Check{OK: true}, m.Plain, len(m.Entries), m.Bytes, env

+ return out, true

+func downloadVersion(ctx context.Context, vs lease.VersionedStore, bucket, key, version, path string) error {

+ rc, err := vs.OpenVersion(ctx, bucket, key, version)

+ if err != nil {

+ return err

+ }

+ defer rc.Close()

+ f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)

+ if err != nil {

+ return err

+ }

+ if _, err := io.Copy(f, io.LimitReader(rc, archive.MaxDecoded+archive.MaxDecoded/16)); err != nil {

+ f.Close()

+ return err

+ }

+ return f.Close()

+// carriedEnv extracts the verified archive into a scratch directory and

+// returns its <host>.env (nil when it carries none).

+func carriedEnv(f *os.File, m archive.Manifest, priv *ecdh.PrivateKey, bind appconfig.Binding, tmp, host string) ([]byte, error) {

+ if _, err := f.Seek(0, io.SeekStart); err != nil {

+ return nil, err

+ }

+ pr, pw := io.Pipe()

+ done := make(chan error, 1)

+ go func() {

+ err := archive.Open(f, m.SealedDataKey, priv, bind, pw)

+ pw.CloseWithError(err)

+ done <- err

+ }()

+ envPath := filepath.Join(tmp, host+".env")

+ xerr := archive.Extract(pr, filepath.Join(tmp, "files"), envPath)

+ pr.CloseWithError(errors.New("extract stopped reading"))

+ if oerr := <-done; oerr != nil && xerr == nil {

+ xerr = oerr

+ }

+ if xerr != nil {

+ return nil, xerr

+ }

+ env, err := os.ReadFile(envPath)

+ if errors.Is(err, os.ErrNotExist) {

+ return nil, nil

+ }

+ return env, err

+// checkEnv is T3: the environment this box would launch the flat held row

+// with, compared by HMAC with the source's launch report. Shadowed names

+// and advisory differences are notes, never failures.

+func (w *movesWorker) checkEnv(ctx context.Context, s release.Store, rec provision.Record, source *moves.SourceReport, art verifiedArchive) (heldEnv, moves.Check, []string) {

+ cfg := w.cfg

+ if source == nil {

+ return heldEnv{}, moves.Check{Reason: t3NoSource}, nil

+ }

+ if !art.check.OK {

+ return heldEnv{}, moves.Check{Reason: t5NoArchive}, nil

+ }

+ key, err := moves.ReportKeyFromSecrets(cfg.SecretsKey)

+ if err != nil {

+ return heldEnv{}, moves.Check{Reason: t2NoKey}, nil

+ }

+ if cfg.ComposeHeldEnv == nil {

+ return heldEnv{}, moves.Check{Reason: t3NoCompose}, nil

+ }

+ env, shadowed, in, err := cfg.ComposeHeldEnv(ctx, s, rec, art.env)

+ if err != nil {

+ log.Printf("moves: %s: T3: %v", rec.Host, err)

+ return heldEnv{}, moves.Check{Reason: t3NoCompose}, nil

+ }

+ held := heldEnv{env: env, shadowed: shadowed, inputs: in}

+ var notes, failed []string

+ for _, name := range shadowed {

+ notes = append(notes, t3Shadowed(name))

+ }

+ for _, d := range moves.CompareEnv(source.Env, moves.ReportEnv(key, env), true) {

+ if d.Note() {

+ notes = append(notes, d.Name+" ("+d.Kind+")")

+ continue

+ }

+ failed = append(failed, d.Name+" ("+d.Kind+")")

+ }

+ if len(failed) > 0 {

+ return held, moves.Check{Reason: t3Differs(failed)}, notes

+ }

+ return held, moves.Check{OK: true}, notes

+// checkStorage is T4: every store bound in the composed environment answers

+// a list, put, get and delete of .carlos-move-probe/<move id> with its own

+// credentials.

+func (w *movesWorker) checkStorage(ctx context.Context, held heldEnv, id string) moves.Check {

+ if held.env == nil {

+ return moves.Check{Reason: t3NoCompose}

+ }

+ if w.cfg.ProbeStore == nil {

+ return moves.Check{Reason: t4Failed("(no probe on this box)")}

+ }

+ if prefix, err := w.cfg.ProbeStore(ctx, held.env, id); err != nil {

+ log.Printf("moves: T4: %s: %v", prefix, err)

+ return moves.Check{Reason: t4Failed(prefix)}

+ }

+ return moves.Check{OK: true}

+// stagedReplicaBytes sums the replica staged in this box's bucket for the

+// move. It prices T6 and stands in for the database size in T11: the

+// staged LTX files hold at least the database's pages, so the estimate errs

+// long, never short.

+func (w *movesWorker) stagedReplicaBytes(ctx context.Context, rec provision.Record) (int64, error) {

+ m, err := w.mirror(ctx, w.cfg.OwnBucketRegion)

+ if err != nil {

+ return 0, err

+ }

+ objs, err := m.List(ctx, w.cfg.OwnBucket, moves.ReplicaPrefix(rec.Host, rec.Move.ID))

+ if err != nil {

+ return 0, err

+ }

+ var n int64

+ for _, o := range objs {

+ n += o.Size

+ }

+ return n, nil

+// checkDisk is T6: free space covers twice the replica, plus the archive,

+// plus 2 GiB, and the disk stays under 85% full after it.

+func (w *movesWorker) checkDisk(replicaBytes int64, replicaErr error, art verifiedArchive) moves.Check {

+ if replicaErr != nil {

+ log.Printf("moves: T6: %v", replicaErr)

+ return moves.Check{Reason: t11NoReplica}

+ }

+ if w.cfg.DiskFree == nil {

+ return moves.Check{Reason: t6Unmeasured}

+ }

+ total, avail, err := w.cfg.DiskFree(w.cfg.DataDir)

+ if err != nil || total == 0 {

+ log.Printf("moves: T6: %v", err)

+ return moves.Check{Reason: t6Unmeasured}

+ }

+ need := uint64(2*replicaBytes+art.bytes) + 2<<30

+ used := total - avail

+ if avail < need || float64(used+need)/float64(total) >= 0.85 {

+ return moves.Check{Reason: t6NoRoom(float64(need) / (1 << 30))}

+ }

+ return moves.Check{OK: true}

+// checkRelease is T7: the source's pinned release materializes for the held

+// row on this box.

+func (w *movesWorker) checkRelease(ctx context.Context, s release.Store, rec provision.Record, source *moves.SourceReport) moves.Check {

+ if source == nil || source.Release == "" {

+ return moves.Check{Reason: t7NoRelease}

+ }

+ if w.cfg.ReleaseReady == nil {

+ return moves.Check{Reason: t7Failed}

+ }

+ return w.cfg.ReleaseReady(ctx, s, rec, source.Release)

+// checkOneOwner is T8: exactly one hosting box publishes the target

+// placement.

+func checkOneOwner(fleet moveFleet, to string) moves.Check {

+ if n := len(fleet.boxes[to]); n != 1 {

+ return moves.Check{Reason: t8Count(n, to)}

+ }

+ return moves.Check{OK: true}

+// checkClaim is T9: the claim is moving under this move, held by the

+// record's incarnation, has touched this region and box, and the staging

+// lane's move.json is this move's (the target owns that validation: the

+// source cannot read the lane, round-2 finding 7).

+func (w *movesWorker) checkClaim(ctx context.Context, obs hostclaim.Observed, rec provision.Record) moves.Check {

+ c := obs.Claim

+ if err := hostclaim.RequireMoveOp(rec.Move.ID, hostclaim.KindMoving, hostclaim.PhaseMoving)(c); err != nil {

+ return moves.Check{Reason: t9NotThisMove}

+ }

+ if c.Holder == nil || c.Holder.Incarnation != rec.Move.Incarnation {

+ return moves.Check{Reason: t9Incarnation}

+ }

+ if !containsString(c.Touched.Regions, rec.Move.To) {

+ return moves.Check{Reason: t9Untouched(rec.Move.To)}

+ }

+ if !containsString(c.Touched.Boxes, w.cfg.Box) {

+ return moves.Check{Reason: t9Untouched(w.cfg.Box)}

+ }

+ m, err := w.mirror(ctx, w.cfg.OwnBucketRegion)

+ if err != nil {

+ log.Printf("moves: %s: T9: %v", rec.Host, err)

+ return moves.Check{Reason: t9NoMoveJSON}

+ }

+ raw, err := m.Get(ctx, w.cfg.OwnBucket, moves.MoveJSONKey(rec.Host, rec.Move.ID))

+ if err != nil {

+ if !errors.Is(err, lease.ErrMirrorNotFound) {

+ log.Printf("moves: %s: T9: %v", rec.Host, err)

+ }

+ return moves.Check{Reason: t9NoMoveJSON}

+ }

+ var mj moves.MoveJSON

+ if err := json.Unmarshal(raw, &mj); err != nil || mj.MoveID != rec.Move.ID || mj.Incarnation != c.Holder.Incarnation ||

+ mj.Host != rec.Host || mj.From != rec.Move.From || mj.To != rec.Move.To {

+ return moves.Check{Reason: t9OtherMove}

+ }

+ return moves.Check{OK: true}

+// checkLane is T10: this box can put and delete moves/_lane/<id> in the

+// source's bucket (M17's return lane).

+func (w *movesWorker) checkLane(ctx context.Context, rec provision.Record, fleet moveFleet) moves.Check {

+ src, ok := fleet.targets[rec.Move.From]

+ if !ok {

+ src, ok = fleet.targets[rec.Region]

+ }

+ if !ok || src.DataBucket == "" {

+ return moves.Check{Reason: t10NoBucket}

+ }

+ if w.cfg.LaneProbe == nil {

+ return moves.Check{Reason: t10Failed}

+ }

+ if err := w.cfg.LaneProbe(ctx, src.DataBucket, src.DataBucketRegion, rec.Move.ID); err != nil {

+ log.Printf("moves: %s: T10: %v", rec.Host, err)

+ return moves.Check{Reason: t10Failed}

+ }

+ return moves.Check{OK: true}

+// checkQuiet is T12: the claim's quiet_after for the target region passed.

+func checkQuiet(obs hostclaim.Observed, region string, now time.Time) moves.Check {

+ if until, ok := obs.Claim.QuietAfter[region]; ok && now.Before(until) {

+ return moves.Check{Reason: t12NotQuiet(region, until)}

+ }

+ return moves.Check{OK: true}

+// measureEvery is how long this box's own measurements are reused: half of

+// moves.MeasurementFreshness, so a report never carries one about to expire,

+// and a pass does not move 80 MiB through S3 and the disk every few seconds.

+const measureEvery = moves.MeasurementFreshness / 2

+// measure is this box's regional S3, disk and per-file sync rates, measured

+// again when older than measureEvery. A failed measurement is logged and

+// left out, which T11 reports as missing.

+func (w *movesWorker) measure(ctx context.Context, rec provision.Record) moves.Measurements {

+ cfg := w.cfg

+ now := cfg.Now()

+ w.mu.Lock()

+ m := w.measured

+ w.mu.Unlock()

+ if cfg.MeasureS3 != nil && (m.S3.At.IsZero() || now.Sub(m.S3.At) > measureEvery) {

+ if r, err := cfg.MeasureS3(ctx, moves.ProbeKey(rec.Host, rec.Move.ID)); err != nil {

+ log.Printf("moves: measure regional S3: %v", err)

+ } else {

+ m.S3 = r

+ }

+ if cfg.MeasureDisk != nil && (m.Disk.At.IsZero() || now.Sub(m.Disk.At) > measureEvery) {

+ if d, f, err := cfg.MeasureDisk(ctx); err != nil {

+ log.Printf("moves: measure the disk: %v", err)

+ } else {

+ m.Disk, m.Fsync = d, f

+ }

+ w.mu.Lock()

+ w.measured = m

+ w.mu.Unlock()

+ return m

+// ---- production hooks (armMovesPass wires them) ----

+// composeHeldEnv is T3's composition for a flat held row (ruling F19): a

+// scratch directory holding the configuration rendered from the merged

+// bundle (appconfig.LoadMergedHostVersions, rendered as RenderHeld renders

+// it) and the carried <host>.env, an ExecBackend over it with this box's

+// region and cache, and a synthetic flat route; LaunchEnvironment composes

+// what the next spawn would get.

+func composeHeldEnv(priv *ecdh.PrivateKey, region string, cache bool) func(context.Context, release.Store, provision.Record, []byte) (map[string]string, []string, provision.MoveInputs, error) {

+ return func(ctx context.Context, s release.Store, rec provision.Record, carried []byte) (map[string]string, []string, provision.MoveInputs, error) {

+ m, v, err := appconfig.LoadMergedHostVersions(ctx, s, rec.Account, rec.App, rec.Environment, rec.Host)

+ if err != nil {

+ return nil, nil, provision.MoveInputs{}, err

+ }

+ var rendered []byte

+ if m != nil {

+ if rendered, err = appconfig.RenderEnv(m, priv, appconfig.Binding{Account: rec.Account, App: rec.App}); err != nil {

+ return nil, nil, provision.MoveInputs{}, err

+ }

+ tmp, err := os.MkdirTemp("", "carlos-move-t3-")

+ if err != nil {

+ return nil, nil, provision.MoveInputs{}, err

+ }

+ defer os.RemoveAll(tmp)

+ if err := os.WriteFile(filepath.Join(tmp, rec.Host+".config.env"), rendered, 0o600); err != nil {

+ return nil, nil, provision.MoveInputs{}, err

+ }

+ if carried != nil {

+ if err := os.WriteFile(filepath.Join(tmp, rec.Host+".env"), carried, 0o600); err != nil {

+ return nil, nil, provision.MoveInputs{}, err

+ }

+ eb := &activator.ExecBackend{DataDir: tmp, Region: region}

+ if cache {

+ eb.Cache = heldEnvCache

+ }

+ env, shadowed, err := eb.LaunchEnvironment(registry.Route{Host: rec.Host, Socket: provision.DefaultSocket(rec.Host)})

+ if err != nil {

+ return nil, nil, provision.MoveInputs{}, err

+ }

+ in := provision.MoveInputs{BundleETags: map[string]string{"default": v.Default, "environment": v.Environment}, OverlayETag: v.Overlay}

+ return env, shadowed, in, nil

+ }

+// laneProbe is T10 in production: put-if-absent then delete-by-version of

+// moves/_lane/<id> in the source's bucket. A probe object left by an earlier

+// pass whose delete failed is removed by version and the check fails this

+// pass; the next one starts clean.

+func laneProbe(versioned func(ctx context.Context, bucket, region string) (lease.VersionedStore, error)) func(context.Context, string, string, string) error {

+ return func(ctx context.Context, bucket, region, id string) error {

+ vs, err := versioned(ctx, bucket, region)

+ if err != nil {

+ return err

+ }

+ key := moves.LaneKey(id)

+ ver, err := vs.PutIfAbsent(ctx, bucket, key, []byte(`{"probe":"`+id+`"}`))

+ if errors.Is(err, lease.ErrExists) {

+ vers, lerr := vs.ListVersions(ctx, bucket, key)

+ if lerr != nil {

+ return fmt.Errorf("a probe object is left and cannot be listed: %w", lerr)

+ }

+ for _, v := range vers {

+ if v.Key == key {

+ if derr := vs.DeleteVersion(ctx, bucket, key, v.VersionID); derr != nil {

+ return derr

+ }

+ return errors.New("a probe object was left by an earlier pass; removed")

+ }

+ if err != nil {

+ return err

+ }

+ return vs.DeleteVersion(ctx, bucket, key, ver)

+ }

+// measureS3 times a 16 MiB put and get of the probe object (moves.ProbeKey,

+// outside moves/, F11) in this box's own bucket with its own credentials,

+// and deletes it.

+func measureS3(mirror func(ctx context.Context) (lease.MirrorStore, error), bucket string, now func() time.Time) func(context.Context, string) (moves.MeasuredRate, error) {

+ return func(ctx context.Context, key string) (moves.MeasuredRate, error) {

+ m, err := mirror(ctx)

+ if err != nil {

+ return moves.MeasuredRate{}, err

+ }

+ const size = 16 << 20

+ body := make([]byte, size)

+ start := now()

+ if err := m.Put(ctx, bucket, key, body); err != nil {

+ return moves.MeasuredRate{}, err

+ }

+ got, err := m.Get(ctx, bucket, key)

+ elapsed := now().Sub(start)

+ if derr := m.Delete(ctx, bucket, key); derr != nil {

+ log.Printf("moves: delete the S3 probe object %s: %v", key, derr)

+ }

+ if err != nil {

+ return moves.MeasuredRate{}, err

+ }

+ if len(got) != size {

+ return moves.MeasuredRate{}, fmt.Errorf("the probe object read back %d bytes of %d", len(got), size)

+ }

+ return moves.RateOf(2*size, elapsed, now().UTC()), nil

+ }

+// measureDiskIn measures this box's disk under <dataDir>/.moves (the

+// platform-specific write and read are movesmeasure_*.go).

+func measureDiskIn(dataDir string, now func() time.Time) func(context.Context) (moves.MeasuredRate, moves.FilesRate, error) {

+ return func(ctx context.Context) (moves.MeasuredRate, moves.FilesRate, error) {

+ dir, err := os.MkdirTemp(filepath.Join(dataDir, ".moves"), "measure-")

+ if err != nil {

+ if err := os.MkdirAll(filepath.Join(dataDir, ".moves"), 0o755); err != nil {

+ return moves.MeasuredRate{}, moves.FilesRate{}, err

+ }

+ if dir, err = os.MkdirTemp(filepath.Join(dataDir, ".moves"), "measure-"); err != nil {

+ return moves.MeasuredRate{}, moves.FilesRate{}, err

+ }

+ defer os.RemoveAll(dir)

+ disk, err := measureDiskRate(dir, now)

+ if err != nil {

+ return moves.MeasuredRate{}, moves.FilesRate{}, err

+ }

+ files, err := measureFsyncRate(dir, now)

+ if err != nil {

+ return moves.MeasuredRate{}, moves.FilesRate{}, err

+ }

+ return disk, files, nil

+ }

+// measureFsyncFiles and measureFsyncWorkers are the per-file sync probe:

+// a few hundred small files synced by the extract's 16-worker pattern

+// (ruling T7-budget).

+const (

+ measureFsyncFiles = 512

+ measureFsyncWorkers = 16

+func measureFsyncRate(dir string, now func() time.Time) (moves.FilesRate, error) {

+ jobs := make(chan int)

+ errs := make(chan error, measureFsyncWorkers)

+ start := now()

+ for i := 0; i < measureFsyncWorkers; i++ {

+ go func() {

+ var first error

+ for n := range jobs {

+ if first != nil {

+ continue

+ }

+ f, err := os.OpenFile(filepath.Join(dir, "f"+strconv.Itoa(n)), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)

+ if err == nil {

+ _, err = f.Write(make([]byte, 4096))

+ if err == nil {

+ err = f.Sync()

+ }

+ if cerr := f.Close(); err == nil {

+ err = cerr

+ }

+ first = err

+ }

+ errs <- first

+ }()

+ }

+ for n := 0; n < measureFsyncFiles; n++ {

+ jobs <- n

+ }

+ close(jobs)

+ for i := 0; i < measureFsyncWorkers; i++ {

+ if err := <-errs; err != nil {

+ return moves.FilesRate{}, err

+ }

+ d := now().Sub(start)

+ if d <= 0 {

+ d = time.Microsecond

+ }

+ return moves.FilesRate{PerS: measureFsyncFiles / d.Seconds(), At: now().UTC(), Files: measureFsyncFiles}, nil

+// heldEnvCache is the non-nil cache T3's synthetic backend carries when the

+// live box has a response cache: LaunchEnvironment reads only whether one

+// is set (CARLOS_PURGE).

+var heldEnvCache = edge.NewCache(1, 1)

+// storePrefixes are the store bindings in an environment: every prefix P

+// with P_BUCKET, P_ACCESS_KEY and P_SECRET_KEY set (objstore.EnvNames).

+func storePrefixes(env map[string]string) []string {

+ var out []string

+ for name := range env {

+ p, ok := strings.CutSuffix(name, "_BUCKET")

+ if !ok || p == "" {

+ continue

+ }

+ _, _, _, ak, sk := objstore.EnvNames(p)

+ if env[ak] != "" && env[sk] != "" {

+ out = append(out, p)

+ }

+ sort.Strings(out)

+ return out

+// movesEdge is the agent's edge, once RunWithActivatorReady has one.

+var movesEdge atomic.Pointer[edge.Edge]

+// setMovesReadyProbe hands the worker's S1 probe the agent's edge

+// (agent.go, RunWithActivatorReady's callback).

+func setMovesReadyProbe(e *edge.Edge) { movesEdge.Store(e) }

+// probeThroughEdge is S1 (§7.3, round-4 finding 11): GET path on host

+// through the edge's uncached path, which wakes a hibernating source and

+// never answers from, or stores into, the response cache. 200 to 399 within

+// 30 s passes.

+func probeThroughEdge(ctx context.Context, host, path string) moves.Check {

+ e := movesEdge.Load()

+ if e == nil {

+ return moves.Check{Reason: s1EdgeNotReady}

+ }

+ return probeWith(ctx, e.ServeUncached, host, path)

+func probeWith(ctx context.Context, serve func(http.ResponseWriter, *http.Request), host, path string) moves.Check {

+ ctx, cancel := context.WithTimeout(ctx, 30*time.Second)

+ defer cancel()

+ if !strings.HasPrefix(path, "/") {

+ path = "/" + path

+ }

+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://"+host+path, nil)

+ if err != nil {

+ return moves.Check{Reason: s1NoAnswer(host)}

+ }

+ req.Host = host

+ req.RemoteAddr = "127.0.0.1:0"

+ rec := &probeRecorder{header: http.Header{}}

+ serve(rec, req)

+ if rec.status == 0 {

+ rec.status = http.StatusOK // a handler that wrote a body without a header

+ }

+ if rec.status < 200 || rec.status > 399 {

+ return moves.Check{Reason: s1NoAnswer(host)}

+ }

+ return moves.Check{OK: true}

+// probeRecorder keeps the status and discards the body.

+type probeRecorder struct {

+ header http.Header

+ status int

+func (r *probeRecorder) Header() http.Header { return r.header }

+func (r *probeRecorder) Write(b []byte) (int, error) {

+ if r.status == 0 {

+ r.status = http.StatusOK

+ }

+ return len(b), nil

+func (r *probeRecorder) WriteHeader(code int) {

+ if r.status == 0 {

+ r.status = code

+ }

diff --git a/internal/host/movesreport_minio_test.go b/internal/host/movesreport_minio_test.go

new file mode 100644

index 000000000..c1ccfc0a5

--- /dev/null

+++ b/internal/host/movesreport_minio_test.go

@@ -0,0 +1,61 @@

+package host

+import (

+ "context"

+ "errors"

+ "os"

+ "testing"

+ "github.com/aws/aws-sdk-go-v2/aws"

+ "github.com/aws/aws-sdk-go-v2/credentials"

+ awss3 "github.com/aws/aws-sdk-go-v2/service/s3"

+ "github.com/aws/smithy-go"

+// TestT4StorageProbeMinio runs T4's store probe (§7.3, §14.1) against a

+// real S3 server: with the delivered variables of a bound store it lists,

+// puts, gets and deletes .carlos-move-probe/<move id> and leaves nothing;

+// with wrong credentials T4 fails with the store's sentence. It runs only

+// where .amadan/ci.d/30-minio-contract stands a MinIO up (CARLOS_MINIO_TEST

+// and the MINIO_* variables, as TestDeliveredContractMinio reads them).

+func TestT4StorageProbeMinio(t *testing.T) {

+ if os.Getenv("CARLOS_MINIO_TEST") != "1" {

+ t.Skip("set CARLOS_MINIO_TEST=1 (with a running MinIO and the MINIO_* env vars) to run the T4 probe test; see .amadan/ci.d/30-minio-contract")

+ }

+ need := func(name string) string {

+ v := os.Getenv(name)

+ if v == "" {

+ t.Fatalf("%s is not set", name)

+ }

+ return v

+ }

+ endpoint, key, secret, bucket := need("MINIO_ENDPOINT"), need("MINIO_SCOPED_KEY"), need("MINIO_SCOPED_SECRET"), need("MINIO_BUCKET")

+ ctx := context.Background()

+ env := map[string]string{

+ "STORE_BUCKET": bucket, "STORE_ENDPOINT": endpoint, "STORE_REGION": "us-east-1",

+ "STORE_ACCESS_KEY": key, "STORE_SECRET_KEY": secret,

+ }

+ if prefix, err := probeStores(ctx, env, p1ID); err != nil {

+ t.Fatalf("probe of %s: %v", prefix, err)

+ }

+ // The probe object is gone.

+ c := awss3.New(awss3.Options{

+ Region: "us-east-1", Credentials: credentials.NewStaticCredentialsProvider(key, secret, ""),

+ BaseEndpoint: aws.String(endpoint), UsePathStyle: true,

+ RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired,

+ ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired,

+ })

+ _, err := c.HeadObject(ctx, &awss3.HeadObjectInput{Bucket: aws.String(bucket), Key: aws.String(".carlos-move-probe/" + p1ID)})

+ var apiErr smithy.APIError

+ if err == nil || !errors.As(err, &apiErr) {

+ t.Fatalf("the probe object was left behind (head: %v)", err)

+ }

+ // Wrong credentials: T4 fails with the store's sentence.

+ env["STORE_SECRET_KEY"] = "not-the-secret"

+ w := &movesWorker{cfg: moveCfg{ProbeStore: probeStores}}

+ got := w.checkStorage(ctx, heldEnv{env: env}, p1ID)

+ if got.OK || got.Reason != t4Failed("STORE") {

+ t.Fatalf("T4 with wrong credentials = %+v", got)

+ }

diff --git a/internal/host/movesreport_test.go b/internal/host/movesreport_test.go

new file mode 100644

index 000000000..e1e4c7218

--- /dev/null

+++ b/internal/host/movesreport_test.go

@@ -0,0 +1,527 @@

+package host

+import (

+ "context"

+ "encoding/json"

+ "errors"

+ "fmt"

+ "io"

+ "net/http"

+ "net/http/httptest"

+ "os"

+ "path/filepath"

+ "runtime"

+ "strings"

+ "syscall"

+ "testing"

+ "time"

+ "github.com/carlosframework/platform/internal/appconfig"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/lease"

+ "github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/provision"

+ "github.com/carlosframework/platform/internal/registry"

+ "github.com/carlosframework/platform/internal/release"

+// failing names the checks that did not pass, with their reasons.

+func failing(r *moves.TargetReport) map[string]string {

+ out := map[string]string{}

+ for _, id := range moves.AllChecks {

+ if c := r.Checks[id]; !c.OK {

+ out[id] = c.Reason

+ }

+ return out

+// The target's report flags each check on its own (§7.3): from a staged

+// move.json, a serving-then-moving claim that has not touched uk, and no

+// archive, T9 fails on touched and T5 on the archive while T1 passes; once

+// the claim touches uk and the source stages an archive, both pass. Each

+// injectable hook, failing, fails its own check alone.

+func TestTargetReportPassFlagsEachCheck(t *testing.T) {

+ ctx := context.Background()

+ h := newPhase1Harness(t)

+ h.start(t, "manual")

+ body, _ := json.Marshal(moves.MoveJSON{MoveID: p1ID, Incarnation: h.inc, Host: p1Host, From: "ie", To: "uk"})

+ h.f.store("data-uk", moves.MoveJSONKey(p1Host, p1ID), fakeMoveObject{lease.MirrorObject{ETag: "m", Size: int64(len(body))}, string(body)})

+ tc := h.targetCfg()

+ h.targetPass(t, tc)

+ r := h.targetReport(t)

+ if r == nil {

+ t.Fatal("no target report")

+ }

+ f := failing(r)

+ if !strings.Contains(f[moves.CheckClaim], "touched") || f[moves.CheckClaim] != t9Untouched("uk") {

+ t.Fatalf("T9 = %q, want the touched sentence for uk (%v)", f[moves.CheckClaim], f)

+ }

+ if !strings.Contains(f[moves.CheckArchive], "no archive") {

+ t.Fatalf("T5 = %q, want the no-archive sentence", f[moves.CheckArchive])

+ }

+ if _, bad := f[moves.CheckProtocol]; bad {

+ t.Fatalf("T1 failed: %v", f)

+ }

+ if r.Box != "uk-box" || r.Region != "uk" || r.MoveID != p1ID {

+ t.Fatalf("report identity %+v", r)

+ }

+ // Touch uk and uk-box, then let the source stage its archive.

+ obs, _ := hostclaim.Read(ctx, h.s, p1Host)

+ if _, err := hostclaim.Update(ctx, h.s, obs, hostclaim.Touch(p1ID, []string{"uk"}, []string{"uk-box"})); err != nil {

+ t.Fatal(err)

+ }

+ h.pass(t)

+ h.targetPass(t, tc)

+ if f := failing(h.targetReport(t)); len(f) != 0 {

+ t.Fatalf("after touch and archive: %v", f)

+ }

+ // Each hook alone.

+ hookErr := errors.New("injected")

+ for name, c := range map[string]struct {

+ check string

+ edit func(*moveCfg)

+ want string

+ }{

+ "ComposeHeldEnv": {moves.CheckEnv, func(c *moveCfg) {

+ c.ComposeHeldEnv = func(context.Context, release.Store, provision.Record, []byte) (map[string]string, []string, provision.MoveInputs, error) {

+ return nil, nil, provision.MoveInputs{}, hookErr

+ }

+ }, t3NoCompose},

+ "ComposeHeldEnv differs": {moves.CheckEnv, func(c *moveCfg) {

+ inner := c.ComposeHeldEnv

+ c.ComposeHeldEnv = func(ctx context.Context, s release.Store, rec provision.Record, carried []byte) (map[string]string, []string, provision.MoveInputs, error) {

+ env, sh, in, err := inner(ctx, s, rec, carried)

+ env["APP_MODE"] = "staging"

+ return env, sh, in, err

+ }

+ }, t3Differs([]string{"APP_MODE (differs)"})},

+ "ProbeStore": {moves.CheckStorage, func(c *moveCfg) {

+ c.ProbeStore = func(context.Context, map[string]string, string) (string, error) { return "MEDIA", hookErr }

+ }, t4Failed("MEDIA")},

+ "DiskFree": {moves.CheckDisk, func(c *moveCfg) {

+ c.DiskFree = func(string) (uint64, uint64, error) { return 0, 0, hookErr }

+ }, t6Unmeasured},

+ "ReleaseReady": {moves.CheckRelease, func(c *moveCfg) {

+ c.ReleaseReady = func(context.Context, release.Store, provision.Record, string) moves.Check {

+ return moves.Check{Reason: t7Waiting}

+ }

+ }, t7Waiting},

+ "LaneProbe": {moves.CheckLane, func(c *moveCfg) {

+ c.LaneProbe = func(context.Context, string, string, string) error { return hookErr }

+ }, t10Failed},

+ } {

+ t.Run(name, func(t *testing.T) {

+ hc := tc

+ c.edit(&hc)

+ h.targetPass(t, hc)

+ f := failing(h.targetReport(t))

+ // T4 probes the stores bound in T3's composition, so without one

+ // it fails closed with T3's reason: the one dependency.

+ if name == "ComposeHeldEnv" && f[moves.CheckStorage] == t3NoCompose {

+ delete(f, moves.CheckStorage)

+ }

+ if len(f) != 1 || f[c.check] != c.want {

+ t.Fatalf("%s failing: %v, want only %s = %q", name, f, c.check, c.want)

+ }

+ })

+ }

+ // T9 owns move.json's validation: one naming another move fails it

+ // with the lane sentence.

+ other, _ := json.Marshal(moves.MoveJSON{MoveID: p1ID2, Incarnation: h.inc, Host: p1Host, From: "ie", To: "uk"})

+ h.f.store("data-uk", moves.MoveJSONKey(p1Host, p1ID), fakeMoveObject{lease.MirrorObject{ETag: "o", Size: int64(len(other))}, string(other)})

+ h.targetPass(t, tc)

+ if f := failing(h.targetReport(t)); f[moves.CheckClaim] != t9OtherMove {

+ t.Fatalf("T9 with another move's move.json: %v", f)

+ }

+ wrongTo, _ := json.Marshal(moves.MoveJSON{MoveID: p1ID, Incarnation: h.inc, Host: p1Host, From: "ie", To: "se"})

+ h.f.store("data-uk", moves.MoveJSONKey(p1Host, p1ID), fakeMoveObject{lease.MirrorObject{ETag: "w", Size: int64(len(wrongTo))}, string(wrongTo)})

+ h.targetPass(t, tc)

+ if f := failing(h.targetReport(t)); f[moves.CheckClaim] != t9OtherMove {

+ t.Fatalf("T9 with a move.json for another region: %v", f)

+ }

+ h.f.store("data-uk", moves.MoveJSONKey(p1Host, p1ID), fakeMoveObject{lease.MirrorObject{ETag: "m", Size: int64(len(body))}, string(body)})

+ // T5 checks the archive against its manifest: a manifest whose entry

+ // digest was changed under the same version is refused.

+ sr := h.sourceReport(t)

+ mkey := sr.Archive.ManifestKey

+ var man map[string]any

+ if err := json.Unmarshal([]byte(h.f.buckets["data-uk"][mkey].body), &man); err != nil {

+ t.Fatal(err)

+ }

+ entries := man["entries"].([]any)

+ entries[0].(map[string]any)["sha256"] = "sha256:" + strings.Repeat("0", 64)

+ tampered, _ := json.Marshal(man)

+ good := h.f.buckets["data-uk"][mkey]

+ h.f.store("data-uk", mkey, fakeMoveObject{good.MirrorObject, string(tampered)})

+ // A version is immutable in S3, so T5 keeps its verdict per version;

+ // the fake rewrote one in place, so the target starts with no verdicts.

+ h.worker2.archives = nil

+ h.targetPass(t, tc)

+ if f := failing(h.targetReport(t)); f[moves.CheckArchive] != t5Refused {

+ t.Fatalf("T5 with a tampered manifest: %v", f)

+ }

+ h.f.store("data-uk", mkey, good)

+ h.worker2.archives = nil

+ // Only the named target writes a report.

+ h3 := newPhase1Harness(t)

+ h3.start(t, "manual")

+ notTarget := h3.targetCfg()

+ notTarget.Box = "uk-box-2"

+ h3.targetPass(t, notTarget)

+ if r := h3.targetReport(t); r != nil {

+ t.Fatalf("a box that is not the move's target wrote a report: %+v", r)

+ }

+// T3 composes the flat held row's environment on a scratch directory

+// (ruling F19): the rendered bundle and the carried .env, through

+// LaunchEnvironment. A carried variable overriding a rendered one is

+// reported as shadowed (round-6 finding 7), with no instance overlay.

+func TestComposeHeldEnvReportsShadowedNames(t *testing.T) {

+ ctx := context.Background()

+ s := release.NewDirStore(t.TempDir())

+ privB64, _, err := appconfig.GenerateKey()

+ if err != nil {

+ t.Fatal(err)

+ }

+ priv, _ := appconfig.ParsePrivateKey(privB64, "test")

+ body, _ := json.Marshal(appconfig.Manifest{Schema: 1, App: "app", Vars: map[string]appconfig.Var{

+ "APP_MODE": {Kind: "env", Value: "rendered"}, "ONLY_RENDERED": {Kind: "env", Value: "1"}}})

+ etag, err := s.Put(ctx, appconfig.Key("bab", "app"), body, "")

+ if err != nil {

+ t.Fatal(err)

+ }

+ rec := provision.Record{Host: p1Host, Account: "bab", App: "app"}

+ env, shadowed, in, err := composeHeldEnv(priv, "uk", true)(ctx, s, rec, []byte("APP_MODE=carried\nCARRIED=1\n"))

+ if err != nil {

+ t.Fatal(err)

+ }

+ if strings.Join(shadowed, ",") != "APP_MODE" {

+ t.Fatalf("shadowed = %v", shadowed)

+ }

+ if env["APP_MODE"] != "carried" || env["ONLY_RENDERED"] != "1" || env["CARRIED"] != "1" || env["CARLOS_REGION"] != "uk" ||

+ env["CARLOS_PURGE"] != "header" || env["CARLOS_CONTROL_SOCKET"] != provision.DefaultSocket(p1Host)+".control" {

+ t.Fatalf("composed env lacks an expected variable (names: %v)", envNames(env))

+ }

+ if in.BundleETags["default"] != etag || in.OverlayETag != "" {

+ t.Fatalf("inputs %+v, want the bundle etag %q", in, etag)

+ }

+ // No cache on the live box: no CARLOS_PURGE.

+ env, _, _, err = composeHeldEnv(priv, "uk", false)(ctx, s, rec, nil)

+ if err != nil {

+ t.Fatal(err)

+ }

+ if _, ok := env["CARLOS_PURGE"]; ok {

+ t.Fatal("CARLOS_PURGE composed for a box with no cache")

+ }

+ // Through the report: the shadowed name is a note, never a failure.

+ h := newPhase1Harness(t)

+ h.start(t, "manual")

+ h.pass(t)

+ tc := h.targetCfg()

+ inner := tc.ComposeHeldEnv

+ tc.ComposeHeldEnv = func(ctx context.Context, s release.Store, rec provision.Record, carried []byte) (map[string]string, []string, provision.MoveInputs, error) {

+ env, _, in, err := inner(ctx, s, rec, carried)

+ return env, []string{"APP_MODE"}, in, err

+ }

+ h.targetPass(t, tc)

+ r := h.targetReport(t)

+ if !r.Checks[moves.CheckEnv].OK || len(r.Notes) != 1 || r.Notes[0] != t3Shadowed("APP_MODE") {

+ t.Fatalf("T3 %+v notes %v", r.Checks[moves.CheckEnv], r.Notes)

+ }

+func envNames(env map[string]string) []string {

+ var out []string

+ for k := range env {

+ out = append(out, k)

+ }

+ return out

+// Round-6 finding 3: on a hardened box the agent cannot write the version

+// store. From a MoveStart'ed record with no target registry row, T7 leaves

+// a held-stage request naming the source report's Release and touches the

+// stage-requested marker; the root pass (ServeHeldStageRequests under

+// heldStageAuthority) materializes it into optDir; the next pass finds

+// LiveVersion equal and T7 passes. A request naming another version, or an

+// op the claim does not name, is refused and live/<host> is untouched.

+func TestPreparingRelaysHeldStageOnAHardenedBox(t *testing.T) {

+ ctx := context.Background()

+ h := newPhase1Harness(t)

+ h.start(t, "manual")

+ plat := runtime.GOOS + "-" + runtime.GOARCH

+ if err := release.Ship(ctx, h.s, "bab", release.Manifest{App: "app", Version: "v7", Kind: "binary", Artifacts: map[string]release.Artifact{plat: {}}},

+ map[string][]byte{plat: []byte("pinned")}); err != nil {

+ t.Fatal(err)

+ }

+ h.pass(t) // the source report names Release v7

+ reg := testRegistry(t)

+ targetData, targetOpt := t.TempDir(), t.TempDir()

+ prev := heldMaterialize

+ t.Cleanup(func() { heldMaterialize = prev })

+ heldMaterialize = func(context.Context, release.Store, string, registry.Route, string, func(func() error) error) (*release.Manifest, error) {

+ return nil, fmt.Errorf("%w: %w", release.ErrStageBlocked, &os.PathError{Op: "mkdir", Path: "/opt/carlos/versions", Err: syscall.EROFS})

+ }

+ tc := h.targetCfg()

+ tc.DataDir = targetData

+ tc.ReleaseReady = heldReleaseReady(reg, targetOpt, targetData, "uk-box", stageRequester(targetData))

+ h.targetPass(t, tc)

+ if f := failing(h.targetReport(t)); f[moves.CheckRelease] != t7Waiting {

+ t.Fatalf("T7 = %q, want %q", f[moves.CheckRelease], t7Waiting)

+ }

+ raw, err := os.ReadFile(filepath.Join(release.HeldStageDir(targetData), p1Host+".json"))

+ if err != nil {

+ t.Fatal(err)

+ }

+ var req release.HeldStageRequest

+ if err := json.Unmarshal(raw, &req); err != nil || req.Version != "v7" || req.Op != p1ID || req.Box != "uk-box" || req.Account != "bab" || req.App != "app" {

+ t.Fatalf("request %s (%v)", raw, err)

+ }

+ if !fileExistsForTest(filepath.Join(targetData, stageRequestMarker)) {

+ t.Fatal("the stage-requested marker was not touched")

+ }

+ // The root pass, with the relay's own authority, materializes it.

+ heldMaterialize = prev

+ if err := release.ServeHeldStageRequests(ctx, h.s, targetOpt, targetData, heldStageAuthority(reg, h.s, "uk-box")); err != nil {

+ t.Fatal(err)

+ }

+ if v, err := release.LiveVersion(targetOpt, p1Host); err != nil || v != "v7" {

+ t.Fatalf("LiveVersion = %q, %v", v, err)

+ }

+ h.targetPass(t, tc)

+ if f := failing(h.targetReport(t)); f[moves.CheckRelease] != "" {

+ t.Fatalf("T7 after the root stage: %q", f[moves.CheckRelease])

+ }

+ // Requests the authority refuses: another version, another op.

+ for _, bad := range []release.HeldStageRequest{

+ {Account: "bab", App: "app", Host: p1Host, Version: "v8", Op: p1ID, Box: "uk-box"},

+ {Account: "bab", App: "app", Host: p1Host, Version: "v7", Op: p1ID2, Box: "uk-box"},

+ } {

+ opt := t.TempDir()

+ if err := writeHeldStageRequest(targetData, bad); err != nil {

+ t.Fatal(err)

+ }

+ if err := release.ServeHeldStageRequests(ctx, h.s, opt, targetData, heldStageAuthority(reg, h.s, "uk-box")); err != nil {

+ t.Fatal(err)

+ }

+ if v, _ := release.LiveVersion(opt, p1Host); v != "" {

+ t.Fatalf("a refused request (%s, %s) repointed live to %q", bad.Version, bad.Op, v)

+ }

+// S1 through the edge's uncached path: 200 to 399 passes, anything else is

+// M-C16; before the edge exists, the probe says so.

+func TestReadyProbeThroughTheEdge(t *testing.T) {

+ ctx := context.Background()

+ for code, ok := range map[int]bool{200: true, 302: true, 399: true, 404: false, 503: false} {

+ serve := func(w http.ResponseWriter, r *http.Request) {

+ if r.Host != p1Host || r.URL.Path != "/ready" {

+ w.WriteHeader(http.StatusTeapot)

+ return

+ }

+ w.WriteHeader(code)

+ }

+ c := probeWith(ctx, serve, p1Host, "/ready")

+ if c.OK != ok || (!ok && c.Reason != s1NoAnswer(p1Host)) {

+ t.Errorf("status %d: %+v", code, c)

+ }

+ movesEdge.Store(nil)

+ if c := probeThroughEdge(ctx, p1Host, "/"); c.OK || c.Reason != s1EdgeNotReady {

+ t.Fatalf("no edge: %+v", c)

+ }

+// T4 finds the store bindings in the composed environment by objstore's

+// variable names; a prefix without its credentials is not a binding.

+func TestStorePrefixesFollowTheDeliveredNames(t *testing.T) {

+ env := map[string]string{

+ "MEDIA_BUCKET": "b", "MEDIA_ACCESS_KEY": "k", "MEDIA_SECRET_KEY": "s",

+ "STORE_BUCKET": "b2", "STORE_ACCESS_KEY": "k2", "STORE_SECRET_KEY": "s2", "STORE_REGION": "eu-west-1",

+ "HALF_BUCKET": "b3", "HALF_ACCESS_KEY": "k3",

+ "_BUCKET": "x",

+ }

+ if got := strings.Join(storePrefixes(env), ","); got != "MEDIA,STORE" {

+ t.Fatalf("prefixes %q", got)

+ }

+// T10 in production: put-if-absent then delete by version; a probe object

+// left behind is removed and the check fails for that pass only.

+func TestLaneProbePutsAndDeletesByVersion(t *testing.T) {

+ ctx := context.Background()

+ f := newFakeMoveMirror()

+ vs := &fakeVersioned{f: f}

+ probe := laneProbe(func(context.Context, string, string) (lease.VersionedStore, error) { return vs, nil })

+ if err := probe(ctx, "data-ie", "eu-west-1", p1ID); err != nil {

+ t.Fatal(err)

+ }

+ if _, ok := f.buckets["data-ie"][moves.LaneKey(p1ID)]; ok {

+ t.Fatal("the probe object was left behind")

+ }

+ f.put("data-ie", moves.LaneKey(p1ID), "left")

+ f.setVersion("data-ie", moves.LaneKey(p1ID), "v-left")

+ if err := probe(ctx, "data-ie", "eu-west-1", p1ID); err == nil {

+ t.Fatal("a left-behind probe object passed")

+ }

+ if err := probe(ctx, "data-ie", "eu-west-1", p1ID); err != nil {

+ t.Fatalf("the next pass did not start clean: %v", err)

+ }

+ vs.deny = func(string, string, string) error { return errors.New("AccessDenied") }

+ if err := probe(ctx, "data-ie", "eu-west-1", p1ID); err == nil {

+ t.Fatal("a denied lane passed")

+ }

+// The disk measurement (O_DIRECT on linux, synced elsewhere) and the

+// per-file sync rate both produce a usable, dated measurement.

+func TestMeasureDiskIn(t *testing.T) {

+ if testing.Short() {

+ t.Skip("writes and reads 64 MiB")

+ }

+ dir := t.TempDir()

+ disk, files, err := measureDiskIn(dir, time.Now)(context.Background())

+ if err != nil {

+ t.Fatal(err)

+ }

+ if disk.MiBPerS <= 0 || disk.At.IsZero() || disk.Bytes != 2*measureDiskSize {

+ t.Fatalf("disk %+v", disk)

+ }

+ if files.PerS <= 0 || files.At.IsZero() || files.Files != measureFsyncFiles {

+ t.Fatalf("fsync %+v", files)

+ }

+ if ents, _ := os.ReadDir(filepath.Join(dir, ".moves")); len(ents) != 0 {

+ t.Fatalf("the measurement left %d entries behind", len(ents))

+ }

+// The regional S3 measurement writes the probe key outside moves/ with

+// this box's own client and deletes it (F11).

+func TestMeasureS3UsesTheProbeKeyAndCleansUp(t *testing.T) {

+ f := newFakeMoveMirror()

+ m := measureS3(func(context.Context) (lease.MirrorStore, error) { return f, nil }, "data-uk", (&stepClock{now: time.Now()}).Now)

+ r, err := m(context.Background(), moves.ProbeKey(p1Host, p1ID))

+ if err != nil {

+ t.Fatal(err)

+ }

+ if r.MiBPerS <= 0 || r.At.IsZero() {

+ t.Fatalf("%+v", r)

+ }

+ if len(f.buckets["data-uk"]) != 0 {

+ t.Fatalf("the probe object was left behind: %v", f.buckets["data-uk"])

+ }

+ if strings.Join(f.ops, ",") != "put data-uk/"+moves.ProbeKey(p1Host, p1ID)+",delete data-uk/"+moves.ProbeKey(p1Host, p1ID) {

+ t.Fatalf("ops %v", f.ops)

+ }

+// T7 on a box that can write its version store: the release is staged and

+// published directly, under the same authority the root relay uses, so a

+// claim that names another op publishes nothing.

+func TestHeldReleaseReadyOnAWritableBox(t *testing.T) {

+ ctx := context.Background()

+ h := newPhase1Harness(t)

+ rec := h.start(t, "manual")

+ plat := runtime.GOOS + "-" + runtime.GOARCH

+ if err := release.Ship(ctx, h.s, "bab", release.Manifest{App: "app", Version: "v7", Kind: "binary", Artifacts: map[string]release.Artifact{plat: {}}},

+ map[string][]byte{plat: []byte("pinned")}); err != nil {

+ t.Fatal(err)

+ }

+ h.pass(t)

+ reg := testRegistry(t)

+ data, opt := t.TempDir(), t.TempDir()

+ ready := heldReleaseReady(reg, opt, data, "uk-box", nil)

+ if c := ready(ctx, h.s, rec, "v7"); !c.OK {

+ t.Fatalf("T7 = %+v", c)

+ }

+ if v, _ := release.LiveVersion(opt, p1Host); v != "v7" {

+ t.Fatalf("live = %q", v)

+ }

+ // The claim moves to another op while the release stages: the

+ // publication's authority refuses, nothing is published.

+ prev := heldMaterialize

+ t.Cleanup(func() { heldMaterialize = prev })

+ heldMaterialize = func(ctx context.Context, s release.Store, optDir string, rt registry.Route, v string, guard func(func() error) error) (*release.Manifest, error) {

+ rewriteClaimOp(t, h.s, func(op *hostclaim.Op) { op.ID = p1ID2 })

+ return prev(ctx, s, optDir, rt, v, guard)

+ }

+ opt2 := t.TempDir()

+ if c := heldReleaseReady(reg, opt2, data, "uk-box", nil)(ctx, h.s, rec, "v7"); c.OK || c.Reason != t7Failed {

+ t.Fatalf("T7 when the op changed during staging = %+v", c)

+ }

+ if v, _ := release.LiveVersion(opt2, p1Host); v != "" {

+ t.Fatalf("published after the op changed: %q", v)

+ }

+ // Under another op from the start: refused before anything is staged.

+ heldMaterialize = prev

+ opt3 := t.TempDir()

+ if c := heldReleaseReady(reg, opt3, data, "uk-box", nil)(ctx, h.s, rec, "v7"); c.OK || c.Reason != t7Failed {

+ t.Fatalf("T7 under another op = %+v", c)

+ }

+ if _, err := os.Stat(filepath.Join(opt3, "versions")); !errors.Is(err, os.ErrNotExist) {

+ t.Fatalf("a release was staged for a stale op: %v", err)

+ }

+// probeStore's SDK path, locally: against a minimal S3 endpoint it lists one

+// key, then puts, gets and deletes .carlos-move-probe/<id>, in that order,

+// path-style at the delivered endpoint, and a refusal fails it. (The real

+// server is TestT4StorageProbeMinio's, in CI.)

+func TestProbeStoreSpeaksS3(t *testing.T) {

+ var seen []string

+ objects := map[string][]byte{}

+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

+ if !strings.Contains(r.Header.Get("Authorization"), "Credential=good/") {

+ w.WriteHeader(http.StatusForbidden)

+ io.WriteString(w, `<?xml version="1.0"?><Error><Code>AccessDenied</Code><Message>no</Message></Error>`)

+ return

+ }

+ key := strings.TrimPrefix(r.URL.Path, "/bkt")

+ switch {

+ case r.Method == http.MethodGet && r.URL.Query().Get("list-type") == "2":

+ seen = append(seen, "list")

+ io.WriteString(w, `<?xml version="1.0"?><ListBucketResult><Name>bkt</Name><KeyCount>0</KeyCount><IsTruncated>false</IsTruncated></ListBucketResult>`)

+ case r.Method == http.MethodPut:

+ seen = append(seen, "put "+key)

+ b, _ := io.ReadAll(r.Body)

+ objects[key] = b

+ case r.Method == http.MethodGet:

+ seen = append(seen, "get "+key)

+ w.Write(objects[key])

+ case r.Method == http.MethodDelete:

+ seen = append(seen, "delete "+key)

+ delete(objects, key)

+ w.WriteHeader(http.StatusNoContent)

+ default:

+ w.WriteHeader(http.StatusMethodNotAllowed)

+ }

+ }))

+ t.Cleanup(srv.Close)

+ env := map[string]string{"STORE_BUCKET": "bkt", "STORE_ENDPOINT": srv.URL, "STORE_ACCESS_KEY": "good", "STORE_SECRET_KEY": "s"}

+ if prefix, err := probeStores(context.Background(), env, p1ID); err != nil {

+ t.Fatalf("%s: %v", prefix, err)

+ }

+ k := "/.carlos-move-probe/" + p1ID

+ if got, want := strings.Join(seen, ","), "list,put "+k+",get "+k+",delete "+k; got != want {

+ t.Fatalf("requests %q, want %q", got, want)

+ }

+ if len(objects) != 0 {

+ t.Fatal("the probe object was left behind")

+ }

+ env["STORE_ACCESS_KEY"] = "bad"

+ if prefix, err := probeStores(context.Background(), env, p1ID); err == nil || prefix != "STORE" {

+ t.Fatalf("wrong credentials: %q %v", prefix, err)

+ }

diff --git a/internal/host/movesstage.go b/internal/host/movesstage.go

new file mode 100644

index 000000000..e77a99abc

--- /dev/null

+++ b/internal/host/movesstage.go

@@ -0,0 +1,173 @@

+package host

+import (

+ "bytes"

+ "context"

+ "encoding/json"

+ "errors"

+ "fmt"

+ "io"

+ "log"

+ "os"

+ "path/filepath"

+ "github.com/aws/aws-sdk-go-v2/aws"

+ "github.com/aws/aws-sdk-go-v2/credentials"

+ awss3 "github.com/aws/aws-sdk-go-v2/service/s3"

+ "github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/objstore"

+ "github.com/carlosframework/platform/internal/provision"

+ "github.com/carlosframework/platform/internal/registry"

+ "github.com/carlosframework/platform/internal/release"

+// probeStores is T4 in production (§7.3): for each store bound in the

+// composed environment (objstore.EnvNames), with that store's own

+// credentials, list one key, then put, get and delete

+// .carlos-move-probe/<move id>. It returns the first failing binding's

+// prefix with its error. The client is built the way a member app builds

+// it from the delivered variables (objstore's delivered contract): static

+// credentials, the delivered region, and the endpoint path-style when one

+// is delivered.

+func probeStores(ctx context.Context, env map[string]string, id string) (string, error) {

+ for _, p := range storePrefixes(env) {

+ if err := probeStore(ctx, env, p, id); err != nil {

+ return p, err

+ }

+ return "", nil

+func probeStore(ctx context.Context, env map[string]string, prefix, id string) error {

+ bn, rn, en, ak, sk := objstore.EnvNames(prefix)

+ region := env[rn]

+ if region == "" {

+ region = "us-east-1"

+ }

+ opts := awss3.Options{

+ Region: region,

+ Credentials: credentials.NewStaticCredentialsProvider(env[ak], env[sk], ""),

+ RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired,

+ ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired,

+ }

+ if ep := env[en]; ep != "" {

+ opts.BaseEndpoint = aws.String(ep)

+ opts.UsePathStyle = true

+ }

+ c := awss3.New(opts)

+ bucket := aws.String(env[bn])

+ key := aws.String(".carlos-move-probe/" + id)

+ body := []byte("carlos move probe " + id)

+ if _, err := c.ListObjectsV2(ctx, &awss3.ListObjectsV2Input{Bucket: bucket, MaxKeys: aws.Int32(1)}); err != nil {

+ return fmt.Errorf("list: %w", err)

+ }

+ if _, err := c.PutObject(ctx, &awss3.PutObjectInput{Bucket: bucket, Key: key, Body: bytes.NewReader(body)}); err != nil {

+ return fmt.Errorf("put: %w", err)

+ }

+ out, err := c.GetObject(ctx, &awss3.GetObjectInput{Bucket: bucket, Key: key})

+ if err != nil {

+ return fmt.Errorf("get: %w", err)

+ }

+ got, err := io.ReadAll(io.LimitReader(out.Body, 1<<16))

+ out.Body.Close()

+ if err != nil {

+ return fmt.Errorf("get: %w", err)

+ }

+ if !bytes.Equal(got, body) {

+ return errors.New("get: the probe object read back different bytes")

+ }

+ if _, err := c.DeleteObject(ctx, &awss3.DeleteObjectInput{Bucket: bucket, Key: key}); err != nil {

+ return fmt.Errorf("delete: %w", err)

+ }

+ return nil

+// heldMaterialize is release.MaterializeHeld as a variable, so a test can

+// make the stage fail the way a read-only version store does on a hardened

+// box (the sandboxed agent cannot write /opt/carlos).

+var heldMaterialize = release.MaterializeHeld

+// heldReleaseReady is T7 in production (§7.3, §9.2, round-4 finding 6): the

+// pinned release (the source report's Release) is staged for the held row

+// and live/<host> points at it, under the same authority the root relay

+// uses (heldStageAuthority: the claim is this move's and targets this box,

+// the row is absent or held by the op, the source reports this release).

+// On a hardened box the stage is blocked, so a held-stage request is left

+// for the root stage pass (release.ServeHeldStageRequests) and its marker

+// touched; T7 passes once LiveVersion equals the pinned version.

+func heldReleaseReady(reg *registry.Registry, optDir, dataDir, box string, requestStage func(host, ver string, err error)) func(context.Context, release.Store, provision.Record, string) moves.Check {

+ return func(ctx context.Context, s release.Store, rec provision.Record, version string) moves.Check {

+ if live, err := release.LiveVersion(optDir, rec.Host); err == nil && live == version {

+ return moves.Check{OK: true}

+ }

+ req := release.HeldStageRequest{Account: rec.Account, App: rec.App, Host: rec.Host, Version: version, Op: rec.Move.ID, Box: box}

+ auth := heldStageAuthority(reg, s, box)

+ if err := auth(req, nil); err != nil {

+ logMoveStage(rec.Host, version, err)

+ return moves.Check{Reason: t7Failed}

+ }

+ rt := registry.Route{Host: rec.Host, Kind: "instance", App: rec.App, Account: rec.Account}

+ _, err := heldMaterialize(ctx, s, optDir, rt, version, func(publish func() error) error { return auth(req, publish) })

+ switch {

+ case errors.Is(err, release.ErrStageBlocked):

+ if werr := writeHeldStageRequest(dataDir, req); werr != nil {

+ logMoveStage(rec.Host, version, werr)

+ return moves.Check{Reason: t7Failed}

+ }

+ if requestStage != nil {

+ requestStage(rec.Host, version, err)

+ }

+ return moves.Check{Reason: t7Waiting}

+ case err != nil:

+ logMoveStage(rec.Host, version, err)

+ return moves.Check{Reason: t7Failed}

+ }

+ if live, err := release.LiveVersion(optDir, rec.Host); err != nil || live != version {

+ return moves.Check{Reason: t7Failed}

+ }

+ return moves.Check{OK: true}

+ }

+func logMoveStage(host, version string, err error) {

+ log.Printf("moves: %s: T7: stage %s: %v", host, version, err)

+// writeHeldStageRequest leaves req for the root stage pass at

+// HeldStageDir/<host>.json, atomically: written under a name the pass skips

+// (a leading dot) and renamed into place.

+func writeHeldStageRequest(dataDir string, req release.HeldStageRequest) error {

+ dir := release.HeldStageDir(dataDir)

+ if err := os.MkdirAll(dir, 0o755); err != nil {

+ return err

+ }

+ body, err := json.Marshal(req)

+ if err != nil {

+ return err

+ }

+ f, err := os.CreateTemp(dir, "."+req.Host+".*")

+ if err != nil {

+ return err

+ }

+ tmp := f.Name()

+ if _, err := f.Write(body); err != nil {

+ f.Close()

+ os.Remove(tmp)

+ return err

+ }

+ if err := f.Sync(); err != nil {

+ f.Close()

+ os.Remove(tmp)

+ return err

+ }

+ if err := f.Close(); err != nil {

+ os.Remove(tmp)

+ return err

+ }

+ if err := os.Rename(tmp, filepath.Join(dir, req.Host+".json")); err != nil {

+ os.Remove(tmp)

+ return err

+ }

+ return nil