11a303996 · carlos/platform

diff --git a/internal/activator/activator.go b/internal/activator/activator.go

index dd5fbbf24..f8c892862 100644

--- a/internal/activator/activator.go

+++ b/internal/activator/activator.go

@@ -260,9 +260,12 @@ type actEntry struct {

route registry.Route

state actState

- ready chan struct{} // waking: closed when the wake resolves; then check err

- err error // waking: the wake's outcome

- stopped chan struct{} // stopping: closed when the stop completes

+ ready chan struct{} // waking: closed when the wake resolves; then check err

+ err error // waking: the wake's outcome

+ stopped chan struct{} // stopping: closed when the stop completes

+ wakeCancel context.CancelFunc

+ stopOutcome *stopObservation

+ sidecarUncertain bool

// Activity tracking: an instance is idle only when nothing is in flight

// AND the idle window has passed since the last request finished. In-flight

@@ -396,8 +399,10 @@ type Service struct {

// codepath was touched at all. Single-box v1 drops the field; it re-enters

// with the mesh port.

- mu sync.Mutex

- entries map[string]*actEntry

+ mu sync.Mutex

+ entries map[string]*actEntry

+ fences map[string]uint64

+ stopHistory map[string]*stopObservation

sweepStop chan struct{}

sweepDone chan struct{}

@@ -454,6 +459,10 @@ func (a *Service) begin(ctx context.Context, rt registry.Route, stamp bool) (rel

defer deadline.Stop()

for {

a.mu.Lock()

+ if a.fences[rt.Host] != 0 {

+ a.mu.Unlock()

+ return nil, ErrFenced

+ }

e := a.entries[rt.Host]

if e == nil {

// Asleep → kick off ONE wake; this request and any latecomers all

@@ -549,6 +558,10 @@ func (a *Service) BeginSidecarRequest(ctx context.Context, rt registry.Route) er

defer deadline.Stop()

for {

a.mu.Lock()

+ if a.fences[rt.Host] != 0 {

+ a.mu.Unlock()

+ return ErrFenced

+ }

e := a.entries[rt.Host]

if e == nil {

a.mu.Unlock()

@@ -628,6 +641,12 @@ func (a *Service) releaseFunc(e *actEntry, stamp bool) func() {

func (a *Service) wake(e *actEntry) {

ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)

defer cancel()

+ a.mu.Lock()

+ e.wakeCancel = cancel

+ if a.fences[e.host] != 0 {

+ cancel()

+ }

+ a.mu.Unlock()

wakeStart := a.now()

w, err := a.doWake(ctx, e.route)

if w != nil {

@@ -646,11 +665,17 @@ func (a *Service) wake(e *actEntry) {

a.mu.Lock()

if err != nil {

e.err = err

+ outcome := &stopObservation{uncertain: true}

+ if w != nil && w.stopOutcome != nil {

+ outcome = w.stopOutcome

+ }

+ a.rememberStopLocked(e.host, outcome)

delete(a.entries, e.host) // next request retries from scratch

} else {

e.state = actAwake

e.rep, e.proc = w.rep, w.proc

e.sidecar, e.sidecarWanted = w.sidecar, w.sidecarWanted

+ e.sidecarUncertain = w.sidecarUncertain

e.sidecarStarted = a.now()

e.sidecarBackoff = 0

e.sidecarRetryAt = time.Time{} // a wake-time spawn failure retries on the next sweep

@@ -662,7 +687,10 @@ func (a *Service) wake(e *actEntry) {

// cycle. Fresh wake, so it can't be stale yet.

e.spawnVersion = w.spawnVersion

e.staleSince = time.Time{}

- if a.OnWake != nil {

+ if a.fences[e.host] != 0 {

+ a.beginStopLocked(e)

+ }

+ if a.OnWake != nil && a.fences[e.host] == 0 {

// Fire-and-forget, off this goroutine: e.host is read once here,

// before this goroutine can be reused for anything else — the

// hook must never share e.host's own goroutine's fate (a wedged

@@ -719,11 +747,13 @@ func wakeEventMessage(w *wokenInstance) string {

// members > bots, so a host whose sidecar won't start still serves (the

// sweeper retries via sidecarWanted).

type wokenInstance struct {

- rep Rep

- proc Proc

- sidecar Proc

- sidecarWanted bool

- spawnVersion string // live/<host> target read at exec — for build convergence

+ stopOutcome *stopObservation

+ sidecarUncertain bool

+ rep Rep

+ proc Proc

+ sidecar Proc

+ sidecarWanted bool

+ spawnVersion string // live/<host> target read at exec — for build convergence

// spawnTook is how long Backend.Start took: exec to the socket accepting,

// which is the instance's OWN boot and nothing of the platform's. wakeTook

// is the whole wake around it, filled in by wake() rather than here

@@ -740,6 +770,10 @@ type wokenInstance struct {

func (a *Service) doWake(ctx context.Context, rt registry.Route) (*wokenInstance, error) {

host := rt.Host

+ w := &wokenInstance{stopOutcome: &stopObservation{}}

+ if ctx.Err() != nil || !a.wakeAllowed(host) {

+ return w, ErrFenced

+ }

// 1. The lease first: everything after this line is done as the prefix's

// single writer. Contention is not retried here — on one box it means a

@@ -747,14 +781,22 @@ func (a *Service) doWake(ctx context.Context, rt registry.Route) (*wokenInstance

// the TTL instead of stalling its request.

lease, err := a.backend.Acquire(ctx, rt)

if err != nil {

+ if lease != nil {

+ w.stopOutcome.uncertain = true

+ w.stopOutcome.leaseReleased = lease.Release(ctx) == nil

+ }

var held *errLeaseHeld

if errors.As(err, &held) {

log.Printf("activator: LEASE HELD for %s by %q — a previous incarnation hasn't released; refusing to wake (retry after %s)",

host, held.holder, held.retryAfter)

}

- return nil, err

+ return w, err

}

+ if ctx.Err() != nil || !a.wakeAllowed(host) {

+ w.stopOutcome.leaseReleased = lease.Release(ctx) == nil

+ return w, ErrFenced

+ }

// 2. Restore the DB from S3 when the local copy is gone — the restore must

// happen *under* the lock (restoring without it could later shadow newer

// S3 state written by the true holder). When it's still on disk it is the

@@ -762,8 +804,8 @@ func (a *Service) doWake(ctx context.Context, rt registry.Route) (*wokenInstance

// what we ourselves checkpointed at the last sleep.

fresh, err := a.backend.Restore(ctx, rt)

if err != nil {

- _ = lease.Release(ctx)

- return nil, fmt.Errorf("restore %s: %w", host, err)

+ w.stopOutcome.leaseReleased = lease.Release(ctx) == nil

+ return w, fmt.Errorf("restore %s: %w", host, err)

}

// 3. First-boot seed, strictly between the restore and the spawn — so it

@@ -774,11 +816,17 @@ func (a *Service) doWake(ctx context.Context, rt registry.Route) (*wokenInstance

// can never run over recovered data. A seed that fails fails the WHOLE

// wake (the Start-failure precedent: release the lease, surface the

// error) — a half-seeded instance must not serve.

+ if ctx.Err() != nil || !a.wakeAllowed(host) {

+ w.stopOutcome.leaseReleased = lease.Release(ctx) == nil

+ return w, ErrFenced

+ }

if fresh && rt.Seed != "" {

+ w.stopOutcome.uncertain = true

if err := a.backend.Seed(ctx, rt); err != nil {

- _ = lease.Release(ctx)

- return nil, fmt.Errorf("seed %s: %w", host, err)

+ w.stopOutcome.leaseReleased = lease.Release(ctx) == nil

+ return w, fmt.Errorf("seed %s: %w", host, err)

}

+ w.stopOutcome.uncertain = false

}

// 4. Spawn before replication starts: a fresh instance creates its own DB

@@ -791,26 +839,38 @@ func (a *Service) doWake(ctx context.Context, rt registry.Route) (*wokenInstance

// parsing templates — and none of the platform's lease, restore or

// replication work. An app measuring its own TTFB starts the clock after

// all of it (discussion #6).

+ if ctx.Err() != nil || !a.wakeAllowed(host) {

+ w.stopOutcome.leaseReleased = lease.Release(ctx) == nil

+ return w, ErrFenced

+ }

+ w.stopOutcome.uncertain = true

spawnStart := a.now()

proc, spawnVer, err := a.backend.Start(ctx, rt)

spawnTook := a.now().Sub(spawnStart)

+ w.proc = proc

if err != nil {

- _ = lease.Release(ctx)

- return nil, fmt.Errorf("start %s: %w", host, err)

+ if proc != nil {

+ a.cleanupFailedWake(ctx, w, lease)

+ } else {

+ w.stopOutcome.leaseReleased = lease.Release(ctx) == nil

+ }

+ return w, fmt.Errorf("start %s: %w", host, err)

}

+ if ctx.Err() != nil || !a.wakeAllowed(host) {

+ a.cleanupFailedWake(ctx, w, lease)

+ return w, ErrFenced

+ }

// 5. Trade the held lease for running replication, at the route's

// replication-class sync interval (the per-plan durability ladder).

// The DB path rides the route; when empty the backend defaults it to

// <DataDir>/<host>.db internally (Eleven defaulted here via a package

// global — base dirs are backend fields now).

rep, err := lease.Replicate(ctx, rt.DBPath, replicationSyncInterval(rt.Replication))

+ w.rep = rep

if err != nil {

- stopCtx, cancel := context.WithTimeout(context.Background(), actStopTimeout)

- defer cancel()

- _ = proc.Stop(stopCtx)

- _ = lease.Release(ctx)

- return nil, fmt.Errorf("replicate %s: %w", host, err)

+ a.cleanupFailedWake(ctx, w, lease)

+ return w, fmt.Errorf("replicate %s: %w", host, err)

}

// 6. Co-spawn the sidecar, last — only once the instance is healthy and

@@ -818,11 +878,21 @@ func (a *Service) doWake(ctx context.Context, rt registry.Route) (*wokenInstance

// marked requests pass through the edge while we're awake). A spawn

// failure is loud but never fails the wake: members come before bots, and

// the sweeper retries with backoff.

- w := &wokenInstance{rep: rep, proc: proc, spawnVersion: spawnVer, spawnTook: spawnTook}

+ w.rep, w.spawnVersion, w.spawnTook = rep, spawnVer, spawnTook

+ w.stopOutcome = nil

+ if ctx.Err() != nil || !a.wakeAllowed(host) {

+ return w, nil

+ }

if a.backend.HasSidecar(rt) {

w.sidecarWanted = true

if sc, err := a.backend.StartSidecar(ctx, rt); err != nil {

- log.Printf("activator: SIDECAR for %s failed to start: %v — serving without it (the sweep retries)", host, err)

+ w.sidecarUncertain = true

+ w.sidecar = sc

+ if sc == nil {

+ log.Printf("activator: SIDECAR for %s failed to start: %v — serving without it (the sweep retries)", host, err)

+ } else {

+ log.Printf("activator: SIDECAR for %s failed to start: %v — retaining returned process", host, err)

+ }

} else {

w.sidecar = sc

}

@@ -889,6 +959,7 @@ func (a *Service) Bounce(host string) bool {

// the entry. The local DB stays in place — it's the warm cache for the next

// wake; S3 is the durable truth under the lease.

func (a *Service) stop(e *actEntry) {

+ outcome := &stopObservation{sidecarStop: e.sidecar == nil, sidecarExited: e.sidecar == nil, uncertain: e.sidecarUncertain}

// The deliberate-stop marker for the member's timeline: hibernation,

// convergence cycling, bounce, shutdown all pass through here, so an

// "exited" line WITHOUT this line before it reads as what it is — a

@@ -903,7 +974,9 @@ func (a *Service) stop(e *actEntry) {

// first, the schedule the next-due probe reads can't shift under it.

if e.sidecar != nil {

scCtx, scCancel := context.WithTimeout(context.Background(), sidecarStopTimeout)

- if err := e.sidecar.Stop(scCtx); err != nil {

+ err := e.sidecar.Stop(scCtx)

+ outcome.sidecarStop, outcome.sidecarExited = err == nil, e.sidecar.Exited()

+ if err != nil {

log.Printf("activator: stop sidecar %s: %v", e.host, err)

}

scCancel()

@@ -939,7 +1012,12 @@ func (a *Service) stop(e *actEntry) {

// Separate budgets: a SIGTERM drain that runs long must not eat the final

// checkpoint's time — the S3 sync is the part that makes sleep durable.

procCtx, cancelProc := context.WithTimeout(context.Background(), actStopTimeout)

- err = e.proc.Stop(procCtx)

+ if e.proc != nil {

+ err = e.proc.Stop(procCtx)

+ outcome.mainStop, outcome.mainExited = err == nil, e.proc.Exited()

+ } else {

+ err = ErrStopUnproven

+ }

cancelProc()

if err != nil {

log.Printf("activator: stop %s: %v", e.host, err)

@@ -949,10 +1027,18 @@ func (a *Service) stop(e *actEntry) {

// a local checkpoint.

ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)

defer cancel()

- if err := e.rep.Close(ctx); err != nil {

+ repErr := ErrStopUnproven

+ if e.rep != nil {

+ repErr = e.rep.Close(ctx)

+ }

+ outcome.replicaClosed = repErr == nil

+ if err := repErr; err != nil {

log.Printf("activator: final checkpoint %s: %v", e.host, err)

}

a.mu.Lock()

+ outcome.uncertain = !outcome.complete()

+ e.stopOutcome = outcome

+ a.rememberStopLocked(e.host, outcome)

delete(a.entries, e.host)

a.mu.Unlock()

close(e.stopped)

@@ -1018,6 +1104,9 @@ func (a *Service) sweepOnce() {

a.mu.Lock()

cycled := 0 // build-convergence cycles started this tick (rate-limited)

for _, e := range a.entries {

+ if a.fences[e.host] != 0 {

+ continue

+ }

if e.state != actAwake {

continue

}

@@ -1105,8 +1194,13 @@ func (a *Service) sweepOnce() {

e.sidecarRetryAt = now.Add(e.sidecarBackoff)

sc, err := a.backend.StartSidecar(context.Background(), e.route)

if err != nil {

- log.Printf("activator: SIDECAR restart for %s failed: %v (next attempt in %s)", e.host, err, e.sidecarBackoff)

- e.sidecar = nil

+ e.sidecarUncertain = true

+ if sc == nil {

+ log.Printf("activator: SIDECAR restart for %s failed: %v (next attempt in %s)", e.host, err, e.sidecarBackoff)

+ } else {

+ log.Printf("activator: SIDECAR restart for %s failed: %v — retaining returned process", e.host, err)

+ }

+ e.sidecar = sc

continue

}

log.Printf("activator: sidecar for %s was down — restarted (backoff now %s)", e.host, e.sidecarBackoff)

@@ -1134,6 +1228,9 @@ func (a *Service) sweepOnce() {

if !rt.Hibernate || rt.Status != "ready" {

continue

}

+ if a.fences[rt.Host] != 0 {

+ continue

+ }

if _, live := a.entries[rt.Host]; live {

continue

}

@@ -1271,6 +1368,13 @@ func (a *Service) ServeScheduled(w http.ResponseWriter, r *http.Request, rt regi

}

func (a *Service) serve(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler, stamp bool) bool {

+ a.mu.Lock()

+ fenced := a.fences[rt.Host] != 0

+ a.mu.Unlock()

+ if fenced {

+ maintenanceResponse(w, r)

+ return true

+ }

// Per-host gating: only a hibernate-flagged instance is activator-managed;

// unflagged instances and sibling services are the edge's to proxy

// straight — the deploy-safety property that lets the bucket env ship

@@ -1296,6 +1400,10 @@ func (a *Service) serve(w http.ResponseWriter, r *http.Request, rt registry.Rout

// could spoof.

r.Header.Del(SidecarHeader)

if err := a.BeginSidecarRequest(r.Context(), rt); err != nil {

+ if errors.Is(err, ErrFenced) {

+ maintenanceResponse(w, r)

+ return true

+ }

if errors.Is(err, context.Canceled) {

return true // the sidecar hung up while queued on a wake

}

@@ -1320,6 +1428,8 @@ func (a *Service) serve(w http.ResponseWriter, r *http.Request, rt registry.Rout

if err != nil {

var held *errLeaseHeld

switch {

+ case errors.Is(err, ErrFenced):

+ maintenanceResponse(w, r)

case errors.As(err, &held):

retry := int(held.retryAfter/time.Second) + 1

w.Header().Set("Retry-After", strconv.Itoa(retry))

diff --git a/internal/activator/maintenance.go b/internal/activator/maintenance.go

new file mode 100644

index 000000000..37c804c81

--- /dev/null

+++ b/internal/activator/maintenance.go

@@ -0,0 +1,166 @@

+package activator

+

+import (

+ "context"

+ "errors"

+ "net/http"

+ "time"

+

+ "github.com/carlosframework/platform/internal/controlname"

+ "github.com/carlosframework/platform/internal/edge"

+)

+

+var (

+ ErrFenced = errors.New("activator: maintenance fence active")

+ ErrFenceGeneration = errors.New("activator: maintenance generation refused")

+ ErrStopUnproven = errors.New("activator: stop evidence unavailable")

+)

+

+// StopEvidence records local process and replica closure. It grants no lease,

+// cgroup, database or resume authority.

+type StopEvidence struct {

+ host string

+ generation uint64

+ observation *stopObservation

+}

+

+func (e *StopEvidence) Matches(host string, generation uint64) bool {

+ return e != nil && e.host == host && e.generation == generation && e.observation != nil && e.observation.complete()

+}

+

+type stopObservation struct {

+ leaseReleased bool

+ mainStop, mainExited bool

+ sidecarStop, sidecarExited bool

+ replicaClosed bool

+ uncertain bool

+}

+

+func (o *stopObservation) complete() bool {

+ return o != nil && !o.uncertain && o.mainStop && o.mainExited && o.sidecarStop && o.sidecarExited && o.replicaClosed

+}

+

+// FenceAndStop keeps admission closed after cancellation or failure. A missing

+// entry is not evidence that an earlier process or replica stopped successfully.

+func (a *Service) FenceAndStop(ctx context.Context, host string, generation uint64) (*StopEvidence, error) {

+ if a == nil || !controlname.ValidHostShape(host) || generation == 0 {

+ return nil, ErrFenceGeneration

+ }

+ if ctx == nil {

+ return nil, ErrStopUnproven

+ }

+ deadline, ok := ctx.Deadline()

+ if !ok || ctx.Err() != nil || !time.Now().Before(deadline) {

+ return nil, ErrStopUnproven

+ }

+ for {

+ if err := a.lockFence(ctx); err != nil {

+ return nil, err

+ }

+ if a.fences == nil {

+ a.fences = make(map[string]uint64)

+ }

+ if prior := a.fences[host]; prior > generation {

+ a.mu.Unlock()

+ return nil, ErrFenceGeneration

+ }

+ a.fences[host] = generation

+ e := a.entries[host]

+ if e == nil {

+ outcome := a.stopHistory[host]

+ a.mu.Unlock()

+ if ctx.Err() != nil || !time.Now().Before(deadline) {

+ return nil, ErrStopUnproven

+ }

+ if !outcome.complete() {

+ return nil, ErrStopUnproven

+ }

+ return &StopEvidence{host: host, generation: generation, observation: outcome}, nil

+ }

+ var wait <-chan struct{}

+ switch e.state {

+ case actWaking:

+ if e.wakeCancel != nil {

+ e.wakeCancel()

+ }

+ wait = e.ready

+ case actAwake:

+ a.beginStopLocked(e)

+ wait = e.stopped

+ case actStopping:

+ wait = e.stopped

+ default:

+ a.mu.Unlock()

+ return nil, ErrStopUnproven

+ }

+ a.mu.Unlock()

+ select {

+ case <-ctx.Done():

+ return nil, ErrStopUnproven

+ case <-wait:

+ }

+}

+

+func (a *Service) lockFence(ctx context.Context) error {

+ timer := time.NewTicker(time.Millisecond)

+ defer timer.Stop()

+ for {

+ if ctx.Err() != nil {

+ return ErrStopUnproven

+ }

+ if a.mu.TryLock() {

+ if ctx.Err() != nil {

+ a.mu.Unlock()

+ return ErrStopUnproven

+ }

+ return nil

+ }

+ select {

+ case <-ctx.Done():

+ return ErrStopUnproven

+ case <-timer.C:

+ }

+}

+func (a *Service) wakeAllowed(host string) bool {

+ a.mu.Lock()

+ defer a.mu.Unlock()

+ return a.fences[host] == 0

+}

+func (a *Service) rememberStopLocked(host string, observation *stopObservation) {

+ if a.stopHistory == nil {

+ a.stopHistory = make(map[string]*stopObservation)

+ }

+ if prior := a.stopHistory[host]; prior != nil && prior.uncertain {

+ retained := *observation

+ retained.uncertain = true

+ observation = &retained

+ }

+ a.stopHistory[host] = observation

+}

+func maintenanceResponse(w http.ResponseWriter, r *http.Request) {

+ edge.MarkAnswered(r)

+ w.Header().Set("Cache-Control", "no-store")

+ w.Header().Set("Retry-After", "5")

+ http.Error(w, "instance failed to wake", http.StatusServiceUnavailable)

+}

+

+func (a *Service) cleanupFailedWake(ctx context.Context, w *wokenInstance, lease Lease) {

+ stopCtx, cancel := context.WithTimeout(context.Background(), actStopTimeout)

+ defer cancel()

+ if w.proc != nil {

+ err := w.proc.Stop(stopCtx)

+ w.stopOutcome.mainStop, w.stopOutcome.mainExited = err == nil, w.proc.Exited()

+ }

+ // A backend failure may conceal resources beyond the returned handle.

+ // Successful cleanup of that handle does not erase the uncertainty.

+ if w.rep != nil {

+ closeCtx, closeCancel := context.WithTimeout(context.Background(), 30*time.Second)

+ defer closeCancel()

+ w.stopOutcome.replicaClosed = w.rep.Close(closeCtx) == nil

+ w.stopOutcome.leaseReleased = w.stopOutcome.replicaClosed

+ return

+ }

+ w.stopOutcome.leaseReleased = lease.Release(ctx) == nil

+}

diff --git a/internal/activator/maintenance_test.go b/internal/activator/maintenance_test.go

new file mode 100644

index 000000000..603fbfc35

--- /dev/null

+++ b/internal/activator/maintenance_test.go

@@ -0,0 +1,615 @@

+package activator

+

+import (

+ "context"

+ "errors"

+ "fmt"

+ "github.com/carlosframework/platform/internal/edge"

+ "github.com/carlosframework/platform/internal/registry"

+ "net/http"

+ "net/http/httptest"

+ "sync/atomic"

+ "testing"

+ "time"

+)

+

+type maintenanceProc struct {

+ calls atomic.Int32

+ exited atomic.Bool

+ gate <-chan struct{}

+ entered chan struct{}

+ err error

+ remains bool

+}

+

+func (p *maintenanceProc) Stop(ctx context.Context) error {

+ if p.calls.Add(1) == 1 && p.entered != nil {

+ close(p.entered)

+ }

+ if p.gate != nil {

+ select {

+ case <-p.gate:

+ case <-ctx.Done():

+ return ctx.Err()

+ }

+ if !p.remains {

+ p.exited.Store(true)

+ }

+ return p.err

+}

+func (p *maintenanceProc) Exited() bool { return p.exited.Load() }

+

+type maintenanceRep struct {

+ gate <-chan struct{}

+ entered chan struct{}

+ calls atomic.Int32

+ err error

+}

+

+func (r *maintenanceRep) Err() error { return nil }

+func (r *maintenanceRep) Close(ctx context.Context) error {

+ if r.calls.Add(1) == 1 && r.entered != nil {

+ close(r.entered)

+ }

+ if r.gate != nil {

+ select {

+ case <-r.gate:

+ case <-ctx.Done():

+ return ctx.Err()

+ }

+ return r.err

+}

+func maintenanceContext(t *testing.T, duration time.Duration) context.Context {

+ t.Helper()

+ ctx, cancel := context.WithTimeout(t.Context(), duration)

+ t.Cleanup(cancel)

+ return ctx

+}

+func maintenanceAwake(t *testing.T) (*Service, *actEntry, *maintenanceProc, *maintenanceProc, *maintenanceRep) {

+ t.Helper()

+ a, _, _, reg := testActivator(t, time.Hour)

+ t.Cleanup(func() { reg.Close() })

+ main, side, rep := &maintenanceProc{}, &maintenanceProc{}, &maintenanceRep{}

+ e := &actEntry{host: "fence.example", route: testRoute("fence.example"), state: actAwake, proc: main, sidecar: side, rep: rep, lastActive: a.now()}

+ a.entries[e.host] = e

+ return a, e, main, side, rep

+}

+func TestMaintenanceFenceChecksEveryStopOutcome(t *testing.T) {

+ for _, scenario := range []string{"success", "main-error", "sidecar-error", "replica-error", "main-alive", "sidecar-alive", "missing-main", "missing-replica", "uncertain-sidecar"} {

+ t.Run(scenario, func(t *testing.T) {

+ a, e, main, side, rep := maintenanceAwake(t)

+ fail := errors.New("private backend detail")

+ switch scenario {

+ case "main-error":

+ main.err = fail

+ case "sidecar-error":

+ side.err = fail

+ case "replica-error":

+ rep.err = fail

+ case "main-alive":

+ main.remains = true

+ case "sidecar-alive":

+ side.remains = true

+ case "missing-main":

+ e.proc = nil

+ case "missing-replica":

+ e.rep = nil

+ case "uncertain-sidecar":

+ e.sidecarUncertain = true

+ }

+ proof, err := a.FenceAndStop(maintenanceContext(t, time.Second), e.host, 1)

+ if scenario == "success" {

+ if err != nil || !proof.Matches(e.host, 1) || proof.Matches(e.host, 2) {

+ t.Fatal("checked stop missing", err)

+ }

+ } else if err != ErrStopUnproven || proof != nil {

+ t.Fatal("failed stop became evidence", err)

+ }

+ if a.fences[e.host] != 1 {

+ t.Fatal("failure removed fence")

+ }

+ if !a.Quiesced(e.host) {

+ t.Fatal("ordinary entry removal changed")

+ }

+ retry, retryErr := a.FenceAndStop(maintenanceContext(t, time.Second), e.host, 1)

+ if scenario != "success" && (retryErr != ErrStopUnproven || retry != nil) {

+ t.Fatal("deleted entry erased failure")

+ }

+ if main.calls.Load() > 1 || side.calls.Load() > 1 || rep.calls.Load() > 1 {

+ t.Fatal("replayed spent stop")

+ }

+ })

+ }

+}

+func TestMaintenanceFenceBlocksEveryAdmissionIncludingStaleRoute(t *testing.T) {

+ a, _, _, reg := testActivator(t, time.Hour)

+ defer reg.Close()

+ host := "absent.example"

+ if proof, err := a.FenceAndStop(maintenanceContext(t, time.Second), host, 3); err != ErrStopUnproven || proof != nil {

+ t.Fatal("absence became stop proof", err)

+ }

+ rt := testRoute(host)

+ if _, err := a.beginRequest(t.Context(), rt); err != ErrFenced {

+ t.Fatal("member wake passed", err)

+ }

+ if err := a.BeginSidecarRequest(t.Context(), rt); err != ErrFenced {

+ t.Fatal("sidecar passed", err)

+ }

+ for _, scheduled := range []bool{false, true} {

+ for _, stale := range []bool{false, true} {

+ for _, sidecar := range []bool{false, true} {

+ route := rt

+ if stale {

+ route.Hibernate = false

+ route.Kind = "service"

+ }

+ req := httptest.NewRequest("GET", "https://"+host+"/", nil)

+ req, answered := edge.WithAnswerFlag(req)

+ if sidecar {

+ req.Header.Set(SidecarHeader, "1")

+ }

+ out := httptest.NewRecorder()

+ proxied := false

+ proxy := http.HandlerFunc(func(http.ResponseWriter, *http.Request) { proxied = true })

+ var handled bool

+ if scheduled {

+ handled = a.ServeScheduled(out, req, route, proxy)

+ } else {

+ handled = a.ServeHTTP(out, req, route, proxy)

+ }

+ if !handled || proxied || !*answered || out.Code != 503 || out.Header().Get("Cache-Control") != "no-store" {

+ t.Fatal("fence bypassed by stale or scheduled route")

+ }

+ if _, err := a.FenceAndStop(maintenanceContext(t, time.Second), host, 2); err != ErrFenceGeneration {

+ t.Fatal("older generation accepted")

+ }

+ // A new process has no closure evidence from the old activator's lifetime.

+ fresh := New(a.cfg, a.backend, reg)

+ if proof, err := fresh.FenceAndStop(maintenanceContext(t, time.Second), host, 3); err != ErrStopUnproven || proof != nil {

+ t.Fatal("restart invented old evidence")

+ }

+}

+func TestMaintenanceFenceContextAndGenerationDoNotRestoreAdmission(t *testing.T) {

+ a, e, main, _, _ := maintenanceAwake(t)

+ gate := make(chan struct{})

+ main.gate = gate

+ main.entered = make(chan struct{})

+ first := make(chan error, 1)

+ go func() { _, err := a.FenceAndStop(maintenanceContext(t, time.Second), e.host, 1); first <- err }()

+ <-main.entered

+ if proof, err := a.FenceAndStop(maintenanceContext(t, 15*time.Millisecond), e.host, 2); err != ErrStopUnproven || proof != nil {

+ t.Fatal("timed-out stop claimed proof")

+ }

+ if _, err := a.beginRequest(t.Context(), e.route); err != ErrFenced {

+ t.Fatal("timeout reopened admission")

+ }

+ close(gate)

+ if err := <-first; err != ErrFenceGeneration {

+ t.Fatal("old waiter survived newer fence", err)

+ }

+ proof, err := a.FenceAndStop(maintenanceContext(t, time.Second), e.host, 2)

+ if err != nil || !proof.Matches(e.host, 2) {

+ t.Fatal("checked outcome lost after timeout", err)

+ }

+ for _, ctx := range []context.Context{nil, context.Background(), maintenanceContext(t, -time.Second)} {

+ if p, err := a.FenceAndStop(ctx, e.host, 2); err == nil || p != nil {

+ t.Fatal("unbounded or expired stop accepted")

+ }

+ a.mu.Lock()

+ started := time.Now()

+ if _, err := a.FenceAndStop(maintenanceContext(t, 15*time.Millisecond), e.host, 2); err != ErrStopUnproven {

+ t.Fatal(err)

+ }

+ a.mu.Unlock()

+ if time.Since(started) > 250*time.Millisecond {

+ t.Fatal("fence mutex escaped deadline")

+ }

+}

+func TestMaintenanceOrdinaryStopFailureRemainsObservable(t *testing.T) {

+ a, e, main, _, _ := maintenanceAwake(t)

+ main.err = errors.New("stop refused")

+ a.mu.Lock()

+ a.beginStopLocked(e)

+ done := e.stopped

+ a.mu.Unlock()

+ <-done

+ if proof, err := a.FenceAndStop(maintenanceContext(t, time.Second), e.host, 4); err != ErrStopUnproven || proof != nil {

+ t.Fatal("ordinary stop erased failure", err)

+ }

+}

+

+func TestMaintenanceLaterOrdinaryStopCannotEraseUnresolvedCleanup(t *testing.T) {

+ a, e, main, _, _ := maintenanceAwake(t)

+ main.err = errors.New("earlier process may still run")

+ a.mu.Lock()

+ a.beginStopLocked(e)

+ done := e.stopped

+ a.mu.Unlock()

+ <-done

+ replacement := &actEntry{host: e.host, route: e.route, state: actAwake, proc: &maintenanceProc{}, sidecar: &maintenanceProc{}, rep: &maintenanceRep{}}

+ a.mu.Lock()

+ a.entries[e.host] = replacement

+ a.beginStopLocked(replacement)

+ done = replacement.stopped

+ a.mu.Unlock()

+ <-done

+ if proof, err := a.FenceAndStop(maintenanceContext(t, time.Second), e.host, 1); err != ErrStopUnproven || proof != nil {

+ t.Fatal("new entry erased unresolved predecessor", err)

+ }

+}

+

+type maintenanceBackend struct {

+ *fakeBackend

+ main *maintenanceProc

+ side *maintenanceProc

+ rep *maintenanceRep

+ acquireGate <-chan struct{}

+ acquireFailure error

+ acquireEntered chan struct{}

+ replicateGate <-chan struct{}

+ replicateEntered chan struct{}

+ sideGate <-chan struct{}

+ sideEntered chan struct{}

+ startFailure error

+ sideFailure error

+ replicateFailure error

+}

+

+func (b *maintenanceBackend) Acquire(ctx context.Context, rt registry.Route) (Lease, error) {

+ if b.acquireEntered != nil {

+ close(b.acquireEntered)

+ }

+ if b.acquireGate != nil {

+ <-b.acquireGate

+ }

+ lease, err := b.fakeBackend.Acquire(ctx, rt)

+ if err != nil {

+ return nil, err

+ }

+ return &maintenanceLease{Lease: lease, b: b}, b.acquireFailure

+}

+func (b *maintenanceBackend) Start(context.Context, registry.Route) (Proc, string, error) {

+ b.mu.Lock()

+ b.starts++

+ b.mu.Unlock()

+ if b.startFailure != nil {

+ if b.main == nil {

+ return nil, "", b.startFailure

+ }

+ return b.main, "", b.startFailure

+ }

+ return b.main, "", nil

+}

+func (b *maintenanceBackend) StartSidecar(context.Context, registry.Route) (Proc, error) {

+ b.mu.Lock()

+ b.sidecarStarts++

+ b.mu.Unlock()

+ if b.sideEntered != nil {

+ close(b.sideEntered)

+ }

+ if b.sideGate != nil {

+ <-b.sideGate

+ }

+ return b.side, b.sideFailure

+}

+

+type maintenanceLease struct {

+ Lease

+ b *maintenanceBackend

+}

+

+func (l *maintenanceLease) Replicate(context.Context, string, time.Duration) (Rep, error) {

+ if l.b.replicateEntered != nil {

+ close(l.b.replicateEntered)

+ }

+ if l.b.replicateGate != nil {

+ <-l.b.replicateGate

+ }

+ if l.b.rep == nil {

+ return nil, l.b.replicateFailure

+ }

+ return l.b.rep, l.b.replicateFailure

+}

+func TestMaintenanceFenceDuringWakeOwnsItsCleanup(t *testing.T) {

+ for _, phase := range []string{"acquire", "replicate"} {

+ t.Run(phase, func(t *testing.T) {

+ a, base, _, reg := testActivator(t, time.Hour)

+ defer reg.Close()

+ gate, entered := make(chan struct{}), make(chan struct{})

+ b := &maintenanceBackend{fakeBackend: base, main: &maintenanceProc{}, rep: &maintenanceRep{}}

+ if phase == "acquire" {

+ b.acquireGate = gate

+ b.acquireEntered = entered

+ } else {

+ b.replicateGate = gate

+ b.replicateEntered = entered

+ }

+ a.backend = b

+ rt := testRoute("waking.example")

+ request := make(chan error, 1)

+ go func() {

+ release, err := a.beginRequest(t.Context(), rt)

+ if release != nil {

+ release()

+ }

+ request <- err

+ }()

+ <-entered

+ if proof, err := a.FenceAndStop(maintenanceContext(t, 15*time.Millisecond), rt.Host, 1); err != ErrStopUnproven || proof != nil {

+ t.Fatal("in-flight wake yielded proof", err)

+ }

+ close(gate)

+ if err := <-request; err != ErrFenced {

+ t.Fatal("waking request admitted after fence", err)

+ }

+ proof, err := a.FenceAndStop(maintenanceContext(t, time.Second), rt.Host, 1)

+ if phase == "acquire" {

+ if err != ErrStopUnproven || proof != nil {

+ t.Fatal("canceled pre-spawn wake invented closure", err)

+ }

+ if b.starts != 0 || b.restores != 0 {

+ t.Fatal("fenced acquisition continued startup")

+ }

+ } else {

+ if err != nil || !proof.Matches(rt.Host, 1) {

+ t.Fatal("started wake did not yield checked cleanup", err)

+ }

+ if b.main.calls.Load() != 1 || b.rep.calls.Load() != 1 {

+ t.Fatal("partial wake resources not closed")

+ }

+ })

+ }

+}

+func TestMaintenanceFailedBackendStartRemainsUnproven(t *testing.T) {

+ a, base, _, reg := testActivator(t, time.Hour)

+ defer reg.Close()

+ a.backend = &maintenanceBackend{fakeBackend: base, startFailure: errors.New("hidden backend cleanup failed")}

+ rt := testRoute("hidden.example")

+ if _, err := a.beginRequest(t.Context(), rt); err == nil {

+ t.Fatal("failed backend started")

+ }

+ if proof, err := a.FenceAndStop(maintenanceContext(t, time.Second), rt.Host, 1); err != ErrStopUnproven || proof != nil {

+ t.Fatal("hidden cleanup failure became proof", err)

+ }

+}

+func TestMaintenanceFenceSerializesSidecarSweepReplacement(t *testing.T) {

+ a, e, _, side, rep := maintenanceAwake(t)

+ gate, entered := make(chan struct{}), make(chan struct{})

+ base := a.backend.(*fakeBackend)

+ base.sidecarStaged = true

+ next := &maintenanceProc{}

+ a.backend = &maintenanceBackend{fakeBackend: base, side: next, sideGate: gate, sideEntered: entered, rep: rep}

+ side.exited.Store(true)

+ e.sidecarWanted = true

+ sweep := make(chan struct{})

+ go func() { a.sweepOnce(); close(sweep) }()

+ <-entered

+ if proof, err := a.FenceAndStop(maintenanceContext(t, 15*time.Millisecond), e.host, 1); err != ErrStopUnproven || proof != nil {

+ t.Fatal("blocked replacement yielded proof", err)

+ }

+ close(gate)

+ <-sweep

+ if proof, err := a.FenceAndStop(maintenanceContext(t, time.Second), e.host, 1); err != nil || !proof.Matches(e.host, 1) {

+ t.Fatal("replacement escaped checked stop", err)

+ }

+ if next.calls.Load() != 1 {

+ t.Fatal("replacement sidecar was not stopped")

+ }

+ before := base.sidecarStartCount()

+ a.sweepOnce()

+ if base.sidecarStartCount() != before {

+ t.Fatal("fenced sweep restarted sidecar")

+ }

+}

+

+type maintenanceDue struct {

+ DueStore

+ route registry.Route

+ entered chan struct{}

+ gate <-chan struct{}

+}

+

+func (d *maintenanceDue) DueInstances(int64) ([]registry.Route, error) {

+ close(d.entered)

+ <-d.gate

+ return []registry.Route{d.route}, nil

+}

+func TestMaintenanceFenceStopsStaleScheduledWake(t *testing.T) {

+ a, b, _, reg := testActivator(t, time.Hour)

+ defer reg.Close()

+ rt := testRoute("scheduled.example")

+ rt.Status = "ready"

+ gate, entered := make(chan struct{}), make(chan struct{})

+ a.due = &maintenanceDue{DueStore: reg, route: rt, entered: entered, gate: gate}

+ swept := make(chan struct{})

+ go func() { a.sweepOnce(); close(swept) }()

+ <-entered

+ if _, err := a.FenceAndStop(maintenanceContext(t, time.Second), rt.Host, 1); err != ErrStopUnproven {

+ t.Fatal(err)

+ }

+ close(gate)

+ <-swept

+ b.mu.Lock()

+ starts, acquires := b.starts, b.acquires

+ b.mu.Unlock()

+ if starts != 0 || acquires != 0 || a.entryCount() != 0 {

+ t.Fatal("stale due query crossed installed fence")

+ }

+}

+func TestMaintenanceAwakeFenceSuppressesSidecarAdoption(t *testing.T) {

+ a, e, _, side, _ := maintenanceAwake(t)

+ base := a.backend.(*fakeBackend)

+ base.sidecarStaged = true

+ side.exited.Store(true)

+ a.fences = map[string]uint64{e.host: 1}

+ a.sweepOnce()

+ if base.sidecarStartCount() != 0 {

+ t.Fatal("awake fence allowed sidecar adoption")

+ }

+ if proof, err := a.FenceAndStop(maintenanceContext(t, time.Second), e.host, 1); err != nil || proof == nil {

+ t.Fatal(err)

+ }

+}

+

+func TestMaintenanceFailedStartCleansReturnedHandleBeforeRelease(t *testing.T) {

+ a, base, _, reg := testActivator(t, time.Hour)

+ defer reg.Close()

+ gate, entered := make(chan struct{}), make(chan struct{})

+ proc := &maintenanceProc{gate: gate, entered: entered}

+ a.backend = &maintenanceBackend{fakeBackend: base, main: proc, startFailure: errors.New("started then failed")}

+ rt := testRoute("partial.example")

+ done := make(chan error, 1)

+ go func() { _, err := a.beginRequest(t.Context(), rt); done <- err }()

+ select {

+ case <-entered:

+ case <-time.After(3 * time.Second):

+ t.Fatal("returned process cleanup was never attempted")

+ }

+ base.mu.Lock()

+ released := base.releases

+ base.mu.Unlock()

+ if released != 0 {

+ t.Fatal("lease released before returned process cleanup")

+ }

+ close(gate)

+ if err := <-done; err == nil {

+ t.Fatal("partial failed start admitted")

+ }

+ if proc.calls.Load() != 1 || !proc.Exited() {

+ t.Fatal("returned process handle was abandoned")

+ }

+ base.mu.Lock()

+ released = base.releases

+ base.mu.Unlock()

+ if released != 1 {

+ t.Fatal("lease release missing or repeated", released)

+ }

+ if proof, err := a.FenceAndStop(maintenanceContext(t, time.Second), rt.Host, 1); err != ErrStopUnproven || proof != nil {

+ t.Fatal("partial failure erased uncertainty", err)

+ }

+}

+func TestMaintenanceFailedSidecarHandlesRemainOwned(t *testing.T) {

+ for _, phase := range []string{"wake", "sweep"} {

+ t.Run(phase, func(t *testing.T) {

+ a, base, _, reg := testActivator(t, time.Hour)

+ defer reg.Close()

+ base.sidecarStaged = true

+ main, partial, rep := &maintenanceProc{}, &maintenanceProc{}, &maintenanceRep{}

+ b := &maintenanceBackend{fakeBackend: base, main: main, side: partial, rep: rep, sideFailure: errors.New("sidecar started then failed")}

+ a.backend = b

+ rt := testRoute("partial-sidecar.example")

+ if phase == "wake" {

+ release, err := a.beginRequest(t.Context(), rt)

+ if err != nil {

+ t.Fatal(err)

+ }

+ release()

+ } else {

+ old := &maintenanceProc{}

+ old.exited.Store(true)

+ a.entries[rt.Host] = &actEntry{host: rt.Host, route: rt, state: actAwake, proc: main, rep: rep, sidecar: old, sidecarWanted: true, lastActive: a.now()}

+ a.sweepOnce()

+ }

+ a.mu.Lock()

+ entry := a.entries[rt.Host]

+ owned := entry.sidecar == partial && entry.sidecarUncertain

+ a.mu.Unlock()

+ if !owned {

+ t.Fatal("failed sidecar handle was discarded")

+ }

+ before := base.sidecarStartCount()

+ a.sweepOnce()

+ if base.sidecarStartCount() != before {

+ t.Fatal("live partial sidecar was replaced")

+ }

+ if proof, err := a.FenceAndStop(maintenanceContext(t, time.Second), rt.Host, 1); err != ErrStopUnproven || proof != nil {

+ t.Fatal("failed sidecar yielded trusted evidence", err)

+ }

+ if partial.calls.Load() != 1 || !partial.Exited() {

+ t.Fatal("checked stop did not own partial sidecar")

+ }

+ })

+ }

+}

+

+func TestMaintenanceFailedReplicationClosesReturnedReplicaWithoutDoubleRelease(t *testing.T) {

+ for _, closeFails := range []bool{false, true} {

+ t.Run(fmt.Sprint(closeFails), func(t *testing.T) {

+ a, base, _, reg := testActivator(t, time.Hour)

+ defer reg.Close()

+ mainGate, mainEntered := make(chan struct{}), make(chan struct{})

+ repGate, repEntered := make(chan struct{}), make(chan struct{})

+ main := &maintenanceProc{gate: mainGate, entered: mainEntered}

+ rep := &maintenanceRep{gate: repGate, entered: repEntered}

+ if closeFails {

+ rep.err = errors.New("replica close failed")

+ }

+ a.backend = &maintenanceBackend{fakeBackend: base, main: main, rep: rep, replicateFailure: errors.New("replica opened then failed")}

+ rt := testRoute("partial-replica.example")

+ done := make(chan error, 1)

+ go func() { _, err := a.beginRequest(t.Context(), rt); done <- err }()

+ select {

+ case <-mainEntered:

+ case <-time.After(3 * time.Second):

+ t.Fatal("returned main cleanup missing")

+ }

+ if rep.calls.Load() != 0 {

+ t.Fatal("replica closed before main stopped")

+ }

+ close(mainGate)

+ select {

+ case <-repEntered:

+ case <-time.After(3 * time.Second):

+ t.Fatal("returned replica cleanup missing")

+ }

+ if !main.Exited() {

+ t.Fatal("replica closed before main exited")

+ }

+ base.mu.Lock()

+ released := base.releases

+ base.mu.Unlock()

+ if released != 0 {

+ t.Fatal("lease released while replica close pending")

+ }

+ close(repGate)

+ if err := <-done; err == nil {

+ t.Fatal("partial replica failure admitted")

+ }

+ base.mu.Lock()

+ released = base.releases

+ base.mu.Unlock()

+ if released != 0 || rep.calls.Load() != 1 {

+ t.Fatal("returned replica lease released twice", released, rep.calls.Load())

+ }

+ if proof, err := a.FenceAndStop(maintenanceContext(t, time.Second), rt.Host, 1); err != ErrStopUnproven || proof != nil {

+ t.Fatal("partial replica uncertainty erased", err)

+ }

+ })

+ }

+}

+

+func TestMaintenanceFailedAcquireReleasesReturnedLease(t *testing.T) {

+ a, base, _, reg := testActivator(t, time.Hour)

+ defer reg.Close()

+ a.backend = &maintenanceBackend{fakeBackend: base, acquireFailure: errors.New("lease returned with acquisition failure")}

+ rt := testRoute("partial-lease.example")

+ if _, err := a.beginRequest(t.Context(), rt); err == nil {

+ t.Fatal("failed acquisition admitted")

+ }

+ base.mu.Lock()

+ released, starts := base.releases, base.starts

+ base.mu.Unlock()

+ if released != 1 || starts != 0 {

+ t.Fatal("returned lease abandoned or wake continued", released, starts)

+ }

+ if proof, err := a.FenceAndStop(maintenanceContext(t, time.Second), rt.Host, 1); err != ErrStopUnproven || proof != nil {

+ t.Fatal("failed acquisition became stop proof", err)

+ }

+}

diff --git a/internal/activator/scheduled_token.go b/internal/activator/scheduled_token.go

new file mode 100644

index 000000000..0074b6320

--- /dev/null

+++ b/internal/activator/scheduled_token.go

@@ -0,0 +1,17 @@

+package activator

+

+// PrepareScheduledToken serializes token preparation with the local maintenance

+// fence. A preparation already in progress finishes before the fence is installed;

+// a fenced host never enters prepare. The callback must not reenter Service.

+// This does not authorize a wake or establish durable maintenance authority.

+func (a *Service) PrepareScheduledToken(host string, prepare func() (string, error)) (string, error) {

+ if a == nil || prepare == nil {

+ return "", ErrFenced

+ }

+ a.mu.Lock()

+ defer a.mu.Unlock()

+ if a.fences[host] != 0 {

+ return "", ErrFenced

+ }

+ return prepare()

+}

diff --git a/internal/activator/scheduled_token_test.go b/internal/activator/scheduled_token_test.go

new file mode 100644

index 000000000..ee2841e43

--- /dev/null

+++ b/internal/activator/scheduled_token_test.go

@@ -0,0 +1,90 @@

+package activator

+

+import (

+ "context"

+ "errors"

+ "testing"

+ "time"

+)

+

+func TestScheduledTokenFenceRefusesPreparation(t *testing.T) {

+ a := &Service{}

+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)

+ defer cancel()

+ if _, err := a.FenceAndStop(ctx, "token.example.com", 1); !errors.Is(err, ErrStopUnproven) {

+ t.Fatalf("missing entry: %v", err)

+ }

+ called := false

+ tok, err := a.PrepareScheduledToken("token.example.com", func() (string, error) {

+ called = true

+ return "secret", nil

+ })

+ if called || tok != "" || !errors.Is(err, ErrFenced) {

+ t.Fatalf("fenced preparation: called=%v token-present=%v err=%v", called, tok != "", err)

+ }

+ want := errors.New("prepare failed")

+ _, err = a.PrepareScheduledToken("sibling.example.com", func() (string, error) { return "", want })

+ if !errors.Is(err, want) {

+ t.Fatalf("unfenced sibling: %v", err)

+ }

+}

+

+func TestScheduledTokenPreparationOrdersFence(t *testing.T) {

+ const host = "token.example.com"

+ a := &Service{stopHistory: map[string]*stopObservation{host: {

+ mainStop: true, mainExited: true, sidecarStop: true, sidecarExited: true, replicaClosed: true,

+ }}}

+ entered, release := make(chan struct{}), make(chan struct{})

+ done := make(chan error, 1)

+ go func() {

+ _, err := a.PrepareScheduledToken(host, func() (string, error) {

+ close(entered)

+ <-release

+ return "token", nil

+ })

+ done <- err

+ }()

+ select {

+ case <-entered:

+ case <-time.After(time.Second):

+ t.Fatal("preparation did not start")

+ }

+ // Even with prior checked-stop history, a deadline cannot establish a

+ // fence while previously admitted token I/O is still running.

+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)

+ proof, err := a.FenceAndStop(ctx, host, 1)

+ cancel()

+ close(release)

+ if proof != nil || !errors.Is(err, ErrStopUnproven) {

+ t.Fatalf("fence crossed active preparation: proof=%v err=%v", proof != nil, err)

+ }

+ select {

+ case err := <-done:

+ if err != nil {

+ t.Fatal(err)

+ }

+ case <-time.After(time.Second):

+ t.Fatal("preparation did not finish")

+ }

+ ctx, cancel = context.WithTimeout(context.Background(), time.Second)

+ defer cancel()

+ proof, err = a.FenceAndStop(ctx, host, 1)

+ if err != nil || !proof.Matches(host, 1) {

+ t.Fatalf("fence after preparation: %v", err)

+ }

+ _, err = a.PrepareScheduledToken(host, func() (string, error) {

+ t.Error("post-fence callback ran")

+ return "", nil

+ })

+ if !errors.Is(err, ErrFenced) {

+ t.Fatal(err)

+ }

+}

+

+func TestScheduledTokenInvalidCallback(t *testing.T) {

+ for _, a := range []*Service{nil, {}} {

+ if _, err := a.PrepareScheduledToken("token.example.com", nil); !errors.Is(err, ErrFenced) {

+ t.Fatalf("nil callback: %v", err)

+ }

+}

diff --git a/internal/cgroup/strict.go b/internal/cgroup/strict.go

new file mode 100644

index 000000000..f8e071ece

--- /dev/null

+++ b/internal/cgroup/strict.go

@@ -0,0 +1,10 @@

+package cgroup

+

+import "errors"

+

+var (

+ ErrStrictUnsupported = errors.New("cgroup: strict containment unsupported")

+ ErrStrictRefused = errors.New("cgroup: strict containment refused")

+ ErrStrictState = errors.New("cgroup: strict containment state refused")

+ ErrStrictDeadline = errors.New("cgroup: strict containment requires a live deadline")

+)

diff --git a/internal/cgroup/strict_kernel_linux_test.go b/internal/cgroup/strict_kernel_linux_test.go

new file mode 100644

index 000000000..922e09764

--- /dev/null

+++ b/internal/cgroup/strict_kernel_linux_test.go

@@ -0,0 +1,343 @@

+//go:build linux

+

+package cgroup

+

+import (

+ "context"

+ "encoding/json"

+ "io"

+ "os"

+ "os/exec"

+ "path/filepath"

+ "strconv"

+ "strings"

+ "testing"

+ "time"

+)

+

+type strictKernelReceipt struct {

+ PID int `json:"pid"`

+ Group string `json:"group"`

+ Child int `json:"child,omitempty"`

+}

+

+func strictKernelGroup(t *testing.T) string {

+ t.Helper()

+ raw, e := os.ReadFile("/proc/self/cgroup")

+ if e != nil {

+ t.Fatal(e)

+ }

+ group, ok := ParseSelfCgroup(string(raw))

+ if !ok {

+ t.Fatal("no unified group")

+ }

+ return "/" + group

+}

+func strictKernelSave(t *testing.T, name string, child int) {

+ t.Helper()

+ raw, e := json.Marshal(strictKernelReceipt{os.Getpid(), strictKernelGroup(t), child})

+ if e != nil {

+ t.Fatal(e)

+ }

+ if e = os.WriteFile(filepath.Join(os.Getenv("CARLOS_STRICT_PROOF_DIR"), name+".json"), raw, 0600); e != nil {

+ t.Fatal(e)

+ }

+}

+func strictKernelCommand(mode string) *exec.Cmd {

+ cmd := exec.Command(os.Args[0], "-test.run=^TestStrictKernelChild$", "-test.count=1")

+ cmd.Env = []string{"CARLOS_STRICT_CHILD=" + mode, "CARLOS_STRICT_PROOF_DIR=" + os.Getenv("CARLOS_STRICT_PROOF_DIR")}

+ return cmd

+}

+func TestStrictKernelChild(t *testing.T) {

+ mode := os.Getenv("CARLOS_STRICT_CHILD")

+ if mode == "" {

+ t.Skip("synthetic child only")

+ }

+ if mode == "grandchild" {

+ strictKernelSave(t, "grandchild", 0)

+ time.Sleep(time.Minute)

+ return

+ }

+ if mode == "marker" {

+ strictKernelSave(t, "marker", 0)

+ return

+ }

+ child := strictKernelCommand("grandchild")

+ child.Stdout = os.Stdout

+ child.Stderr = os.Stderr

+ if e := child.Start(); e != nil {

+ t.Fatal(e)

+ }

+ strictKernelSave(t, "worker", child.Process.Pid)

+ if mode == "exit-with-descendant" {

+ return

+ }

+ time.Sleep(time.Minute)

+}

+func strictKernelAwait(t *testing.T, name string) strictKernelReceipt {

+ t.Helper()

+ path := filepath.Join(os.Getenv("CARLOS_STRICT_PROOF_DIR"), name+".json")

+ deadline := time.Now().Add(3 * time.Second)

+ for time.Now().Before(deadline) {

+ raw, e := os.ReadFile(path)

+ if e == nil {

+ var r strictKernelReceipt

+ if json.Unmarshal(raw, &r) == nil {

+ return r

+ }

+ time.Sleep(5 * time.Millisecond)

+ }

+ t.Fatal("kernel child receipt missing", name)

+ return strictKernelReceipt{}

+}

+

+// The local user unit delegates memory but not io. This scaffold moves the

+// probe into a supervisor leaf and enables only the controller strict mode uses.

+func strictKernelManager(t *testing.T) *Manager {

+ t.Helper()

+ root := filepath.Join("/sys/fs/cgroup", strictKernelGroup(t))

+ leaf := filepath.Join(root, "supervise")

+ if e := os.Mkdir(leaf, 0755); e != nil {

+ return nil

+ }

+ if e := os.WriteFile(filepath.Join(leaf, "cgroup.procs"), []byte(strconv.Itoa(os.Getpid())), 0); e != nil {

+ return nil

+ }

+ if e := os.WriteFile(filepath.Join(root, "cgroup.subtree_control"), []byte("+memory"), 0); e != nil {

+ return nil

+ }

+ return &Manager{root: root, logf: t.Logf}

+}

+

+func TestStrictKernel(t *testing.T) {

+ mode := os.Getenv("CARLOS_STRICT_KERNEL")

+ if mode == "" {

+ t.Skip("requires disposable delegated user-systemd service")

+ }

+ m := strictKernelManager(t)

+ if mode == "permission-denied" {

+ if m != nil {

+ t.Fatal("read-only cgroup delegation accepted")

+ }

+ if p, e := m.PrepareStrict("denied.example", TierDefault); e == nil || p != nil {

+ t.Fatal("missing delegation became ordinary spawn")

+ }

+ return

+ }

+ if m == nil {

+ t.Fatal("disposable service has no cgroup delegation")

+ }

+ if mode == "unsupported" {

+ p, e := m.PrepareStrict("unsupported.example", TierDefault)

+ if e != nil {

+ t.Fatal(e)

+ }

+ if e = p.Start(strictKernelCommand("marker")); e != ErrStrictUnsupported {

+ t.Fatal("blocked clone3 was not unsupported", e)

+ }

+ if e = p.Start(strictKernelCommand("marker")); e != ErrStrictState {

+ t.Fatal("blocked clone retried", e)

+ }

+ if _, e = os.Stat(filepath.Join(os.Getenv("CARLOS_STRICT_PROOF_DIR"), "marker.json")); e == nil {

+ t.Fatal("fallback executed marker")

+ }

+ if e = p.Close(); e != nil {

+ t.Fatal(e)

+ }

+ return

+ }

+ t.Run("concurrent-reservation", func(t *testing.T) {

+ type result struct {

+ p *Prepared

+ err error

+ }

+ results := make(chan result, 12)

+ begin := make(chan struct{})

+ for range 12 {

+ go func() {

+ <-begin

+ p, err := (&Manager{root: m.root}).PrepareStrict("concurrent.example", TierDefault)

+ results <- result{p, err}

+ }()

+ }

+ close(begin)

+ var winner *Prepared

+ for range 12 {

+ r := <-results

+ if r.err == nil {

+ if winner != nil {

+ t.Fatal("two concurrent preparations won")

+ }

+ winner = r.p

+ } else if r.err != ErrStrictState {

+ t.Fatal(r.err)

+ }

+ if winner == nil {

+ t.Fatal("no preparation won")

+ }

+ if err := winner.Close(); err != nil {

+ t.Fatal(err)

+ }

+ })

+ t.Run("descendant-pipe-and-reservation", func(t *testing.T) {

+ host := "tree.example"

+ p, e := m.PrepareStrict(host, TierDefault)

+ if e != nil {

+ t.Fatal(e)

+ }

+ other := &Manager{root: m.root}

+ if q, e := other.PrepareStrict(host, TierDefault); e != ErrStrictState || q != nil {

+ t.Fatal("second manager reserved same group", e)

+ }

+ read, write, e := os.Pipe()

+ if e != nil {

+ t.Fatal(e)

+ }

+ defer read.Close()

+ defer write.Close()

+ cmd := strictKernelCommand("exit-with-descendant")

+ cmd.Stdout = write

+ if e = p.Start(cmd); e != nil {

+ t.Fatal(e)

+ }

+ write.Close()

+ worker := strictKernelAwait(t, "worker")

+ child := strictKernelAwait(t, "grandchild")

+ expected := strings.TrimPrefix(m.Root(), "/sys/fs/cgroup") + "/" + ExecName(host)

+ if worker.Group != expected || child.Group != expected || worker.Child != child.PID {

+ t.Fatal("child escaped prepared cgroup")

+ }

+ if e = cmd.Wait(); e != nil {

+ t.Fatal(e)

+ }

+ drained := make(chan struct{})

+ go func() { io.Copy(io.Discard, read); close(drained) }()

+ select {

+ case <-drained:

+ t.Fatal("descendant did not retain output pipe")

+ case <-time.After(20 * time.Millisecond):

+ }

+ if e = p.Close(); e != ErrStrictRefused {

+ t.Fatal("live descendant handle discarded", e)

+ }

+ ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)

+ defer cancel()

+ if e = p.KillAndEmpty(ctx); e != nil {

+ t.Fatal(e)

+ }

+ select {

+ case <-drained:

+ case <-ctx.Done():

+ t.Fatal("empty proof left descendant output blocked")

+ }

+ if populated, e := strictPopulation(p.events); e != nil || populated {

+ t.Fatal("kernel population remained", e)

+ }

+ if e = p.Close(); e != nil {

+ t.Fatal(e)

+ }

+ q, e := other.PrepareStrict(host, TierDefault)

+ if e != nil {

+ t.Fatal("closed reservation was not released", e)

+ }

+ if e = q.Close(); e != nil {

+ t.Fatal(e)

+ }

+ })

+ t.Run("kernel-limit-readback", func(t *testing.T) {

+ for i, limit := range []Limit{TierDefault, TierChildren} {

+ p, e := m.PrepareStrict("limit"+strconv.Itoa(i)+".example", limit)

+ if e != nil {

+ t.Fatal(e)

+ }

+ if e = p.limitsOK(); e != nil {

+ t.Fatal(e)

+ }

+ if e = strictWrite(p.limits["memory.oom.group"], "0"); e != nil {

+ t.Fatal(e)

+ }

+ if e = p.Start(strictKernelCommand("marker")); e != ErrStrictRefused {

+ t.Fatal("changed actual kernel limit reached spawn", e)

+ }

+ if e = p.Close(); e != nil {

+ t.Fatal(e)

+ }

+ })

+

+ t.Run("permission-denied-controls", func(t *testing.T) {

+ host := "permissions.example"

+ p, e := m.PrepareStrict(host, TierDefault)

+ if e != nil {

+ t.Fatal(e)

+ }

+ if e = p.Close(); e != nil {

+ t.Fatal(e)

+ }

+ for _, name := range []string{"cgroup.kill", "cgroup.events", "memory.max"} {

+ path := filepath.Join(m.root, ExecName(host), name)

+ info, e := os.Stat(path)

+ if e != nil {

+ t.Fatal(e)

+ }

+ if e = os.Chmod(path, 0); e != nil {

+ t.Fatal(e)

+ }

+ candidate, prepareErr := m.PrepareStrict(host, TierDefault)

+ if e = os.Chmod(path, info.Mode().Perm()); e != nil {

+ t.Fatal(e)

+ }

+ if prepareErr == nil || candidate != nil {

+ t.Fatal("unusable required control accepted", name)

+ }

+ })

+ t.Run("missing-memory-controller", func(t *testing.T) {

+ path := filepath.Join(m.root, "cgroup.subtree_control")

+ if e := os.WriteFile(path, []byte("-memory"), 0); e != nil {

+ t.Fatal(e)

+ }

+ defer func() {

+ if e := os.WriteFile(path, []byte("+memory"), 0); e != nil {

+ t.Error(e)

+ }

+ }()

+ if p, e := m.PrepareStrict("missing.example", TierDefault); e == nil || p != nil {

+ t.Fatal("missing memory limit interface accepted")

+ }

+ })

+}

+func TestStrictKernelSupervisor(t *testing.T) {

+ if os.Getenv("CARLOS_STRICT_SUPERVISOR") != "1" {

+ t.Skip("disposable unit parent-death proof")

+ }

+ if _, e := os.Stat(filepath.Join(os.Getenv("CARLOS_STRICT_PROOF_DIR"), "supervisor.json")); e == nil {

+ strictKernelSave(t, "restarted", 0)

+ time.Sleep(time.Minute)

+ return

+ }

+ m := strictKernelManager(t)

+ if m == nil {

+ t.Fatal("no delegation")

+ }

+ p, e := m.PrepareStrict("parent.example", TierDefault)

+ if e != nil {

+ t.Fatal(e)

+ }

+ cmd := strictKernelCommand("tree")

+ read, write, e := os.Pipe()

+ if e != nil {

+ t.Fatal(e)

+ }

+ defer read.Close()

+ cmd.Stdout = write

+ if e = p.Start(cmd); e != nil {

+ t.Fatal(e)

+ }

+ write.Close()

+ strictKernelSave(t, "supervisor", cmd.Process.Pid)

+ io.Copy(io.Discard, read)

+ t.Fatal("parent-death probe returned before supervisor was killed")

+}

diff --git a/internal/cgroup/strict_linux.go b/internal/cgroup/strict_linux.go

new file mode 100644

index 000000000..7d365038e

--- /dev/null

+++ b/internal/cgroup/strict_linux.go

@@ -0,0 +1,356 @@

+//go:build linux

+

+package cgroup

+

+import (

+ "context"

+ "errors"

+ "io"

+ "os"

+ "os/exec"

+ "path/filepath"

+ "strconv"

+ "strings"

+ "sync"

+ "syscall"

+ "time"

+

+ "github.com/carlosframework/platform/internal/controlname"

+ "golang.org/x/sys/unix"

+)

+

+type strictIdentity struct{ device, inode uint64 }

+

+var strictReservations = struct {

+ sync.Mutex

+ held map[strictIdentity]bool

+}{held: make(map[strictIdentity]bool)}

+

+// Prepared pins one empty host cgroup. It grants no lease, fence or database

+// authority. The caller must also obtain Cmd.Wait evidence after cleanup.

+type Prepared struct{ *strictPreparedState }

+

+type strictPreparedState struct {

+ mu sync.Mutex

+ root, dir, kill, events *os.File

+ limits map[string]*os.File

+ expected map[string]string

+ identity strictIdentity

+ host string

+ attempted, empty, closed bool

+ start func(*exec.Cmd) error

+}

+

+func strictOpen(dir *os.File, name string, flags int) (*os.File, error) {

+ fd, e := unix.Openat(int(dir.Fd()), name, flags|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0)

+ if e != nil {

+ return nil, e

+ }

+ return os.NewFile(uintptr(fd), name), nil

+}

+func strictID(f *os.File) (strictIdentity, error) {

+ var st unix.Stat_t

+ if e := unix.Fstat(int(f.Fd()), &st); e != nil {

+ return strictIdentity{}, e

+ }

+ return strictIdentity{uint64(st.Dev), st.Ino}, nil

+}

+func strictRead(f *os.File) (string, error) {

+ if _, e := f.Seek(0, 0); e != nil {

+ return "", e

+ }

+ b, e := io.ReadAll(io.LimitReader(f, 4097))

+ if e != nil || len(b) > 4096 {

+ return "", ErrStrictRefused

+ }

+ return strings.TrimSpace(string(b)), nil

+}

+func strictWrite(f *os.File, value string) error {

+ if _, e := f.Seek(0, 0); e != nil {

+ return e

+ }

+ n, e := f.WriteString(value)

+ if e != nil {

+ return e

+ }

+ if n != len(value) {

+ return io.ErrShortWrite

+ }

+ return nil

+}

+func strictPopulation(f *os.File) (bool, error) {

+ raw, e := strictRead(f)

+ if e != nil {

+ return false, e

+ }

+ seen := false

+ populated := false

+ for _, line := range strings.Split(raw, "\n") {

+ parts := strings.Fields(line)

+ if len(parts) != 2 {

+ return false, ErrStrictRefused

+ }

+ if parts[0] == "populated" {

+ if seen || (parts[1] != "0" && parts[1] != "1") {

+ return false, ErrStrictRefused

+ }

+ seen = true

+ populated = parts[1] == "1"

+ }

+ if !seen {

+ return false, ErrStrictRefused

+ }

+ return populated, nil

+}

+func (p *Prepared) identityOK() error {

+ if p.closed {

+ return ErrStrictState

+ }

+ id, e := strictID(p.dir)

+ if e != nil || id != p.identity {

+ return ErrStrictRefused

+ }

+ var st unix.Stat_t

+ if e = unix.Fstatat(int(p.root.Fd()), ExecName(p.host), &st, unix.AT_SYMLINK_NOFOLLOW); e != nil || st.Mode&unix.S_IFMT != unix.S_IFDIR || (strictIdentity{uint64(st.Dev), st.Ino}) != p.identity {

+ return ErrStrictRefused

+ }

+ return nil

+}

+func (p *Prepared) limitsOK() error {

+ for name, want := range p.expected {

+ got, e := strictRead(p.limits[name])

+ if e != nil || got != want {

+ return ErrStrictRefused

+ }

+ return nil

+}

+func (p *Prepared) closeFiles() {

+ for _, f := range p.limits {

+ f.Close()

+ }

+ for _, f := range []*os.File{p.events, p.kill, p.dir, p.root} {

+ if f != nil {

+ f.Close()

+ }

+}

+

+// PrepareStrict never uses accounting's fallback. Only the two existing tiers

+// are accepted; a zero or missing limit cannot create an uncapped command.

+func (m *Manager) PrepareStrict(host string, limit Limit) (out *Prepared, err error) {

+ if m == nil || m.root == "" {

+ return nil, ErrStrictUnsupported

+ }

+ if !controlname.ValidHostShape(host) || (limit != TierDefault && limit != TierChildren) {

+ return nil, ErrStrictRefused

+ }

+ own, e := os.ReadFile("/proc/self/cgroup")

+ rel, ok := ParseSelfCgroup(string(own))

+ if e != nil || !ok {

+ return nil, ErrStrictUnsupported

+ }

+ current := filepath.Join("/sys/fs/cgroup", rel)

+ rootPath := filepath.Clean(m.root)

+ if current != rootPath && current != filepath.Join(rootPath, "supervise") {

+ return nil, ErrStrictRefused

+ }

+ root, e := os.OpenFile(filepath.Clean(m.root), os.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW, 0)

+ if e != nil {

+ return nil, ErrStrictUnsupported

+ }

+ p := &Prepared{strictPreparedState: &strictPreparedState{root: root, host: host, limits: map[string]*os.File{}, start: func(cmd *exec.Cmd) error { return cmd.Start() }}}

+ reserved := false

+ defer func() {

+ if err != nil {

+ p.closeFiles()

+ if reserved {

+ strictReservations.Lock()

+ delete(strictReservations.held, p.identity)

+ strictReservations.Unlock()

+ }

+ }()

+ var fs unix.Statfs_t

+ if e = unix.Fstatfs(int(root.Fd()), &fs); e != nil || uint64(fs.Type) != uint64(unix.CGROUP2_SUPER_MAGIC) {

+ return nil, ErrStrictUnsupported

+ }

+ if e = unix.Mkdirat(int(root.Fd()), ExecName(host), 0755); e != nil && !errors.Is(e, unix.EEXIST) {

+ return nil, ErrStrictRefused

+ }

+ p.dir, e = strictOpen(root, ExecName(host), unix.O_RDONLY|unix.O_DIRECTORY)

+ if e != nil {

+ return nil, ErrStrictRefused

+ }

+ p.identity, e = strictID(p.dir)

+ if e != nil {

+ return nil, ErrStrictRefused

+ }

+ strictReservations.Lock()

+ if strictReservations.held[p.identity] {

+ strictReservations.Unlock()

+ return nil, ErrStrictState

+ }

+ strictReservations.held[p.identity] = true

+ reserved = true

+ strictReservations.Unlock()

+ p.events, e = strictOpen(p.dir, "cgroup.events", unix.O_RDONLY)

+ if e != nil {

+ return nil, ErrStrictUnsupported

+ }

+ p.kill, e = strictOpen(p.dir, "cgroup.kill", unix.O_WRONLY)

+ if e != nil {

+ return nil, ErrStrictUnsupported

+ }

+ if populated, e := strictPopulation(p.events); e != nil || populated {

+ return nil, ErrStrictRefused

+ }

+ high := highOf(limit.MemoryMax) / int64(os.Getpagesize()) * int64(os.Getpagesize())

+ p.expected = map[string]string{"memory.max": strconv.FormatInt(limit.MemoryMax, 10), "memory.high": strconv.FormatInt(high, 10), "memory.swap.max": strconv.FormatInt(limit.MemoryMax, 10), "memory.oom.group": "1"}

+ for name := range p.expected {

+ p.limits[name], e = strictOpen(p.dir, name, unix.O_RDWR)

+ if e != nil {

+ return nil, ErrStrictUnsupported

+ }

+ for _, name := range []string{"memory.max", "memory.high", "memory.swap.max", "memory.oom.group"} {

+ if e = strictWrite(p.limits[name], p.expected[name]); e != nil {

+ return nil, ErrStrictRefused

+ }

+ if e = p.limitsOK(); e != nil {

+ return nil, e

+ }

+ if e = p.identityOK(); e != nil {

+ return nil, e

+ }

+ return p, nil

+}

+

+// Start consumes one attempt even if exec or containment fails. The caller

+// supplies the complete command, including its approved executable descriptor.

+func (p *Prepared) Start(cmd *exec.Cmd) error {

+ if p == nil || p.strictPreparedState == nil {

+ return ErrStrictState

+ }

+ p.mu.Lock()

+ defer p.mu.Unlock()

+ if p.closed || p.attempted || p.empty {

+ return ErrStrictState

+ }

+ p.attempted = true

+ if cmd == nil || cmd.Process != nil || cmd.ProcessState != nil {

+ return ErrStrictState

+ }

+ if e := p.identityOK(); e != nil {

+ return e

+ }

+ if e := p.limitsOK(); e != nil {

+ return e

+ }

+ if populated, e := strictPopulation(p.events); e != nil || populated {

+ return ErrStrictRefused

+ }

+ setInto(cmd, int(p.dir.Fd()))

+ cmd.SysProcAttr.Pdeathsig = syscall.SIGKILL

+ if e := p.start(cmd); e != nil {

+ if shouldLatch(e) {

+ return ErrStrictUnsupported

+ }

+ return ErrStrictRefused

+ }

+ if cmd.Process == nil {

+ return ErrStrictRefused

+ }

+ return nil

+}

+

+// KillAndEmpty proves kernel population is zero before the caller's deadline.

+// Failure retains the pinned cleanup handle and permits another bounded attempt.

+func (p *Prepared) KillAndEmpty(ctx context.Context) error {

+ if p == nil || p.strictPreparedState == nil {

+ return ErrStrictState

+ }

+ if ctx == nil {

+ return ErrStrictDeadline

+ }

+ deadline, ok := ctx.Deadline()

+ if !ok || ctx.Err() != nil || !time.Now().Before(deadline) {

+ return ErrStrictDeadline

+ }

+ lockTick := time.NewTicker(time.Millisecond)

+ defer lockTick.Stop()

+ for !p.mu.TryLock() {

+ select {

+ case <-ctx.Done():

+ return ErrStrictDeadline

+ case <-lockTick.C:

+ }

+ defer p.mu.Unlock()

+ if p.closed || p.empty {

+ return ErrStrictState

+ }

+ if ctx.Err() != nil {

+ return ErrStrictDeadline

+ }

+ if e := p.identityOK(); e != nil {

+ return e

+ }

+ p.attempted = true

+ if e := strictWrite(p.kill, "1"); e != nil {

+ return ErrStrictRefused

+ }

+ timer := time.NewTicker(5 * time.Millisecond)

+ defer timer.Stop()

+ for {

+ if ctx.Err() != nil || !time.Now().Before(deadline) {

+ return ErrStrictDeadline

+ }

+ if e := p.identityOK(); e != nil {

+ return e

+ }

+ populated, e := strictPopulation(p.events)

+ if e != nil {

+ return ErrStrictRefused

+ }

+ if !populated {

+ if ctx.Err() != nil || !time.Now().Before(deadline) {

+ return ErrStrictDeadline

+ }

+ p.empty = true

+ return nil

+ }

+ select {

+ case <-ctx.Done():

+ return ErrStrictDeadline

+ case <-timer.C:

+ }

+}

+

+// Close never removes a path. A populated or unverifiable group retains its

+// descriptors and reservation so the same handle can still attempt cleanup.

+func (p *Prepared) Close() error {

+ if p == nil || p.strictPreparedState == nil {

+ return ErrStrictState

+ }

+ p.mu.Lock()

+ defer p.mu.Unlock()

+ if p.closed {

+ return ErrStrictState

+ }

+ if e := p.identityOK(); e != nil {

+ return e

+ }

+ if populated, e := strictPopulation(p.events); e != nil || populated {

+ return ErrStrictRefused

+ }

+ p.closed = true

+ p.closeFiles()

+ strictReservations.Lock()

+ delete(strictReservations.held, p.identity)

+ strictReservations.Unlock()

+ return nil

+}

diff --git a/internal/cgroup/strict_linux_test.go b/internal/cgroup/strict_linux_test.go

new file mode 100644

index 000000000..0d6124a94

--- /dev/null

+++ b/internal/cgroup/strict_linux_test.go

@@ -0,0 +1,246 @@

+//go:build linux

+

+package cgroup

+

+import (

+ "context"

+ "errors"

+ "os"

+ "os/exec"

+ "path/filepath"

+ "sync/atomic"

+ "syscall"

+ "testing"

+ "time"

+)

+

+// Filesystem fixtures test handle state only. PrepareStrict must reject them;

+// the delegated-kernel suite separately proves actual containment and controls.

+func strictFixture(t *testing.T) *Prepared {

+ t.Helper()

+ root := t.TempDir()

+ host := "fixture.example"

+ dir := filepath.Join(root, ExecName(host))

+ if e := os.Mkdir(dir, 0700); e != nil {

+ t.Fatal(e)

+ }

+ for name, value := range map[string]string{"cgroup.events": "populated 0\nfrozen 0\n", "cgroup.kill": "", "memory.max": "4096", "memory.high": "4096", "memory.swap.max": "4096", "memory.oom.group": "1"} {

+ if e := os.WriteFile(filepath.Join(dir, name), []byte(value), 0600); e != nil {

+ t.Fatal(e)

+ }

+ open := func(path string) *os.File {

+ flags := os.O_RDWR

+ if info, e := os.Stat(path); e == nil && info.IsDir() {

+ flags = os.O_RDONLY

+ }

+ f, e := os.OpenFile(path, flags, 0)

+ if e != nil {

+ t.Fatal(e)

+ }

+ return f

+ }

+ p := &Prepared{strictPreparedState: &strictPreparedState{root: open(root), dir: open(dir), host: host, limits: map[string]*os.File{}, expected: map[string]string{}, start: func(*exec.Cmd) error { return syscall.ENOSYS }}}

+ p.kill = open(filepath.Join(dir, "cgroup.kill"))

+ p.events = open(filepath.Join(dir, "cgroup.events"))

+ p.identity, _ = strictID(p.dir)

+ for _, name := range []string{"memory.max", "memory.high", "memory.swap.max", "memory.oom.group"} {

+ p.limits[name] = open(filepath.Join(dir, name))

+ p.expected[name], _ = strictRead(p.limits[name])

+ }

+ strictReservations.Lock()

+ strictReservations.held[p.identity] = true

+ strictReservations.Unlock()

+ t.Cleanup(func() {

+ p.closeFiles()

+ strictReservations.Lock()

+ delete(strictReservations.held, p.identity)

+ strictReservations.Unlock()

+ })

+ return p

+}

+func strictContext(t *testing.T, d time.Duration) context.Context {

+ t.Helper()

+ ctx, cancel := context.WithTimeout(t.Context(), d)

+ t.Cleanup(cancel)

+ return ctx

+}

+func TestStrictRejectsUnavailableManagerHostAndTier(t *testing.T) {

+ for _, m := range []*Manager{nil, {root: t.TempDir()}} {

+ for _, host := range []string{"ok.example", "../outside", "Upper.example", "", "a/b"} {

+ if p, e := m.PrepareStrict(host, TierDefault); e == nil || p != nil {

+ t.Fatal("unsupported filesystem or host accepted")

+ }

+ m := &Manager{root: "/sys/fs/cgroup"}

+ for _, limit := range []Limit{{}, {MemoryMax: -1}, {MemoryMax: 1}} {

+ if p, e := m.PrepareStrict("x.example", limit); e == nil || p != nil {

+ t.Fatal("missing or foreign tier accepted")

+ }

+}

+func TestStrictStartConsumesOneAttemptWithoutFallback(t *testing.T) {

+ p := strictFixture(t)

+ var calls atomic.Int32

+ p.start = func(cmd *exec.Cmd) error {

+ calls.Add(1)

+ if cmd.SysProcAttr == nil || !cmd.SysProcAttr.UseCgroupFD || cmd.SysProcAttr.CgroupFD != int(p.dir.Fd()) || cmd.SysProcAttr.Pdeathsig != syscall.SIGKILL || !cmd.SysProcAttr.Setpgid {

+ t.Error("strict spawn attributes missing or clobbered")

+ }

+ return syscall.ENOSYS

+ }

+ copied := *p

+ cmd := exec.Command("/not-executed")

+ cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}

+ if e := p.Start(cmd); !errors.Is(e, ErrStrictUnsupported) {

+ t.Fatal("unsupported clone accepted", e)

+ }

+ if e := p.Start(exec.Command("/not-executed")); !errors.Is(e, ErrStrictState) {

+ t.Fatal("second spawn accepted", e)

+ }

+ if e := copied.Start(exec.Command("/not-executed")); !errors.Is(e, ErrStrictState) {

+ t.Fatal("copied handle restored spawn authority", e)

+ }

+ if calls.Load() != 1 {

+ t.Fatal("strict start retried", calls.Load())

+ }

+ if raw, e := strictRead(p.kill); e != nil || raw != "" {

+ t.Fatal("failed spawn invented cleanup", raw, e)

+ }

+}

+func TestStrictCleanupRequiresDeadlineAndSurvivesTimeout(t *testing.T) {

+ p := strictFixture(t)

+ for _, ctx := range []context.Context{nil, context.Background(), strictContext(t, -time.Second)} {

+ if e := p.KillAndEmpty(ctx); !errors.Is(e, ErrStrictDeadline) {

+ t.Fatal("unbounded/expired cleanup accepted", e)

+ }

+ if e := os.WriteFile(filepath.Join(p.root.Name(), ExecName(p.host), "cgroup.events"), []byte("populated 1\n"), 0600); e != nil {

+ t.Fatal(e)

+ }

+ if e := p.Close(); !errors.Is(e, ErrStrictRefused) {

+ t.Fatal("populated group handle discarded", e)

+ }

+ if e := p.KillAndEmpty(strictContext(t, 15*time.Millisecond)); !errors.Is(e, ErrStrictDeadline) {

+ t.Fatal("populated group claimed empty", e)

+ }

+ if e := p.Start(exec.Command("/not-executed")); !errors.Is(e, ErrStrictState) {

+ t.Fatal("cleanup timeout restored spawn authority", e)

+ }

+ if e := os.WriteFile(filepath.Join(p.root.Name(), ExecName(p.host), "cgroup.events"), []byte("populated 0\n"), 0600); e != nil {

+ t.Fatal(e)

+ }

+ if e := p.KillAndEmpty(strictContext(t, time.Second)); e != nil {

+ t.Fatal("same-handle cleanup retry failed", e)

+ }

+ if e := p.KillAndEmpty(strictContext(t, time.Second)); !errors.Is(e, ErrStrictState) {

+ t.Fatal("second cleanup reused proof", e)

+ }

+ if e := p.Close(); e != nil {

+ t.Fatal(e)

+ }

+ if e := p.Close(); !errors.Is(e, ErrStrictState) {

+ t.Fatal("double close accepted")

+ }

+ if e := p.Start(exec.Command("/not-executed")); !errors.Is(e, ErrStrictState) {

+ t.Fatal("closed spawn accepted")

+ }

+ if e := p.KillAndEmpty(strictContext(t, time.Second)); !errors.Is(e, ErrStrictState) {

+ t.Fatal("closed cleanup accepted")

+ }

+}

+func TestStrictCleanupMutexWaitHonorsDeadline(t *testing.T) {

+ p := strictFixture(t)

+ p.mu.Lock()

+ defer p.mu.Unlock()

+ start := time.Now()

+ if e := p.KillAndEmpty(strictContext(t, 15*time.Millisecond)); !errors.Is(e, ErrStrictDeadline) {

+ t.Fatal(e)

+ }

+ if time.Since(start) > 250*time.Millisecond {

+ t.Fatal("mutex wait escaped caller deadline")

+ }

+}

+func TestStrictStaleHandleCannotKillReplacement(t *testing.T) {

+ p := strictFixture(t)

+ old := filepath.Join(p.root.Name(), ExecName(p.host))

+ if e := os.Rename(old, old+"-old"); e != nil {

+ t.Fatal(e)

+ }

+ if e := os.Mkdir(old, 0700); e != nil {

+ t.Fatal(e)

+ }

+ kill := filepath.Join(old, "cgroup.kill")

+ if e := os.WriteFile(kill, []byte("replacement"), 0600); e != nil {

+ t.Fatal(e)

+ }

+ if e := p.KillAndEmpty(strictContext(t, time.Second)); !errors.Is(e, ErrStrictRefused) {

+ t.Fatal("wrong identity accepted", e)

+ }

+ if e := p.Close(); !errors.Is(e, ErrStrictRefused) {

+ t.Fatal("stale close claimed proof", e)

+ }

+ raw, e := os.ReadFile(kill)

+ if e != nil || string(raw) != "replacement" {

+ t.Fatal("replacement group touched", e)

+ }

+ if e := p.Start(exec.Command("/not-executed")); !errors.Is(e, ErrStrictRefused) {

+ t.Fatal("stale spawn accepted", e)

+ }

+}

+func TestStrictChangedLimitsAndMalformedEventsRefuse(t *testing.T) {

+ for _, name := range []string{"memory.max", "memory.high", "memory.swap.max", "memory.oom.group", "cgroup.events"} {

+ t.Run(name, func(t *testing.T) {

+ p := strictFixture(t)

+ if e := os.WriteFile(filepath.Join(p.root.Name(), ExecName(p.host), name), []byte("missing\n"), 0600); e != nil {

+ t.Fatal(e)

+ }

+ called := false

+ p.start = func(*exec.Cmd) error { called = true; return nil }

+ if e := p.Start(exec.Command("/not-executed")); e == nil || called {

+ t.Fatal("unchecked controls reached spawn", e)

+ }

+ if name == "cgroup.events" {

+ if e := p.Close(); e == nil {

+ t.Fatal("unknown emptiness discarded handle")

+ }

+ })

+ }

+}

+

+func TestStrictConcurrentAndInvalidStartsConsumeAttempt(t *testing.T) {

+ p := strictFixture(t)

+ var calls atomic.Int32

+ p.start = func(*exec.Cmd) error { calls.Add(1); return syscall.ENOSYS }

+ results := make(chan error, 16)

+ for range 16 {

+ go func() { results <- p.Start(exec.Command("/not-executed")) }()

+ }

+ unsupported := 0

+ for range 16 {

+ switch err := <-results; err {

+ case ErrStrictUnsupported:

+ unsupported++

+ case ErrStrictState:

+ default:

+ t.Fatal(err)

+ }

+ if calls.Load() != 1 || unsupported != 1 {

+ t.Fatal("concurrent starts repeated attempt")

+ }

+ q := strictFixture(t)

+ if err := q.Start(nil); err != ErrStrictState {

+ t.Fatal(err)

+ }

+ if err := q.Start(exec.Command("/not-executed")); err != ErrStrictState {

+ t.Fatal("invalid command restored attempt", err)

+ }

+ for _, zero := range []*Prepared{nil, {}} {

+ if zero.Start(nil) != ErrStrictState || zero.Close() != ErrStrictState || zero.KillAndEmpty(context.Background()) != ErrStrictState {

+ t.Fatal("zero handle accepted")

+ }

+}

diff --git a/internal/cgroup/strict_other.go b/internal/cgroup/strict_other.go

new file mode 100644

index 000000000..0a1b0ed76

--- /dev/null

+++ b/internal/cgroup/strict_other.go

@@ -0,0 +1,17 @@

+//go:build !linux

+

+package cgroup

+

+import (

+ "context"

+ "os/exec"

+)

+

+type Prepared struct{}

+

+func (m *Manager) PrepareStrict(host string, limit Limit) (*Prepared, error) {

+ return nil, ErrStrictUnsupported

+}

+func (p *Prepared) Start(cmd *exec.Cmd) error { return ErrStrictUnsupported }

+func (p *Prepared) KillAndEmpty(ctx context.Context) error { return ErrStrictUnsupported }

+func (p *Prepared) Close() error { return ErrStrictUnsupported }

diff --git a/internal/controlname/names.go b/internal/controlname/names.go

new file mode 100644

index 000000000..51c0876fa

--- /dev/null

+++ b/internal/controlname/names.go

@@ -0,0 +1,28 @@

+// Package controlname validates identifiers used as control-plane key segments.

+package controlname

+

+import (

+ "regexp"

+ "strings"

+)

+

+var (

+ account = regexp.MustCompile(`^[a-z]{3,16}$`)

+ app = regexp.MustCompile(`^[a-z][a-z0-9-]{0,31}$`)

+ label = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$`)

+)

+

+func ValidAccount(s string) bool { return account.MatchString(s) }

+func ValidApp(s string) bool { return app.MatchString(s) }

+func ValidLabel(s string) bool { return label.MatchString(s) }

+func ValidHostShape(host string) bool {

+ if host == "" || len(host) > 253 {

+ return false

+ }

+ for _, part := range strings.Split(host, ".") {

+ if !ValidLabel(part) {

+ return false

+ }

+ return true

+}

diff --git a/internal/host/schedule_token_fence_test.go b/internal/host/schedule_token_fence_test.go

new file mode 100644

index 000000000..a902c1bfe

--- /dev/null

+++ b/internal/host/schedule_token_fence_test.go

@@ -0,0 +1,102 @@

+package host

+

+import (

+ "bytes"

+ "context"

+ "errors"

+ "os"

+ "path/filepath"

+ "strings"

+ "testing"

+ "time"

+

+ "github.com/carlosframework/platform/internal/activator"

+)

+

+func TestScheduleTokenFencePreservesEnvironment(t *testing.T) {

+ for _, content := range []string{"", "APP_SETTING=retained\n", "APP_SETTING=retained\nCARLOS_ADMIN_TOKEN=" + strings.Repeat("a", 64) + "\n"} {

+ name := "missing-file"

+ if content != "" {

+ name = "existing-without-token"

+ }

+ if strings.Contains(content, "CARLOS_ADMIN_TOKEN") {

+ name = "existing-token"

+ }

+ t.Run(name, func(t *testing.T) {

+ dd := t.TempDir()

+ const host = "fenced.example.com"

+ path := filepath.Join(dd, host+".env")

+ if content != "" {

+ if err := os.WriteFile(path, []byte(content), 0o644); err != nil {

+ t.Fatal(err)

+ }

+ if err := os.Chmod(path, 0o644); err != nil {

+ t.Fatal(err)

+ }

+ svc := &activator.Service{}

+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)

+ defer cancel()

+ if _, err := svc.FenceAndStop(ctx, host, 1); !errors.Is(err, activator.ErrStopUnproven) {

+ t.Fatal(err)

+ }

+ // Exercise the actual runner wiring, including the nil-store arm.

+ sd := newScheduleDeps(nil, nil, svc, nil, dd)

+ if token, err := sd.runner.Token(host); token != "" || !errors.Is(err, activator.ErrFenced) {

+ t.Fatalf("fenced token: present=%v err=%v", token != "", err)

+ }

+ raw, err := os.ReadFile(path)

+ if content == "" {

+ if !os.IsNotExist(err) {

+ t.Fatalf("fence created env: %v", err)

+ }

+ return

+ }

+ if err != nil || !bytes.Equal(raw, []byte(content)) {

+ t.Fatalf("fence changed bytes: %v", err)

+ }

+ fi, err := os.Stat(path)

+ if err != nil {

+ t.Fatal(err)

+ }

+ if fi.Mode().Perm() != 0o644 {

+ t.Fatalf("fence changed mode: %o", fi.Mode().Perm())

+ }

+ })

+ }

+}

+

+func TestScheduleTokenUnfencedManagedPreparation(t *testing.T) {

+ dd := t.TempDir()

+ const host = "ordinary.example.com"

+ path := filepath.Join(dd, host+".env")

+ if err := os.WriteFile(path, []byte("APP_SETTING=retained\n"), 0o644); err != nil {

+ t.Fatal(err)

+ }

+ sd := newScheduleDeps(nil, nil, &activator.Service{}, nil, dd)

+ first, err := sd.runner.Token(host)

+ if err != nil || len(first) != 64 {

+ t.Fatalf("mint: present=%v err=%v", first != "", err)

+ }

+ if err := os.Chmod(path, 0o644); err != nil {

+ t.Fatal(err)

+ }

+ second, err := sd.runner.Token(host)

+ if err != nil || first != second {

+ t.Fatalf("retained token: %v", err)

+ }

+ raw, err := os.ReadFile(path)

+ if err != nil {

+ t.Fatal(err)

+ }

+ if !bytes.HasPrefix(raw, []byte("APP_SETTING=retained\n")) || bytes.Count(raw, []byte("CARLOS_ADMIN_TOKEN=")) != 1 {

+ t.Fatal("ordinary token preparation lost environment or duplicated token")

+ }

+ fi, err := os.Stat(path)

+ if err != nil {

+ t.Fatal(err)

+ }

+ if fi.Mode().Perm() != 0o600 {

+ t.Fatalf("ordinary mode: %o", fi.Mode().Perm())

+ }

+}

diff --git a/internal/host/schedule_wake_test.go b/internal/host/schedule_wake_test.go

index ce1ee966d..691c80e62 100644

--- a/internal/host/schedule_wake_test.go

+++ b/internal/host/schedule_wake_test.go

@@ -238,7 +238,7 @@ func TestScheduledTickWakesASleepingInstanceAndItHibernatesAgain(t *testing.T) {

now := time.Now().UTC()

occurrence := now.Unix() - 5

r := &schedule.Runner{

- Reg: reg, Deliver: &activatorDeliverer{svc: svc}, Token: scheduleToken(dd),

+ Reg: reg, Deliver: &activatorDeliverer{svc: svc}, Token: scheduleToken(dd, svc),

Now: func() time.Time { return now }, Timeout: 30 * time.Second,

Box: "box1", Build: "test",

}

@@ -257,7 +257,7 @@ func TestScheduledTickWakesASleepingInstanceAndItHibernatesAgain(t *testing.T) {

case <-time.After(30 * time.Second):

t.Fatal("the handler never ran — the tick did not wake the sleeping instance")

}

- tok, err := scheduleToken(dd)(host)

+ tok, err := scheduleToken(dd, svc)(host)

if err != nil {

t.Fatal(err)

}

diff --git a/internal/host/schedule_wiring.go b/internal/host/schedule_wiring.go

index 266f3a434..6774ca4c1 100644

--- a/internal/host/schedule_wiring.go

+++ b/internal/host/schedule_wiring.go

@@ -84,7 +84,7 @@ func newScheduleDeps(reg *registry.Registry, store release.Store, svc *activator

runner: &schedule.Runner{

Reg: reg,

Deliver: &activatorDeliverer{svc: svc},

- Token: scheduleToken(dd),

+ Token: scheduleToken(dd, svc),

Event: event,

Box: box,

Build: version.Build,

@@ -157,9 +157,9 @@ func armSchedule(reg *registry.Registry, store release.Store, storeErr error, sv

// The error is redacted: it reaches the row's last_status and from there the

// bucket status object and `carlos schedule ls`, and a member must not learn

// the box's filesystem layout from it.

-func scheduleToken(dd string) func(host string) (string, error) {

+func scheduleToken(dd string, svc *activator.Service) func(host string) (string, error) {

read := schedule.TokenFromEnvFile(dd)

- return func(host string) (string, error) {

+ prepare := func(host string) (string, error) {

if err := activator.EnsureAdminToken(filepath.Join(dd, host+".env")); err != nil {

return "", errors.New(activator.RedactPaths(err.Error()))

}

@@ -169,6 +169,14 @@ func scheduleToken(dd string) func(host string) (string, error) {

}

return tok, nil

}

+ return func(host string) (string, error) {

+ if svc == nil {

+ return prepare(host)

+ }

+ return svc.PrepareScheduledToken(host, func() (string, error) {

+ return prepare(host)

+ })

+ }

}

// convergePassTimeout bounds ONE converge pass. The agent's three call sites

diff --git a/internal/host/schedule_wiring_test.go b/internal/host/schedule_wiring_test.go

index 8e5940f75..ea89a293e 100644

--- a/internal/host/schedule_wiring_test.go

+++ b/internal/host/schedule_wiring_test.go

@@ -121,7 +121,7 @@ func TestScheduleTokenMintsForASleepingInstance(t *testing.T) {

if _, err := os.Stat(envPath); !os.IsNotExist(err) {

t.Fatalf("fixture: env file already exists (%v)", err)

}

- tok, err := scheduleToken(dd)(host)

+ tok, err := scheduleToken(dd, nil)(host)

if err != nil {

t.Fatalf("Token on a host with no env file: %v", err)

}

@@ -135,7 +135,7 @@ func TestScheduleTokenMintsForASleepingInstance(t *testing.T) {

if err := activator.EnsureAdminToken(envPath); err != nil {

t.Fatal(err)

}

- again, err := scheduleToken(dd)(host)

+ again, err := scheduleToken(dd, nil)(host)

if err != nil || again != tok {

t.Fatalf("second read = %q (%v), want the first token %q", again, err, tok)

}

@@ -150,7 +150,7 @@ func TestScheduleTokenMintsForASleepingInstance(t *testing.T) {

// filesystem layout.

func TestScheduleTokenErrorCarriesNoAbsolutePath(t *testing.T) {

dd := filepath.Join(t.TempDir(), "missing-data-dir")

- _, err := scheduleToken(dd)("x.bab.oncarlos.com")

+ _, err := scheduleToken(dd, nil)("x.bab.oncarlos.com")

if err == nil {

t.Fatal("Token succeeded with no data dir")

}

diff --git a/internal/lease/checkpoint.go b/internal/lease/checkpoint.go

new file mode 100644

index 000000000..a7d7d1ec9

--- /dev/null

+++ b/internal/lease/checkpoint.go

@@ -0,0 +1,56 @@

+package lease

+

+import (

+ "context"

+ "errors"

+ "time"

+)

+

+// CheckpointReceipt describes a completed sync and close under a healthy

+// lease. It does not prove restored database semantics or authorize a command.

+// SafeUntil is useful only in-process, with its monotonic clock component.

+type CheckpointReceipt struct {

+ CompletedAt time.Time

+ SafeUntil time.Time

+}

+

+// Checkpoint keeps the same renewal running while one temporary replica store

+// opens, syncs and closes. Failure poisons the handle but does not release it;

+// the caller must retain its fence, record the failure and explicitly Release.

+func (h *Held) Checkpoint(ctx context.Context, dbPath string) (receipt CheckpointReceipt, err error) {

+ if err = h.begin(false); err != nil {

+ return receipt, err

+ }

+ defer func() { h.finish(false, err) }()

+ opCtx, cancel := context.WithCancel(ctx)

+ stop := context.AfterFunc(h.rn.ctx, cancel)

+ defer stop()

+ defer cancel()

+ store, db := h.newStore(dbPath, 0)

+ // Short checkpoints need only the explicit L0 sync. Background snapshots

+ // are unnecessary and v0.5.14 closes the DB before joining those monitors.

+ db.MonitorInterval = 0

+ db.Replica.MonitorEnabled = false

+ store.CompactionMonitorEnabled = false

+ store.L0RetentionCheckInterval = 0

+ store.SetRetentionEnabled(false)

+ openErr := store.Open(opCtx)

+ var syncErr error

+ if openErr == nil {

+ syncErr = db.SyncAndWait(opCtx)

+ }

+ if openErr != nil || syncErr != nil {

+ cancel()

+ }

+ closeErr := store.Close(opCtx)

+ until, healthErr := h.rn.SafeUntil()

+ err = errors.Join(openErr, syncErr, closeErr, healthErr, opCtx.Err())

+ if err != nil {

+ return CheckpointReceipt{}, err

+ }

+ completed := time.Now()

+ if !completed.Before(until) {

+ return CheckpointReceipt{}, ErrLost

+ }

+ return CheckpointReceipt{CompletedAt: completed, SafeUntil: until}, nil

+}

diff --git a/internal/lease/checkpoint_test.go b/internal/lease/checkpoint_test.go

new file mode 100644

index 000000000..2727cfda4

--- /dev/null

+++ b/internal/lease/checkpoint_test.go

@@ -0,0 +1,691 @@

+package lease

+

+import (

+ "bytes"

+ "context"

+ "database/sql"

+ "encoding/json"

+ "errors"

+ "fmt"

+ "io"

+ "net/http"

+ "net/http/httptest"

+ "path/filepath"

+ "sync"

+ "testing"

+ "time"

+

+ "github.com/aws/aws-sdk-go-v2/aws"

+ s3 "github.com/aws/aws-sdk-go-v2/service/s3"

+ "github.com/aws/smithy-go"

+ "github.com/benbjohnson/litestream"

+ lsfile "github.com/benbjohnson/litestream/file"

+ lss3 "github.com/benbjohnson/litestream/s3"

+ "github.com/superfly/ltx"

+)

+

+type lockStore struct {

+ mu sync.Mutex

+ body []byte

+ etag string

+ puts, deletes int

+ beforePut func(context.Context, *s3.PutObjectInput) error

+}

+

+func (s *lockStore) GetObject(ctx context.Context, p *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) {

+ s.mu.Lock()

+ defer s.mu.Unlock()

+ if s.body == nil {

+ return nil, &smithy.GenericAPIError{Code: "NoSuchKey"}

+ }

+ return &s3.GetObjectOutput{Body: io.NopCloser(bytes.NewReader(bytes.Clone(s.body))), ETag: aws.String(s.etag)}, nil

+}

+func (s *lockStore) PutObject(ctx context.Context, p *s3.PutObjectInput, _ ...func(*s3.Options)) (*s3.PutObjectOutput, error) {

+ raw, e := io.ReadAll(p.Body)

+ if e != nil {

+ return nil, e

+ }

+ if s.beforePut != nil {

+ if e := s.beforePut(ctx, p); e != nil {

+ return nil, e

+ }

+ s.mu.Lock()

+ defer s.mu.Unlock()

+ if (aws.ToString(p.IfNoneMatch) == "*" && s.body != nil) || (p.IfMatch != nil && aws.ToString(p.IfMatch) != s.etag) {

+ return nil, &smithy.GenericAPIError{Code: "PreconditionFailed"}

+ }

+ s.puts++

+ s.etag = fmt.Sprintf("\"etag-%d\"", s.puts)

+ s.body = raw

+ return &s3.PutObjectOutput{ETag: aws.String(s.etag)}, nil

+}

+func (s *lockStore) DeleteObject(ctx context.Context, p *s3.DeleteObjectInput, _ ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) {

+ s.mu.Lock()

+ defer s.mu.Unlock()

+ if p.IfMatch == nil || aws.ToString(p.IfMatch) != s.etag {

+ return nil, &smithy.GenericAPIError{Code: "PreconditionFailed"}

+ }

+ s.deletes++

+ s.body = nil

+ s.etag = ""

+ return &s3.DeleteObjectOutput{}, nil

+}

+func (s *lockStore) counts() (int, int) { s.mu.Lock(); defer s.mu.Unlock(); return s.puts, s.deletes }

+func (s *lockStore) ServeHTTP(w http.ResponseWriter, r *http.Request) {

+ var err error

+ switch r.Method {

+ case "GET":

+ var out *s3.GetObjectOutput

+ out, err = s.GetObject(r.Context(), &s3.GetObjectInput{})

+ if err == nil {

+ w.Header().Set("ETag", aws.ToString(out.ETag))

+ io.Copy(w, out.Body)

+ return

+ }

+ case "PUT":

+ p := &s3.PutObjectInput{Body: r.Body}

+ if v := r.Header.Get("If-Match"); v != "" {

+ p.IfMatch = aws.String(v)

+ }

+ if v := r.Header.Get("If-None-Match"); v != "" {

+ p.IfNoneMatch = aws.String(v)

+ }

+ var out *s3.PutObjectOutput

+ out, err = s.PutObject(r.Context(), p)

+ if err == nil {

+ w.Header().Set("ETag", aws.ToString(out.ETag))

+ return

+ }

+ case "DELETE":

+ _, err = s.DeleteObject(r.Context(), &s3.DeleteObjectInput{IfMatch: aws.String(r.Header.Get("If-Match"))})

+ if err == nil {

+ w.WriteHeader(204)

+ return

+ }

+ var api *smithy.GenericAPIError

+ if errors.As(err, &api) {

+ code := 412

+ if api.Code == "NoSuchKey" {

+ code = 404

+ }

+ w.Header().Set("Content-Type", "application/xml")

+ w.WriteHeader(code)

+ fmt.Fprintf(w, "<Error><Code>%s</Code></Error>", api.Code)

+ return

+ }

+ w.WriteHeader(500)

+}

+func localHeld(t *testing.T, backend *lockStore, ttl time.Duration) *Held {

+ t.Helper()

+ srv := httptest.NewServer(backend)

+ t.Cleanup(srv.Close)

+ fakeCreds(t)

+ t.Setenv("CARLOS_S3_ENDPOINT", srv.URL)

+ h, e := acquireTTL(context.Background(), "test-bucket", "one.example", "test-region", ttl)

+ if e != nil {

+ t.Fatal(e)

+ }

+ t.Cleanup(func() { _ = h.Release(context.Background()) })

+ return h

+}

+func await(t *testing.T, ch <-chan struct{}) {

+ t.Helper()

+ select {

+ case <-ch:

+ case <-time.After(5 * time.Second):

+ t.Fatal("timed out")

+ }

+}

+func restoreCount(t *testing.T, replica, out string) int {

+ t.Helper()

+ r := litestream.NewReplicaWithClient(nil, lsfile.NewReplicaClient(replica))

+ opt := litestream.NewRestoreOptions()

+ opt.OutputPath = out

+ if e := r.Restore(context.Background(), opt); e != nil {

+ t.Fatal(e)

+ }

+ d, e := sql.Open("sqlite", "file:"+out+"?mode=ro")

+ if e != nil {

+ t.Fatal(e)

+ }

+ defer d.Close()

+ var n int

+ var integrity string

+ if e = d.QueryRow("PRAGMA integrity_check").Scan(&integrity); e != nil || integrity != "ok" {

+ t.Fatalf("integrity %q %v", integrity, e)

+ }

+ if e = d.QueryRow("SELECT count(*) FROM t").Scan(&n); e != nil {

+ t.Fatal(e)

+ }

+ var contents string

+ if e = d.QueryRow("SELECT group_concat(body, '|') FROM (SELECT body FROM t ORDER BY id)").Scan(&contents); e != nil {

+ t.Fatal(e)

+ }

+ if n == 5 && contents != "post-vacuum|post-vacuum|post-vacuum|post-vacuum|post-vacuum" {

+ t.Fatalf("restored content receipt differs: %q", contents)

+ }

+ return n

+}

+func TestHeldCheckpointVacuumRestoreAndTransfer(t *testing.T) {

+ backend := &lockStore{}

+ h := localHeld(t, backend, 10*time.Second)

+ path := filepath.Join(t.TempDir(), "source.db")

+ seedDB(t, path)

+ replica := t.TempDir()

+ h.replicaClient = lsfile.NewReplicaClient(replica)

+ rn := h.rn

+ receipt, e := h.Checkpoint(context.Background(), path)

+ if e != nil {

+ t.Fatal(e)

+ }

+ if receipt.CompletedAt.IsZero() || !receipt.CompletedAt.Before(receipt.SafeUntil) || h.rn != rn {

+ t.Fatal("invalid checkpoint receipt/ownership")

+ }

+ if n := restoreCount(t, replica, filepath.Join(t.TempDir(), "old.db")); n != 50 {

+ t.Fatal(n)

+ }

+ d, e := sql.Open("sqlite", path)

+ if e != nil {

+ t.Fatal(e)

+ }

+ _, e = d.Exec("DELETE FROM t WHERE id>5; UPDATE t SET body='post-vacuum'; PRAGMA wal_checkpoint(TRUNCATE); VACUUM; PRAGMA wal_checkpoint(TRUNCATE)")

+ if e != nil {

+ t.Fatal(e)

+ }

+ if e = d.Close(); e != nil {

+ t.Fatal(e)

+ }

+ if _, e = h.Checkpoint(context.Background(), path); e != nil {

+ t.Fatal(e)

+ }

+ if n := restoreCount(t, replica, filepath.Join(t.TempDir(), "new.db")); n != 5 {

+ t.Fatalf("stale replica: %d", n)

+ }

+ if _, deletes := backend.counts(); deletes != 0 {

+ t.Fatal("checkpoint released lock")

+ }

+ // The actual leaser sees the same still-owned lock after both checkpoints.

+ if peer, e := acquireTTL(context.Background(), "test-bucket", "one.example", "test-region", 10*time.Second); e == nil {

+ peer.Release(context.Background())

+ t.Fatal("acquisition gap")

+ }

+ r, e := h.Replicate(context.Background(), path, 0)

+ if e != nil {

+ t.Fatal(e)

+ }

+ defer r.Close(context.Background())

+ if r.Lost() != h.Lost() {

+ t.Fatal("transfer replaced loss channel")

+ }

+ select {

+ case <-r.Lost():

+ t.Fatal("healthy transfer signaled loss")

+ default:

+ }

+ if r.rn != rn {

+ t.Fatal("renewer replaced")

+ }

+ if e = h.Release(context.Background()); !errors.Is(e, ErrSpent) {

+ t.Fatalf("spent release %v", e)

+ }

+ if _, e = h.SafeUntil(); !errors.Is(e, ErrSpent) {

+ t.Fatal("spent deadline accepted")

+ }

+ if _, e = r.SafeUntil(); e != nil {

+ t.Fatal(e)

+ }

+ if _, deletes := backend.counts(); deletes != 0 {

+ t.Fatal("spent Held released successor")

+ }

+ if e = r.Close(context.Background()); e != nil {

+ t.Fatal(e)

+ }

+ if _, e = r.SafeUntil(); e == nil {

+ t.Fatal("stopped deadline accepted")

+ }

+ select {

+ case <-r.Lost():

+ t.Fatal("normal Close signaled loss")

+ default:

+ }

+}

+

+type blockedReplica struct {

+ litestream.ReplicaClient

+ entered chan struct{}

+ release chan struct{}

+ once sync.Once

+ fail error

+}

+

+func (b *blockedReplica) WriteLTXFile(ctx context.Context, level int, min, max ltx.TXID, r io.Reader) (*ltx.FileInfo, error) {

+ b.once.Do(func() { close(b.entered) })

+ select {

+ case <-b.release:

+ case <-ctx.Done():

+ return nil, ctx.Err()

+ }

+ if b.fail != nil {

+ return nil, b.fail

+ }

+ return b.ReplicaClient.WriteLTXFile(ctx, level, min, max, r)

+}

+func TestCheckpointExclusionAndPoison(t *testing.T) {

+ for _, fail := range []bool{false, true} {

+ t.Run(fmt.Sprint(fail), func(t *testing.T) {

+ backend := &lockStore{}

+ h := localHeld(t, backend, 10*time.Second)

+ path := filepath.Join(t.TempDir(), "source.db")

+ seedDB(t, path)

+ b := &blockedReplica{ReplicaClient: lsfile.NewReplicaClient(t.TempDir()), entered: make(chan struct{}), release: make(chan struct{})}

+ if fail {

+ b.fail = errors.New("synthetic replica write refusal")

+ }

+ h.replicaClient = b

+ opCtx, cancel := context.WithTimeout(context.Background(), time.Second)

+ defer cancel()

+ done := make(chan error, 1)

+ go func() { _, e := h.Checkpoint(opCtx, path); done <- e }()

+ await(t, b.entered)

+ if _, e := h.Checkpoint(context.Background(), path); !errors.Is(e, ErrBusy) {

+ t.Fatal(e)

+ }

+ if _, e := h.Replicate(context.Background(), path, 0); !errors.Is(e, ErrBusy) {

+ t.Fatal(e)

+ }

+ if e := h.Release(context.Background()); !errors.Is(e, ErrBusy) {

+ t.Fatal(e)

+ }

+ close(b.release)

+ e := <-done

+ if fail {

+ if e == nil {

+ t.Fatal("failed sync accepted")

+ }

+ if _, e = h.Checkpoint(context.Background(), path); e == nil {

+ t.Fatal("poisoned checkpoint")

+ }

+ if _, e = h.Replicate(context.Background(), path, 0); e == nil {

+ t.Fatal("poisoned replicate")

+ }

+ if _, e = h.SafeUntil(); e == nil {

+ t.Fatal("poisoned command deadline")

+ }

+ } else if e != nil {

+ t.Fatal(e)

+ }

+ if e = h.rn.Err(); e != nil {

+ t.Fatalf("ownership released on checkpoint outcome: %v", e)

+ }

+ if _, deletes := backend.counts(); deletes != 0 {

+ t.Fatal("unexpected release")

+ }

+ if e = h.Release(context.Background()); e != nil {

+ t.Fatal(e)

+ }

+ })

+ }

+}

+

+func TestBlockedRenewalDeadlinePrecedesTakeover(t *testing.T) {

+ entered := make(chan struct{})

+ var once sync.Once

+ backend := &lockStore{beforePut: func(ctx context.Context, p *s3.PutObjectInput) error {

+ if p.IfMatch != nil {

+ once.Do(func() { close(entered) })

+ <-ctx.Done()

+ return ctx.Err()

+ }

+ return nil

+ }}

+ ttl := 900 * time.Millisecond

+ h := localHeld(t, backend, ttl)

+ until, e := h.SafeUntil()

+ if e != nil {

+ t.Fatal(e)

+ }

+ await(t, entered)

+ // A worker observes cancellation without waiting for the blocked remote call.

+ await(t, h.Lost())

+ if e = h.Err(); e == nil {

+ t.Fatal("blocked renewal remained healthy")

+ }

+ if time.Since(until) > ttl/10 {

+ t.Fatal("loss notification consumed termination margin")

+ }

+ backend.mu.Lock()

+ var original litestream.Lease

+ json.Unmarshal(backend.body, &original)

+ backend.mu.Unlock()

+ if !time.Now().Before(original.ExpiresAt) {

+ t.Fatal("termination did not precede lease expiry")

+ }

+ if peer, e := acquireTTL(context.Background(), "test-bucket", "one.example", "test-region", ttl); e == nil {

+ peer.Release(context.Background())

+ t.Fatal("peer took live lock")

+ }

+ time.Sleep(time.Until(original.ExpiresAt) + 20*time.Millisecond)

+ // A separate real leaser can take over the expired CAS object. Its acquisition

+ // uses If-Match, so disable the synthetic blocked-renewal hook first.

+ backend.beforePut = nil

+ peer, e := acquireTTL(context.Background(), "test-bucket", "one.example", "test-region", ttl)

+ if e != nil {

+ t.Fatal(e)

+ }

+ defer peer.Release(context.Background())

+ if e = h.Release(context.Background()); e == nil {

+ t.Fatal("stale release erased peer")

+ }

+ if e = peer.Err(); e != nil {

+ t.Fatal(e)

+ }

+}

+

+func TestLateRenewalCannotResurrect(t *testing.T) {

+ gate := make(chan struct{})

+ entered := make(chan struct{})

+ var once sync.Once

+ backend := &lockStore{beforePut: func(ctx context.Context, p *s3.PutObjectInput) error {

+ if p.IfMatch != nil {

+ once.Do(func() { close(entered) })

+ <-gate

+ }

+ return nil

+ }}

+ leaser := lss3.NewLeaser()

+ leaser.SetClient(backend)

+ leaser.TTL = 300 * time.Millisecond

+ started := time.Now()

+ lock, e := leaser.AcquireLease(context.Background())

+ if e != nil {

+ t.Fatal(e)

+ }

+ rn := startRenewer(leaser, lock, started, 30*time.Millisecond)

+ await(t, entered)

+ await(t, rn.ctx.Done())

+ if !errors.Is(rn.Err(), ErrLost) {

+ t.Fatal("deadline not latched")

+ }

+ close(gate)

+ await(t, rn.done)

+ if !errors.Is(rn.Err(), ErrLost) {

+ t.Fatal("late acknowledgement resurrected ownership")

+ }

+ if _, e = rn.SafeUntil(); e == nil {

+ t.Fatal("late acknowledgement extended deadline")

+ }

+ leaser.ReleaseLease(context.Background(), rn.stop())

+}

+

+func TestAcquireDeadlineStartsBeforeRemoteRequest(t *testing.T) {

+ entered := make(chan time.Time, 1)

+ backend := &lockStore{beforePut: func(ctx context.Context, p *s3.PutObjectInput) error {

+ if p.IfMatch == nil {

+ entered <- time.Now()

+ time.Sleep(120 * time.Millisecond)

+ }

+ return nil

+ }}

+ ttl := time.Second

+ h := localHeld(t, backend, ttl)

+ beforeAck := <-entered

+ until, e := h.SafeUntil()

+ if e != nil {

+ t.Fatal(e)

+ }

+ if until.After(beforeAck.Add(ttl - marginForTTL(ttl))) {

+ t.Fatal("acquisition latency extended safe deadline")

+ }

+ if until == until.Round(0) {

+ t.Fatal("deadline lost monotonic clock")

+ }

+}

+

+func TestAcquireLateSuccessGrantsNoHandle(t *testing.T) {

+ backend := &lockStore{beforePut: func(ctx context.Context, p *s3.PutObjectInput) error {

+ if p.IfMatch == nil {

+ time.Sleep(350 * time.Millisecond)

+ }

+ return nil

+ }}

+ srv := httptest.NewServer(backend)

+ defer srv.Close()

+ fakeCreds(t)

+ t.Setenv("CARLOS_S3_ENDPOINT", srv.URL)

+ h, e := acquireTTL(context.Background(), "test-bucket", "one.example", "test-region", 300*time.Millisecond)

+ if e == nil || h != nil {

+ if h != nil {

+ h.Release(context.Background())

+ }

+ t.Fatal("late acquisition granted authority")

+ }

+ if _, deletes := backend.counts(); deletes != 1 {

+ t.Fatal("known late acquisition was not conditionally cleaned up")

+ }

+}

+

+func TestCheckpointLossCancelsBlockedSync(t *testing.T) {

+ path := filepath.Join(t.TempDir(), "source.db")

+ seedDB(t, path)

+ b := &blockedReplica{ReplicaClient: lsfile.NewReplicaClient(t.TempDir()), entered: make(chan struct{}), release: make(chan struct{})}

+ entered := make(chan struct{})

+ var once sync.Once

+ backend := &lockStore{beforePut: func(ctx context.Context, p *s3.PutObjectInput) error {

+ if p.IfMatch != nil {

+ // Keep renewal healthy until the upload is actually blocked.

+ // Setup and filesystem latency must not consume the test lease.

+ select {

+ case <-b.entered:

+ default:

+ return nil

+ }

+ once.Do(func() { close(entered) })

+ <-ctx.Done()

+ return ctx.Err()

+ }

+ return nil

+ }}

+ h := localHeld(t, backend, 600*time.Millisecond)

+ h.replicaClient = b

+ done := make(chan error, 1)

+ go func() { _, e := h.Checkpoint(context.Background(), path); done <- e }()

+ await(t, b.entered)

+ await(t, entered)

+ select {

+ case e := <-done:

+ if e == nil {

+ t.Fatal("lost lease returned receipt")

+ }

+ case <-time.After(2 * time.Second):

+ t.Fatal("loss did not cancel checkpoint")

+ }

+ if _, deletes := backend.counts(); deletes != 0 {

+ t.Fatal("checkpoint released after loss")

+ }

+ if _, e := h.SafeUntil(); e == nil {

+ t.Fatal("failed checkpoint usable")

+ }

+}

+

+func TestCheckpointOpenFailureLeavesOwnedPoisonedHandle(t *testing.T) {

+ backend := &lockStore{}

+ h := localHeld(t, backend, 10*time.Second)

+ // A directory is never a SQLite database; the real store Open must refuse.

+ if receipt, e := h.Checkpoint(context.Background(), t.TempDir()); e == nil || !receipt.CompletedAt.IsZero() {

+ t.Fatal("failed open returned receipt")

+ }

+ if e := h.rn.Err(); e != nil {

+ t.Fatal("open failure lost held renewal")

+ }

+ if _, e := h.Replicate(context.Background(), "unused", 0); e == nil {

+ t.Fatal("open failure did not poison")

+ }

+ if _, deletes := backend.counts(); deletes != 0 {

+ t.Fatal("open failure released lease")

+ }

+}

+

+func TestRenewDeadlineStartsBeforeRemoteRequest(t *testing.T) {

+ entered := make(chan time.Time, 1)

+ backend := &lockStore{beforePut: func(ctx context.Context, p *s3.PutObjectInput) error {

+ if p.IfMatch != nil {

+ select {

+ case entered <- time.Now():

+ default:

+ }

+ time.Sleep(100 * time.Millisecond)

+ }

+ return nil

+ }}

+ leaser := lss3.NewLeaser()

+ leaser.SetClient(backend)

+ leaser.TTL = time.Second

+ started := time.Now()

+ lock, e := leaser.AcquireLease(context.Background())

+ if e != nil {

+ t.Fatal(e)

+ }

+ rn := startRenewer(leaser, lock, started, 100*time.Millisecond)

+ defer func() { leaser.ReleaseLease(context.Background(), rn.stop()) }()

+ initial, e := rn.SafeUntil()

+ if e != nil {

+ t.Fatal(e)

+ }

+ var began time.Time

+ select {

+ case began = <-entered:

+ case <-time.After(2 * time.Second):

+ t.Fatal("renewal never started")

+ }

+ timeout := time.Now().Add(time.Second)

+ for {

+ until, e := rn.SafeUntil()

+ if e != nil {

+ t.Fatal(e)

+ }

+ if until.After(initial) {

+ if until.After(began.Add(900 * time.Millisecond)) {

+ t.Fatal("renewal latency extended deadline")

+ }

+ if until == until.Round(0) {

+ t.Fatal("renewed deadline lost monotonic clock")

+ }

+ break

+ }

+ if time.Now().After(timeout) {

+ t.Fatal("renewal never completed")

+ }

+ time.Sleep(time.Millisecond)

+ }

+}

+

+type closeFailureReplica struct {

+ litestream.ReplicaClient

+ path string

+ writes int

+ failure error

+}

+

+func (c *closeFailureReplica) WriteLTXFile(ctx context.Context, level int, min, max ltx.TXID, r io.Reader) (*ltx.FileInfo, error) {

+ c.writes++

+ if c.writes > 1 {

+ return nil, c.failure

+ }

+ info, e := c.ReplicaClient.WriteLTXFile(ctx, level, min, max, r)

+ if e != nil {

+ return nil, e

+ }

+ // Test-only interference after the explicit sync's last upload gives the

+ // real Close a fresh tail to flush. Maintenance callers must prove stopped

+ // writers; this deliberately hostile fixture isolates Close error handling.

+ d, e := sql.Open("sqlite", c.path)

+ if e != nil {

+ return nil, e

+ }

+ _, e = d.Exec("INSERT INTO t(body) VALUES('late fixture tail')")

+ closeErr := d.Close()

+ if e != nil {

+ return nil, e

+ }

+ if closeErr != nil {

+ return nil, closeErr

+ }

+ return info, nil

+}

+func TestCheckpointChecksCloseAfterSuccessfulSync(t *testing.T) {

+ backend := &lockStore{}

+ h := localHeld(t, backend, 10*time.Second)

+ path := filepath.Join(t.TempDir(), "source.db")

+ seedDB(t, path)

+ sentinel := errors.New("synthetic final tail upload failure")

+ client := &closeFailureReplica{ReplicaClient: lsfile.NewReplicaClient(t.TempDir()), path: path, failure: sentinel}

+ h.replicaClient = client

+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)

+ defer cancel()

+ receipt, e := h.Checkpoint(ctx, path)

+ if !errors.Is(e, sentinel) || !receipt.CompletedAt.IsZero() || client.writes < 2 {

+ t.Fatalf("close error not preserved: %v writes=%d receipt=%+v", e, client.writes, receipt)

+ }

+ if e = h.rn.Err(); e != nil {

+ t.Fatalf("close failure stopped ownership: %v", e)

+ }

+}

+

+func TestRenewalFailureSignalsLostBeforeDeadline(t *testing.T) {

+ backend := &lockStore{beforePut: func(ctx context.Context, p *s3.PutObjectInput) error {

+ if p.IfMatch != nil {

+ return &smithy.GenericAPIError{Code: "PreconditionFailed"}

+ }

+ return nil

+ }}

+ h := localHeld(t, backend, 3*time.Second)

+ until, e := h.SafeUntil()

+ if e != nil {

+ t.Fatal(e)

+ }

+ loss := h.Lost()

+ await(t, loss)

+ if !time.Now().Before(until) {

+ t.Fatal("renewal refusal was signaled only at deadline")

+ }

+ if e = h.Err(); e == nil {

+ t.Fatal("loss notification without failed health")

+ }

+ if _, e = h.SafeUntil(); e == nil {

+ t.Fatal("loss notification left safe deadline usable")

+ }

+ if h.Lost() != loss {

+ t.Fatal("loss channel replaced")

+ }

+}

+

+func TestHealthyReleaseDoesNotSignalLost(t *testing.T) {

+ h := localHeld(t, &lockStore{}, 300*time.Millisecond)

+ until, e := h.SafeUntil()

+ if e != nil {

+ t.Fatal(e)

+ }

+ loss := h.Lost()

+ if e = h.Release(context.Background()); e != nil {

+ t.Fatal(e)

+ }

+ if _, e = h.SafeUntil(); !errors.Is(e, ErrSpent) {

+ t.Fatal("released handle remained usable")

+ }

+ time.Sleep(time.Until(until) + 30*time.Millisecond)

+ select {

+ case <-loss:

+ t.Fatal("normal release masqueraded as loss")

+ default:

+ }

+ if h.Err() == nil {

+ t.Fatal("spent handle remained healthy")

+ }

+ select {

+ case <-loss:

+ t.Fatal("late health check invented loss after release")

+ default:

+ }

+}

diff --git a/internal/lease/health.go b/internal/lease/health.go

new file mode 100644

index 000000000..69814f15c

--- /dev/null

+++ b/internal/lease/health.go

@@ -0,0 +1,170 @@

+package lease

+

+import (

+ "context"

+ "errors"

+ "sync"

+ "time"

+

+ "github.com/benbjohnson/litestream"

+ lss3 "github.com/benbjohnson/litestream/s3"

+)

+

+var (

+ ErrLost = errors.New("lease safety deadline elapsed")

+ ErrSpent = errors.New("lease handle is spent")

+ ErrBusy = errors.New("lease operation already running")

+)

+

+const terminationMargin = 5 * time.Second

+

+// Production uses the fixed margin. Short TTLs exist only through acquireTTL,

+// the package's private test seam.

+func marginForTTL(ttl time.Duration) time.Duration {

+ if ttl < 3*terminationMargin {

+ return ttl / 10

+ }

+ return terminationMargin

+}

+

+type renewer struct {

+ ctx context.Context

+ leaser *lss3.Leaser

+ mu sync.Mutex

+ lease *litestream.Lease

+ err error

+ until time.Time

+ margin time.Duration

+ stopped bool

+ cancel context.CancelFunc

+ done chan struct{}

+ watchDone chan struct{}

+ changed chan struct{}

+ lost chan struct{}

+}

+

+func startRenewer(leaser *lss3.Leaser, lease *litestream.Lease, started time.Time, margin time.Duration) *renewer {

+ ctx, cancel := context.WithCancel(context.Background())

+ rn := &renewer{ctx: ctx, leaser: leaser, lease: lease, until: started.Add(leaser.TTL - margin), margin: margin, cancel: cancel, done: make(chan struct{}), watchDone: make(chan struct{}), changed: make(chan struct{}, 1), lost: make(chan struct{})}

+ go rn.watch(ctx)

+ go rn.renew(ctx)

+ return rn

+}

+

+func (rn *renewer) loseLocked(err error) {

+ if rn.err == nil {

+ rn.err = err

+ close(rn.lost)

+ }

+}

+

+// checkLocked latches loss permanently; an acknowledgement arriving after a

+// blocked renewal cannot resurrect ownership, even before the timer runs.

+func (rn *renewer) checkLocked() error {

+ if rn.err != nil {

+ return rn.err

+ }

+ if rn.stopped {

+ return ErrSpent

+ }

+ if !time.Now().Before(rn.until) {

+ rn.loseLocked(ErrLost)

+ }

+ return rn.err

+}

+func (rn *renewer) Err() error {

+ rn.mu.Lock()

+ defer rn.mu.Unlock()

+ return rn.checkLocked()

+}

+func (rn *renewer) SafeUntil() (time.Time, error) {

+ rn.mu.Lock()

+ defer rn.mu.Unlock()

+ if err := rn.checkLocked(); err != nil {

+ return time.Time{}, err

+ }

+ return rn.until, nil

+}

+func (rn *renewer) watch(ctx context.Context) {

+ defer close(rn.watchDone)

+ for {

+ until, err := rn.SafeUntil()

+ if err != nil {

+ rn.cancel()

+ return

+ }

+ timer := time.NewTimer(time.Until(until))

+ select {

+ case <-ctx.Done():

+ timer.Stop()

+ return

+ case <-rn.changed:

+ timer.Stop()

+ case <-timer.C:

+ }

+}

+func (rn *renewer) renew(ctx context.Context) {

+ defer close(rn.done)

+ every := rn.leaser.TTL / 3

+ if every <= 0 {

+ every = time.Nanosecond

+ }

+ ticker := time.NewTicker(every)

+ defer ticker.Stop()

+ for {

+ select {

+ case <-ctx.Done():

+ return

+ case <-ticker.C:

+ rn.mu.Lock()

+ err := rn.checkLocked()

+ cur := rn.lease

+ rn.mu.Unlock()

+ if err != nil {

+ rn.cancel()

+ return

+ }

+ started := time.Now()

+ renewed, err := rn.leaser.RenewLease(ctx, cur)

+ rn.mu.Lock()

+ health := rn.checkLocked()

+ // Retain the newest ETag solely for conditional cleanup, including a late

+ // success. It never repairs a latched loss or extends a stopped handle.

+ if err == nil {

+ rn.lease = renewed

+ }

+ if health == nil {

+ if err != nil {

+ rn.loseLocked(err)

+ } else {

+ rn.until = started.Add(rn.leaser.TTL - rn.margin)

+ rn.checkLocked()

+ }

+ failed := rn.err != nil || rn.stopped

+ rn.mu.Unlock()

+ if failed {

+ rn.cancel()

+ return

+ }

+ select {

+ case rn.changed <- struct{}{}:

+ default:

+ }

+}

+

+func (rn *renewer) stop() *litestream.Lease {

+ rn.mu.Lock()

+ rn.checkLocked()

+ rn.stopped = true

+ rn.mu.Unlock()

+ rn.cancel()

+ <-rn.done

+ <-rn.watchDone

+ rn.mu.Lock()

+ defer rn.mu.Unlock()

+ return rn.lease

+}

diff --git a/internal/lease/lease.go b/internal/lease/lease.go

index 3a108ed81..8755c2ed3 100644

--- a/internal/lease/lease.go

+++ b/internal/lease/lease.go

@@ -92,80 +92,6 @@ func (e *HeldError) Error() string {

return fmt.Sprintf("lease already held until %s", e.ExpiresAt.Format(time.RFC3339))

}

-// renewer keeps a held lease alive on a ticker. A failed renewal is recorded

-// and renewal stops — leadership is lost and the owner must stand down;

-// retrying could fence with a new claimant who took over after expiry.

-type renewer struct {

- leaser *lss3.Leaser

-

- mu sync.Mutex

- lease *litestream.Lease

- err error // set if a renewal ever fails (leadership lost)

-

- cancel context.CancelFunc

- done chan struct{}

-}

-

-func startRenewer(leaser *lss3.Leaser, lease *litestream.Lease) *renewer {

- ctx, cancel := context.WithCancel(context.Background())

- rn := &renewer{leaser: leaser, lease: lease, cancel: cancel, done: make(chan struct{})}

- // Renew at a third of the lease's OWN TTL, not the package constant: the

- // leaser is the one that stamps expiry (leaser.TTL), so reading the

- // interval off it keeps renewal and expiry in step for any TTL. In

- // production leaser.TTL is always LeaseTTL, so this is LeaseTTL/3 as

- // before; under a test's short TTL it is what makes "stopped renewing"

- // distinguishable from "never renewed at all". The fallback only guards

- // NewTicker's panic on a non-positive duration.

- renewEvery := leaser.TTL / 3

- if renewEvery <= 0 {

- renewEvery = LeaseTTL / 3

- }

- go func() {

- defer close(rn.done)

- t := time.NewTicker(renewEvery)

- defer t.Stop()

- for {

- select {

- case <-ctx.Done():

- return

- case <-t.C:

- rn.mu.Lock()

- cur := rn.lease

- rn.mu.Unlock()

- renewed, err := rn.leaser.RenewLease(ctx, cur)

- rn.mu.Lock()

- if err != nil {

- rn.err = err

- } else {

- rn.lease = renewed

- }

- rn.mu.Unlock()

- if err != nil {

- return

- }

- }()

- return rn

-}

-

-// Err reports whether a renewal has failed (leadership lost).

-func (rn *renewer) Err() error {

- rn.mu.Lock()

- defer rn.mu.Unlock()

- return rn.err

-}

-

-// stop halts renewal and returns the most recent lease (for release, or to

-// hand renewal duty to a successor renewer).

-func (rn *renewer) stop() *litestream.Lease {

- rn.cancel()

- <-rn.done

- rn.mu.Lock()

- defer rn.mu.Unlock()

- return rn.lease

-}

-

// Held is an acquired single-writer lease that is not yet coupled to

// replication. The wake path needs this split: the DB restore must happen

// *under* the lock (restoring without it could later shadow newer S3 state

@@ -173,9 +99,14 @@ func (rn *renewer) stop() *litestream.Lease {

// file exists. Renewal runs from acquisition, so a slow restore can't silently

// outlive the TTL.

type Held struct {

- leaser *lss3.Leaser

- region string // the leaser's client keeps its region private; carry it for Replicate

- rn *renewer

+ leaser *lss3.Leaser

+ region string // the leaser's client keeps its region private; carry it for Replicate

+ rn *renewer

+ mu sync.Mutex

+ busy bool

+ spent bool

+ poison error

+ replicaClient litestream.ReplicaClient // private local-replica test seam

}

// Acquire claims the single-writer lease at s3://bucket/prefix/lock.json

@@ -212,6 +143,7 @@ func acquireTTL(ctx context.Context, bucket, prefix, region string, ttl time.Dur

leaser.Path = prefix // lock lives at <prefix>/lock.json, colocated with the replica

leaser.TTL = ttl

+ started := time.Now()

lease, err := leaser.AcquireLease(ctx)

if err != nil {

// A *litestream.LeaseExistsError means someone else holds it —

@@ -222,15 +154,86 @@ func acquireTTL(ctx context.Context, bucket, prefix, region string, ttl time.Dur

}

return nil, fmt.Errorf("acquire lease: %w", err)

}

- return &Held{leaser: leaser, region: region, rn: startRenewer(leaser, lease)}, nil

+ rn := startRenewer(leaser, lease, started, marginForTTL(ttl))

+ if err := rn.Err(); err != nil {

+ cur := rn.stop()

+ relCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)

+ defer cancel()

+ _ = leaser.ReleaseLease(relCtx, cur)

+ return nil, err

+ }

+ return &Held{leaser: leaser, region: region, rn: rn}, nil

}

-// Err reports whether lease renewal has failed (leadership lost).

-func (h *Held) Err() error { return h.rn.Err() }

+// Err refuses a spent or failed handle as well as lost ownership.

+func (h *Held) Err() error {

+ h.mu.Lock()

+ defer h.mu.Unlock()

+ if h.spent {

+ return ErrSpent

+ }

+ if h.poison != nil {

+ return h.poison

+ }

+ return h.rn.Err()

+}

-// Release stops renewal and releases the lease — the bail-out when a wake

-// fails between acquisition and replication.

+// Lost closes once on renewal failure or the safety deadline. Healthy transfer

+// and explicit release do not close it. A worker must recheck Err/SafeUntil and

+// prove its process tree stopped before the deadline; notification is not proof.

+func (h *Held) Lost() <-chan struct{} { return h.rn.lost }

+

+// SafeUntil retains Go's monotonic clock component. It must not be serialized

+// and later treated as ownership evidence; callers must keep checking health.

+func (h *Held) SafeUntil() (time.Time, error) {

+ h.mu.Lock()

+ defer h.mu.Unlock()

+ if h.spent {

+ return time.Time{}, ErrSpent

+ }

+ if h.poison != nil {

+ return time.Time{}, h.poison

+ }

+ return h.rn.SafeUntil()

+}

+

+func (h *Held) begin(release bool) error {

+ h.mu.Lock()

+ defer h.mu.Unlock()

+ if h.spent {

+ return ErrSpent

+ }

+ if h.busy {

+ return ErrBusy

+ }

+ if !release {

+ if h.poison != nil {

+ return h.poison

+ }

+ if err := h.rn.Err(); err != nil {

+ return err

+ }

+ h.busy = true

+ return nil

+}

+func (h *Held) finish(spent bool, poison error) {

+ h.mu.Lock()

+ defer h.mu.Unlock()

+ h.busy = false

+ h.spent = spent

+ if poison != nil {

+ h.poison = poison

+ }

+}

+

+// Release is the only operation allowed after a failed checkpoint. A failed

+// remote release still spends this handle; it cannot authorize more work.

func (h *Held) Release(ctx context.Context) error {

+ if err := h.begin(true); err != nil {

+ return err

+ }

+ defer h.finish(true, nil)

lease := h.rn.stop()

if lease == nil {

return nil

@@ -252,36 +255,47 @@ func (h *Held) Release(ctx context.Context) error {

// litestream's 1s default (the stream class): idle ticks make zero S3 calls,

// so stream's request bill is change-driven, not clock-driven.

func (h *Held) Replicate(ctx context.Context, dbPath string, syncInterval time.Duration) (*Replicator, error) {

+ if err := h.begin(false); err != nil {

+ return nil, err

+ }

+ transferred := false

+ defer func() { h.finish(transferred, nil) }()

+ opCtx, cancel := context.WithCancel(ctx)

+ stop := context.AfterFunc(h.rn.ctx, cancel)

+ defer stop()

+ defer cancel()

+ store, db := h.newStore(dbPath, syncInterval)

+ if err := store.Open(opCtx); err != nil {

+ cancel()

+ closeErr := store.Close(opCtx)

+ return nil, errors.Join(fmt.Errorf("open litestream store: %w", err), closeErr)

+ }

if err := h.rn.Err(); err != nil {

- return nil, fmt.Errorf("lease already lost: %w", err)

+ cancel()

+ closeErr := store.Close(opCtx)

+ return nil, errors.Join(err, closeErr)

}

+ transferred = true

+ return &Replicator{leaser: h.leaser, store: store, db: db, rn: h.rn}, nil

+}

+func (h *Held) newStore(dbPath string, syncInterval time.Duration) (*litestream.Store, *litestream.DB) {

db := litestream.NewDB(dbPath)

- // The activator's replication state lives in its OWN meta directory —

- // litestream derives ".<db>-litestream" beside the DB by default, which any

- // other litestream process over the same file (the box-wide backup

- // replicator, before or after a handover) also uses. Sharing corrupts LTX

- // state even SEQUENTIALLY: the fleet-wide flip left a dozen hosts in an

- // error loop until their meta dirs were cleared by hand. A distinct

- // namespace ends the class.

+ // Preserve the activator namespace: sharing the default metadata directory

+ // with another replicator corrupts shadow state even across sequential runs.

db.SetMetaPath(filepath.Join(filepath.Dir(dbPath), ".activator-"+filepath.Base(dbPath)+"-litestream"))

db.MonitorInterval = dbMonitorInterval

- client := newReplicaClient(h.leaser.Bucket, h.leaser.Path, h.region)

+ client := h.replicaClient

+ if client == nil {

+ client = newReplicaClient(h.leaser.Bucket, h.leaser.Path, h.region)

+ }

db.Replica = litestream.NewReplicaWithClient(db, client)

if syncInterval > 0 {

db.Replica.SyncInterval = syncInterval

}

-

store := litestream.NewStore([]*litestream.DB{db}, compactionLevels)

store.L0RetentionCheckInterval = l0RetentionCheckInterval

- if err := store.Open(ctx); err != nil {

- return nil, fmt.Errorf("open litestream store: %w", err)

- }

-

- // Hand renewal to the replicator. The handoff gap between stopping one

- // renewer and starting the next is microseconds against a 30s TTL.

- lease := h.rn.stop()

- return &Replicator{leaser: h.leaser, store: store, db: db, rn: startRenewer(h.leaser, lease)}, nil

+ return store, db

}

// Replicator couples an exclusive S3 lease with a running litestream

@@ -289,10 +303,15 @@ func (h *Held) Replicate(ctx context.Context, dbPath string, syncInterval time.D

// acquired *before* replication starts and renewed on a ticker; Close stops

// replication and releases the lease.

type Replicator struct {

- leaser *lss3.Leaser

- store *litestream.Store

- db *litestream.DB

- rn *renewer

+ leaser *lss3.Leaser

+ store *litestream.Store

+ db *litestream.DB

+ rn *renewer

+ mu sync.Mutex

+ busy bool

+ spent bool

+ poison error

+ replicaClient litestream.ReplicaClient // private local-replica test seam

}

// ReplicateWithLease acquires the single-writer lease at

@@ -324,6 +343,12 @@ func (r *Replicator) SyncAndWait(ctx context.Context) error {

// Err reports whether lease renewal has failed (leadership lost).

func (r *Replicator) Err() error { return r.rn.Err() }

+func (r *Replicator) SafeUntil() (time.Time, error) { return r.rn.SafeUntil() }

+

+// Lost is the same loss notification channel held before replication began.

+// Normal Close does not signal loss; SafeUntil refuses the stopped handle.

+func (r *Replicator) Lost() <-chan struct{} { return r.rn.lost }

+

// Abandon stops lease renewal and replication WITHOUT the final sync or the

// lease release — the wreckage a crashed box leaves behind: lock.json simply

// expires on its TTL, and the replica holds whatever was last synced. It

diff --git a/internal/maintenance/result.go b/internal/maintenance/result.go

new file mode 100644

index 000000000..6374ad256

--- /dev/null

+++ b/internal/maintenance/result.go

@@ -0,0 +1,90 @@

+package maintenance

+

+import (

+ "context"

+ "crypto/ecdh"

+ "encoding/base64"

+ "errors"

+ "io"

+ "os"

+ "time"

+)

+

+const MaxResultBytes = 64 << 10

+const MaxExecutionDuration = 10 * time.Minute

+

+var (

+ ErrResult = errors.New("maintenance: private result unavailable")

+ ErrResultOverflow = errors.New("maintenance: private result exceeds limit")

+)

+

+// CaptureResult takes ownership of a dedicated stdout read pipe and closes it

+// on every return. It returns complete bytes only after EOF within the deadline.

+// Closing this pipe does not prove the command or its descendants have stopped;

+// the worker must separately check Wait and containment before publishing success.

+func CaptureResult(ctx context.Context, pipe *os.File) ([]byte, error) {

+ if pipe == nil {

+ return nil, ErrResult

+ }

+ defer pipe.Close()

+ if ctx == nil {

+ return nil, ErrResult

+ }

+ deadline, ok := ctx.Deadline()

+ remaining := time.Until(deadline)

+ if !ok || remaining <= 0 || remaining > MaxExecutionDuration || ctx.Err() != nil {

+ return nil, ErrResult

+ }

+ info, err := pipe.Stat()

+ if err != nil || info.Mode()&os.ModeNamedPipe == 0 {

+ return nil, ErrResult

+ }

+ stop := context.AfterFunc(ctx, func() { _ = pipe.Close() })

+ defer stop()

+ body, err := io.ReadAll(io.LimitReader(pipe, MaxResultBytes+1))

+ if err != nil || ctx.Err() != nil {

+ clear(body)

+ return nil, ErrResult

+ }

+ if len(body) > MaxResultBytes {

+ clear(body)

+ return nil, ErrResultOverflow

+ }

+ if err := pipe.Close(); err != nil || ctx.Err() != nil {

+ clear(body)

+ return nil, ErrResult

+ }

+ return body, nil

+}

+

+func SealResult(pub *ecdh.PublicKey, binding Binding, body []byte) (string, error) {

+ if binding.Direction != DirectionResult || len(body) > MaxResultBytes {

+ return "", ErrResult

+ }

+ sealed, err := Seal(pub, binding, body)

+ if err != nil {

+ return "", ErrResult

+ }

+ return sealed, nil

+}

+

+// OpenResult decrypts bytes for the recipient. Encryption alone does not prove

+// which worker produced them; callers must verify the authenticated operation

+// outcome before treating these bytes as execution or completion evidence.

+func OpenResult(priv *ecdh.PrivateKey, binding Binding, sealed string) ([]byte, error) {

+ const overhead = len(envelopePrefix) + ephemeralBytes + nonceBytes + tagBytes

+ if binding.Direction != DirectionResult || len(sealed) > base64.StdEncoding.EncodedLen(overhead+MaxResultBytes) {

+ return nil, ErrResult

+ }

+ body, err := Open(priv, binding, sealed)

+ if err != nil {

+ return nil, ErrResult

+ }

+ // The decoded bound is necessary even after the encoded-length check:

+ // base64 padding can give adjacent raw lengths the same encoded length.

+ if len(body) > MaxResultBytes {

+ clear(body)

+ return nil, ErrResult

+ }

+ return body, nil

+}

diff --git a/internal/maintenance/result_test.go b/internal/maintenance/result_test.go

new file mode 100644

index 000000000..5d6d83521

--- /dev/null

+++ b/internal/maintenance/result_test.go

@@ -0,0 +1,195 @@

+package maintenance

+

+import (

+ "bytes"

+ "context"

+ "crypto/ecdh"

+ "crypto/rand"

+ "os"

+ "strconv"

+ "testing"

+ "time"

+)

+

+func resultPipe(t *testing.T) (*os.File, *os.File) {

+ t.Helper()

+ r, w, err := os.Pipe()

+ if err != nil {

+ t.Fatal(err)

+ }

+ t.Cleanup(func() { _ = r.Close(); _ = w.Close() })

+ return r, w

+}

+func resultContext(t *testing.T) context.Context {

+ t.Helper()

+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)

+ t.Cleanup(cancel)

+ return ctx

+}

+

+func TestCaptureResultCompleteBoundaries(t *testing.T) {

+ for _, n := range []int{0, 1, MaxResultBytes} {

+ t.Run(strconv.Itoa(n), func(t *testing.T) {

+ r, w := resultPipe(t)

+ body := bytes.Repeat([]byte{'x'}, n)

+ done := make(chan error, 1)

+ go func() { _, e := w.Write(body); _ = w.Close(); done <- e }()

+ got, err := CaptureResult(resultContext(t), r)

+ if err != nil || !bytes.Equal(got, body) {

+ t.Fatalf("size%d capture failed: %v", n, err)

+ }

+ if err := <-done; err != nil {

+ t.Fatal(err)

+ }

+ if _, err := r.Stat(); err == nil {

+ t.Fatal("capture retained its owned read pipe")

+ }

+ })

+ }

+}

+

+func TestCaptureResultOverflowWithoutEOF(t *testing.T) {

+ r, w := resultPipe(t)

+ done := make(chan error, 1)

+ go func() { _, err := w.Write(bytes.Repeat([]byte{'x'}, MaxResultBytes+1)); done <- err }()

+ body, err := CaptureResult(resultContext(t), r)

+ if body != nil || err != ErrResultOverflow {

+ t.Fatalf("overflow = %d bytes, %v", len(body), err)

+ }

+ if err := <-done; err != nil {

+ t.Fatal(err)

+ }

+ // The write end is deliberately still open: overflow cannot require EOF.

+ if _, err := w.Stat(); err != nil {

+ t.Fatal("writer closed before overflow observation")

+ }

+}

+

+func TestCaptureResultCancellationDiscardsPartialBytes(t *testing.T) {

+ r, w := resultPipe(t)

+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)

+ defer cancel()

+ type answer struct {

+ body []byte

+ err error

+ }

+ done := make(chan answer, 1)

+ go func() { b, e := CaptureResult(ctx, r); done <- answer{b, e} }()

+ if _, err := w.Write([]byte("synthetic-private-fragment")); err != nil {

+ t.Fatal(err)

+ }

+ cancel()

+ select {

+ case got := <-done:

+ if got.body != nil || got.err != ErrResult {

+ t.Fatal("cancel returned partial bytes or success")

+ }

+ case <-time.After(time.Second):

+ t.Fatal("cancellation did not unblock the actual pipe")

+ }

+ if _, err := r.Stat(); err == nil {

+ t.Fatal("cancel retained read descriptor")

+ }

+ if _, err := w.Stat(); err != nil {

+ t.Fatal("capture claimed to terminate independent writer")

+ }

+}

+

+func TestCaptureResultRequiresEOFBeforeDeadline(t *testing.T) {

+ r, _ := resultPipe(t)

+ ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)

+ defer cancel()

+ done := make(chan error, 1)

+ go func() {

+ b, e := CaptureResult(ctx, r)

+ if b != nil {

+ done <- nil

+ return

+ }

+ done <- e

+ }()

+ select {

+ case err := <-done:

+ if err != ErrResult {

+ t.Fatal("open writer accepted without EOF")

+ }

+ case <-time.After(time.Second):

+ t.Fatal("deadline did not close pipe")

+ }

+}

+

+func TestCaptureResultRefusesUnboundedOrNonpipeInput(t *testing.T) {

+ long, cancelLong := context.WithTimeout(context.Background(), MaxExecutionDuration+time.Minute)

+ defer cancelLong()

+ expired, cancelExpired := context.WithCancel(resultContext(t))

+ cancelExpired()

+ for _, ctx := range []context.Context{nil, context.Background(), long, expired} {

+ r, _ := resultPipe(t)

+ if b, e := CaptureResult(ctx, r); b != nil || e != ErrResult {

+ t.Fatal("invalid context accepted")

+ }

+ if _, e := r.Stat(); e == nil {

+ t.Fatal("refusal retained read pipe")

+ }

+ f, e := os.CreateTemp(t.TempDir(), "regular")

+ if e != nil {

+ t.Fatal(e)

+ }

+ if b, e := CaptureResult(resultContext(t), f); b != nil || e != ErrResult {

+ t.Fatal("regular file accepted as private stdout")

+ }

+ if b, e := CaptureResult(resultContext(t), nil); b != nil || e != ErrResult {

+ t.Fatal("nil pipe accepted")

+ }

+}

+

+func TestResultEnvelopePurposeAndLimits(t *testing.T) {

+ key, err := ecdh.P256().GenerateKey(rand.Reader)

+ if err != nil {

+ t.Fatal(err)

+ }

+ fingerprint, err := RecipientFingerprint(key.PublicKey())

+ if err != nil {

+ t.Fatal(err)

+ }

+ binding := Binding{Version: 1, Direction: DirectionResult, Account: "account", App: "home", Host: "test.example", RequestID: "request", AttemptID: "attempt", Recipient: fingerprint}

+ for _, n := range []int{0, 1, MaxResultBytes} {

+ body := bytes.Repeat([]byte{'r'}, n)

+ sealed, err := SealResult(key.PublicKey(), binding, body)

+ if err != nil {

+ t.Fatal(err)

+ }

+ got, err := OpenResult(key, binding, sealed)

+ if err != nil || !bytes.Equal(body, got) {

+ t.Fatalf("result roundtrip size%d: %v", n, err)

+ }

+ tooLarge := bytes.Repeat([]byte{'r'}, MaxResultBytes+1)

+ if _, err := SealResult(key.PublicKey(), binding, tooLarge); err != ErrResult {

+ t.Fatal("oversized seal accepted")

+ }

+ // Generic crypto accepts this size. Base64's rounding also fits the result

+ // encoded-length bound, so the separate decoded-length check must reject it.

+ sealed, err := Seal(key.PublicKey(), binding, tooLarge)

+ if err != nil {

+ t.Fatal(err)

+ }

+ if b, err := OpenResult(key, binding, sealed); b != nil || err != ErrResult {

+ t.Fatal("oversized decoded result accepted")

+ }

+ for _, direction := range []Direction{DirectionInput, DirectionEnvironment} {

+ wrong := binding

+ wrong.Direction = direction

+ if _, err := SealResult(key.PublicKey(), wrong, []byte("ok")); err != ErrResult {

+ t.Fatal("wrong seal purpose accepted")

+ }

+ sealed, err := Seal(key.PublicKey(), wrong, []byte("ok"))

+ if err != nil {

+ t.Fatal(err)

+ }

+ if b, err := OpenResult(key, wrong, sealed); b != nil || err != ErrResult {

+ t.Fatal("wrong open purpose accepted")

+ }

+}

diff --git a/internal/maintenanceguard/guard.go b/internal/maintenanceguard/guard.go

new file mode 100644

index 000000000..0db0bc804

--- /dev/null

+++ b/internal/maintenanceguard/guard.go

@@ -0,0 +1,174 @@

+package maintenanceguard

+

+import (

+ "context"

+ "errors"

+ "math"

+ "os"

+ "slices"

+)

+

+type GuardSnapshot struct {

+ repository *Repository

+ value Guard

+ etag string

+ exists bool

+}

+

+func (s GuardSnapshot) Exists() bool { return s.exists }

+func (s GuardSnapshot) Value() Guard { return cloneGuard(s.value) }

+func (s GuardSnapshot) Projection() (Projection, error) {

+ if !s.exists {

+ return Projection{}, ErrInvalid

+ }

+ raw, e := guardBytes(s.value)

+ if e != nil {

+ return Projection{}, e

+ }

+ return Projection{Version, s.value.Generation, digest(raw)}, nil

+}

+func (r *Repository) ReadGuard(ctx context.Context, host string) (GuardSnapshot, error) {

+ key, e := GuardKey(host)

+ if e != nil {

+ return GuardSnapshot{}, e

+ }

+ raw, etag, e := r.store.Get(ctx, key)

+ if errors.Is(e, os.ErrNotExist) {

+ return GuardSnapshot{repository: r}, nil

+ }

+ if e != nil {

+ return GuardSnapshot{}, e

+ }

+ var g Guard

+ if decode(raw, &g) != nil || !g.valid() || g.Identity.Host != host || !cleanETag(etag) {

+ return GuardSnapshot{}, ErrInvalid

+ }

+ if e = r.checkGuardReferences(ctx, g); e != nil {

+ return GuardSnapshot{}, e

+ }

+ return GuardSnapshot{r, cloneGuard(g), etag, true}, nil

+}

+func (r *Repository) guardClaim(ctx context.Context, c ClaimSnapshot, g Guard) error {

+ if c.value.Kind != MaintenanceOperation || c.value.Account != g.Identity.Account || c.value.App != g.Identity.App || c.value.OperationID != g.OperationID {

+ return ErrConflict

+ }

+ return r.currentClaim(ctx, c)

+}

+func (r *Repository) writeGuard(ctx context.Context, c ClaimSnapshot, g Guard, etag string) (GuardSnapshot, error) {

+ if e := r.guardClaim(ctx, c, g); e != nil {

+ return GuardSnapshot{}, e

+ }

+ raw, e := guardBytes(g)

+ if e != nil {

+ return GuardSnapshot{}, e

+ }

+ if e = r.checkGuardReferences(ctx, g); e != nil {

+ return GuardSnapshot{}, e

+ }

+ key, _ := GuardKey(g.Identity.Host)

+ next, e := r.store.Put(ctx, key, raw, etag)

+ if e != nil {

+ return GuardSnapshot{}, e

+ }

+ if !cleanETag(next) {

+ return GuardSnapshot{}, ErrUnavailable

+ }

+ // There is no cross-object atomicity. A lost claim leaves the durable guard

+ // in place, but the caller receives no successful execution/resume handoff.

+ if e = r.currentClaim(ctx, c); e != nil {

+ return GuardSnapshot{}, e

+ }

+ return GuardSnapshot{r, cloneGuard(g), next, true}, nil

+}

+func (r *Repository) CreateGuard(ctx context.Context, c ClaimSnapshot, g Guard) (GuardSnapshot, error) {

+ if !g.valid() || g.Generation != 1 || g.Phase != Requested || g.Evidence != nil {

+ return GuardSnapshot{}, ErrInvalid

+ }

+ return r.writeGuard(ctx, c, g, "")

+}

+func newAttempt(old, next Guard) bool {

+ return old.Request.ID != next.Request.ID && old.Attempt.ID != next.Attempt.ID && next.Evidence == nil

+}

+func samePins(old, next Guard) bool {

+ return old.RecordRevision == next.RecordRevision && old.ArtifactVersion == next.ArtifactVersion && old.ArtifactDigest == next.ArtifactDigest && old.ConfigDigest == next.ConfigDigest && slices.Equal(old.RequiredContracts, next.RequiredContracts)

+}

+func validAdvance(old, next Guard) bool {

+ if !next.valid() || old.Generation == math.MaxInt64 || next.Generation != old.Generation+1 || next.Identity != old.Identity || next.OwnerBox != old.OwnerBox || !containsAll(next.RequiredContracts, old.RequiredContracts) {

+ return false

+ }

+ if old.Phase == Resumed {

+ return next.Phase == Requested && next.OperationID != old.OperationID && newAttempt(old, next)

+ }

+ if next.OperationID != old.OperationID {

+ return false

+ }

+ if next.Phase == Requested {

+ return (old.Phase == Succeeded || old.Phase == Failed || old.Phase == Ambiguous) && newAttempt(old, next)

+ }

+ if next.Phase == ResumeRequested {

+ return (old.Phase == Fenced || old.Phase == Succeeded || old.Phase == Failed || old.Phase == Ambiguous) && newAttempt(old, next)

+ }

+ if old.Request != next.Request || old.Attempt != next.Attempt || !samePins(old, next) {

+ return false

+ }

+ switch old.Phase {

+ case Requested:

+ return next.Phase == Quiescing || next.Phase == Failed || next.Phase == Ambiguous

+ case Quiescing:

+ return next.Phase == Fenced || next.Phase == Failed || next.Phase == Ambiguous

+ case Fenced:

+ return next.Phase == Executing || next.Phase == Failed || next.Phase == Ambiguous

+ case Executing:

+ return next.Phase == Succeeded || next.Phase == Failed || next.Phase == Ambiguous

+ case ResumeRequested:

+ return next.Phase == Resumed || next.Phase == Failed || next.Phase == Ambiguous

+ }

+ return false

+}

+

+// AdvanceGuard validates continuity and state progression, then performs one

+// CAS. Fresh owner approval, stopped-process/lease proofs and checker semantics

+// remain caller preconditions; valid references alone do not prove them.

+func (r *Repository) AdvanceGuard(ctx context.Context, c ClaimSnapshot, old GuardSnapshot, next Guard) (GuardSnapshot, error) {

+ if old.repository != r || !old.exists || !cleanETag(old.etag) || !validAdvance(old.value, next) {

+ return GuardSnapshot{}, ErrConflict

+ }

+ return r.writeGuard(ctx, c, next, old.etag)

+}

+

+type ServingDecision string

+

+const (

+ Unmanaged ServingDecision = "unmanaged"

+ Managed ServingDecision = "managed"

+ Denied ServingDecision = "denied"

+)

+

+// CheckServing is a denial check, not a lease or process-health permit. The

+// caller supplies contracts from an already verified signed release manifest,

+// verifies the current record identity, and repeats this check after lease

+// acquisition before starting or seeding anything. Legacy unmanaged records may

+// omit Record: both barriers must be absent before that omission is accepted.

+func (r *Repository) CheckServing(ctx context.Context, identity Identity, projection *Projection, verifiedContracts []string) (ServingDecision, error) {

+ if !identity.locationValid() || !ValidContracts(verifiedContracts) {

+ return Denied, ErrDenied

+ }

+ s, e := r.ReadGuard(ctx, identity.Host)

+ if e != nil {

+ return Denied, ErrDenied

+ }

+ if !s.exists {

+ if projection == nil {

+ return Unmanaged, nil

+ }

+ return Denied, ErrDenied

+ }

+ if !identity.valid() || projection == nil || !projection.valid() || s.value.Identity != identity || s.value.Phase != Resumed || !containsAll(verifiedContracts, s.value.RequiredContracts) {

+ return Denied, ErrDenied

+ }

+ expected, e := s.Projection()

+ if e != nil || *projection != expected {

+ return Denied, ErrDenied

+ }

+ return Managed, nil

+}

diff --git a/internal/maintenanceguard/guard_test.go b/internal/maintenanceguard/guard_test.go

new file mode 100644

index 000000000..92a8712ce

--- /dev/null

+++ b/internal/maintenanceguard/guard_test.go

@@ -0,0 +1,337 @@

+package maintenanceguard_test

+

+import (

+ "bytes"

+ "context"

+ "crypto/sha256"

+ "encoding/base64"

+ "encoding/hex"

+ "errors"

+ "fmt"

+ "os"

+ "sync"

+ "testing"

+ "time"

+

+ guard "github.com/carlosframework/platform/internal/maintenanceguard"

+ "github.com/carlosframework/platform/internal/release"

+)

+

+var ctx = context.Background()

+

+func id(b byte) string { return base64.RawURLEncoding.EncodeToString(bytes.Repeat([]byte{b}, 32)) }

+func hash(s string) string { x := sha256.Sum256([]byte(s)); return hex.EncodeToString(x[:]) }

+

+type fixture struct {

+ store *release.DirStore

+ repo *guard.Repository

+ identity guard.Identity

+}

+

+func setup(t *testing.T) fixture {

+ t.Helper()

+ s := release.NewDirStore(t.TempDir())

+ identity, e := guard.RecordIdentity("aaa", "home", "one.home.example", time.Date(2026, 9, 8, 0, 0, 0, 1, time.UTC))

+ if e != nil {

+ t.Fatal(e)

+ }

+ return fixture{s, guard.New(s), guard.Identity{Account: "aaa", App: "home", Host: "one.home.example", Record: identity}}

+}

+func (f fixture) ref(t *testing.T, kind guard.ReferenceKind, n byte) guard.Reference {

+ t.Helper()

+ r, e := f.repo.PutReference(ctx, f.identity.Host, kind, id(n), []byte(fmt.Sprintf(`{"metadata":"%s-%d"}`, kind, n)))

+ if e != nil {

+ t.Fatal(e)

+ }

+ return r

+}

+func (f fixture) claim(t *testing.T, n byte) guard.ClaimSnapshot {

+ t.Helper()

+ s, e := f.repo.AcquireClaim(ctx, "aaa", "home", id(n), guard.MaintenanceOperation)

+ if e != nil {

+ t.Fatal(e)

+ }

+ return s

+}

+func (f fixture) initial(t *testing.T, c guard.ClaimSnapshot) guard.Guard {

+ t.Helper()

+ return guard.Guard{Version: 1, Identity: f.identity, Generation: 1, Phase: guard.Requested, OperationID: c.Value().OperationID, OwnerBox: "box-one", RecordRevision: `"original-record-etag"`, Request: f.ref(t, guard.RequestReference, 2), Attempt: f.ref(t, guard.AttemptReference, 3), ArtifactVersion: "v1", ArtifactDigest: hash("artifact"), ConfigDigest: hash("randomized-config-envelope"), RequiredContracts: []string{}}

+}

+func (f fixture) create(t *testing.T, c guard.ClaimSnapshot) guard.GuardSnapshot {

+ t.Helper()

+ g, e := f.repo.CreateGuard(ctx, c, f.initial(t, c))

+ if e != nil {

+ t.Fatal(e)

+ }

+ return g

+}

+func (f fixture) step(t *testing.T, c guard.ClaimSnapshot, s guard.GuardSnapshot, phase guard.Phase) guard.GuardSnapshot {

+ t.Helper()

+ g := s.Value()

+ g.Generation++

+ g.Phase = phase

+ if phase == guard.ResumeRequested {

+ g.Request = f.ref(t, guard.RequestReference, 4)

+ g.Attempt = f.ref(t, guard.AttemptReference, 5)

+ g.RequiredContracts = []string{"home.operator.v1"}

+ g.Evidence = nil

+ }

+ if phase == guard.Succeeded || phase == guard.Resumed {

+ ref := f.ref(t, guard.ReceiptReference, byte(g.Generation+10))

+ g.Evidence = &ref

+ }

+ next, e := f.repo.AdvanceGuard(ctx, c, s, g)

+ if e != nil {

+ t.Fatal(phase, e)

+ }

+ return next

+}

+func (f fixture) resumed(t *testing.T, c guard.ClaimSnapshot) guard.GuardSnapshot {

+ s := f.create(t, c)

+ for _, p := range []guard.Phase{guard.Quiescing, guard.Fenced, guard.Executing, guard.Succeeded, guard.ResumeRequested, guard.Resumed} {

+ s = f.step(t, c, s, p)

+ }

+ return s

+}

+func denied(t *testing.T, f fixture, p *guard.Projection, contracts []string) {

+ t.Helper()

+ d, e := f.repo.CheckServing(ctx, f.identity, p, contracts)

+ if d != guard.Denied || !errors.Is(e, guard.ErrDenied) {

+ t.Fatal(d, e)

+ }

+}

+

+func TestGuardUnmanagedAndAbsenceRecheck(t *testing.T) {

+ f := setup(t)

+ if d, e := f.repo.CheckServing(ctx, f.identity, nil, nil); e != nil || d != guard.Unmanaged {

+ t.Fatal(d, e)

+ }

+ p := guard.Projection{Version: 1, Generation: 1, GuardDigest: hash("guard")}

+ denied(t, f, &p, nil)

+ c := f.claim(t, 1)

+ // A claim alone is not a host fence. This models the crash before guard creation.

+ if d, e := f.repo.CheckServing(ctx, f.identity, nil, nil); e != nil || d != guard.Unmanaged {

+ t.Fatal(d, e)

+ }

+ s := f.create(t, c)

+ p, e := s.Projection()

+ if e != nil {

+ t.Fatal(e)

+ }

+ // A worker that observed absence before acquiring its lease must recheck.

+ denied(t, f, nil, nil)

+ denied(t, f, &p, nil)

+}

+func TestGuardResumeCrashProjectionAndContracts(t *testing.T) {

+ f := setup(t)

+ c := f.claim(t, 1)

+ s := f.create(t, c)

+ for _, phase := range []guard.Phase{guard.Quiescing, guard.Fenced, guard.Executing, guard.Succeeded, guard.ResumeRequested} {

+ s = f.step(t, c, s, phase)

+ p, _ := s.Projection()

+ denied(t, f, &p, []string{"home.operator.v1"})

+ }

+ oldProjection, _ := s.Projection()

+ s = f.step(t, c, s, guard.Resumed)

+ // Durable guard publication without the record CAS must not open admission.

+ denied(t, f, &oldProjection, []string{"home.operator.v1"})

+ denied(t, f, nil, []string{"home.operator.v1"})

+ reopened := guard.New(f.store)

+ loaded, e := reopened.ReadGuard(ctx, f.identity.Host)

+ if e != nil {

+ t.Fatal(e)

+ }

+ p, _ := loaded.Projection()

+ if d, e := reopened.CheckServing(ctx, f.identity, &p, []string{"home.operator.v1"}); e != nil || d != guard.Managed {

+ t.Fatal(d, e)

+ }

+ denied(t, f, &p, nil)

+ denied(t, f, &p, []string{"home.operator.v0"})

+ foreign := f.identity

+ foreign.Account = "bbb"

+ if d, e := f.repo.CheckServing(ctx, foreign, &p, []string{"home.operator.v1"}); d != guard.Denied || e == nil {

+ t.Fatal("hostname reassigned")

+ }

+ foreign = f.identity

+ foreign.Record = hash("replacement")

+ if d, e := f.repo.CheckServing(ctx, foreign, &p, []string{"home.operator.v1"}); d != guard.Denied || e == nil {

+ t.Fatal("record replaced")

+ }

+ if _, e = f.repo.ReleaseClaim(ctx, c); e != nil {

+ t.Fatal(e)

+ }

+ held, e := f.repo.ReadLifecycle(ctx, "aaa", "home")

+ if e != nil || held.Value().State != "idle" || held.Value().Generation != 2 {

+ t.Fatal(held.Value(), e)

+ }

+}

+func TestGuardStaleWorkersAndExecutingRecovery(t *testing.T) {

+ f := setup(t)

+ c := f.claim(t, 1)

+ first := f.create(t, c)

+ second := f.step(t, c, first, guard.Quiescing)

+ stale := first.Value()

+ stale.Generation++

+ stale.Phase = guard.Quiescing

+ if _, e := f.repo.AdvanceGuard(ctx, c, first, stale); !errors.Is(e, release.ErrCASConflict) {

+ t.Fatal("stale write did not preserve CAS conflict", e)

+ }

+ s := f.step(t, c, second, guard.Fenced)

+ s = f.step(t, c, s, guard.Executing)

+ afterCrash, e := f.repo.ReadGuard(ctx, f.identity.Host)

+ if e != nil {

+ t.Fatal(e)

+ }

+ repeat := afterCrash.Value()

+ repeat.Generation++

+ if _, e = f.repo.AdvanceGuard(ctx, c, afterCrash, repeat); !errors.Is(e, guard.ErrConflict) {

+ t.Fatal("executing automatically repeated", e)

+ }

+ s = f.step(t, c, afterCrash, guard.Ambiguous)

+ retry := s.Value()

+ retry.Generation++

+ retry.Phase = guard.Requested

+ retry.Request = f.ref(t, guard.RequestReference, 8)

+ retry.Attempt = f.ref(t, guard.AttemptReference, 9)

+ retry.Evidence = nil

+ if _, e = f.repo.AdvanceGuard(ctx, c, s, retry); e != nil {

+ t.Fatal("explicit new attempt refused", e)

+ }

+}

+func TestGuardContinuityAndContractFloorCannotShrink(t *testing.T) {

+ f := setup(t)

+ c := f.claim(t, 1)

+ s := f.create(t, c)

+ for _, change := range []func(*guard.Guard){func(g *guard.Guard) { g.Generation-- }, func(g *guard.Guard) { g.Generation++ }, func(g *guard.Guard) { g.Identity.Record = hash("other") }, func(g *guard.Guard) { g.OwnerBox = "box-two" }, func(g *guard.Guard) { g.Request.Digest = hash("other") }, func(g *guard.Guard) { g.ArtifactDigest = hash("other") }} {

+ g := s.Value()

+ g.Generation++

+ g.Phase = guard.Quiescing

+ change(&g)

+ if _, e := f.repo.AdvanceGuard(ctx, c, s, g); !errors.Is(e, guard.ErrConflict) {

+ t.Fatal("continuity changed", e)

+ }

+ for _, p := range []guard.Phase{guard.Quiescing, guard.Fenced, guard.ResumeRequested, guard.Resumed} {

+ s = f.step(t, c, s, p)

+ }

+ if _, e := f.repo.ReleaseClaim(ctx, c); e != nil {

+ t.Fatal(e)

+ }

+ c = f.claim(t, 20)

+ g := s.Value()

+ g.Generation++

+ g.Phase = guard.Requested

+ g.OperationID = c.Value().OperationID

+ g.Request = f.ref(t, guard.RequestReference, 21)

+ g.Attempt = f.ref(t, guard.AttemptReference, 22)

+ g.Evidence = nil

+ g.RequiredContracts = nil

+ if _, e := f.repo.AdvanceGuard(ctx, c, s, g); !errors.Is(e, guard.ErrConflict) {

+ t.Fatal("floor removed", e)

+ }

+}

+func TestImmutableReferenceAndMissingEvidence(t *testing.T) {

+ f := setup(t)

+ r := f.ref(t, guard.RequestReference, 2)

+ if _, e := f.repo.PutReference(ctx, f.identity.Host, guard.RequestReference, r.ID, []byte("changed")); !errors.Is(e, guard.ErrConflict) {

+ t.Fatal(e)

+ }

+ c := f.claim(t, 1)

+ s := f.resumed(t, c)

+ p, _ := s.Projection()

+ // Private payload expiry is separate and cannot remove permanent metadata.

+ private := "private/expired-payload"

+ if _, e := f.store.Put(ctx, private, []byte("synthetic-private"), ""); e != nil {

+ t.Fatal(e)

+ }

+ if e := f.store.Delete(ctx, private); e != nil {

+ t.Fatal(e)

+ }

+ if d, e := f.repo.CheckServing(ctx, f.identity, &p, []string{"home.operator.v1"}); e != nil || d != guard.Managed {

+ t.Fatal(d, e)

+ }

+ key, _ := guard.ReceiptKey(f.identity.Host, s.Value().Evidence.ID)

+ if e := f.store.Delete(ctx, key); e != nil {

+ t.Fatal(e)

+ }

+ denied(t, f, &p, []string{"home.operator.v1"})

+}

+

+// absenceBarrier forces both contenders to observe absence before either CAS.

+type absenceBarrier struct {

+ release.Store

+ key string

+ mu sync.Mutex

+ n int

+ ready chan struct{}

+}

+

+func (s *absenceBarrier) Get(ctx context.Context, key string) ([]byte, string, error) {

+ b, etag, e := s.Store.Get(ctx, key)

+ if key == s.key && errors.Is(e, os.ErrNotExist) {

+ s.mu.Lock()

+ s.n++

+ if s.n == 2 {

+ close(s.ready)

+ }

+ s.mu.Unlock()

+ <-s.ready

+ }

+ return b, etag, e

+}

+func TestLifecycleSimultaneousAbsenceMaintenanceVersusPurge(t *testing.T) {

+ for i := 0; i < 20; i++ {

+ f := setup(t)

+ key, _ := guard.LifecycleKey("aaa", "home")

+ barrier := &absenceBarrier{Store: f.store, key: key, ready: make(chan struct{})}

+ r := guard.New(barrier)

+ record := "apps/aaa/home/instances/one.home.example.json"

+ if _, e := f.store.Put(ctx, record, []byte("existing-record"), ""); e != nil {

+ t.Fatal(e)

+ }

+ type result struct {

+ kind guard.OperationKind

+ claim guard.ClaimSnapshot

+ err error

+ }

+ out := make(chan result, 2)

+ for n, kind := range []guard.OperationKind{guard.MaintenanceOperation, guard.PurgeOperation} {

+ go func(n int, kind guard.OperationKind) {

+ c, e := r.AcquireClaim(ctx, "aaa", "home", id(byte(n+40)), kind)

+ out <- result{kind, c, e}

+ }(n, kind)

+ }

+ a, b := <-out, <-out

+ if (a.err == nil) == (b.err == nil) {

+ t.Fatal("two absence readers both won or lost", a.err, b.err)

+ }

+ winner, loser := a, b

+ if winner.err != nil {

+ winner, loser = b, a

+ }

+ if !errors.Is(loser.err, release.ErrCASConflict) {

+ t.Fatal(loser.err)

+ }

+ if winner.kind == guard.PurgeOperation {

+ if e := f.store.Delete(ctx, record); e != nil {

+ t.Fatal(e)

+ }

+ } else {

+ g := f.initial(t, winner.claim)

+ if _, e := r.CreateGuard(ctx, winner.claim, g); e != nil {

+ t.Fatal(e)

+ }

+ s, e := r.ReadGuard(ctx, f.identity.Host)

+ if e != nil {

+ t.Fatal(e)

+ }

+ _, _, recordErr := f.store.Get(ctx, record)

+ if winner.kind == guard.MaintenanceOperation && (!s.Exists() || recordErr != nil) {

+ t.Fatal("purge crossed maintenance claim")

+ }

+ if winner.kind == guard.PurgeOperation && (s.Exists() || !errors.Is(recordErr, os.ErrNotExist)) {

+ t.Fatal("maintenance crossed purge claim")

+ }

+}

diff --git a/internal/maintenanceguard/json.go b/internal/maintenanceguard/json.go

new file mode 100644

index 000000000..a737b38bc

--- /dev/null

+++ b/internal/maintenanceguard/json.go

@@ -0,0 +1,106 @@

+package maintenanceguard

+

+import (

+ "bytes"

+ "encoding/json"

+ "errors"

+ "io"

+ "reflect"

+ "strings"

+)

+

+const MaxWireBytes = 16 << 10

+

+var ErrInvalid = errors.New("maintenance guard: invalid record")

+

+// decode checks exact field names and duplicates before typed decoding. The

+// expected schema bounds nesting; arbitrary JSON trees are never traversed.

+func decode(raw []byte, dst any) error {

+ if len(raw) == 0 || len(raw) > MaxWireBytes {

+ return ErrInvalid

+ }

+ typ := reflect.TypeOf(dst)

+ if typ == nil || typ.Kind() != reflect.Pointer {

+ return ErrInvalid

+ }

+ dec := json.NewDecoder(bytes.NewReader(raw))

+ dec.UseNumber()

+ if !jsonShape(dec, typ.Elem()) {

+ return ErrInvalid

+ }

+ if _, e := dec.Token(); e != io.EOF {

+ return ErrInvalid

+ }

+ if json.Unmarshal(raw, dst) != nil {

+ return ErrInvalid

+ }

+ return nil

+}

+func jsonShape(dec *json.Decoder, typ reflect.Type) bool {

+ for typ.Kind() == reflect.Pointer {

+ typ = typ.Elem()

+ }

+ token, e := dec.Token()

+ if e != nil || token == nil {

+ return false

+ }

+ if delim, ok := token.(json.Delim); ok {

+ switch delim {

+ case '{':

+ if typ.Kind() != reflect.Struct {

+ return false

+ }

+ fields := map[string]reflect.Type{}

+ for i := 0; i < typ.NumField(); i++ {

+ field := typ.Field(i)

+ name := strings.Split(field.Tag.Get("json"), ",")[0]

+ if name != "" && name != "-" {

+ fields[name] = field.Type

+ }

+ seen := map[string]bool{}

+ for dec.More() {

+ key, e := dec.Token()

+ if e != nil {

+ return false

+ }

+ name, ok := key.(string)

+ field, known := fields[name]

+ if !ok || !known || seen[name] {

+ return false

+ }

+ seen[name] = true

+ if !jsonShape(dec, field) {

+ return false

+ }

+ end, e := dec.Token()

+ return e == nil && end == json.Delim('}')

+ case '[':

+ if typ.Kind() != reflect.Slice {

+ return false

+ }

+ for dec.More() {

+ if !jsonShape(dec, typ.Elem()) {

+ return false

+ }

+ end, e := dec.Token()

+ return e == nil && end == json.Delim(']')

+ default:

+ return false

+ }

+ switch typ.Kind() {

+ case reflect.String:

+ _, ok := token.(string)

+ return ok

+ case reflect.Int, reflect.Int64:

+ _, ok := token.(json.Number)

+ return ok

+ case reflect.Bool:

+ _, ok := token.(bool)

+ return ok

+ }

+ return false

+}

diff --git a/internal/maintenanceguard/lifecycle.go b/internal/maintenanceguard/lifecycle.go

new file mode 100644

index 000000000..9331a6f7b

--- /dev/null

+++ b/internal/maintenanceguard/lifecycle.go

@@ -0,0 +1,213 @@

+package maintenanceguard

+

+import (

+ "context"

+ "encoding/json"

+ "errors"

+ "math"

+ "os"

+)

+

+type OperationKind string

+

+const (

+ MaintenanceOperation OperationKind = "maintenance"

+ TrashOperation OperationKind = "trash"

+ RestoreOperation OperationKind = "restore"

+ PurgeOperation OperationKind = "purge"

+)

+

+func (k OperationKind) valid() bool {

+ return k == MaintenanceOperation || k == TrashOperation || k == RestoreOperation || k == PurgeOperation

+}

+

+type Lifecycle struct {

+ Version int `json:"version"`

+ Account string `json:"account"`

+ App string `json:"app"`

+ Generation int64 `json:"generation"`

+ State string `json:"state"`

+ OperationID string `json:"operation_id"`

+ Kind OperationKind `json:"kind"`

+ ClaimGeneration int64 `json:"claim_generation"`

+ CompletionDigest string `json:"completion_digest,omitempty"`

+}

+

+func (l Lifecycle) valid() bool {

+ if _, e := appBase(l.Account, l.App); e != nil {

+ return false

+ }

+ if l.Version != Version || l.Generation <= 0 || !ValidID(l.OperationID) || !l.Kind.valid() || l.ClaimGeneration <= 0 {

+ return false

+ }

+ switch l.State {

+ case "claimed":

+ return l.Generation == l.ClaimGeneration && l.CompletionDigest == ""

+ case "idle":

+ return l.ClaimGeneration < math.MaxInt64 && l.Generation == l.ClaimGeneration+1 && ValidDigest(l.CompletionDigest)

+ }

+ return false

+}

+

+type claimIntent struct {

+ Version int `json:"version"`

+ Account string `json:"account"`

+ App string `json:"app"`

+ OperationID string `json:"operation_id"`

+ Kind OperationKind `json:"kind"`

+ Generation int64 `json:"generation"`

+}

+

+func intent(l Lifecycle) claimIntent {

+ return claimIntent{Version, l.Account, l.App, l.OperationID, l.Kind, l.ClaimGeneration}

+}

+func claimKey(l Lifecycle, completed bool) string {

+ p, _ := appBase(l.Account, l.App)

+ suffix := "claim"

+ if completed {

+ suffix = "completion"

+ }

+ return p + "operations/" + l.OperationID + "/" + suffix + ".json"

+}

+

+type claimCompletion struct {

+ Claim claimIntent `json:"claim"`

+ Generation int64 `json:"generation"`

+}

+

+type ClaimSnapshot struct {

+ repository *Repository

+ value Lifecycle

+ etag string

+ exists bool

+}

+

+func (s ClaimSnapshot) Exists() bool { return s.exists }

+func (s ClaimSnapshot) Value() Lifecycle { return s.value }

+func (r *Repository) ReadLifecycle(ctx context.Context, account, app string) (ClaimSnapshot, error) {

+ key, e := LifecycleKey(account, app)

+ if e != nil {

+ return ClaimSnapshot{}, e

+ }

+ raw, etag, e := r.store.Get(ctx, key)

+ if errors.Is(e, os.ErrNotExist) {

+ return ClaimSnapshot{repository: r, value: Lifecycle{Account: account, App: app}}, nil

+ }

+ if e != nil {

+ return ClaimSnapshot{}, e

+ }

+ var l Lifecycle

+ if decode(raw, &l) != nil || !l.valid() || l.Account != account || l.App != app || !cleanETag(etag) {

+ return ClaimSnapshot{}, ErrInvalid

+ }

+ b, _, e := r.store.Get(ctx, claimKey(l, false))

+ var i claimIntent

+ if e != nil || decode(b, &i) != nil || i != intent(l) {

+ return ClaimSnapshot{}, ErrUnavailable

+ }

+ if l.State == "idle" {

+ b, _, e = r.store.Get(ctx, claimKey(l, true))

+ var c claimCompletion

+ if e != nil || decode(b, &c) != nil || c.Claim != i || c.Generation != l.Generation || digest(b) != l.CompletionDigest {

+ return ClaimSnapshot{}, ErrUnavailable

+ }

+ return ClaimSnapshot{r, l, etag, true}, nil

+}

+

+// AcquireClaim excludes competing app lifecycle operations. The caller must

+// authorize the operation first; acquisition alone cannot start maintenance.

+// Immutable intent may survive a lost CAS, but grants no claim or side effect.

+func (r *Repository) AcquireClaim(ctx context.Context, account, app, id string, kind OperationKind) (ClaimSnapshot, error) {

+ if !ValidID(id) || !kind.valid() {

+ return ClaimSnapshot{}, ErrInvalid

+ }

+ old, e := r.ReadLifecycle(ctx, account, app)

+ if e != nil {

+ return ClaimSnapshot{}, e

+ }

+ if old.exists && old.value.State != "idle" {

+ return ClaimSnapshot{}, ErrBusy

+ }

+ generation := int64(1)

+ if old.exists {

+ if old.value.Generation == math.MaxInt64 {

+ return ClaimSnapshot{}, ErrConflict

+ }

+ generation = old.value.Generation + 1

+ }

+ l := Lifecycle{Version: Version, Account: account, App: app, Generation: generation, State: "claimed", OperationID: id, Kind: kind, ClaimGeneration: generation}

+ b, _ := json.Marshal(intent(l))

+ if e = r.putImmutable(ctx, claimKey(l, false), b); e != nil {

+ return ClaimSnapshot{}, e

+ }

+ raw, _ := json.Marshal(l)

+ key, _ := LifecycleKey(account, app)

+ etag, e := r.store.Put(ctx, key, raw, old.etag)

+ if e != nil {

+ return ClaimSnapshot{}, e

+ }

+ if !cleanETag(etag) {

+ return ClaimSnapshot{}, ErrUnavailable

+ }

+ return ClaimSnapshot{r, l, etag, true}, nil

+}

+

+// RecoverClaim observes only the exact still-held operation. It does not expire

+// a claim, release it, reapprove it, or rerun any command.

+func (r *Repository) RecoverClaim(ctx context.Context, account, app, id string, kind OperationKind) (ClaimSnapshot, error) {

+ if !ValidID(id) || !kind.valid() {

+ return ClaimSnapshot{}, ErrInvalid

+ }

+ s, e := r.ReadLifecycle(ctx, account, app)

+ if e != nil {

+ return ClaimSnapshot{}, e

+ }

+ if !s.exists || s.value.State != "claimed" || s.value.OperationID != id || s.value.Kind != kind {

+ return ClaimSnapshot{}, ErrConflict

+ }

+ return s, nil

+}

+func (r *Repository) currentClaim(ctx context.Context, s ClaimSnapshot) error {

+ if s.repository != r || !s.exists || s.value.State != "claimed" || !s.value.valid() || !cleanETag(s.etag) {

+ return ErrInvalid

+ }

+ now, e := r.ReadLifecycle(ctx, s.value.Account, s.value.App)

+ if e != nil {

+ return e

+ }

+ if !now.exists || now.etag != s.etag || now.value != s.value {

+ return ErrConflict

+ }

+ return nil

+}

+

+// ReleaseClaim requires the caller's completed-operation proof. Maintenance

+// callers must first verify both resumed projections and healthy serving under

+// the reviewed lease boundary. This CAS does not establish those facts itself.

+func (r *Repository) ReleaseClaim(ctx context.Context, s ClaimSnapshot) (ClaimSnapshot, error) {

+ if e := r.currentClaim(ctx, s); e != nil {

+ return ClaimSnapshot{}, e

+ }

+ if s.value.Generation == math.MaxInt64 {

+ return ClaimSnapshot{}, ErrConflict

+ }

+ l := s.value

+ l.Generation++

+ l.State = "idle"

+ completion, _ := json.Marshal(claimCompletion{intent(l), l.Generation})

+ if e := r.putImmutable(ctx, claimKey(l, true), completion); e != nil {

+ return ClaimSnapshot{}, e

+ }

+ l.CompletionDigest = digest(completion)

+ raw, _ := json.Marshal(l)

+ key, _ := LifecycleKey(l.Account, l.App)

+ etag, e := r.store.Put(ctx, key, raw, s.etag)

+ if e != nil {

+ return ClaimSnapshot{}, e

+ }

+ if !cleanETag(etag) {

+ return ClaimSnapshot{}, ErrUnavailable

+ }

+ return ClaimSnapshot{r, l, etag, true}, nil

+}

diff --git a/internal/maintenanceguard/store.go b/internal/maintenanceguard/store.go

new file mode 100644

index 000000000..9d35a06e0

--- /dev/null

+++ b/internal/maintenanceguard/store.go

@@ -0,0 +1,89 @@

+package maintenanceguard

+

+import (

+ "bytes"

+ "context"

+ "errors"

+ "os"

+)

+

+// Store is structural so release, provision and host can share these rules

+// without importing each other. Put uses empty ETag for create-only writes.

+type Store interface {

+ Get(context.Context, string) ([]byte, string, error)

+ Put(context.Context, string, []byte, string) (string, error)

+}

+type Repository struct{ store Store }

+

+func New(store Store) *Repository { return &Repository{store: store} }

+

+type ReferenceKind string

+

+const (

+ RequestReference ReferenceKind = "requests"

+ AttemptReference ReferenceKind = "attempts"

+ ReceiptReference ReferenceKind = "receipts"

+)

+

+func (k ReferenceKind) valid() bool {

+ return k == RequestReference || k == AttemptReference || k == ReceiptReference

+}

+

+// PutReference stores permanent, nonsecret metadata. Expiring private payloads

+// belong elsewhere: active guards must keep resolving these immutable bytes.

+// The caller validates metadata, ownership and any encrypted payload bindings.

+func (r *Repository) PutReference(ctx context.Context, host string, kind ReferenceKind, id string, raw []byte) (Reference, error) {

+ if !kind.valid() || len(raw) == 0 || len(raw) > MaxReferenceBytes {

+ return Reference{}, ErrInvalid

+ }

+ key, e := refKey(host, string(kind), id)

+ if e != nil {

+ return Reference{}, e

+ }

+ if e = r.putImmutable(ctx, key, raw); e != nil {

+ return Reference{}, e

+ }

+ return Reference{ID: id, Digest: digest(raw)}, nil

+}

+func (r *Repository) putImmutable(ctx context.Context, key string, raw []byte) error {

+ old, _, e := r.store.Get(ctx, key)

+ if e == nil {

+ if bytes.Equal(old, raw) {

+ return nil

+ }

+ return ErrConflict

+ }

+ if !errors.Is(e, os.ErrNotExist) {

+ return e

+ }

+ _, e = r.store.Put(ctx, key, raw, "")

+ return e

+}

+func (r *Repository) checkReference(ctx context.Context, host string, kind ReferenceKind, ref Reference) error {

+ if !ref.valid() {

+ return ErrInvalid

+ }

+ key, e := refKey(host, string(kind), ref.ID)

+ if e != nil {

+ return e

+ }

+ raw, _, e := r.store.Get(ctx, key)

+ if e != nil || len(raw) == 0 || len(raw) > MaxReferenceBytes || digest(raw) != ref.Digest {

+ return ErrUnavailable

+ }

+ return nil

+}

+func (r *Repository) checkGuardReferences(ctx context.Context, g Guard) error {

+ for _, v := range []struct {

+ kind ReferenceKind

+ ref Reference

+ }{{RequestReference, g.Request}, {AttemptReference, g.Attempt}} {

+ if e := r.checkReference(ctx, g.Identity.Host, v.kind, v.ref); e != nil {

+ return e

+ }

+ if g.Evidence != nil {

+ return r.checkReference(ctx, g.Identity.Host, ReceiptReference, *g.Evidence)

+ }

+ return nil

+}

diff --git a/internal/maintenanceguard/types.go b/internal/maintenanceguard/types.go

new file mode 100644

index 000000000..a8d6911fb

--- /dev/null

+++ b/internal/maintenanceguard/types.go

@@ -0,0 +1,241 @@

+// Package maintenanceguard stores denial state and lifecycle exclusion. Its

+// records do not prove human approval, process health, or lease ownership.

+package maintenanceguard

+

+import (

+ "crypto/sha256"

+ "encoding/base64"

+ "encoding/binary"

+ "encoding/hex"

+ "encoding/json"

+ "errors"

+ "github.com/carlosframework/platform/internal/controlname"

+ "regexp"

+ "sort"

+ "strings"

+ "time"

+)

+

+const Version = 1

+const MaxContracts = 16

+const MaxReferenceBytes = 2 << 20

+

+var (

+ ErrUnavailable = errors.New("maintenance guard: unavailable")

+ ErrConflict = errors.New("maintenance guard: conflict")

+ ErrBusy = errors.New("maintenance guard: lifecycle claimed")

+ ErrDenied = errors.New("maintenance guard: serving denied")

+ atomRE = regexp.MustCompile(`^[a-zA-Z0-9._-]{1,64}$`)

+ contractRE = regexp.MustCompile(`^[a-z][a-z0-9]*([.-][a-z0-9]+)*$`)

+)

+

+func ValidID(s string) bool {

+ if len(s) != 43 {

+ return false

+ }

+ b, e := base64.RawURLEncoding.Strict().DecodeString(s)

+ return e == nil && len(b) == 32 && base64.RawURLEncoding.EncodeToString(b) == s

+}

+func ValidDigest(s string) bool {

+ if len(s) != 64 {

+ return false

+ }

+ b, e := hex.DecodeString(s)

+ return e == nil && len(b) == 32 && hex.EncodeToString(b) == s

+}

+func ValidContract(s string) bool { return len(s) > 0 && len(s) <= 64 && contractRE.MatchString(s) }

+func ValidContracts(set []string) bool {

+ if len(set) > MaxContracts {

+ return false

+ }

+ for i, s := range set {

+ if !ValidContract(s) || (i > 0 && set[i-1] >= s) {

+ return false

+ }

+ return true

+}

+func atom(s string) bool { return s != "." && s != ".." && atomRE.MatchString(s) }

+func revision(s string) bool {

+ if s == "" || len(s) > 256 {

+ return false

+ }

+ for _, c := range []byte(s) {

+ if c <= 32 || c > 126 || c == '*' {

+ return false

+ }

+ return true

+}

+func digest(raw []byte) string { sum := sha256.Sum256(raw); return hex.EncodeToString(sum[:]) }

+

+// RecordIdentity binds immutable creation fields. It is continuity metadata,

+// not proof against a privileged writer restoring or forging those fields.

+func RecordIdentity(account, app, host string, created time.Time) (string, error) {

+ if !controlname.ValidAccount(account) || !controlname.ValidApp(app) || !controlname.ValidHostShape(host) || created.IsZero() {

+ return "", ErrInvalid

+ }

+ stamp := created.UTC().Format(time.RFC3339Nano)

+ if parsed, e := time.Parse(time.RFC3339Nano, stamp); e != nil || !parsed.Equal(created) {

+ return "", ErrInvalid

+ }

+ raw := []byte("carlos-maintenance-record-identity-v1\x00")

+ for _, s := range []string{account, app, host, stamp} {

+ raw = binary.BigEndian.AppendUint32(raw, uint32(len(s)))

+ raw = append(raw, s...)

+ }

+ return digest(raw), nil

+}

+

+type Identity struct {

+ Account string `json:"account"`

+ App string `json:"app"`

+ Host string `json:"host"`

+ Record string `json:"record"`

+}

+

+func (i Identity) valid() bool {

+ return i.locationValid() && ValidDigest(i.Record)

+}

+func (i Identity) locationValid() bool {

+ return controlname.ValidAccount(i.Account) && controlname.ValidApp(i.App) && controlname.ValidHostShape(i.Host)

+}

+

+type Reference struct {

+ ID string `json:"id"`

+ Digest string `json:"digest"`

+}

+

+func (r Reference) valid() bool { return ValidID(r.ID) && ValidDigest(r.Digest) }

+

+type Phase string

+

+const (

+ Requested Phase = "requested"

+ Quiescing Phase = "quiescing"

+ Fenced Phase = "fenced"

+ Executing Phase = "executing"

+ Succeeded Phase = "succeeded"

+ Failed Phase = "failed"

+ Ambiguous Phase = "ambiguous"

+ ResumeRequested Phase = "resume-requested"

+ Resumed Phase = "resumed"

+)

+

+func (p Phase) valid() bool {

+ switch p {

+ case Requested, Quiescing, Fenced, Executing, Succeeded, Failed, Ambiguous, ResumeRequested, Resumed:

+ return true

+ }

+ return false

+}

+

+type Guard struct {

+ Version int `json:"version"`

+ Identity Identity `json:"identity"`

+ Generation int64 `json:"generation"`

+ Phase Phase `json:"phase"`

+ OperationID string `json:"operation_id"`

+ OwnerBox string `json:"owner_box"`

+ RecordRevision string `json:"record_revision"`

+ Request Reference `json:"request"`

+ Attempt Reference `json:"attempt"`

+ ArtifactVersion string `json:"artifact_version"`

+ ArtifactDigest string `json:"artifact_digest"`

+ ConfigDigest string `json:"config_digest"`

+ RequiredContracts []string `json:"required_contracts"`

+ Evidence *Reference `json:"evidence,omitempty"`

+}

+

+func (g Guard) valid() bool {

+ if g.Version != Version || !g.Identity.valid() || g.Generation <= 0 || !g.Phase.valid() || !ValidID(g.OperationID) || !atom(g.OwnerBox) || !revision(g.RecordRevision) || !g.Request.valid() || !g.Attempt.valid() || !atom(g.ArtifactVersion) || !ValidDigest(g.ArtifactDigest) || !ValidDigest(g.ConfigDigest) || !ValidContracts(g.RequiredContracts) {

+ return false

+ }

+ if g.Evidence != nil && !g.Evidence.valid() {

+ return false

+ }

+ return (g.Phase != Resumed && g.Phase != Succeeded) || (g.Evidence != nil && (g.Phase != Resumed || len(g.RequiredContracts) > 0))

+}

+func cloneGuard(g Guard) Guard {

+ g.RequiredContracts = append([]string{}, g.RequiredContracts...)

+ if g.Evidence != nil {

+ r := *g.Evidence

+ g.Evidence = &r

+ }

+ return g

+}

+func guardBytes(g Guard) ([]byte, error) {

+ if !g.valid() {

+ return nil, ErrInvalid

+ }

+ return json.Marshal(cloneGuard(g))

+}

+

+type Projection struct {

+ Version int `json:"version"`

+ Generation int64 `json:"generation"`

+ GuardDigest string `json:"guard_digest"`

+}

+

+func (p Projection) valid() bool {

+ return p.Version == Version && p.Generation > 0 && ValidDigest(p.GuardDigest)

+}

+func EncodeProjection(p Projection) ([]byte, error) {

+ if !p.valid() {

+ return nil, ErrInvalid

+ }

+ return json.Marshal(p)

+}

+func DecodeProjection(raw []byte) (Projection, error) {

+ var p Projection

+ if decode(raw, &p) != nil || !p.valid() {

+ return Projection{}, ErrInvalid

+ }

+ return p, nil

+}

+func containsAll(superset, subset []string) bool {

+ for _, s := range subset {

+ i := sort.SearchStrings(superset, s)

+ if i == len(superset) || superset[i] != s {

+ return false

+ }

+ return true

+}

+func hostBase(host string) (string, error) {

+ if !controlname.ValidHostShape(host) {

+ return "", ErrInvalid

+ }

+ return "control/maintenance/hosts/" + host + "/", nil

+}

+func GuardKey(host string) (string, error) {

+ p, e := hostBase(host)

+ if e != nil {

+ return "", e

+ }

+ return p + "guard.json", nil

+}

+func refKey(host, kind, id string) (string, error) {

+ p, e := hostBase(host)

+ if e != nil || !ValidID(id) {

+ return "", ErrInvalid

+ }

+ return p + kind + "/" + id + ".json", nil

+}

+func RequestKey(host, id string) (string, error) { return refKey(host, "requests", id) }

+func AttemptKey(host, id string) (string, error) { return refKey(host, "attempts", id) }

+func ReceiptKey(host, id string) (string, error) { return refKey(host, "receipts", id) }

+func appBase(account, app string) (string, error) {

+ if !controlname.ValidAccount(account) || !controlname.ValidApp(app) {

+ return "", ErrInvalid

+ }

+ return "control/maintenance/apps/" + account + "/" + app + "/", nil

+}

+func LifecycleKey(account, app string) (string, error) {

+ p, e := appBase(account, app)

+ if e != nil {

+ return "", e

+ }

+ return p + "lifecycle.json", nil

+}

+func cleanETag(s string) bool { return s != "" && len(s) <= 512 && !strings.ContainsAny(s, "\r\n\x00") }

diff --git a/internal/maintenanceguard/wire_test.go b/internal/maintenanceguard/wire_test.go

new file mode 100644

index 000000000..ca5f1a3ae

--- /dev/null

+++ b/internal/maintenanceguard/wire_test.go

@@ -0,0 +1,282 @@

+package maintenanceguard_test

+

+import (

+ "encoding/json"

+ "errors"

+ "math"

+ "strings"

+ "testing"

+ "time"

+

+ guard "github.com/carlosframework/platform/internal/maintenanceguard"

+)

+

+func replace(t *testing.T, f fixture, key string, raw []byte) {

+ t.Helper()

+ _, etag, e := f.store.Get(ctx, key)

+ if e != nil {

+ t.Fatal(e)

+ }

+ if _, e = f.store.Put(ctx, key, raw, etag); e != nil {

+ t.Fatal(e)

+ }

+}

+func TestWireMalformedGuardAlwaysDenies(t *testing.T) {

+ cases := map[string]func([]byte) []byte{

+ "unknown": func(b []byte) []byte { return append(b[:len(b)-1], []byte(`,"unknown":1}`)...) },

+ "duplicate": func(b []byte) []byte { return append(b[:len(b)-1], []byte(`,"version":1}`)...) },

+ "case alias": func(b []byte) []byte { return []byte(strings.Replace(string(b), `"version"`, `"Version"`, 1)) },

+ "unknown phase": func(b []byte) []byte {

+ return []byte(strings.Replace(string(b), `"requested"`, `"awaiting-approval"`, 1))

+ },

+ "zero generation": func(b []byte) []byte {

+ return []byte(strings.Replace(string(b), `"generation":1`, `"generation":0`, 1))

+ },

+ "negative generation": func(b []byte) []byte {

+ return []byte(strings.Replace(string(b), `"generation":1`, `"generation":-1`, 1))

+ },

+ "overflow generation": func(b []byte) []byte {

+ return []byte(strings.Replace(string(b), `"generation":1`, `"generation":9223372036854775808`, 1))

+ },

+ "fraction": func(b []byte) []byte {

+ return []byte(strings.Replace(string(b), `"generation":1`, `"generation":1.5`, 1))

+ },

+ "version": func(b []byte) []byte { return []byte(strings.Replace(string(b), `"version":1`, `"version":2`, 1)) },

+ "trailing": func(b []byte) []byte { return append(b, []byte(` {}`)...) },

+ "null": func([]byte) []byte { return []byte(`null`) },

+ "deep": func([]byte) []byte { return []byte(strings.Repeat("[", 1000) + strings.Repeat("]", 1000)) },

+ "oversize": func([]byte) []byte { return []byte(strings.Repeat(" ", guard.MaxWireBytes+1)) },

+ "null reference": func(b []byte) []byte {

+ return []byte(strings.Replace(string(b), `"required_contracts":[]`, `"required_contracts":null`, 1))

+ },

+ }

+ for name, mutate := range cases {

+ t.Run(name, func(t *testing.T) {

+ f := setup(t)

+ c := f.claim(t, 1)

+ s := f.create(t, c)

+ p, _ := s.Projection()

+ key, _ := guard.GuardKey(f.identity.Host)

+ raw, _, e := f.store.Get(ctx, key)

+ if e != nil {

+ t.Fatal(e)

+ }

+ replace(t, f, key, mutate(raw))

+ if _, e = f.repo.ReadGuard(ctx, f.identity.Host); e == nil {

+ t.Fatal("invalid guard parsed")

+ }

+ denied(t, f, &p, nil)

+ })

+ }

+}

+func TestProjectionClosedAndBounded(t *testing.T) {

+ p := guard.Projection{Version: 1, Generation: 3, GuardDigest: hash("guard")}

+ b, e := guard.EncodeProjection(p)

+ if e != nil {

+ t.Fatal(e)

+ }

+ if got, e := guard.DecodeProjection(b); e != nil || got != p {

+ t.Fatal(got, e)

+ }

+ for _, raw := range []string{`null`, `{}`, `{"version":2,"generation":3,"guard_digest":"` + p.GuardDigest + `"}`, string(b[:len(b)-1]) + `,"version":1}`, string(b[:len(b)-1]) + `,"other":true}`, strings.Replace(string(b), `"version"`, `"Version"`, 1), strings.Repeat("x", guard.MaxWireBytes+1)} {

+ if _, e := guard.DecodeProjection([]byte(raw)); e == nil {

+ t.Fatal("invalid projection parsed")

+ }

+}

+func TestLifecycleRecoveryNoTimeoutAndRetainedHistory(t *testing.T) {

+ f := setup(t)

+ c := f.claim(t, 1)

+ restarted := guard.New(f.store)

+ recovered, e := restarted.RecoverClaim(ctx, "aaa", "home", id(1), guard.MaintenanceOperation)

+ if e != nil || recovered.Value() != c.Value() {

+ t.Fatal(e)

+ }

+ for _, kind := range []guard.OperationKind{guard.TrashOperation, guard.RestoreOperation, guard.PurgeOperation, guard.MaintenanceOperation} {

+ if _, e = restarted.AcquireClaim(ctx, "aaa", "home", id(2), kind); !errors.Is(e, guard.ErrBusy) {

+ t.Fatal("occupied claim bypassed", e)

+ }

+ if _, e = restarted.RecoverClaim(ctx, "aaa", "home", id(2), guard.MaintenanceOperation); !errors.Is(e, guard.ErrConflict) {

+ t.Fatal(e)

+ }

+ if _, e = restarted.RecoverClaim(ctx, "aaa", "home", id(1), guard.PurgeOperation); !errors.Is(e, guard.ErrConflict) {

+ t.Fatal(e)

+ }

+ released, e := restarted.ReleaseClaim(ctx, recovered)

+ if e != nil || released.Value().Generation != 2 || released.Value().State != "idle" {

+ t.Fatal(e)

+ }

+ next, e := restarted.AcquireClaim(ctx, "aaa", "home", id(2), guard.PurgeOperation)

+ if e != nil || next.Value().Generation != 3 {

+ t.Fatal(e)

+ }

+ if _, e = restarted.ReleaseClaim(ctx, recovered); !errors.Is(e, guard.ErrConflict) {

+ t.Fatal("stale release cleared new claim", e)

+ }

+ if _, e = restarted.ReleaseClaim(ctx, next); e != nil {

+ t.Fatal(e)

+ }

+ if _, e = restarted.AcquireClaim(ctx, "aaa", "home", id(1), guard.MaintenanceOperation); !errors.Is(e, guard.ErrConflict) {

+ t.Fatal("old operation ID reused", e)

+ }

+ keys, e := f.store.List(ctx, "control/maintenance/apps/aaa/home/operations/")

+ if e != nil || len(keys) != 4 {

+ t.Fatal("immutable history missing", len(keys), e)

+ }

+}

+func TestMalformedLifecycleAndOverflowRefuse(t *testing.T) {

+ f := setup(t)

+ c := f.claim(t, 1)

+ key, _ := guard.LifecycleKey("aaa", "home")

+ raw, _, e := f.store.Get(ctx, key)

+ if e != nil {

+ t.Fatal(e)

+ }

+ for _, bad := range []string{strings.Replace(string(raw), `"claimed"`, `"expired"`, 1), strings.Replace(string(raw), `"version":1`, `"version":2`, 1), string(raw[:len(raw)-1]) + `,"expires_at":1}`} {

+ replace(t, f, key, []byte(bad))

+ if _, e = f.repo.AcquireClaim(ctx, "aaa", "home", id(2), guard.PurgeOperation); e == nil {

+ t.Fatal("malformed claim acquired")

+ }

+ replace(t, f, key, raw)

+ l := c.Value()

+ l.Generation = math.MaxInt64

+ l.ClaimGeneration = math.MaxInt64

+ b, _ := json.Marshal(l)

+ replace(t, f, key, b)

+ intentKey := "control/maintenance/apps/aaa/home/operations/" + id(1) + "/claim.json"

+ var intent map[string]json.RawMessage

+ old, _, e := f.store.Get(ctx, intentKey)

+ if e != nil {

+ t.Fatal(e)

+ }

+ if e = json.Unmarshal(old, &intent); e != nil {

+ t.Fatal(e)

+ }

+ intent["generation"] = []byte("9223372036854775807")

+ b, _ = json.Marshal(intent)

+ replace(t, f, intentKey, b)

+ loaded, e := f.repo.ReadLifecycle(ctx, "aaa", "home")

+ if e != nil {

+ t.Fatal(e)

+ }

+ if _, e = f.repo.ReleaseClaim(ctx, loaded); !errors.Is(e, guard.ErrConflict) {

+ t.Fatal("generation overflow", e)

+ }

+}

+func TestGuardGenerationOverflowRefuses(t *testing.T) {

+ f := setup(t)

+ c := f.claim(t, 1)

+ s := f.create(t, c)

+ g := s.Value()

+ g.Generation = math.MaxInt64

+ b, _ := json.Marshal(g)

+ key, _ := guard.GuardKey(g.Identity.Host)

+ replace(t, f, key, b)

+ s, e := f.repo.ReadGuard(ctx, g.Identity.Host)

+ if e != nil {

+ t.Fatal(e)

+ }

+ g.Phase = guard.Quiescing

+ if _, e = f.repo.AdvanceGuard(ctx, c, s, g); !errors.Is(e, guard.ErrConflict) {

+ t.Fatal("guard generation overflow", e)

+ }

+}

+func TestIdentityFramingKeysAndContracts(t *testing.T) {

+ at := time.Date(2026, 9, 8, 0, 1, 2, 3, time.FixedZone("offset", 3600))

+ a, e := guard.RecordIdentity("aaa", "bc", "one.home.example", at)

+ if e != nil {

+ t.Fatal(e)

+ }

+ same, e := guard.RecordIdentity("aaa", "bc", "one.home.example", at.UTC())

+ if e != nil || same != a {

+ t.Fatal("UTC identity differs")

+ }

+ other, _ := guard.RecordIdentity("aaab", "c", "one.home.example", at)

+ if other == a {

+ t.Fatal("unframed identity collision")

+ }

+ if _, e = guard.RecordIdentity("aaa", "home", "one.home.example", time.Time{}); e == nil {

+ t.Fatal("zero creation")

+ }

+ if _, e = guard.RecordIdentity("aaa", "home", "one.home.example", time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC)); e == nil {

+ t.Fatal("non-RFC3339 creation")

+ }

+ for _, host := range []string{"", "../other", "A.example", "a.example:443", "a.example.", strings.Repeat("a", 254)} {

+ if _, e := guard.GuardKey(host); e == nil {

+ t.Fatal("host key accepted", host)

+ }

+ for _, bad := range []string{"*", "Doc.v1", "docs..v1", "docs/v1", "docs-", "docs_v1", strings.Repeat("a", 65)} {

+ if guard.ValidContract(bad) {

+ t.Fatal("contract accepted", bad)

+ }

+ set := []string{}

+ for i := 0; i < 16; i++ {

+ set = append(set, "docs.v"+string(rune('a'+i)))

+ }

+ if !guard.ValidContracts(set) || guard.ValidContracts(append(set, "docs.vz")) || guard.ValidContracts([]string{"docs.v1", "docs.v1"}) || guard.ValidContracts([]string{"z.v1", "a.v1"}) {

+ t.Fatal("contract set bound/order")

+ }

+ if !guard.ValidID(id(1)) || guard.ValidID(id(1)+"=") || guard.ValidDigest(strings.ToUpper(hash("digest"))) {

+ t.Fatal("canonical digest/id")

+ }

+}

+

+func TestPermanentReferenceDigestMismatchDenies(t *testing.T) {

+ f := setup(t)

+ c := f.claim(t, 1)

+ s := f.resumed(t, c)

+ p, _ := s.Projection()

+ key, _ := guard.RequestKey(f.identity.Host, s.Value().Request.ID)

+ replace(t, f, key, []byte(`{"metadata":"changed outside immutable writer"}`))

+ denied(t, f, &p, []string{"home.operator.v1"})

+}

+

+func TestDirectHelpersBoundBeforeAllocation(t *testing.T) {

+ raw := strings.Repeat("a", 1<<20)

+ for name, fn := range map[string]func(string) bool{"id": guard.ValidID, "digest": guard.ValidDigest} {

+ t.Run(name, func(t *testing.T) {

+ if allocations := testing.AllocsPerRun(10, func() {

+ if fn(raw) {

+ panic("oversized helper input accepted")

+ }

+ }); allocations != 0 {

+ t.Fatalf("oversized direct input allocated before refusal: %v", allocations)

+ }

+ })

+ }

+}

+func TestLegacyUnmanagedIdentityNeedsNoCreationDigest(t *testing.T) {

+ f := setup(t)

+ legacy := f.identity

+ legacy.Record = ""

+ if d, e := f.repo.CheckServing(ctx, legacy, nil, nil); e != nil || d != guard.Unmanaged {

+ t.Fatalf("legacy route: %v %v", d, e)

+ }

+ bogus := legacy

+ bogus.Host = "../foreign"

+ if d, e := f.repo.CheckServing(ctx, bogus, nil, nil); e == nil || d != guard.Denied {

+ t.Fatal("invalid location accepted")

+ }

+ p := guard.Projection{Version: 1, Generation: 1, GuardDigest: hash("absent")}

+ if d, e := f.repo.CheckServing(ctx, legacy, &p, nil); e == nil || d != guard.Denied {

+ t.Fatal("projection alone accepted")

+ }

+ c := f.claim(t, 1)

+ s := f.create(t, c)

+ for _, phase := range []guard.Phase{guard.Quiescing, guard.Fenced, guard.ResumeRequested, guard.Resumed} {

+ s = f.step(t, c, s, phase)

+ }

+ p, e := s.Projection()

+ if e != nil {

+ t.Fatal(e)

+ }

+ for _, projection := range []*guard.Projection{nil, &p} {

+ if d, e := f.repo.CheckServing(ctx, legacy, projection, []string{"home.operator.v1"}); e == nil || d != guard.Denied {

+ t.Fatal("managed record lacking identity accepted")

+ }

+}

diff --git a/internal/privatefile/io_linux.go b/internal/privatefile/io_linux.go

new file mode 100644

index 000000000..206989a3e

--- /dev/null

+++ b/internal/privatefile/io_linux.go

@@ -0,0 +1,114 @@

+//go:build linux

+

+package privatefile

+

+import (

+ "io"

+ "os"

+ "path/filepath"

+ "strings"

+

+ "golang.org/x/sys/unix"

+)

+

+// Walk using retained directory descriptors so a renamed ancestor cannot turn

+// a checked path into a symlink traversal.

+func parent(path string) (*os.File, string, error) {

+ if !filepath.IsAbs(path) || filepath.Clean(path) != path || path == "/" {

+ return nil, "", ErrUnavailable

+ }

+ parts := strings.Split(strings.TrimPrefix(path, "/"), "/")

+ fd, err := unix.Open("/", unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC, 0)

+ if err != nil {

+ return nil, "", ErrUnavailable

+ }

+ for _, part := range parts[:len(parts)-1] {

+ next, e := unix.Openat(fd, part, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0)

+ unix.Close(fd)

+ if e != nil {

+ return nil, "", ErrUnavailable

+ }

+ fd = next

+ }

+ return os.NewFile(uintptr(fd), "private directory"), parts[len(parts)-1], nil

+}

+func valid(f *os.File, max int) bool {

+ var st unix.Stat_t

+ return unix.Fstat(int(f.Fd()), &st) == nil && st.Mode&unix.S_IFMT == unix.S_IFREG && st.Mode&07777 == 0600 && st.Uid == uint32(os.Geteuid()) && st.Nlink == 1 && st.Size >= 0 && st.Size <= int64(max)

+}

+func read(path string, max int) ([]byte, error) {

+ if max <= 0 || max == int(^uint(0)>>1) {

+ return nil, ErrUnavailable

+ }

+ p, name, e := parent(path)

+ if e != nil {

+ return nil, ErrUnavailable

+ }

+ defer p.Close()

+ fd, e := unix.Openat(int(p.Fd()), name, unix.O_RDONLY|unix.O_NOFOLLOW|unix.O_NONBLOCK|unix.O_CLOEXEC, 0)

+ if e != nil {

+ return nil, ErrUnavailable

+ }

+ f := os.NewFile(uintptr(fd), "private input")

+ if !valid(f, max) {

+ f.Close()

+ return nil, ErrUnavailable

+ }

+ b, e := io.ReadAll(io.LimitReader(f, int64(max)+1))

+ ok := valid(f, max)

+ ce := f.Close()

+ if e != nil || ce != nil || !ok || len(b) > max {

+ clear(b)

+ return nil, ErrUnavailable

+ }

+ return b, nil

+}

+

+type durableFile interface {

+ Write([]byte) (int, error)

+ Sync() error

+ Close() error

+}

+

+func finish(f durableFile, p interface{ Sync() error }, b []byte) error {

+ n, e := f.Write(b)

+ if e != nil || n != len(b) {

+ f.Close()

+ return ErrUnavailable

+ }

+ if f.Sync() != nil {

+ f.Close()

+ return ErrUnavailable

+ }

+ if f.Close() != nil {

+ return ErrUnavailable

+ }

+ if p.Sync() != nil {

+ return ErrUnavailable

+ }

+ return nil

+}

+func create(path string, b []byte, max int) error {

+ if max <= 0 || len(b) > max {

+ return ErrUnavailable

+ }

+ p, name, e := parent(path)

+ if e != nil {

+ return ErrUnavailable

+ }

+ defer p.Close()

+ var st unix.Stat_t

+ if unix.Fstat(int(p.Fd()), &st) != nil || st.Uid != uint32(os.Geteuid()) || st.Mode&0077 != 0 {

+ return ErrUnavailable

+ }

+ fd, e := unix.Openat(int(p.Fd()), name, unix.O_WRONLY|unix.O_CREAT|unix.O_EXCL|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0600)

+ if e != nil {

+ return ErrUnavailable

+ }

+ f := os.NewFile(uintptr(fd), "private output")

+ if !valid(f, max) {

+ f.Close()

+ return ErrUnavailable

+ }

+ return finish(f, p, b)

+}

diff --git a/internal/privatefile/io_linux_test.go b/internal/privatefile/io_linux_test.go

new file mode 100644

index 000000000..f72daf427

--- /dev/null

+++ b/internal/privatefile/io_linux_test.go

@@ -0,0 +1,207 @@

+//go:build linux

+

+package privatefile

+

+import (

+ "bytes"

+ "errors"

+ "os"

+ "path/filepath"

+ "sync"

+ "sync/atomic"

+ "testing"

+

+ "golang.org/x/sys/unix"

+)

+

+func privateDir(t *testing.T) string {

+ t.Helper()

+ d := t.TempDir()

+ // The test runner may put TMPDIR behind a symlink. Positive fixtures

+ // use its real directory; symlink-refusal cases add their own links.

+ d, err := filepath.EvalSymlinks(d)

+ if err != nil {

+ t.Fatal(err)

+ }

+ if e := os.Chmod(d, 0700); e != nil {

+ t.Fatal(e)

+ }

+ return d

+}

+func TestExactAndExclusive(t *testing.T) {

+ p := filepath.Join(privateDir(t), "result")

+ b := bytes.Repeat([]byte{0, 255, 10, 32}, 16384)

+ if e := CreateExclusiveDurable(p, b, 65536); e != nil {

+ t.Fatal(e)

+ }

+ got, e := Read(p, 65536)

+ if e != nil || !bytes.Equal(got, b) {

+ t.Fatal("exact read failed", e)

+ }

+ if CreateExclusiveDurable(p, []byte("replacement"), 65536) != ErrUnavailable {

+ t.Fatal("overwrote")

+ }

+ if _, e = Read(p, 65535); e != ErrUnavailable {

+ t.Fatal("accepted oversize")

+ }

+ if CreateExclusiveDurable(p+"big", append(b, 0), 65536) != ErrUnavailable {

+ t.Fatal("accepted oversize create")

+ }

+}

+func TestUnsafeFiles(t *testing.T) {

+ for _, kind := range []string{"mode", "symlink", "ancestor", "hardlink", "fifo", "directory", "relative", "missing", "parent"} {

+ t.Run(kind, func(t *testing.T) {

+ d := privateDir(t)

+ p := filepath.Join(d, "input")

+ if e := os.WriteFile(p, []byte("private"), 0600); e != nil {

+ t.Fatal(e)

+ }

+ switch kind {

+ case "mode":

+ os.Chmod(p, 0640)

+ case "symlink":

+ q := p + "link"

+ os.Symlink(p, q)

+ p = q

+ case "ancestor":

+ q := d + "link"

+ os.Symlink(d, q)

+ t.Cleanup(func() { os.Remove(q) })

+ p = filepath.Join(q, "input")

+ case "hardlink":

+ os.Link(p, p+"link")

+ case "fifo":

+ os.Remove(p)

+ if e := unix.Mkfifo(p, 0600); e != nil {

+ t.Fatal(e)

+ }

+ case "directory":

+ p = d

+ case "relative":

+ p = "relative"

+ case "missing":

+ p += "missing"

+ case "parent":

+ os.Chmod(d, 0755)

+ if CreateExclusiveDurable(p+"new", nil, 10) != ErrUnavailable {

+ t.Fatal("public parent")

+ }

+ return

+ }

+ if _, e := Read(p, 100); e != ErrUnavailable {

+ t.Fatal("unsafe read accepted", e)

+ }

+ if kind == "missing" {

+ return

+ }

+ if e := CreateExclusiveDurable(p, nil, 100); e != ErrUnavailable {

+ t.Fatal("unsafe create accepted", e)

+ }

+ })

+ }

+}

+func TestConcurrentCreate(t *testing.T) {

+ p := filepath.Join(privateDir(t), "race")

+ var wg sync.WaitGroup

+ var successes atomic.Int32

+ for i := 0; i < 32; i++ {

+ wg.Add(1)

+ go func() {

+ defer wg.Done()

+ if CreateExclusiveDurable(p, []byte("exact"), 100) == nil {

+ successes.Add(1)

+ }

+ }()

+ }

+ wg.Wait()

+ if successes.Load() != 1 {

+ t.Fatal(successes.Load())

+ }

+}

+

+type failingFile struct {

+ *os.File

+ stage string

+}

+

+func (f failingFile) Write(b []byte) (int, error) {

+ if f.stage == "write" {

+ n, _ := f.File.Write(b[:1])

+ return n, errors.New("private diagnostic")

+ }

+ return f.File.Write(b)

+}

+func (f failingFile) Sync() error {

+ if f.stage == "sync" {

+ return errors.New("private diagnostic")

+ }

+ return f.File.Sync()

+}

+func (f failingFile) Close() error {

+ e := f.File.Close()

+ if f.stage == "close" {

+ return errors.New("private diagnostic")

+ }

+ return e

+}

+

+type failingParent struct{}

+

+func (failingParent) Sync() error { return errors.New("private diagnostic") }

+func TestFailurePreservesFileWithoutCompletionClaim(t *testing.T) {

+ for _, stage := range []string{"write", "sync", "close", "parent"} {

+ t.Run(stage, func(t *testing.T) {

+ d := privateDir(t)

+ p := filepath.Join(d, "interrupted")

+ f, e := os.OpenFile(p, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)

+ if e != nil {

+ t.Fatal(e)

+ }

+ parent, e := os.Open(d)

+ if e != nil {

+ t.Fatal(e)

+ }

+ defer parent.Close()

+ var barrier interface{ Sync() error } = parent

+ if stage == "parent" {

+ barrier = failingParent{}

+ }

+ if e = finish(failingFile{f, stage}, barrier, []byte("private")); e != ErrUnavailable {

+ t.Fatal(e)

+ }

+ if _, e = os.Stat(p); e != nil {

+ t.Fatal("failed file removed")

+ }

+ if CreateExclusiveDurable(p, []byte("replacement"), 100) != ErrUnavailable {

+ t.Fatal("failed file overwritten")

+ }

+ // A read can succeed even after failed sync; it never proves durability.

+ if _, e = Read(p, 100); e != nil {

+ t.Fatal(e)

+ }

+ })

+ }

+}

+

+func TestCreateThroughSymlinkAndBounds(t *testing.T) {

+ d := privateDir(t)

+ alias := d + "-alias"

+ if e := os.Symlink(d, alias); e != nil {

+ t.Fatal(e)

+ }

+ t.Cleanup(func() { os.Remove(alias) })

+ if CreateExclusiveDurable(filepath.Join(alias, "new"), []byte("secret"), 10) != ErrUnavailable {

+ t.Fatal("ancestor symlink followed")

+ }

+ if _, e := os.Stat(filepath.Join(d, "new")); !os.IsNotExist(e) {

+ t.Fatal("created through alias")

+ }

+ for _, max := range []int{0, -1} {

+ if CreateExclusiveDurable(filepath.Join(d, "new"), nil, max) != ErrUnavailable {

+ t.Fatal("invalid bound")

+ }

+ if _, e := Read(filepath.Join(d, "new"), max); e != ErrUnavailable {

+ t.Fatal("invalid read bound")

+ }

+}

diff --git a/internal/privatefile/io_other.go b/internal/privatefile/io_other.go

new file mode 100644

index 000000000..a07694fd8

--- /dev/null

+++ b/internal/privatefile/io_other.go

@@ -0,0 +1,6 @@

+//go:build !linux

+

+package privatefile

+

+func read(string, int) ([]byte, error) { return nil, ErrUnsupported }

+func create(string, []byte, int) error { return ErrUnsupported }

diff --git a/internal/privatefile/privatefile.go b/internal/privatefile/privatefile.go

new file mode 100644

index 000000000..048f7317e

--- /dev/null

+++ b/internal/privatefile/privatefile.go

@@ -0,0 +1,25 @@

+// Package privatefile handles bounded private files. Runtime support is Linux

+// only; other platforms refuse operations until their privacy guarantees exist.

+package privatefile

+

+import "errors"

+

+var ErrUnavailable = errors.New("private file unavailable")

+var ErrUnsupported = errors.New("private files unsupported on this platform")

+

+// Read returns exact bytes from an owned, single-link, regular 0600 file.

+// Paths must be absolute and already clean. It rejects symlinks in every path component. The caller validates payload

+// syntax and completeness: Read cannot distinguish an interrupted write from

+// intentionally short content. max must be positive.

+func Read(path string, max int) ([]byte, error) { return read(path, max) }

+

+// CreateExclusiveDurable creates a new 0600 file and checks file and parent

+// synchronization before returning success. The parent must already exist and

+// be owned and private. The path must be absolute and already clean. Failures preserve any created file, which prevents

+// retry from overwriting it. The caller must retain its recovery material and

+// must not treat a later Read as proof that this operation completed. A future

+// resume caller needs its own checked durability barrier after validating the

+// saved payload; reopening a file does not repair a failed synchronization.

+func CreateExclusiveDurable(path string, data []byte, max int) error {

+ return create(path, data, max)

+}

diff --git a/internal/provision/maintenance.go b/internal/provision/maintenance.go

new file mode 100644

index 000000000..dc8b6aa90

--- /dev/null

+++ b/internal/provision/maintenance.go

@@ -0,0 +1,94 @@

+package provision

+

+import (

+ "bytes"

+ "encoding/json"

+ "errors"

+ "io"

+ "strings"

+

+ "github.com/carlosframework/platform/internal/maintenanceguard"

+)

+

+var ErrMaintenanceProjection = errors.New("provision: invalid maintenance projection or protected fields")

+

+// MaintenanceProjection retains a present projection, including invalid input.

+// Its value receiver validates on marshal, so an ignored decode error cannot

+// turn a present invalid projection into an absent one. An empty slice is absent.

+type MaintenanceProjection []byte

+

+func (p *MaintenanceProjection) UnmarshalJSON(raw []byte) error {

+ *p = append((*p)[:0], raw...)

+ _, err := maintenanceguard.DecodeProjection(raw)

+ if err != nil {

+ return ErrMaintenanceProjection

+ }

+ return nil

+}

+func (p MaintenanceProjection) MarshalJSON() ([]byte, error) {

+ if _, err := maintenanceguard.DecodeProjection(p); err != nil {

+ return nil, ErrMaintenanceProjection

+ }

+ return append([]byte(nil), p...), nil

+}

+

+// DecodeRecord is the bucket-record parser. API response views may use ordinary

+// JSON decoding; they do not authorize writes. Keep this off Record's method

+// set so embedding Record cannot swallow a wrapper's fields or Gate shadow.

+func DecodeRecord(raw []byte) (Record, error) {

+ invalid := Record{Maintenance: MaintenanceProjection("null")}

+ dec := json.NewDecoder(bytes.NewReader(raw))

+ first, e := dec.Token()

+ if e != nil || first != json.Delim('{') {