34406a1b7 · carlos/platform

diff --git a/internal/activator/wakecheck.go b/internal/activator/wakecheck.go

index 265a7ee11..803840d9b 100644

--- a/internal/activator/wakecheck.go

+++ b/internal/activator/wakecheck.go

@@ -36,6 +36,16 @@ type HeldWake struct {

Held bool // false: root follows the channel for it; today's wake

Unknown bool // rule 3's "build unknown"

Failed bool // root reports Target failed on this box (its decision, or a refusal record)

+ // AskRoot: root has made no decision for the instance's current

+ // settings (ruling R-HELD-PURE), so whether it is held is not known

+ // here. The wake holds: it runs Target only if the live link already

+ // names it, and otherwise asks root for a pass and waits for its

+ // decision. Target is the build state's build ("" when there is none).

+ AskRoot bool

+ // Unread: the build state could not be read for this wake (its required

+ // refresh failed). Refused: never decided from a stale copy (Astra round

+ // 1 finding 1).

+ Unread bool

// Staged is a retained copy of Target on this box whose bytes match its

// staged manifest; nil when there is none.

Staged *release.StagedIdentity

@@ -49,6 +59,14 @@ const heldPoll = 250 * time.Millisecond

// rule so it holds with CARLOS_SWAP off and no planner (Astra I3 #2, #5):

//

// - not held: "" and nothing to do, today's wake;

+// - unread (the build state could not be read for this wake):

+// errBuildStateUnwritten, nothing asked or run (Astra round 1 finding 1);

+// - root has not decided for the current settings (AskRoot, ruling

+// R-HELD-PURE): the build state's build runs if the live link already

+// names it; otherwise root is asked for a pass (RequestStage naming no

+// build) and the wake waits for its decision, within the same bound,

+// then goes on below with that answer. It never runs a live link it

+// does not know it may run;

// - unknown: errBuildUnknown, and root is not asked (it installs nothing);

// - a target root reports failed: refused (errTargetFailed) with no

// planner, even while it is still the live link; with a planner the

@@ -78,6 +96,23 @@ func (a *Service) wakeCheck(ctx context.Context, host string, direct bool) (stri

if !h.Held {

return "", nil, nil

}

+ if h.Unread {

+ return "", nil, fmt.Errorf("%w: the build state of %s could not be read for its wake", errBuildStateUnwritten, host)

+ }

+ wait, cancel := context.WithTimeout(ctx, wakeWaitBound)

+ defer cancel()

+ if h.AskRoot {

+ var err error

+ if h, err = a.awaitRootDecision(wait, host, h); err != nil {

+ return "", nil, err

+ }

+ switch {

+ case !h.Held:

+ return "", nil, nil

+ case h.Unread:

+ return "", nil, fmt.Errorf("%w: the build state of %s could not be read for its wake", errBuildStateUnwritten, host)

+ }

if h.Unknown || h.Target == "" {

return "", nil, errBuildUnknown

}

@@ -106,8 +141,6 @@ func (a *Service) wakeCheck(ctx context.Context, host string, direct bool) (stri

if a.RequestStage != nil {

a.RequestStage(host, h.Target)

}

- wait, cancel := context.WithTimeout(ctx, wakeWaitBound)

- defer cancel()

tick := time.NewTicker(heldPoll)

defer tick.Stop()

for n := 1; ; n++ {

@@ -148,6 +181,47 @@ func (a *Service) wakeCheck(ctx context.Context, host string, direct bool) (stri

// HeldTarget again (which may read the bucket).

const heldReask = 8

+// awaitRootDecision is wakeCheck's wait for root when root has made no

+// decision for the instance's current settings (h.AskRoot; ruling

+// R-HELD-PURE). It returns h itself when the wake may go on now: its target

+// failed (the failure rule decides, below) or the live link already names

+// it, the build the data last ran. Otherwise it asks root for a pass,

+// naming no build (it does not know which root will install), and asks

+// HeldTarget again every heldReask polls until root's answer is no longer

+// AskRoot, which it returns; or the live link names the target (root decided

+// to hold it and installed it). A wait that ends first is errStaleLink.

+func (a *Service) awaitRootDecision(wait context.Context, host string, h HeldWake) (HeldWake, error) {

+ if h.Failed || (h.Target != "" && a.liveVersion(host) == h.Target) {

+ return h, nil

+ }

+ if a.RequestStage != nil {

+ a.RequestStage(host, "")

+ }

+ tick := time.NewTicker(heldPoll)

+ defer tick.Stop()

+ for n := 1; ; n++ {

+ if h.Target != "" && a.liveVersion(host) == h.Target {

+ return h, nil

+ }

+ if n%heldReask == 0 && wait.Err() == nil {

+ next, ok := a.reaskHeld(wait, host)

+ if !ok {

+ break

+ }

+ if !next.AskRoot {

+ return next, nil

+ }

+ h = next

+ }

+ select {

+ case <-wait.Done():

+ return HeldWake{}, fmt.Errorf("%w: root has not decided for %s's current update settings", errStaleLink, host)

+ case <-tick.C:

+ }

+ return HeldWake{}, fmt.Errorf("%w: root has not decided for %s's current update settings", errStaleLink, host)

+}

+

// reaskHeld is HeldTarget(host) bounded by wait: false when wait ends first.

// The call itself is not cancelled (HeldTarget takes no context); it finishes

// within its own budget into a buffered channel nobody reads.

@@ -164,7 +238,8 @@ func (a *Service) reaskHeld(wait context.Context, host string) (HeldWake, bool)

// sameHeld reports whether two answers send the wake to the same place.

func sameHeld(x, y HeldWake) bool {

- return x.Held == y.Held && x.Target == y.Target && x.Unknown == y.Unknown && x.Failed == y.Failed

+ return x.Held == y.Held && x.Target == y.Target && x.Unknown == y.Unknown && x.Failed == y.Failed &&

+ x.AskRoot == y.AskRoot && x.Unread == y.Unread

}

// liveVersion is what live/<host> names now, "" when it cannot be read.

diff --git a/internal/activator/wakecheck_test.go b/internal/activator/wakecheck_test.go

index 837a8b4ff..0fea754d5 100644

--- a/internal/activator/wakecheck_test.go

+++ b/internal/activator/wakecheck_test.go

@@ -576,3 +576,94 @@ func TestAHeldWakesReaskNeverOutlastsTheWait(t *testing.T) {

t.Fatal("a wake that never saw its target ran something")

}

+

+// Astra round 1 finding 1: a wake whose build state could not be read for it

+// is refused before anything is acquired, and root is not asked.

+func TestAnUnreadBuildStateRefusesTheWake(t *testing.T) {

+ a, b := identityActivator(t, "v1")

+ a.HeldTarget = func(string) HeldWake { return HeldWake{Held: true, Unread: true} }

+ a.RequestStage = func(string, string) { t.Error("asked root for a wake whose build state is unread") }

+ _, err := a.beginRequest(context.Background(), identityRoute(t, "one.example.com"))

+ if !errors.Is(err, errBuildStateUnwritten) {

+ t.Fatalf("err = %v, want errBuildStateUnwritten", err)

+ }

+ if acq, _, starts := b.counts(); acq != 0 || starts != 0 {

+ t.Fatalf("acquired %d, started %d: a wake decided from a stale copy", acq, starts)

+ }

+}

+

+// Ruling R-HELD-PURE (Astra round 1 finding 4): with no decision from root

+// for the current settings the wake holds. Its build state's build runs at

+// once when the live link names it, and root is not asked.

+func TestAWakeRootHasNotDecidedForRunsItsInstalledBuild(t *testing.T) {

+ a, b := identityActivator(t, "v1")

+ a.HeldTarget = func(string) HeldWake { return HeldWake{Held: true, AskRoot: true, Target: "v1"} }

+ a.RequestStage = func(string, string) { t.Error("asked root although the build state's build is installed") }

+ done, err := a.beginRequest(context.Background(), identityRoute(t, "one.example.com"))

+ if err != nil {

+ t.Fatal(err)

+ }

+ done()

+ if b.lastExeVersion() != "v1" || b.startCount() != 1 {

+ t.Fatalf("started %d of %q, want v1", b.startCount(), b.lastExeVersion())

+ }

+}

+

+// Otherwise it never runs the live link it does not know it may run: it asks

+// root for a pass (no build named) and waits for root's decision, then

+// follows it: a hold on v1 runs v1 once root installs it; a decision that the

+// instance is not held is today's wake; no decision within the bound fails

+// the wake with nothing run.

+func TestAWakeRootHasNotDecidedForWaitsForRoot(t *testing.T) {

+ cases := []struct {

+ name string

+ decided HeldWake

+ install string

+ want string // the build started, "" for none

+ }{

+ {"held on its build", HeldWake{Held: true, Target: "v1"}, "v1", "v1"},

+ {"not held", HeldWake{}, "", "v3"},

+ {"never decides", HeldWake{Held: true, AskRoot: true, Target: "v1"}, "", ""},

+ }

+ for _, c := range cases {

+ t.Run(c.name, func(t *testing.T) {

+ shrinkWakeWait(t, 3*time.Second)

+ a, b := identityActivator(t, "v3")

+ var asked atomic.Bool

+ a.HeldTarget = func(string) HeldWake {

+ if asked.Load() {

+ return c.decided

+ }

+ return HeldWake{Held: true, AskRoot: true, Target: "v1"}

+ }

+ r := &asks{then: func(host, build string) {

+ asked.Store(true) // root's pass decides

+ if c.install != "" && build == c.install {

+ b.setLive(c.install)

+ }

+ }}

+ a.RequestStage = r.ask

+ done, err := a.beginRequest(context.Background(), identityRoute(t, "one.example.com"))

+ if c.want == "" {

+ if !errors.Is(err, errStaleLink) && !errors.Is(err, errWakeTimeout) {

+ t.Fatalf("err = %v; want the wake to fail", err)

+ }

+ waitFor(t, "the wake to end", func() bool { return a.entryGone("one.example.com") })

+ if b.startCount() != 0 {

+ t.Fatalf("started %q with no decision from root", b.lastExeVersion())

+ }

+ return

+ }

+ if err != nil {

+ t.Fatal(err)

+ }

+ done()

+ if b.lastExeVersion() != c.want || b.startCount() != 1 {

+ t.Fatalf("started %d of %q, want %s", b.startCount(), b.lastExeVersion(), c.want)

+ }

+ if got := r.seen(); len(got) == 0 || got[0] != "one.example.com@" {

+ t.Fatalf("asked %v; want a pass asked for first, naming no build", got)

+ }

+ })

+ }

+}

diff --git a/internal/host/agent.go b/internal/host/agent.go

index 9c58b3494..54eccff12 100644

--- a/internal/host/agent.go

+++ b/internal/host/agent.go

@@ -461,7 +461,17 @@ func runAgent(args []string, buildClaims ClaimFactory) {

svc.HeldTarget = func(h string) activator.HeldWake { return bs.wakeTarget(context.Background(), h) }

if !cfg.HostWrites.Enabled {

askStage := stageRequester(dataDir())

- svc.RequestStage = func(host, build string) { askStage(host, build, nil) }

+ // A wake waiting for root's decision on new settings names

+ // no build (ruling R-HELD-PURE): one marker for a root pass,

+ // nothing the relay would keep asking for.

+ askPass := stageMarkerRequester(dataDir())

+ svc.RequestStage = func(host, build string) {

+ if build == "" {

+ askPass(host, build, nil)

+ return

+ }

+ askStage(host, build, nil)

+ }

}

// Every worker's captured serving build, published for root's pin

@@ -503,7 +513,7 @@ func runAgent(args []string, buildClaims ClaimFactory) {

} else {

eb.Mirror = m

}

- armMovesPass(*cfg, dd, reg, svc, eb, claims.Writes, claims.Providers, rolloutSink)

+ armMovesPass(*cfg, dd, reg, svc, eb, claims.Writes, claims.Providers, rolloutSink, moveCarry(bs, bsErr))

// The startup sweep (design §6.7): a leftover instance unit from a

// previous agent process, one this box's registry no longer holds

// a live wake for, is stopped before this process starts spawning

diff --git a/internal/host/buildstate.go b/internal/host/buildstate.go

index b68d89c72..81dfa2e41 100644

--- a/internal/host/buildstate.go

+++ b/internal/host/buildstate.go

@@ -428,6 +428,16 @@ const updateTries = 3

// set under the same lock, so a concurrent refresh cannot put older state

// over it. A failed update writes nothing the caller may run on.

func (b *buildStates) update(ctx context.Context, host string, merge func(cur *release.BuildState) release.BuildState) (release.BuildState, error) {

+ return b.write(ctx, host, func(cur *release.BuildState) (release.BuildState, bool, error) { return merge(cur), true, nil }, nil)

+}

+

+// write is update with a merge that may refuse (an error: nothing is

+// written) or find nothing to write (false: the object and the registry copy

+// are left as they are, and the state as read is returned). replica says

+// whether the instance has a replica, for the registry copy; nil is yes (a

+// process is about to run against the database).

+func (b *buildStates) write(ctx context.Context, host string, merge func(cur *release.BuildState) (release.BuildState, bool, error),

+ replica func(ctx context.Context, store release.Store, prefix string) (bool, error)) (release.BuildState, error) {

m := b.hostLock(host)

m.Lock()

defer m.Unlock()

@@ -441,7 +451,16 @@ func (b *buildStates) update(ctx context.Context, host string, merge func(cur *r

if err != nil {

return release.BuildState{}, err

}

- next := merge(cur)

+ next, put, err := merge(cur)

+ if err != nil {

+ return release.BuildState{}, err

+ }

+ if !put {

+ if cur == nil {

+ return release.BuildState{}, nil

+ }

+ return *cur, nil

+ }

next.Schema, next.Host = 1, host

next.At = b.now().UTC().Truncate(time.Second)

raw, err := json.MarshalIndent(next, "", " ")

@@ -456,9 +475,15 @@ func (b *buildStates) update(ctx context.Context, host string, merge func(cur *r

} else if err != nil {

return release.BuildState{}, err

}

- // A process is about to run against the database, so it has (or is

- // about to have) a replica.

- c := release.BuildStateCopy{Known: true, Replica: true, State: &next, ReadAt: b.now().UTC().Truncate(time.Second)}

+ // A process about to run against the database has (or is about to

+ // have) a replica; any other writer says.

+ hasRep := true

+ if replica != nil {

+ if hasRep, err = replica(ctx, store, prefix); err != nil {

+ return next, fmt.Errorf("build state of %s written, its replica not listed: %w", host, err)

+ }

+ c := release.BuildStateCopy{Known: true, Replica: hasRep, State: &next, ReadAt: b.now().UTC().Truncate(time.Second)}

if err := b.reg.SetBuildState(host, c.Column()); err != nil {

return next, fmt.Errorf("build state of %s written, registry copy not: %w", host, err)

}

@@ -476,7 +501,9 @@ func (b *buildStates) update(ctx context.Context, host string, merge func(cur *r

// It fails closed. A consumed file the agent cannot read refuses the exec:

// a write without it could miss a resolution and leave R uncompleted after

// its T ran. update sets the registry copy itself, and refuses an object it

-// cannot parse rather than overwrite it (ruling R-BS-CORRUPT).

+// cannot parse rather than overwrite it (ruling R-BS-CORRUPT). For a row

+// settings apply to, a state that now names another held target than build

+// (release.HeldExecStale) refuses it too: the wake decided on a stale copy.

func (b *buildStates) beforeExec(ctx context.Context, host, build string) error {

if build == "" {

return fmt.Errorf("no build to record for %s", host)

@@ -485,12 +512,64 @@ func (b *buildStates) beforeExec(ctx context.Context, host, build string) error

if err != nil {

return fmt.Errorf("read root's consumed file for %s: %w", host, err)

}

- _, err = b.update(ctx, host, func(cur *release.BuildState) release.BuildState {

- return release.NextBuildState(cur, consumed, build)

- })

+ // The held target is revalidated against the state read under the lease

+ // (Astra round 1 finding 1): a wake that decided on a stale copy is

+ // refused here, before anything is written or run.

+ rt, _ := b.reg.Get(host)

+ held := rt != nil && settingsApply(*rt)

+ _, err = b.write(ctx, host, func(cur *release.BuildState) (release.BuildState, bool, error) {

+ if held {

+ if err := release.HeldExecStale(release.DecideHeldWake(b.heldWakeInput(*rt, cur)), cur, build); err != nil {

+ return release.BuildState{}, false, err

+ }

+ return release.NextBuildState(cur, consumed, build), true, nil

+ }, nil)

+ return err

+}

+

+// carryRequest is moveCfg.CarryRequest (ruling R-SPEC-43A-2; Astra round 1

+// finding 3): before a cut seals host's build state at bucket/prefix (the

+// lease's location: the hibernate prefix for a legacy cut, the claim

+// holder's location for a Phase 1 cut or a return), root's consumed record

+// on this box is merged into it with release.CarryRequest: a resolution and

+// a failed completion, never a successful completion, never last_executed.

+// The object is written only when that changes its request state, under the

+// host's lock and against the etag it read (write), so it never overwrites a

+// newer record. A consumed file that does not read refuses the carry, and so

+// the cut: a request this box closed would reopen on the destination.

+func (b *buildStates) carryRequest(ctx context.Context, host, bucket, prefix string) error {

+ consumed, err := release.ReadConsumed(b.optDir, host)

+ if err != nil {

+ return fmt.Errorf("read root's consumed file for %s: %w", host, err)

+ }

+ if consumed == nil {

+ return nil

+ }

+ ctx = activator.WithBuildStateLocation(ctx, activator.BuildStateLocation{Bucket: bucket, Prefix: prefix})

+ _, err = b.write(ctx, host, func(cur *release.BuildState) (release.BuildState, bool, error) {

+ next := release.CarryRequest(cur, consumed)

+ return next, !release.SameRequestState(cur, &next), nil

+ }, hasReplica)

return err

}

+// moveCarry is moveCfg.CarryRequest for this box: the build-state store's

+// carry, a refusal when the store did not start (a cut must not seal a

+// state it could not bring up to date), and nil on a box with no replica

+// bucket (no build state to carry).

+func moveCarry(bs *buildStates, initErr error) func(ctx context.Context, host, bucket, prefix string) error {

+ if initErr != nil {

+ return func(context.Context, string, string, string) error {

+ return fmt.Errorf("the build state store did not start on this box (%v)", initErr)

+ }

+ if bs == nil {

+ return nil

+ }

+ return bs.carryRequest

+}

+

// buildStateHook is Service.BeforeExec for this box (settings spec §4.2,

// Astra I3 #1). Only a box with no replica bucket configured (bs nil, no

// error) keeps the nil hook: it runs no hibernating instance, so it has no

@@ -510,20 +589,18 @@ func buildStateHook(bs *buildStates, initErr error) func(ctx context.Context, ho

}

// heldTarget is the held wake check's answer (activator.Service.HeldTarget;

-// settings spec §4.3 rule 3): the target a held instance must wake onto, in

-// root's rule order (Target): a pin, then a pending request's T, then

-// last_executed (Astra I2 #7). Held is false for an instance root follows the

-// channel for. Failed and Staged feed the wake's own admission rule, which

-// holds with no planner (Astra I3 #2, #5).

+// settings spec §4.3 rule 3): release.DecideHeldWake (ruling R-HELD-PURE)

+// on what root reads and what root wrote: the route's settings column, its

+// build-state copy, root's consumed file and refusal record, and the build

+// the live link names. The decision is root's rule order (Target): a pin,

+// then a pending request's T, then rule 3's target (last_executed, else the

+// installed build, each failure-checked). Root's consumed reason counts only

+// for the settings column it was decided for: with none, a wake that could

+// be held holds and asks root, never "not held" (Astra round 1 finding 4).

+// A copy whose last refresh failed is refused as unread (finding 1).

//

-// It reads what root reads and what root wrote. The column is parsed as root

-// parses it (ParseUpdateSettingsLenient). One that does not parse even then

-// is held exactly as root holds it: root records reason unknown for it

-// (ruling R-SETTINGS-UNREADABLE), and a wake holds as unknown while that is

-// so; with no such word from root it is not held. Root's consumed reason

-// decides whether a major hold is in force (the agent does not read tags).

-// A pending request's T is followed only with the build state read (Known):

-// with it unread root does no request bookkeeping and will not install T.

+// A held, decided target that root does not report failed is offered with

+// this box's verified retained copy of it, if any (Staged).

//

// Settings hold only the rows root applies them to (settingsApply; ruling

// R-HELD-SCOPE): every other row, a named unit or an exec worker under an

@@ -533,76 +610,29 @@ func (b *buildStates) heldTarget(host string) activator.HeldWake {

if err != nil || rt == nil || !settingsApply(*rt) {

return activator.HeldWake{}

}

- c, _ := release.ReadConsumed(b.optDir, host)

- rootUnknown := c != nil && c.Reason == string(release.ReasonUnknown)

- u, err := release.ParseUpdateSettingsLenient(rt.UpdateSettings)

- if err != nil {

- if rootUnknown {

- return activator.HeldWake{Held: true, Unknown: true}

- }

- return activator.HeldWake{}

+ d := release.DecideHeldWake(b.heldWakeInput(*rt, nil))

+ h := activator.HeldWake{Target: d.Target, Held: d.Held, Unknown: d.Unknown, Failed: d.Failed, AskRoot: d.AskRoot, Unread: d.Unread}

+ if h.Held && h.Target != "" && !h.Failed && !h.AskRoot && !h.Unknown && !h.Unread {

+ h.Staged = b.retained(*rt, h.Target)

}

- if u.Default() {

- return activator.HeldWake{}

- }

- bs, _ := release.ParseBuildStateCopy(rt.BuildState)

- refusal, _ := release.ReadRefusal(b.bakeDir, host)

- // rootFailed: root's own decision found v failed, or root's refusal

- // record says v did not come up here. A wrong-program or not-proven

- // refusal is not a failure of the build (ruling R-REFUSALKIND).

- rootFailed := func(v string) bool {

- return (c != nil && c.Reason == string(release.ReasonFailed) && c.Detail == v) ||

- (refusal != nil && refusal.Kind == release.RefusalNotUp && refusal.Version == v)

- }

- held := func(v string) activator.HeldWake {

- h := activator.HeldWake{Target: v, Held: true, Failed: rootFailed(v)}

- if !h.Failed {

- h.Staged = b.retained(*rt, v)

- }

- return h

- }

- // Root's order (Target, §4.3): a pin outranks a pending request (I2 #7).

- if u.Pin != "" {

- return held(u.Pin)

- }

- // The build state's resolution before the consumed file's, as root's

- // resolvedFor (ruling R-RESOLUTION-STATE-WINS): when both name the same

- // request, root installs the build state's T, so the wake targets it.

- if bs.Known && u.InstallRequest > release.Watermark(c, bs.State) {

- if bs.State != nil && bs.State.Resolved != nil && bs.State.Resolved.ID == u.InstallRequest {

- return held(bs.State.Resolved.Target)

- }

- if c != nil && c.Resolved != nil && c.Resolved.ID == u.InstallRequest {

- return held(c.Resolved.Target)

- }

- le := ""

- if bs.State != nil {

- le = bs.State.LastExecuted

- }

- // Rule 3 (held): manual, or a major hold in force by root's word. Root

- // rewrites a major hold whose last_executed it holds as failed to

- // Reason failed with that build as Detail, so that decision is a major

- // hold too (ruling R-HELD-FAILED): held on it, and Failed (rootFailed).

- majorFailed := c != nil && u.Major != nil && le != "" && c.Reason == string(release.ReasonFailed) && c.Detail == le

- heldByRoot := c != nil && (c.Reason == string(release.ReasonManual) || c.Reason == string(release.ReasonMajor) || rootUnknown) || majorFailed

- if u.Updates != release.UpdatesManual && !heldByRoot {

- return activator.HeldWake{}

+ return h

+}

+

+// heldWakeInput reads DecideHeldWake's input for rt. state, when set, is

+// the build state just read under the lease (beforeExec's revalidation), in

+// place of the registry copy. A consumed file or refusal record that does

+// not read is no word from root (nil); a copy that does not parse is unread.

+func (b *buildStates) heldWakeInput(rt registry.Route, state *release.BuildState) release.HeldWakeInput {

+ in := release.HeldWakeInput{Settings: rt.UpdateSettings}

+ in.Consumed, _ = release.ReadConsumed(b.optDir, rt.Host)

+ in.Refusal, _ = release.ReadRefusal(b.bakeDir, rt.Host)

+ in.Installed, _ = release.LiveVersion(b.optDir, rt.Host)

+ if state != nil {

+ in.Build = release.BuildStateCopy{Known: true, Replica: true, State: state}

+ } else {

+ in.Build, _ = release.ParseBuildStateCopy(rt.BuildState)

}

- // Ruling R-HELD-UNKNOWN, as root's Target decides rule 3: an unread copy

- // is unknown, whatever root last wrote; a read one with last_executed

- // targets it, even over a stale unknown of root's (root reaches the same

- // answer from the same copy, and the wake asks it to). Root's unknown

- // stands only for a read copy with no last_executed.

- switch {

- case !bs.Known:

- return activator.HeldWake{Held: true, Unknown: true}

- case le != "":

- return held(le)

- case rootUnknown:

- return activator.HeldWake{Held: true, Unknown: true}

- }

- return activator.HeldWake{} // no build state: whatever root installed is the instance's

+ return in

}

// settingsApply reports a row root applies update settings to: Kind

@@ -626,8 +656,8 @@ const wakeHeldBudget = 5 * time.Second

// at every reconcile and before every wake"), for a route whose settings

// could hold it. A route with default settings, or one root applies no

// settings to (settingsApply), is today's wake: nothing is read for it. A

-// refresh that fails keeps the last copy (refresh records the failure), and

-// the answer is read from that, as root reads it.

+// refresh that fails refuses the wake as unread (Astra round 1 finding 1):

+// the last copy may name a build the data has since moved past.

func (b *buildStates) wakeTarget(ctx context.Context, host string) activator.HeldWake {

rt, err := b.reg.Get(host)

if err != nil || rt == nil || !settingsApply(*rt) {

@@ -638,10 +668,16 @@ func (b *buildStates) wakeTarget(ctx context.Context, host string) activator.Hel

}

if keepsBuildState(*rt, b.claims != nil) {

ctx, cancel := context.WithTimeout(ctx, wakeHeldBudget)

- if err := b.refresh(ctx, host); err != nil {

- log.Printf("agent: build state of %s before its wake: %v", host, err)

- }

+ err := b.refresh(ctx, host)

cancel()

+ if err != nil {

+ // Refused, never decided from the last copy (Astra round 1

+ // finding 1): the durable state may name another build by now.

+ // refresh recorded the failure in the copy, which DecideHeldWake

+ // refuses too, unless the registry write itself failed.

+ log.Printf("agent: %s not woken: its build state could not be read before the wake: %v", host, err)

+ return activator.HeldWake{Held: true, Unread: true}

+ }

}

return b.heldTarget(host)

}

diff --git a/internal/host/buildstate_astra_test.go b/internal/host/buildstate_astra_test.go

new file mode 100644

index 000000000..5b5416890

--- /dev/null

+++ b/internal/host/buildstate_astra_test.go

@@ -0,0 +1,174 @@

+package host

+

+import (

+ "context"

+ "errors"

+ "os"

+ "strings"

+ "testing"

+

+ "github.com/carlosframework/platform/internal/release"

+)

+

+// Settings plan 4, Astra round 1 findings 1, 4 and 5: the held wake's answer

+// (heldTarget, wakeTarget) and the pre-exec write's revalidation.

+

+const astraHost = "a.example.com"

+

+// decided writes root's consumed file as a decision made for the route's

+// current update_settings column (Consumed.Settings, ruling R-HELD-PURE).

+func decided(t *testing.T, b *buildStates, c release.Consumed) {

+ t.Helper()

+ rt, err := b.reg.Get(astraHost)

+ if err != nil || rt == nil {

+ t.Fatalf("route: %v", err)

+ }

+ c.Settings = release.SettingsDigest(rt.UpdateSettings)

+ writeConsumed(t, b, c)

+}

+

+// installLive stages ver of km and points the host's live link at it, as

+// root's install leaves a box.

+func installLive(t *testing.T, b *buildStates, ver string) {

+ t.Helper()

+ if err := release.StageArtifact(b.optDir, "bab", ver, "km", []byte(ver)); err != nil {

+ t.Fatal(err)

+ }

+ if err := release.PointLive(b.optDir, astraHost, "bab", ver, "km"); err != nil {

+ t.Fatal(err)

+ }

+ if v, _ := release.LiveVersion(b.optDir, astraHost); v != ver {

+ t.Fatalf("live = %q, want %s", v, ver)

+ }

+}

+

+// Finding 4 (§4.3 rule 3; ruling R-HELD-PURE): a sleeper last ran B (major

+// 1); under auto, root cold-installed A (major 2). The member sets major 1.

+// Before root decides for the new settings, its consumed file still says

+// adopt, for the old column. The wake must not read that as "not held" and

+// run A against B's data: it holds on B and asks root. Root's decision for

+// the new column then decides: a major hold holds on B; an adoption (the head

+// is in major 1) is not held. No consumed file at all holds and asks too.

+func TestAMajorHoldIsNotBypassedBeforeRootDecidesForTheSettings(t *testing.T) {

+ b, _, reg := testBuildStates(t)

+ installLive(t, b, "A")

+ setCopy(t, reg, release.BuildStateCopy{Known: true, Replica: true, State: &release.BuildState{LastExecuted: "B"}})

+ setSettings(t, reg, release.UpdateSettings{Updates: release.UpdatesAuto, Channel: "stable"})

+ decided(t, b, release.Consumed{Reason: string(release.ReasonAdopt), Desired: "A", Detail: "A"})

+ if h := b.heldTarget(astraHost); h.Held {

+ t.Fatalf("auto, root decided adoption for these settings: %+v; want today's wake", h)

+ }

+ one := 1

+ setSettings(t, reg, release.UpdateSettings{Updates: release.UpdatesAuto, Channel: "stable", Major: &one})

+ if h := b.heldTarget(astraHost); !h.Held || !h.AskRoot || h.Target != "B" || h.Staged != nil {

+ t.Fatalf("major 1, root's decision is for the old settings: %+v; want held on B, asking root, never A", h)

+ }

+ decided(t, b, release.Consumed{Reason: string(release.ReasonMajor), Desired: "B", Detail: "B"})

+ if h := b.heldTarget(astraHost); !h.Held || h.AskRoot || h.Target != "B" {

+ t.Fatalf("root's major hold for these settings: %+v; want held on B", h)

+ }

+ decided(t, b, release.Consumed{Reason: string(release.ReasonAdopt), Desired: "C", Detail: "C"})

+ if h := b.heldTarget(astraHost); h.Held {

+ t.Fatalf("root's adoption for these settings: %+v; want today's wake", h)

+ }

+ if err := os.Remove(release.ConsumedPath(b.optDir, astraHost)); err != nil {

+ t.Fatal(err)

+ }

+ if h := b.heldTarget(astraHost); !h.Held || !h.AskRoot || h.Target != "B" {

+ t.Fatalf("no decision from root at all: %+v; want held on B, asking root", h)

+ }

+}

+

+// Finding 5 (§4.3 failure precedence, §6): a legacy manual instance with its

+// build state read and absent, T installed, and root's decision that T

+// failed. The installed-build fallback is failure-checked like

+// last_executed: held on T and failed, never "not held" (which ran T).

+func TestAFailedInstalledBuildWithNoBuildStateIsHeldFailed(t *testing.T) {

+ b, _, reg := testBuildStates(t)

+ installLive(t, b, "T")

+ setSettings(t, reg, release.UpdateSettings{Updates: release.UpdatesManual, Channel: "stable"})

+ setCopy(t, reg, release.BuildStateCopy{Known: true, Replica: true})

+ decided(t, b, release.Consumed{Reason: string(release.ReasonFailed), Detail: "T"})

+ if h := b.heldTarget(astraHost); !h.Held || !h.Failed || h.Target != "T" || h.Staged != nil {

+ t.Fatalf("heldTarget = %+v; want T held and failed", h)

+ }

+ // Not failed: held on the installed build, as root's Target holds it.

+ decided(t, b, release.Consumed{Reason: string(release.ReasonManual), Desired: "T", Detail: "T"})

+ if h := b.heldTarget(astraHost); !h.Held || h.Failed || h.Target != "T" {

+ t.Fatalf("heldTarget = %+v; want held on the installed T", h)

+ }

+}

+

+// stateGetFails fails every Get of a build state, as a bucket blip.

+type stateGetFails struct {

+ release.Store

+ err error

+}

+

+func (f stateGetFails) Get(ctx context.Context, key string) ([]byte, string, error) {

+ if strings.HasSuffix(key, "/build-state.json") {

+ return nil, "", f.err

+ }

+ return f.Store.Get(ctx, key)

+}

+

+// Finding 1, the wake's half (§4.3 stale-link refusal): a recovered

+// instance's copy says last_executed A, and A is live, while its durable

+// build state names B. The wake's refresh fails: the wake is refused, never

+// decided from the old copy (which runs A).

+func TestAHeldWakeWhoseRefreshFailedIsRefused(t *testing.T) {

+ b, s, reg := testBuildStates(t)

+ ctx := context.Background()

+ installLive(t, b, "A")

+ if err := put(t, b, ctx, astraHost, release.BuildState{LastExecuted: "B"}); err != nil {

+ t.Fatal(err)

+ }

+ setCopy(t, reg, release.BuildStateCopy{Known: true, Replica: true, State: &release.BuildState{LastExecuted: "A"}})

+ setSettings(t, reg, release.UpdateSettings{Updates: release.UpdatesManual, Channel: "stable"})

+ decided(t, b, release.Consumed{Reason: string(release.ReasonManual), Desired: "A", Detail: "A"})

+ b.storeFor = storesOf(map[string]release.Store{"hib": stateGetFails{Store: s, err: errors.New("s3: 503")}})

+ if h := b.wakeTarget(ctx, astraHost); !h.Held || !h.Unread || h.Target == "A" {

+ t.Fatalf("wakeTarget after a failed refresh = %+v; want refused as unread, never A", h)

+ }

+ // The registry keeps the failure, so the answer read again says the same.

+ if h := b.heldTarget(astraHost); !h.Unread {

+ t.Fatalf("heldTarget after the failed refresh = %+v; want unread", h)

+ }

+ // With the bucket back the wake targets the durable B.

+ b.storeFor = storesOf(map[string]release.Store{"hib": s})

+ if h := b.wakeTarget(ctx, astraHost); !h.Held || h.Unread || h.Target != "B" {

+ t.Fatalf("wakeTarget with the bucket back = %+v; want held on B", h)

+ }

+}

+

+// Finding 1, the write's half: the wake decided on a stale copy (A), and the

+// pre-exec write reads the durable state under the lease (B). It names a

+// held target other than the build about to run, so the record is refused

+// as stale and the object is not touched; nothing runs. B itself records,

+// and a route with default settings records whatever runs, as today.

+func TestThePreExecWriteRevalidatesTheHeldTarget(t *testing.T) {

+ b, s, reg := testBuildStates(t)

+ ctx := context.Background()

+ installLive(t, b, "A")

+ if err := put(t, b, ctx, astraHost, release.BuildState{LastExecuted: "B"}); err != nil {

+ t.Fatal(err)

+ }

+ setCopy(t, reg, release.BuildStateCopy{Known: true, Replica: true, State: &release.BuildState{LastExecuted: "A"}})

+ setSettings(t, reg, release.UpdateSettings{Updates: release.UpdatesManual, Channel: "stable"})

+ decided(t, b, release.Consumed{Reason: string(release.ReasonManual), Desired: "A", Detail: "A"})

+ err := b.beforeExec(ctx, astraHost, "A")

+ if err == nil || !strings.Contains(err.Error(), "B") {

+ t.Fatalf("beforeExec(A) over a durable B = %v; want a stale refusal naming B", err)

+ }

+ st, _, _ := getState(ctx, s, release.BuildStateKey("inst/"+astraHost), astraHost)

+ if st == nil || st.LastExecuted != "B" {

+ t.Fatalf("state after the refusal = %+v; want B untouched", st)

+ }

+ if err := b.beforeExec(ctx, astraHost, "B"); err != nil {

+ t.Fatalf("beforeExec(B) = %v; the held target records", err)

+ }

+ setSettings(t, reg, release.UpdateSettings{})

+ if err := b.beforeExec(ctx, astraHost, "A"); err != nil {

+ t.Fatalf("default settings: beforeExec(A) = %v; want today's record", err)

+ }

+}

diff --git a/internal/host/buildstate_carry_test.go b/internal/host/buildstate_carry_test.go

new file mode 100644

index 000000000..282a7241f

--- /dev/null

+++ b/internal/host/buildstate_carry_test.go

@@ -0,0 +1,206 @@

+package host

+

+import (

+ "context"

+ "errors"

+ "os"

+ "path/filepath"

+ "strings"

+ "testing"

+ "time"

+

+ "github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/provision"

+ "github.com/carlosframework/platform/internal/release"

+)

+

+// Ruling R-SPEC-43A-2 (Astra round 1 finding 3): before a cut seals the build

+// state, the source box's consumed request record is merged into it under

+// the lease (release.CarryRequest), on every lane: the legacy cut, the Phase

+// 1 cut's final round and a dirty return's lane.

+

+// carryCall is one CarryRequest call, host bucket/prefix.

+type carryCall struct{ host, bucket, prefix string }

+

+// The legacy cut carries the request before its byte-final round, after the

+// final checkpoint, at the hibernate prefix the round copies; the carried

+// state is what lands in staging.

+func TestTheLegacyCutCarriesTheRequestBeforeItsFinalRound(t *testing.T) {

+ h := newMoveHarness(t)

+ host := "a.go.example.com"

+ h.record(t, host, provision.MoveRequest("uk", time.Now()), provision.MovePromoteReady(time.Now()), provision.MoveMarkCutting())

+ h.f.put("data-ie", "instances/"+host+"/generations/g/snapshot", "e1")

+ h.f.put("data-ie", "instances/"+host+"/build-state.json", "before")

+ var calls []carryCall

+ h.cfg.CarryRequest = func(_ context.Context, host, bucket, prefix string) error {

+ calls = append(calls, carryCall{host, bucket, prefix})

+ h.f.ops = append(h.f.ops, "carry")

+ h.f.put(bucket, prefix+"/build-state.json", "carried")

+ return nil

+ }

+ if err := movesPass(context.Background(), h.s, h.worker()); err != nil {

+ t.Fatal(err)

+ }

+ if len(calls) != 1 || calls[0] != (carryCall{host, "data-ie", "instances/" + host}) {

+ t.Fatalf("carry calls %+v; want one at data-ie/instances/%s", calls, host)

+ }

+ ck, carry, cp := -1, -1, -1

+ for i, op := range h.f.ops {

+ switch {

+ case strings.HasPrefix(op, "checkpoint ") && ck == -1:

+ ck = i

+ case op == "carry":

+ carry = i

+ case strings.HasPrefix(op, "copy ") && cp == -1:

+ cp = i

+ }

+ if !(ck < carry && carry < cp) {

+ t.Fatalf("ops %v: want checkpoint, then the carry, then the final round", h.f.ops)

+ }

+ if got := h.f.bodies("data-uk", "moves/"+host+"/")["build-state.json"]; got != "carried" {

+ t.Fatalf("staged build state %q, want the carried one", got)

+ }

+ if got := h.read(t, host); got.Move != nil {

+ t.Fatalf("the cut did not complete: %+v", got.Move)

+ }

+}

+

+// A carry that fails refuses the cut (the pass logs the round's error and

+// goes on to other hosts): nothing is copied, the warm cache is not purged,

+// and the move stays cutting for the next pass.

+func TestAFailedCarryRefusesTheLegacyCut(t *testing.T) {

+ h := newMoveHarness(t)

+ host := "a.go.example.com"

+ h.record(t, host, provision.MoveRequest("uk", time.Now()), provision.MovePromoteReady(time.Now()), provision.MoveMarkCutting())

+ h.f.put("data-ie", "instances/"+host+"/generations/g/snapshot", "e1")

+ db := filepath.Join(h.cfg.DataDir, host+".db")

+ if err := writeFileForTest(db, "warm"); err != nil {

+ t.Fatal(err)

+ }

+ carried := 0

+ h.cfg.CarryRequest = func(context.Context, string, string, string) error { carried++; return errors.New("s3: 503") }

+ if err := movesPass(context.Background(), h.s, h.worker()); err != nil {

+ t.Fatal(err)

+ }

+ if carried != 1 {

+ t.Fatalf("carry asked %d times, want 1", carried)

+ }

+ if _, err := os.Stat(db); err != nil {

+ t.Fatalf("the warm cache was purged past a failed carry: %v", err)

+ }

+ for _, op := range h.f.ops {

+ if strings.HasPrefix(op, "copy ") {

+ t.Fatalf("ops %v: the final round ran past a failed carry", h.f.ops)

+ }

+ if got := h.read(t, host); got.Move == nil || got.Move.State != provision.MoveCutting {

+ t.Fatalf("move %+v, want still cutting", got.Move)

+ }

+}

+

+// The Phase 1 cut's final round carries the request at the claim holder's

+// location before it copies the state for the seal.

+func TestThePhase1CutCarriesTheRequestBeforeItsSeal(t *testing.T) {

+ h := newCutHarness(t)

+ storeState(h.phase1Harness, "data-ie", cutRoot+"/build-state.json", "v6")

+ h.ready(t)

+ var calls []carryCall

+ h.cfg.CarryRequest = func(_ context.Context, host, bucket, prefix string) error {

+ calls = append(calls, carryCall{host, bucket, prefix})

+ storeState(h.phase1Harness, bucket, prefix+"/build-state.json", "carried")

+ return nil

+ }

+ h.cut(t)

+ h.pass(t)

+ if m := h.move(t); m.State != provision.MoveLanding {

+ t.Fatalf("cut did not finish: %+v", m)

+ }

+ if len(calls) == 0 || calls[len(calls)-1] != (carryCall{p1Host, "data-ie", cutRoot}) {

+ t.Fatalf("carry calls %+v; want one at data-ie/%s", calls, cutRoot)

+ }

+ key := moves.BuildStateKey(p1Host, p1ID)

+ if got := h.f.buckets["data-uk"][key].body; got != p1StateBody("carried") {

+ t.Fatalf("the sealed lane holds %q, want the carried state", got)

+ }

+ if seal := h.seal(t); len(seal.State) != 1 || seal.State[0].VersionID != h.f.versionOf("data-uk", key) {

+ t.Fatalf("seal state %+v", seal.State)

+ }

+}

+

+// A dirty return carries the target's request record into the state beside

+// the target's lease before its lane copies it back.

+func TestTheReturnLaneCarriesTheRequest(t *testing.T) {

+ h := &rollbackHarness{landingHarness: newLandingHarness(t)}

+ storeState(h.phase1Harness, "data-ie", cutRoot+"/build-state.json", "v6")

+ h.flip(t)

+ h.land(t)

+ h.installed(t)

+ h.step(t, provision.LandRestored, provision.LandProbing)

+ h.tpos = h.n

+ storeState(h.phase1Harness, "data-uk", cutRoot+"/build-state.json", "v8")

+ h.uncover(t)

+ h.rollBack(t)

+ var calls []carryCall

+ h.carry = func(_ context.Context, host, bucket, prefix string) error {

+ calls = append(calls, carryCall{host, bucket, prefix})

+ storeState(h.phase1Harness, bucket, prefix+"/build-state.json", "carried")

+ return nil

+ }

+ h.rollbackPass(t)

+ rb := h.rb(t)

+ if !rb.Dirty || rb.TargetPhase != provision.RbTargetReturned {

+ t.Fatalf("rollback %+v", rb)

+ }

+ if len(calls) == 0 || calls[len(calls)-1] != (carryCall{p1Host, "data-uk", cutRoot}) {

+ t.Fatalf("carry calls %+v; want one at data-uk/%s", calls, cutRoot)

+ }

+ if _, body := returnSealState(t, h, rb.ID); body != p1StateBody("carried") {

+ t.Fatalf("the return lane holds %q, want the carried state", body)

+ }

+}

+

+// buildStates.carryRequest, the hook's production body: under the host's

+// lock it merges root's consumed record into the object at the location

+// given (release.CarryRequest), last_executed untouched, and writes only

+// when something changes.

+func TestCarryRequestWritesTheConsumedRecordAtTheLocation(t *testing.T) {

+ b, s, _ := testBuildStates(t)

+ ctx := context.Background()

+ if err := put(t, b, ctx, astraHost, release.BuildState{LastExecuted: "A"}); err != nil {

+ t.Fatal(err)

+ }

+ key := release.BuildStateKey("inst/" + astraHost)

+ etag := mustEtag(t, s, key)

+ // No consumed file: nothing to carry, nothing written.

+ if err := b.carryRequest(ctx, astraHost, "hib", "inst/"+astraHost); err != nil {

+ t.Fatal(err)

+ }

+ if mustEtag(t, s, key) != etag {

+ t.Fatal("a carry with no consumed file rewrote the object")

+ }

+ writeConsumed(t, b, release.Consumed{Resolved: &release.ResolvedRequest{ID: 1, Target: "T"}, Completed: 1, CompletedFailed: true})

+ if err := b.carryRequest(ctx, astraHost, "hib", "inst/"+astraHost); err != nil {

+ t.Fatal(err)

+ }

+ st, _, err := getState(ctx, s, key, astraHost)

+ if err != nil || st == nil || st.LastExecuted != "A" || st.Completed != 1 || !st.CompletedFailed {

+ t.Fatalf("state %+v, %v; want A with R1 completed failed", st, err)

+ }

+ etag = mustEtag(t, s, key)

+ if err := b.carryRequest(ctx, astraHost, "hib", "inst/"+astraHost); err != nil {

+ t.Fatal(err)

+ }

+ if mustEtag(t, s, key) != etag {

+ t.Fatal("a carry with nothing new rewrote the object")

+ }

+ // The location given wins over the hibernate prefix (a claim holder's).

+ other := release.NewDirStore(t.TempDir())

+ b.storeFor = storesOf(map[string]release.Store{"hib": s, "data": other})

+ if err := b.carryRequest(ctx, astraHost, "data", "instances/"+astraHost+"/inc1"); err != nil {

+ t.Fatal(err)

+ }

+ if st, _, _ := getState(ctx, other, release.BuildStateKey("instances/"+astraHost+"/inc1"), astraHost); st == nil || st.Completed != 1 {

+ t.Fatalf("state at the location = %+v", st)

+ }

+}

diff --git a/internal/host/buildstate_test.go b/internal/host/buildstate_test.go

index f4017014a..67da7fbab 100644

--- a/internal/host/buildstate_test.go

+++ b/internal/host/buildstate_test.go

@@ -830,7 +830,7 @@ func TestAMajorHoldStaysHeldWhileARequestBakes(t *testing.T) {

two := 2

setSettings(t, reg, release.UpdateSettings{Updates: release.UpdatesAuto, Channel: "stable", Major: &two, InstallRequest: 1})

setCopy(t, reg, release.BuildStateCopy{Known: true, Replica: true, State: &release.BuildState{LastExecuted: "v2"}})

- writeConsumed(t, b, release.Consumed{Reason: string(release.ReasonMajor), Desired: "v2", Baking: "v3"})

+ decided(t, b, release.Consumed{Reason: string(release.ReasonMajor), Desired: "v2", Baking: "v3"})

if h := b.heldTarget("a.example.com"); !h.Held || h.Target != "v2" {

t.Fatalf("heldTarget = %+v; want v2, held", h)

}

@@ -870,35 +870,43 @@ func TestAHeldWakeFollowsTheBuildStatesResolutionOverTheFile(t *testing.T) {

// Task 2 review: with the build state unread (Known false) root does no

// request bookkeeping and will not install T, so neither does the wake

-// follow it. Held settings then read root's unknown; auto ones are not held.

+// follow it. Held settings then read root's unknown; auto ones are not held

+// (root's decisions here are for the current column, ruling R-HELD-PURE). A

+// copy whose last refresh failed is refused as unread, whatever the settings

+// (Astra round 1 finding 1), and never follows T either.

func TestAnUnreadBuildStateDoesNotFollowAPendingRequest(t *testing.T) {

b, _, reg := testBuildStates(t)

setSettings(t, reg, release.UpdateSettings{Updates: release.UpdatesManual, Channel: "stable", InstallRequest: 2})

- setCopy(t, reg, release.BuildStateCopy{Known: false, ReadErr: "s3: timeout"})

- writeConsumed(t, b, release.Consumed{Reason: string(release.ReasonUnknown), Resolved: &release.ResolvedRequest{ID: 2, Target: "v4"}})

+ setCopy(t, reg, release.BuildStateCopy{Known: false})

+ decided(t, b, release.Consumed{Reason: string(release.ReasonUnknown), Resolved: &release.ResolvedRequest{ID: 2, Target: "v4"}})

if h := b.heldTarget("a.example.com"); !h.Held || !h.Unknown || h.Target != "" {

t.Fatalf("heldTarget = %+v; want held, unknown, never the request's v4", h)

}

setSettings(t, reg, release.UpdateSettings{Updates: release.UpdatesAuto, Channel: "stable", InstallRequest: 2})

- writeConsumed(t, b, release.Consumed{Reason: string(release.ReasonAdopt), Desired: "v5", Resolved: &release.ResolvedRequest{ID: 2, Target: "v4"}})

+ decided(t, b, release.Consumed{Reason: string(release.ReasonAdopt), Desired: "v5", Resolved: &release.ResolvedRequest{ID: 2, Target: "v4"}})

if h := b.heldTarget("a.example.com"); h.Held {

t.Fatalf("heldTarget = %+v; an auto instance with an unread state follows root's adoption", h)

}

+ setCopy(t, reg, release.BuildStateCopy{Known: false, ReadErr: "s3: timeout"})

+ if h := b.heldTarget("a.example.com"); !h.Held || !h.Unread || h.Target != "" {

+ t.Fatalf("heldTarget = %+v; a failed refresh is refused as unread, never the request's v4", h)

+ }

}

// Ruling R-SETTINGS-UNREADABLE (Task 6): a column that does not parse even

-// leniently holds the wake as unknown when root says unknown, and is not

-// held otherwise (root's own reading). A Control name this binary does not

-// know is dropped, not read as unreadable, so the hold stays.

+// leniently holds the wake as unknown when root says unknown for it. With no

+// word from root for that column the wake holds and asks root (ruling

+// R-HELD-PURE: never "not held"). A Control name this binary does not know

+// is dropped, not read as unreadable, so the hold stays.

func TestAnUnreadableColumnHoldsOnlyAsRootDoes(t *testing.T) {

b, _, reg := testBuildStates(t)

if err := reg.SetUpdateSettings("a.example.com", `{"updates":"sometimes"}`); err != nil {

t.Fatal(err)

}

- if h := b.heldTarget("a.example.com"); h.Held {

- t.Fatalf("heldTarget = %+v with no word from root; want root's reading (not held)", h)

+ if h := b.heldTarget("a.example.com"); !h.Held || !h.AskRoot || h.Unknown {

+ t.Fatalf("heldTarget = %+v with no word from root; want held, asking root", h)

}

- writeConsumed(t, b, release.Consumed{Reason: string(release.ReasonUnknown), Detail: "update settings: updates \"sometimes\""})

+ decided(t, b, release.Consumed{Reason: string(release.ReasonUnknown), Detail: "update settings: updates \"sometimes\""})

if h := b.heldTarget("a.example.com"); !h.Held || !h.Unknown {

t.Fatalf("heldTarget = %+v; want held as unknown, as root holds it", h)

}

@@ -906,7 +914,7 @@ func TestAnUnreadableColumnHoldsOnlyAsRootDoes(t *testing.T) {

t.Fatal(err)

}

setCopy(t, reg, release.BuildStateCopy{Known: true, Replica: true, State: &release.BuildState{LastExecuted: "v2"}})

- writeConsumed(t, b, release.Consumed{Reason: string(release.ReasonManual), Desired: "v2"})

+ decided(t, b, release.Consumed{Reason: string(release.ReasonManual), Desired: "v2"})

if h := b.heldTarget("a.example.com"); !h.Held || h.Target != "v2" {

t.Fatalf("heldTarget = %+v; want the manual hold on v2 through an unknown Control name", h)

}

@@ -946,6 +954,7 @@ func TestARetainedCopyIsOfferedOnlyWhenVerified(t *testing.T) {

b, _, reg := testBuildStates(t)

setSettings(t, reg, release.UpdateSettings{Updates: release.UpdatesManual, Channel: "stable"})

setCopy(t, reg, release.BuildStateCopy{Known: true, Replica: true, State: &release.BuildState{LastExecuted: "v2"}})

+ decided(t, b, release.Consumed{Reason: string(release.ReasonManual), Desired: "v2"})

stageWithManifest(t, b.optDir, "bab", "km", "v2", []byte("two"))

h := b.heldTarget("a.example.com")

if !h.Held || h.Target != "v2" || h.Staged == nil || h.Staged.Version != "v2" || h.Staged.StagedPath != release.VersionPath(b.optDir, "bab", "v2", "km") || h.Staged.Digest != fileSHA(t, h.Staged.StagedPath) {

@@ -1058,13 +1067,13 @@ func TestAMajorHoldOnAFailedBuildStaysHeldAndFailed(t *testing.T) {

two := 2

setSettings(t, reg, release.UpdateSettings{Updates: release.UpdatesAuto, Channel: "stable", Major: &two})

setCopy(t, reg, release.BuildStateCopy{Known: true, Replica: true, State: &release.BuildState{LastExecuted: "v2"}})

- writeConsumed(t, b, release.Consumed{Reason: string(release.ReasonFailed), Detail: "v2"})

+ decided(t, b, release.Consumed{Reason: string(release.ReasonFailed), Detail: "v2"})

if h := b.heldTarget("a.example.com"); !h.Held || !h.Failed || h.Target != "v2" {

t.Fatalf("heldTarget = %+v; want v2 held and failed", h)

}

// Root's failed head under rule 4 (Detail is the head, not

// last_executed) is not a hold.

- writeConsumed(t, b, release.Consumed{Reason: string(release.ReasonFailed), Detail: "v3"})

+ decided(t, b, release.Consumed{Reason: string(release.ReasonFailed), Detail: "v3"})

if h := b.heldTarget("a.example.com"); h.Held {

t.Fatalf("heldTarget = %+v; a failed head under adoption is not a hold", h)

}

@@ -1078,12 +1087,12 @@ func TestAHeldWakeReadsTheCopyAsRootsTargetDoes(t *testing.T) {

b, _, reg := testBuildStates(t)

setSettings(t, reg, release.UpdateSettings{Updates: release.UpdatesManual, Channel: "stable"})

setCopy(t, reg, release.BuildStateCopy{}) // never read

- writeConsumed(t, b, release.Consumed{Reason: string(release.ReasonManual), Desired: "v2"})

+ decided(t, b, release.Consumed{Reason: string(release.ReasonManual), Desired: "v2"})

if h := b.heldTarget("a.example.com"); !h.Held || !h.Unknown {

t.Fatalf("an unread copy under manual = %+v; want held, unknown", h)

}

setCopy(t, reg, release.BuildStateCopy{Known: true, Replica: true, State: &release.BuildState{LastExecuted: "v2"}})

- writeConsumed(t, b, release.Consumed{Reason: string(release.ReasonUnknown)}) // written while the copy was unread

+ decided(t, b, release.Consumed{Reason: string(release.ReasonUnknown)}) // written while the copy was unread

if h := b.heldTarget("a.example.com"); !h.Held || h.Unknown || h.Target != "v2" {

t.Fatalf("a read copy with last_executed under root's stale unknown = %+v; want held on v2", h)

}

diff --git a/internal/host/movescut.go b/internal/host/movescut.go

index 4dc6d690a..f7b513c5d 100644

--- a/internal/host/movescut.go

+++ b/internal/host/movescut.go

@@ -983,6 +983,13 @@ func (c *cutRun) finalRound(ctx context.Context, holder hostclaim.Holder, report

// final state and not a round-old one (settings spec rev 11 §4.3a). A

// state that changed under the copy is not final: the pass retries.

// The seal names the version this round copied (ruling R-SEAL-STATE).

+ // This box's request record goes into it first (ruling R-SPEC-43A-2):

+ // execution is given up, so no wake writes it now.

+ if cfg.CarryRequest != nil {

+ if err := cfg.CarryRequest(ctx, host, cfg.OwnBucket, strings.TrimSuffix(holder.Location, "/")); err != nil {

+ return false, fmt.Errorf("final round: carry the request record into the build state: %w", err)

+ }

stRound, stateObj, err := mirrorBuildState(ctx, c.gm, cfg.OwnBucket, holder.Location, dst, host, id)

if err != nil {

return false, fmt.Errorf("final round: build state lane: %w", err)

diff --git a/internal/host/movesland_test.go b/internal/host/movesland_test.go

index e4c13a06d..851882a29 100644

--- a/internal/host/movesland_test.go

+++ b/internal/host/movesland_test.go

@@ -53,6 +53,9 @@ import (

type landingHarness struct {

*cutHarness

+ // carry is the target's moveCfg.CarryRequest (nil: none), for the return

+ // lane's request carry (buildstate_carry_test.go).

+ carry func(ctx context.Context, host, bucket, prefix string) error

tdir string // the target's data directory

leases *fakeParkLeases

lw *movesWorker

@@ -337,6 +340,7 @@ func (h *landingHarness) landingCfg(t *testing.T) moveCfg {

t.Helper()

c := h.target()

c.DataDir = h.tdir

+ c.CarryRequest = h.carry

c.OwnsRoute = func(host string) bool { return host == p1Host }

c.Route = func(host string) (*registry.Route, error) {

if host != p1Host {

diff --git a/internal/host/movespass.go b/internal/host/movespass.go

index ad6e3589f..1d3ca644f 100644

--- a/internal/host/movespass.go

+++ b/internal/host/movespass.go

@@ -196,7 +196,7 @@ func (w *movesWorker) mirror(ctx context.Context, dstRegion string) (lease.Mirro

// Plan 5 Task 9a), behind the claim Plan 12 guard. events is the agent's

// rollout sink, which the move log tells the app's members through (Plan 6

// Task 3).

-func armMovesPass(hib activator.Config, dataDir string, reg *registry.Registry, svc *activator.Service, eb *activator.ExecBackend, writes hostwrite.Config, providers ClaimProviders, events activator.EventSink) {

+func armMovesPass(hib activator.Config, dataDir string, reg *registry.Registry, svc *activator.Service, eb *activator.ExecBackend, writes hostwrite.Config, providers ClaimProviders, events activator.EventSink, carry func(ctx context.Context, host, bucket, prefix string) error) {

dataDir = movesDataDir(dataDir)

readyDelta, ok := moveReadyDeltaFromEnv(os.Getenv)

if !ok {

@@ -224,8 +224,9 @@ func armMovesPass(hib activator.Config, dataDir string, reg *registry.Registry,

}

return nil

},

- LiftFence: svc.LiftFence,

- SetStatus: reg.SetStatus,

+ LiftFence: svc.LiftFence,

+ SetStatus: reg.SetStatus,

+ CarryRequest: carry,

OwnsRoute: func(host string) bool {

rt, err := reg.Get(host)

return err == nil && rt != nil && rt.Provisioned

@@ -512,6 +513,17 @@ type moveCfg struct {

// armMovesPass). Off, the worker runs the legacy protocol only.

Enabled bool

+ // CarryRequest merges this box's consumed request record for host into

+ // the build state at bucket/prefix (release.CarryRequest: root's

+ // resolution and a failed completion; never a successful completion,

+ // never last_executed), under the cut's fence or lease, before a cut's

+ // final round copies the state for the seal: the legacy cut, the Phase 1

+ // cut's final round and a dirty return's lane (ruling R-SPEC-43A-2; Astra

+ // round 1 finding 3). A request this box closed as failed then stays

+ // closed on the destination. Nil carries nothing (a box with no

+ // build-state store, tests).

+ CarryRequest func(ctx context.Context, host, bucket, prefix string) error

+

// Phase 1 (moves design §7.3), wired by armMovesPass only with the

// switch on. A nil hook refuses its step, never falls back.

//

@@ -1137,6 +1149,15 @@ func legacyMoveRound(ctx context.Context, s release.Store, w *movesWorker, rec p

}

+ // Step 3b: this box's request record into the build state, so the

+ // final round carries a request it closed as failed (ruling

+ // R-SPEC-43A-2). The instance is quiesced and fenced: no wake writes

+ // the state now.

+ if cfg.CarryRequest != nil {

+ if err := cfg.CarryRequest(ctx, rec.Host, cfg.OwnBucket, srcPrefix); err != nil {

+ return fmt.Errorf("carry the request record into the build state: %w", err)

+ }

// Step 4: the byte-final mirror round.

m, err := mirror()

if err != nil {

diff --git a/internal/host/movespass_test.go b/internal/host/movespass_test.go

index c28339cf9..f72175a64 100644

--- a/internal/host/movespass_test.go

+++ b/internal/host/movespass_test.go

@@ -932,7 +932,7 @@ func TestArmMovesPassReadsTheReadyDelta(t *testing.T) {

{"lots", defaultMoveReadyDelta},

} {

t.Setenv("CARLOS_MOVE_READY_DELTA_BYTES", tc.raw)

- armMovesPass(activator.Config{}, t.TempDir(), nil, nil, nil, hostwrite.Config{}, ClaimProviders{}, nil)

+ armMovesPass(activator.Config{}, t.TempDir(), nil, nil, nil, hostwrite.Config{}, ClaimProviders{}, nil, nil)

if got := armedMovesWorker.Load().cfg.ReadyDelta; got != tc.want {

t.Errorf("CARLOS_MOVE_READY_DELTA_BYTES=%q armed a bar of %d, want %d", tc.raw, got, tc.want)

}

@@ -951,7 +951,7 @@ func TestArmMovesPassReadsTheSwitch(t *testing.T) {

{"", false}, {"on", true}, {"1", true}, {"true", true}, {"off", false}, {"yes", false},

} {

t.Setenv(moves.SwitchEnv, tc.raw)

- armMovesPass(activator.Config{}, t.TempDir(), nil, nil, nil, hostwrite.Config{}, ClaimProviders{}, nil)

+ armMovesPass(activator.Config{}, t.TempDir(), nil, nil, nil, hostwrite.Config{}, ClaimProviders{}, nil, nil)

if got := armedMovesWorker.Load().cfg.Enabled; got != tc.want {

t.Errorf("%s=%q armed Enabled=%v, want %v", moves.SwitchEnv, tc.raw, got, tc.want)

}

diff --git a/internal/host/movesphase1_test.go b/internal/host/movesphase1_test.go

index b253dd422..6a683861c 100644

--- a/internal/host/movesphase1_test.go

+++ b/internal/host/movesphase1_test.go

@@ -144,7 +144,7 @@ func TestPhase1IsInertWithTheSwitchOff(t *testing.T) {

}

t.Setenv(moves.SwitchEnv, v)

eb := &activator.ExecBackend{}

- armMovesPass(activator.Config{}, t.TempDir(), nil, nil, eb, hostwrite.Config{}, ClaimProviders{}, nil)

+ armMovesPass(activator.Config{}, t.TempDir(), nil, nil, eb, hostwrite.Config{}, ClaimProviders{}, nil, nil)

if (eb.Strict != nil) != on {

t.Fatalf("switch %v: eb.Strict set = %v", on, eb.Strict != nil)

}

diff --git a/internal/host/movesrollback.go b/internal/host/movesrollback.go

index d621985da..cc7fa5539 100644

--- a/internal/host/movesrollback.go

+++ b/internal/host/movesrollback.go

@@ -862,6 +862,13 @@ func (r *rollbackRun) returnState(ctx context.Context, gm *guardedMirror, h *hos

if h == nil {

return nil, fmt.Errorf("move %s: the claim has no holder to return the build state from", r.g.ID)

}

+ // This box's request record goes back too (ruling R-SPEC-43A-2): a

+ // request the target closed as failed stays closed on the source.

+ if cfg.CarryRequest != nil {

+ if err := cfg.CarryRequest(ctx, host, cfg.OwnBucket, strings.TrimSuffix(h.Location, "/")); err != nil {

+ return nil, fmt.Errorf("carry the request record into the returned build state: %w", err)

+ }

_, o, err := mirrorBuildState(ctx, gm, cfg.OwnBucket, h.Location, gm.dst, host, r.rb)

if err != nil {

return nil, fmt.Errorf("return the build state: %w", err)

diff --git a/internal/host/versionsweep.go b/internal/host/versionsweep.go

index 0d5c65a1a..a64771128 100644

--- a/internal/host/versionsweep.go

+++ b/internal/host/versionsweep.go

@@ -1,7 +1,6 @@

package host

import (

- "fmt"

"log"

"os"

"time"

@@ -127,43 +126,28 @@ func sweepStagedVersionsPass(reg routeLister) {

}

// settingsPins is every build the rows' update settings and build-state

-// copies name: a pin, last_executed and a resolved request's T. It reads

-// the registry columns root reads (ParseUpdateSettingsLenient,

-// ParseBuildStateCopy); a copy that kept its last known state after a

-// failed refresh still names it. A column that does not parse, or a held

-// instance (settings other than default) whose build state is not read

-// yet, is an incomplete pin set: its last_executed is unknown, so the

-// sweep deletes nothing. Only a row whose copy the agent refreshes counts:

-// one keepsBuildState never selects (a named unit, the platform's own row)

+// copies name: a pin, last_executed and a resolved request's T

+// (release.SettingsRefs, the reading the restaging pin set uses too). A copy

+// that kept its last known state after a failed refresh still names it. Any

+// row whose references are incomplete makes the whole set incomplete, so the

+// sweep deletes nothing: a column that does not parse, or a held instance

+// (settings other than default) whose build state is not read yet or whose

+// last refresh failed (ReadErr; Astra round 1 finding 6), whose

+// last_executed may name a build only its staged copy here still holds.

+// Version trees hold every app, so this refusal is box-wide. Only a row

+// whose copy the agent refreshes counts (release.BuildStateKept): one

+// keepsBuildState never selects (a named unit, the platform's own row)

// stays unread forever and must not stop the sweep for good.

func settingsPins(routes []registry.Route) (map[string]bool, error) {

out := map[string]bool{}

for _, r := range routes {

- u, err := release.ParseUpdateSettingsLenient(r.UpdateSettings)

+ refs, err := release.SettingsRefs(r)

if err != nil {

- return nil, fmt.Errorf("update settings of %s: %w", r.Host, err)

+ return nil, err

}

- if u.Pin != "" {

- out[u.Pin] = true

- }

- c, err := release.ParseBuildStateCopy(r.BuildState)

- if err != nil {

- return nil, fmt.Errorf("build state of %s: %w", r.Host, err)

- }

- if !c.Known && !u.Default() && heldWithBuildState(r) {

- return nil, fmt.Errorf("the build state of held %s is not read yet", r.Host)

- }

- for _, v := range buildStateBuilds(c.State) {

+ for _, v := range refs {

out[v] = true

}

return out, nil

}

-

-// heldWithBuildState is a row settings can hold whose build state the agent

-// refreshes: keepsBuildState's rows (with host claims off, the wider set:

-// root does not know the agent's claims mode) less workers, which settings

-// never apply to, so no held recovery reads their last_executed.

-func heldWithBuildState(r registry.Route) bool {

- return keepsBuildState(r, false) && !r.IsSidecar()

-}

diff --git a/internal/host/versionsweep_test.go b/internal/host/versionsweep_test.go

index 2942f1d7a..b647c0374 100644

--- a/internal/host/versionsweep_test.go

+++ b/internal/host/versionsweep_test.go

@@ -315,6 +315,35 @@ func TestSweepStagedVersionsKeepsALastKnownState(t *testing.T) {

}

+// Astra round 1 finding 6: a held instance's copy that kept A after a failed

+// refresh is not a complete pin set. The durable state may name B since (a

+// recovery onto B on another box, then a return), and B's staged copy here is

+// its only recovery copy, named by no live link, column or generation. The

+// sweep deletes nothing, and so keeps A too. A default-settings row's failed

+// refresh holds nothing on its last_executed, so it does not stop the sweep.

+func TestSweepStagedVersionsRefusesAHeldCopyWithAReadError(t *testing.T) {

+ opt := t.TempDir()

+ for _, v := range []string{"A", "B", "v9"} {

+ stageSweepVersion(t, opt, "acct1", v, "blog", 48*time.Hour)

+ }

+ manual := release.UpdateSettings{Updates: release.UpdatesManual, Channel: "stable"}.Column()

+ col := release.BuildStateCopy{Known: true, Replica: true, ReadErr: "s3: timeout", State: &release.BuildState{LastExecuted: "A"}}.Column()

+ sweepStagedVersions(fakeLister{routes: []registry.Route{settingsRow("a.example.com", col, manual)}}, opt, noGenerations(t), 6*time.Hour, time.Now())

+ for _, v := range []string{"A", "B"} {

+ if !staged(t, opt, v) {

+ t.Errorf("%s was removed: the sweep ran on a held copy whose last refresh failed", v)

+ }

+ opt = t.TempDir()

+ for _, v := range []string{"A", "B", "v9"} {

+ stageSweepVersion(t, opt, "acct1", v, "blog", 48*time.Hour)

+ }

+ sweepStagedVersions(fakeLister{routes: []registry.Route{settingsRow("a.example.com", col, "")}}, opt, noGenerations(t), 6*time.Hour, time.Now())

+ if !staged(t, opt, "A") || staged(t, opt, "B") {

+ t.Errorf("default settings: A staged %v, B staged %v; want A kept (named) and B swept", staged(t, opt, "A"), staged(t, opt, "B"))

+ }

+}

+

// Any column the sweep cannot read is an incomplete pin set: an unparseable

// build-state copy or update-settings column, or a held instance whose build

// state is not read yet (its last_executed is unknown). Delete nothing.

diff --git a/internal/release/adopt.go b/internal/release/adopt.go

index 37e54ae71..09993da94 100644

--- a/internal/release/adopt.go

+++ b/internal/release/adopt.go

@@ -466,9 +466,13 @@ func PassFilter(ctx context.Context, reg *registry.Registry, s Store, pub ed2551

if err != nil {

return nil, fmt.Errorf("list routes: %w", err)

}

- pins, err := BoxPins(optDir, rows)

- if err != nil {

- return nil, err

+ // An app whose settings references are incomplete comes back

+ // with the pins (*IncompleteRefsError); stageFrom refuses that

+ // app's restage only.

+ pins, incomplete := BoxPins(optDir, rows)

+ var inc *IncompleteRefsError

+ if incomplete != nil && (pins == nil || !errors.As(incomplete, &inc)) {

+ return nil, incomplete

}

// Live and pending generations, from the file the agent's

// activator writes into the data dir (index amendments; its

@@ -494,6 +498,9 @@ func PassFilter(ctx context.Context, reg *registry.Registry, s Store, pub ed2551

pins.add(p)

}

+ if inc != nil {

+ return pins, inc

+ }

return pins, nil

},

}

@@ -1986,6 +1993,12 @@ func stageFrom(ctx context.Context, s Store, optDir, acct, app, ver, host string

return nil, fmt.Errorf("%s/%s: the staged copy differs from the release and what refers to it is unknown, so it was not restaged", app, ver)

}

pins, perr := chk.refs()

+ // References another app could not complete (an unread build state

+ // of one of its held instances) do not stop this one; this app's

+ // do (Astra round 1 finding 2).

+ if inc := (*IncompleteRefsError)(nil); errors.As(perr, &inc) && pins != nil && !inc.Covers(acct, app) {

+ perr = nil

+ }

if perr != nil {

return nil, fmt.Errorf("%s/%s: the staged copy differs from the release and what refers to it is unknown (%v), so it was not restaged", app, ver, perr)

}

@@ -2242,26 +2255,36 @@ func orDetail(d string) string {

// stageAuthorized stages a build settings spec §4.1 authorizes by its

// promotion record: the record's digest binds the manifest before anything

-// is downloaded (ensureStagedDigest). A build with no record (promoted

-// before records existed) is recovered from this box's own staged copy when

-// its bytes still hash to the artifact digest in the copy's sibling, the

-// manifest it was verified against when it was staged; anywhere else it is

-// refused. Any other failure to read the record is a refusal too, never read

-// as "no record". Nothing is backfilled.

+// is downloaded or replaced (ensureStagedDigest). This box's own staged copy

+// is evidence on its own (§4.1's retained identity; Astra round 1 finding

+// 7): when its bytes still hash to the artifact digest in the copy's

+// sibling, the manifest it was verified against when it was staged, it is

+// recovered whatever the remote side says: no record (promoted before

+// records existed), a record or manifest that cannot be read now, or a

+// staging the record path refused. Without such a copy the build is

+// refused, and nothing is downloaded on anything weaker than the record's

+// digest. A failure to read the record is never read as "no record".

+// Nothing is backfilled.

func stageAuthorized(ctx context.Context, s Store, pub ed25519.PublicKey, optDir, acct, app, ver, host string, chk stageCheck) (*Manifest, error) {

rec, err := ReadPromotionRecord(ctx, s, pub, acct, app, ver)

- if err == nil {

- if rec.ManifestDigest == "" {

- return nil, fmt.Errorf("%s: promotion record carries no manifest digest", ver)

+ var refused error

+ switch {

+ case err == nil && rec.ManifestDigest == "":

+ refused = fmt.Errorf("%s: promotion record carries no manifest digest", ver)

+ case err == nil:

+ m, serr := ensureStagedDigest(ctx, s, optDir, acct, app, ver, host, rec.ManifestDigest, chk)

+ if serr == nil {

+ return m, nil

}

- return ensureStagedDigest(ctx, s, optDir, acct, app, ver, host, rec.ManifestDigest, chk)

- }

- if !errors.Is(err, ErrNoPromotionRecord) {

- return nil, fmt.Errorf("%s: promotion record: %w", ver, err)

+ refused = serr

+ case errors.Is(err, ErrNoPromotionRecord):

+ refused = fmt.Errorf("%s: %s", ver, updatewords.Text("settings.evidence.no-record"))

+ default:

+ refused = fmt.Errorf("%s: promotion record: %w", ver, err)

}

if id, m := RetainedStaged(optDir, acct, app, ver); id != nil {

- fmt.Printf(" %s: %s has no promotion record; recovered from this box's staged copy\n", host, ver)

+ fmt.Printf(" %s: %s recovered from this box's verified staged copy (%v)\n", host, ver, refused)

return m, nil

}

- return nil, fmt.Errorf("%s: %s", ver, updatewords.Text("settings.evidence.no-record"))

+ return nil, refused

}

diff --git a/internal/release/adoptsettings_test.go b/internal/release/adoptsettings_test.go

index 3da350580..64f468b5b 100644

--- a/internal/release/adoptsettings_test.go

+++ b/internal/release/adoptsettings_test.go

@@ -650,7 +650,9 @@ func TestNextConsumed(t *testing.T) {

}

for _, tc := range cases {

t.Run(tc.name, func(t *testing.T) {

- tc.in.Route = registry.Route{Host: "a.example.com"}

+ tc.in.Route = registry.Route{Host: "a.example.com", UpdateSettings: tc.in.Settings.Column()}

+ // Every decision names the column it was made for (R-HELD-PURE).

+ tc.want.Settings = SettingsDigest(tc.in.Settings.Column())

var before *Consumed

if tc.in.Consumed != nil {

cp := *tc.in.Consumed

@@ -859,3 +861,82 @@ func TestTheLiveBuildsRefusalIsJudgedOnTheSelectedBuild(t *testing.T) {

t.Fatalf("live %q", v)

}

+

+// failingGets is a Store whose Get of any key in fail returns err, as a

+// bucket that cannot be read at that moment.

+type failingGets struct {

+ Store

+ fail map[string]bool

+ err error

+}

+

+func (f failingGets) Get(ctx context.Context, key string) ([]byte, string, error) {

+ if f.fail[key] {

+ return nil, "", f.err

+ }

+ return f.Store.Get(ctx, key)

+}

+

+// Astra round 1 finding 7 (§4.1): this box's staged copy whose bytes still

+// hash to its sibling manifest is recovered without remote evidence, so a

+// promotion record GET or a manifest GET that fails does not block it. With

+// no verified copy the same failures still refuse, and nothing is downloaded:

+// bytes that are fetched or replaced need the record's digest.

+func TestAVerifiedLocalCopyNeedsNoRemoteEvidence(t *testing.T) {

+ ctx := context.Background()

+ priv, pub := testKeys(t)

+ base := NewDirStore(t.TempDir())

+ shipRelease(t, base, "bab", "km", "v1", adoptPlatform, []byte("v1"))

+ if err := Promote(ctx, base, priv, "bab", "km", "canary/x", "v1"); err != nil {

+ t.Fatal(err)

+ }

+ blip := errors.New("s3: 503 slow down")

+ for name, fail := range map[string]map[string]bool{

+ "record GET fails": {PromotionRecordKey("bab", "km", "v1"): true},

+ "manifest GET fails": {ManifestKey("bab", "km", "v1"): true},

+ } {

+ t.Run(name, func(t *testing.T) {

+ opt := t.TempDir()

+ s := failingGets{Store: base, fail: fail, err: blip}

+ // No copy on this box: refused, nothing staged.

+ if _, err := stageAuthorized(ctx, s, pub, opt, "bab", "km", "v1", "a.example.com", stageCheck{}); err == nil {

+ t.Fatal("staged with no verified copy and no readable remote evidence")

+ }

+ if _, err := os.Stat(VersionPath(opt, "bab", "v1", "km")); err == nil {

+ t.Fatal("bytes were downloaded without remote evidence")

+ }

+ // The box's verified copy (staged earlier, sibling beside it).

+ if _, err := ensureStaged(ctx, base, opt, "bab", "km", "v1", "seed"); err != nil {

+ t.Fatal(err)

+ }

+ m, err := stageAuthorized(ctx, s, pub, opt, "bab", "km", "v1", "a.example.com", stageCheck{})

+ if err != nil || m == nil || m.Version != "v1" {

+ t.Fatalf("stageAuthorized = %+v, %v; want the verified local copy recovered", m, err)

+ }

+ // A copy whose bytes no longer match its sibling is not recovered.

+ if err := os.WriteFile(VersionPath(opt, "bab", "v1", "km"), []byte("tampered"), 0o755); err != nil {

+ t.Fatal(err)

+ }

+ if _, err := stageAuthorized(ctx, s, pub, opt, "bab", "km", "v1", "a.example.com", stageCheck{}); err == nil {

+ t.Fatal("recovered a staged copy that no longer matches its sibling")

+ }

+ })

+ }

+}

+

+// Ruling R-HELD-PURE: root's consumed decision records the settings column

+// it was made for, so the held wake check can tell a decision for the

+// current settings from a stale one. A different column has another digest.

+func TestNextConsumedRecordsTheSettingsItDecidedFor(t *testing.T) {

+ one := 1

+ u := UpdateSettings{Updates: UpdatesAuto, Channel: "stable", Major: &one}

+ in := TargetInput{Route: registry.Route{Host: "a.example.com", UpdateSettings: u.Column()}, Settings: u, Build: BuildStateCopy{Known: true}}

+ got := NextConsumed(in, TargetDecision{Reason: ReasonMajor})

+ if got.Settings == "" || got.Settings != SettingsDigest(u.Column()) {

+ t.Fatalf("Settings = %q, want the digest of %s", got.Settings, u.Column())

+ }

+ two := 2

+ if other := (UpdateSettings{Updates: UpdatesAuto, Channel: "stable", Major: &two}).Column(); SettingsDigest(other) == got.Settings {

+ t.Fatal("two columns share a digest")

+ }

+}

diff --git a/internal/release/buildstate.go b/internal/release/buildstate.go

index a8370147f..08680432d 100644

--- a/internal/release/buildstate.go

+++ b/internal/release/buildstate.go

@@ -1,6 +1,8 @@

package release

import (

+ "crypto/sha256"

+ "encoding/hex"

"encoding/json"

"errors"

"fmt"

@@ -51,6 +53,14 @@ type BuildStateCopy struct {

ReadErr string `json:"read_err,omitempty"`

}

+// Read reports a copy root and the wake may decide on: read, and its latest

+// refresh did not fail. A copy that kept its last known state after a failed

+// refresh (ReadErr) may be stale (a box the instance returned to, whose row's

+// refreshes failed while the instance was away), so its watermark and

+// last_executed are not decided on: it reads as unread (R-TARGET-UNKNOWN).

+// Retention still protects the builds it names (SettingsRefs).

+func (c BuildStateCopy) Read() bool { return c.Known && c.ReadErr == "" }

+

// ParseBuildStateCopy reads the registry column. An empty column is the zero

// copy: Known false, nothing read yet.

func ParseBuildStateCopy(col string) (BuildStateCopy, error) {

@@ -86,8 +96,21 @@ type Consumed struct {

// Baking is the head a pending, unresolved request waits on (this box's

// bake window). Reported beside Reason, never in place of it: Reason is

// what the held wake check reads (Astra I1 #8).

- Baking string `json:"baking,omitempty"`

- At time.Time `json:"at"`

+ Baking string `json:"baking,omitempty"`

+ // Settings is SettingsDigest of the update_settings column this decision

+ // was made for (ruling R-HELD-PURE). The held wake check trusts Reason

+ // only for the column it was decided for: after a member's change, a

+ // missing or stale decision never makes a wake unheld. Empty in a file

+ // from before this field: no decision for any column.

+ Settings string `json:"settings,omitempty"`

+ At time.Time `json:"at"`

+}

+

+// SettingsDigest names one update_settings column exactly (the registry's

+// text, as root and the agent both read it), for Consumed.Settings.

+func SettingsDigest(col string) string {

+ sum := sha256.Sum256([]byte(col))

+ return "sha256:" + hex.EncodeToString(sum[:])

}

// ConsumedPath is the path for host's consumed file. The caller must pass a

@@ -186,6 +209,63 @@ func NextBuildState(cur *BuildState, c *Consumed, build string) BuildState {

return next

}

+// CarryRequest is a graceful move's merge (settings spec §4.3, §4.3a; ruling

+// R-SPEC-43A-2, Astra round 1 finding 3). Root records a resolution, and a

+// request whose T failed as completed-failed, in its own consumed file; the

+// build state learns them only at the next pre-exec write. A request closed

+// as failed on a sleeper runs nothing, so no such write comes. Before a cut

+// seals the build state (the legacy cut, the Phase 1 cut's final round, a

+// dirty return's lane), the source box carries its record into the state

+// under the lease with this merge, so the destination holds "the same

+// request state" (§4.3a) and a completed request never enters rule 2 again.

+//

+// It carries root's resolution on NextBuildState's terms (newer than both

+// sources' watermark and the state's own resolution), and a failed

+// completion: one above the state's watermark becomes it, one equal to it

+// marks it failed. Never a successful completion (only the pre-exec write

+// completes a request successfully, when its T runs) and never

+// last_executed (only a run changes it). A lost box carries nothing: what it

+// held is the spec's lost request (§4.2). The host and the update

+// simulation both call this function.

+func CarryRequest(state *BuildState, c *Consumed) BuildState {

+ next := BuildState{}

+ if state != nil {

+ next = *state

+ next.Resolved = copyResolved(state.Resolved)

+ }

+ if c == nil {

+ return next

+ }

+ if c.Resolved != nil && c.Resolved.ID > Watermark(c, &next) &&

+ (next.Resolved == nil || c.Resolved.ID > next.Resolved.ID) {

+ next.Resolved = copyResolved(c.Resolved)

+ }

+ if c.CompletedFailed && c.Completed != 0 {

+ switch {

+ case c.Completed > next.Completed:

+ next.Completed, next.CompletedFailed = c.Completed, true

+ case c.Completed == next.Completed:

+ next.CompletedFailed = true

+ }

+ return next

+}

+

+// SameRequestState reports whether a and b hold the same request state:

+// resolution, completed id and its outcome. A nil state holds none.

+func SameRequestState(a, b *BuildState) bool {

+ var x, y BuildState

+ if a != nil {

+ x = *a

+ }

+ if b != nil {

+ y = *b

+ }

+ rx, ry := x.Resolved, y.Resolved

+ sameRes := (rx == nil && ry == nil) || (rx != nil && ry != nil && *rx == *ry)

+ return sameRes && x.Completed == y.Completed && x.CompletedFailed == y.CompletedFailed

+}

+

func copyResolved(r *ResolvedRequest) *ResolvedRequest {

if r == nil {

return nil

diff --git a/internal/release/buildstate_test.go b/internal/release/buildstate_test.go

index 366d4d687..224bc33f1 100644

--- a/internal/release/buildstate_test.go

+++ b/internal/release/buildstate_test.go

@@ -6,6 +6,8 @@ import (

"reflect"

"testing"

"time"

+

+ "github.com/carlosframework/platform/internal/registry"

)

func TestBuildStateKeySitsBesideTheLock(t *testing.T) {

@@ -161,3 +163,94 @@ func mustRead(t *testing.T, path string) []byte {

}

return b

}

+

+// Ruling R-SPEC-43A-2 (Astra round 1 finding 3): a graceful move carries the

+// source box's consumed request record into the build state: root's

+// resolution when it is newer than everything the state knows, and a failed

+// completion; never a successful completion, and never last_executed (only a

+// run changes that).

+func TestCarryRequestCarriesTheResolutionAndAFailedCompletion(t *testing.T) {

+ cur := &BuildState{Schema: 1, Host: "h", LastExecuted: "A", Completed: 1}

+ c := &Consumed{Host: "h", Resolved: &ResolvedRequest{ID: 3, Target: "T"}, Completed: 2, CompletedFailed: true}

+ got := CarryRequest(cur, c)

+ if got.LastExecuted != "A" || got.Completed != 2 || !got.CompletedFailed || got.Resolved == nil || got.Resolved.ID != 3 || got.Resolved.Target != "T" {

+ t.Fatalf("carried %+v; want A, completed 2 failed, {3 T}", got)

+ }

+ if cur.Completed != 1 || cur.Resolved != nil {

+ t.Fatalf("the input was changed: %+v", cur)

+ }

+ // A successful completion is the pre-exec write's alone.

+ if got := CarryRequest(cur, &Consumed{Host: "h", Completed: 5}); got.Completed != 1 || got.CompletedFailed {

+ t.Fatalf("a successful completion was carried: %+v", got)

+ }

+ // An older resolution never replaces a newer one; a same-id failure marks

+ // the completed request failed without reopening it.

+ newer := &BuildState{LastExecuted: "A", Completed: 2, Resolved: &ResolvedRequest{ID: 4, Target: "U"}}

+ if got := CarryRequest(newer, &Consumed{Resolved: &ResolvedRequest{ID: 3, Target: "T"}, Completed: 2, CompletedFailed: true}); got.Resolved.ID != 4 || got.Completed != 2 || !got.CompletedFailed {

+ t.Fatalf("carried %+v; want {4 U} kept, R2 completed failed", got)

+ }

+ // No state yet: the record alone makes one, with no last_executed.

+ if got := CarryRequest(nil, c); got.LastExecuted != "" || got.Completed != 2 || !got.CompletedFailed || got.Resolved == nil {

+ t.Fatalf("carried onto no state: %+v", got)

+ }

+}

+

+// The scenario: a sleeping manual instance last ran A. Root resolves R1 to

+// T, which this box holds as failed, so R1 completes failed in root's file

+// and nothing runs. A graceful move carries that into the build state, and

+// the destination never resolves R1 again (rule 2: a completed request never

+// re-enters), so it installs nothing new without another Update now. A box

+// that is lost carries nothing: the destination then resolves R1 again, the

+// spec's lost request (§4.2).

+func TestAFailedCompletionSurvivesAGracefulMove(t *testing.T) {

+ u := UpdateSettings{Updates: UpdatesManual, Channel: "stable", InstallRequest: 1}

+ route := registry.Route{Host: "h", Kind: "instance", UpdateSettings: u.Column()}

+ state := &BuildState{Schema: 1, Host: "h", LastExecuted: "A"}

+ src := TargetInput{Route: route, Pointer: &ChannelPointer{Version: "T"}, Settings: u,

+ Build: BuildStateCopy{Known: true, Replica: true, State: state}, Installed: "A",

+ Failed: func(v string) bool { return v == "T" }}

+ dec := Target(src)

+ if dec.Resolve == nil || dec.Fail != 1 {

+ t.Fatalf("source decision %+v; want R1 resolved to T and completed failed", dec)

+ }

+ consumed := NextConsumed(src, dec)

+ dst := func(st *BuildState) TargetDecision {

+ return Target(TargetInput{Route: route, Pointer: &ChannelPointer{Version: "H"}, Settings: u,

+ Build: BuildStateCopy{Known: true, Replica: true, State: st}})

+ }

+ carried := CarryRequest(state, &consumed)

+ if d := dst(&carried); d.Resolve != nil || d.Reason == ReasonRequest || d.Version != "A" {

+ t.Fatalf("destination after a graceful move: %+v; want R1 closed, held on A", d)

+ }

+ if d := dst(state); d.Resolve == nil || d.Resolve.ID != 1 || d.Version != "H" {

+ t.Fatalf("destination after a lost box: %+v; want R1 resolved again (a lost request)", d)

+ }

+}

+

+// Found by the simulation (seed 164 after Astra round 1's changes): a box

+// the instance returns to keeps its old row, and that copy's refreshes

+// failed while the instance was away (ReadErr), so it still says request R1

+// completed while the bucket says R5 completed failed. Root must read such a

+// copy as unread (R-TARGET-UNKNOWN): no request bookkeeping, so R5 is not

+// resolved again from a stale watermark, and a held instance is unknown

+// rather than recovered onto a stale last_executed.

+func TestTargetReadsACopyWhoseRefreshFailedAsUnread(t *testing.T) {

+ stale := BuildStateCopy{Known: true, Replica: true, ReadErr: "s3: timeout", State: &BuildState{LastExecuted: "b1", Completed: 1, CompletedFailed: true}}

+ for _, u := range []UpdateSettings{

+ {Updates: UpdatesAuto, Channel: "stable", InstallRequest: 5},

+ {Updates: UpdatesManual, Channel: "stable", InstallRequest: 5},

+ } {

+ in := TargetInput{Route: registry.Route{Host: "h", UpdateSettings: u.Column()}, Pointer: &ChannelPointer{Version: "b6"}, Settings: u,

+ Build: stale, Consumed: &Consumed{Host: "h", Completed: 1, CompletedFailed: true}, Installed: "b1"}

+ d := Target(in)

+ if d.Resolve != nil || d.Reason == ReasonRequest {

+ t.Fatalf("%s: %+v; a stale copy's watermark resolved R5 again", u.Updates, d)

+ }

+ if u.Updates == UpdatesManual && (d.Reason != ReasonUnknown || d.Version != "") {

+ t.Fatalf("manual: %+v; want unknown, nothing installed", d)

+ }

+ if next := NextConsumed(in, d); next.Baking != "" || (next.Resolved != nil && next.Resolved.ID == 5) {

+ t.Fatalf("%s: consumed %+v", u.Updates, next)

+ }

+}

diff --git a/internal/release/heldwake.go b/internal/release/heldwake.go

new file mode 100644

index 000000000..58dd25342

--- /dev/null

+++ b/internal/release/heldwake.go

@@ -0,0 +1,143 @@

+package release

+

+import "fmt"

+

+// HeldWakeInput is what the held wake check decides on (settings spec §4.3

+// rule 3, "The wake checks too"; ruling R-HELD-PURE): the registry columns

+// and root's records the agent reads for the wake, read by the caller so the

+// decision stays pure. The host's heldTarget and the update simulation's

+// wake both call DecideHeldWake with it.

+type HeldWakeInput struct {

+ Settings string // the route's update_settings column, as the registry holds it

+ Build BuildStateCopy // the registry copy, refreshed for the wake

+ Consumed *Consumed // root's file for the host; nil when it has none (or it does not read)

+ Refusal *Refusal // root's refusal record for the host; nil when none

+ Installed string // the build the live link names on this box; "" when none

+}

+

+// HeldWakeDecision is the held wake check's answer. The zero value is "not

+// held": today's wake.

+type HeldWakeDecision struct {

+ Target string // the build the instance must run

+ Held bool

+ // Unknown is rule 3's "build unknown": refused, root installs nothing.

+ Unknown bool

+ // Failed: root reports Target failed on this box (its decision, or a

+ // not-up refusal record).

+ Failed bool

+ // AskRoot: root has made no decision for the current settings, so the

+ // wake cannot know whether the instance is held. It holds: Target (the

+ // build state's build, "" when there is none) runs only if it is

+ // already installed; otherwise the wake asks root and waits for its

+ // decision. Never "not held" (Astra round 1 finding 4).

+ AskRoot bool

+ // Unread: the copy's latest refresh failed (ReadErr). Refused: a held

+ // wake is never decided from a stale copy (Astra round 1 finding 1).

+ Unread bool

+}

+

+// DecideHeldWake is the held wake check's decision, in root's rule order

+// (Target, §4.3) so root and the wake agree:

+//

+// - default settings: not held (today's wake), nothing else read;

+// - the copy's latest refresh failed: Unread, refused;

+// - a column that does not parse even leniently: held as unknown once

+// root has decided unknown for exactly this column (ruling

+// R-SETTINGS-UNREADABLE), otherwise AskRoot;

+// - a pin: held on it (rule 1);

+// - with the copy read, a pending request resolved to T, the build

+// state's resolution first (R-RESOLUTION-STATE-WINS): held on T (rule

+// 2; manual and major are ignored for it);

+// - otherwise rule 3's target as Target computes it: an unread copy is

+// unknown; else last_executed; else the installed build (Astra round 1

+// finding 5); else, with a replica, unknown; else a new instance, which

+// root gives the head, so not held. Each named build is failure-checked

+// against root's word (Failed);

+// - whether rule 3 holds at all needs root when the setting is a major

+// hold (the agent does not read tags). Root's consumed Reason is read

+// only when it was decided for the current column (Consumed.Settings);

+// with no such decision the wake holds and asks root (AskRoot), carrying

+// rule 3's target as the one build it may run if installed. Manual

+// holds by rule 3; a major hold holds when root's decision for this

+// column is major, unknown, or failed on rule 3's target (ruling

+// R-HELD-FAILED); any other decision for this column is not held.

+func DecideHeldWake(in HeldWakeInput) HeldWakeDecision {

+ c := in.Consumed

+ decided := c != nil && c.Settings != "" && c.Settings == SettingsDigest(in.Settings)

+ u, uerr := ParseUpdateSettingsLenient(in.Settings)

+ if uerr == nil && u.Default() {

+ return HeldWakeDecision{}

+ }

+ if in.Build.ReadErr != "" {

+ return HeldWakeDecision{Held: true, Unread: true}

+ }

+ if uerr != nil {

+ if decided && c.Reason == string(ReasonUnknown) {

+ return HeldWakeDecision{Held: true, Unknown: true}

+ }

+ return HeldWakeDecision{Held: true, AskRoot: true}

+ }

+ rootFailed := func(v string) bool {

+ return v != "" && ((c != nil && c.Reason == string(ReasonFailed) && c.Detail == v) ||

+ (in.Refusal != nil && in.Refusal.Kind == RefusalNotUp && in.Refusal.Version == v))

+ }

+ held := func(v string) HeldWakeDecision {

+ return HeldWakeDecision{Target: v, Held: true, Failed: rootFailed(v)}

+ }

+ if u.Pin != "" {

+ return held(u.Pin)

+ }

+ state := in.Build.State

+ if in.Build.Read() && u.InstallRequest > Watermark(c, state) {

+ if res := resolvedFor(c, state, u.InstallRequest); res != nil {

+ return held(res.Target)

+ }

+ var r3 HeldWakeDecision

+ switch {

+ case !in.Build.Read():

+ r3 = HeldWakeDecision{Held: true, Unknown: true}

+ case state != nil && state.LastExecuted != "":

+ r3 = held(state.LastExecuted)

+ case in.Installed != "":

+ r3 = held(in.Installed)

+ case in.Build.Replica:

+ r3 = HeldWakeDecision{Held: true, Unknown: true}

+ }

+ if !decided {

+ return HeldWakeDecision{Held: true, AskRoot: true, Target: r3.Target, Failed: r3.Failed}

+ }

+ if u.Updates == UpdatesManual {

+ return r3

+ }

+ if u.Major != nil {

+ switch c.Reason {

+ case string(ReasonMajor), string(ReasonUnknown):

+ return r3

+ case string(ReasonFailed):

+ if r3.Target != "" && c.Detail == r3.Target {

+ return r3

+ }

+ return HeldWakeDecision{}

+}

+

+// HeldExecStale is the pre-exec write's revalidation (Astra round 1 finding

+// 1): d is DecideHeldWake on the build state read under the lease. A held

+// target other than the build about to run means the wake decided on a

+// stale copy, and the record is refused, so nothing runs. The build the

+// state's last_executed names is never stale (the data last ran it: a

+// restart or recycle of it is not a move onto another build), and a target

+// root reports failed is left to the planner's recovery (ruling

+// R-HELD-FAILED), as an unknown target is to the wake's own refusal. nil

+// when the build may run.

+func HeldExecStale(d HeldWakeDecision, st *BuildState, build string) error {

+ if !d.Held || d.Unknown || d.Unread || d.Failed || d.Target == "" || d.Target == build {

+ return nil

+ }

+ if st != nil && st.LastExecuted == build {

+ return nil

+ }

+ return fmt.Errorf("the build state names %s as this instance's build, not %s: the wake decided on a stale copy", d.Target, build)

+}

diff --git a/internal/release/pins.go b/internal/release/pins.go

index dea262942..a9927edf1 100644

--- a/internal/release/pins.go

+++ b/internal/release/pins.go

@@ -12,6 +12,7 @@ import (

"fmt"

"os"

"path/filepath"

+ "sort"

"strconv"

"strings"

@@ -90,8 +91,18 @@ func (f PinSourceFunc) StagedPins(ctx context.Context, optDir string) (StagedPat

// first wake falls back to, spec §8.5). routes is every registry row, unfiltered: a hibernating route,

// an expired canary and a mid-provision row all still refer to their builds.

//

+// It also holds every build a row's update settings and build-state copy

+// name (SettingsRefs: a pin, last_executed, a resolved request's T), under

+// the row's own account and app (settings spec §4.1 retained identity;

+// Astra round 1 finding 2): a sleeper whose build state alone names a build

+// keeps that copy, executable and sibling, when another instance adopts a

+// reuse of its name.

+//

// An entry that cannot be read is an error, never a skipped entry: a pin set

-// that quietly narrowed is how a running build's bytes get replaced.

+// that quietly narrowed is how a running build's bytes get replaced. A row

+// whose settings references cannot be read completely is reported apart, as

+// an *IncompleteRefsError returned WITH the pins: it refuses a restage of its

+// own app only, so one app's unread build state does not stop another's.

func BoxPins(optDir string, routes []registry.Route) (StagedPaths, error) {

pins := StagedPaths{}

dir := liveDir(optDir)

@@ -117,19 +128,114 @@ func BoxPins(optDir string, routes []registry.Route) (StagedPaths, error) {

return nil, fmt.Errorf("live link %s: %w", filepath.Join(dir, e.Name()), err)

}

+ var incomplete *IncompleteRefsError

for _, rt := range routes {

if rt.App == "" {

continue

}

- for _, v := range []string{rt.Version, rt.PrevVersion, rt.GoodVersion} {

+ acct := RouteAccount(rt)

+ refs, rerr := SettingsRefs(rt)

+ for _, v := range append([]string{rt.Version, rt.PrevVersion, rt.GoodVersion}, refs...) {

if ValidVersion(v) {

- pins.add(VersionPath(optDir, RouteAccount(rt), v, rt.App))

+ pins.add(VersionPath(optDir, acct, v, rt.App))

+ }

+ if rerr != nil {

+ if incomplete == nil {

+ incomplete = &IncompleteRefsError{apps: map[string]string{}}

}

+ incomplete.apps[acct+"/"+rt.App] = rerr.Error()

}

+ if incomplete != nil {

+ return pins, incomplete

+ }

return pins, nil

}

+// IncompleteRefsError is BoxPins' report of apps whose settings references

+// (SettingsRefs) could not be read completely. BoxPins returns it together

+// with the pins it did read: a restage of a copy of a named app is refused,

+// because a held instance's last_executed may be the very name being reused;

+// any other app restages on the pins as usual.

+type IncompleteRefsError struct {

+ apps map[string]string // "<account>/<app>" -> why

+}

+

+func (e *IncompleteRefsError) Error() string {

+ keys := make([]string, 0, len(e.apps))

+ for k := range e.apps {

+ keys = append(keys, k)

+ }

+ sort.Strings(keys)

+ parts := make([]string, 0, len(keys))

+ for _, k := range keys {

+ parts = append(parts, k+": "+e.apps[k])

+ }

+ return "settings references incomplete for " + strings.Join(parts, "; ")

+}

+

+// Covers reports whether acct/app's references are incomplete.

+func (e *IncompleteRefsError) Covers(acct, app string) bool {

+ _, ok := e.apps[acct+"/"+app]

+ return ok

+}

+

+// SettingsRefs is every build a row's update settings and build-state copy

+// name (settings spec §4.4, §4.1): its pin, last_executed and a resolved

+// request's T, read as root reads the columns (ParseUpdateSettingsLenient,

+// ParseBuildStateCopy). A copy that kept its last known state after a failed

+// refresh still names it.

+//

+// The error says the list is incomplete, and comes with what could be read:

+// a column that does not parse; or a row settings hold (non-default settings

+// on a row whose build state the agent refreshes, BuildStateKept) whose copy

+// is not read yet, or whose latest refresh failed (ReadErr; Astra round 1

+// finding 6): its last_executed may name another build by now. A row the

+// agent never refreshes (a named unit, the platform's own row, a worker)

+// stays unread forever, and must not make every set incomplete for good.

+func SettingsRefs(rt registry.Route) ([]string, error) {

+ var out []string

+ u, uerr := ParseUpdateSettingsLenient(rt.UpdateSettings)

+ if uerr == nil && u.Pin != "" {

+ out = append(out, u.Pin)

+ }

+ c, cerr := ParseBuildStateCopy(rt.BuildState)

+ if cerr == nil && c.State != nil {

+ if c.State.LastExecuted != "" {

+ out = append(out, c.State.LastExecuted)

+ }

+ if r := c.State.Resolved; r != nil && r.Target != "" {

+ out = append(out, r.Target)

+ }

+ switch {

+ case uerr != nil:

+ return out, fmt.Errorf("update settings of %s: %w", rt.Host, uerr)

+ case cerr != nil:

+ return out, fmt.Errorf("build state of %s: %w", rt.Host, cerr)

+ case u.Default() || !BuildStateKept(rt):

+ return out, nil

+ case !c.Known:

+ return out, fmt.Errorf("the build state of held %s is not read yet", rt.Host)

+ case c.ReadErr != "":

+ return out, fmt.Errorf("the build state of held %s could not be read at its last refresh (%s)", rt.Host, c.ReadErr)

+ }

+ return out, nil

+}

+

+// BuildStateKept is a row update settings can hold whose build state the

+// agent keeps and refreshes at every reconcile: an activator-managed exec

+// instance (Kind instance, hibernating, a socket, no Backing, no Tunnel), not

+// a named unit and not the platform's own row. It is the host's

+// keepsBuildState less workers, which settings never apply to; this package

+// cannot import the activator, so the condition is spelled out here and the

+// host's sweep uses this one.

+func BuildStateKept(rt registry.Route) bool {

+ return rt.Kind == "instance" && rt.Unit == "" && !rt.Platform &&

+ rt.Hibernate && rt.Socket != "" && rt.Backing == "" && rt.Tunnel == ""

+}

+

// Generations is the generations file: for each host with a live or pending

// generation (current, successor, retiring, spawning: spec §8.5), the build

// each one runs. Live generations exist only in the agent's memory, while the

diff --git a/internal/release/stage_reuse_test.go b/internal/release/stage_reuse_test.go

index 31ec8093f..00070365b 100644

--- a/internal/release/stage_reuse_test.go

+++ b/internal/release/stage_reuse_test.go

@@ -352,3 +352,77 @@ func TestStaleCopyTwoRelativeHopsUnderASymlinkedOptDirNameIsNotOverwritten(t *te

f.opt = real

f.assertRefused(t, err.Error(), "record bake clock")

}

+

+// sleeperRow is a held sleeper of app the pass does not adopt (mid

+// provision), whose only reference to its builds is its settings and build

+// state copy: no Version, PrevVersion, GoodVersion, live link or generation.

+func (f reuseFixture) sleeperRow(t *testing.T, host, app string, u UpdateSettings, bs BuildStateCopy) {

+ t.Helper()

+ if err := f.reg.Upsert(registry.Route{Host: host, Kind: "instance", App: app, Channel: "canary/x", Status: "creating",

+ Hibernate: true, Socket: "/run/carlos/" + host + ".sock"}); err != nil {

+ t.Fatal(err)

+ }

+ if err := f.reg.SetUpdateSettings(host, u.Column()); err != nil {

+ t.Fatal(err)

+ }

+ if err := f.reg.SetBuildState(host, bs.Column()); err != nil {

+ t.Fatal(err)

+ }

+}

+

+// Astra round 1 finding 2 (§4.1 retained identity): a sleeper last ran v2,

+// then received cold installs of later builds, so only its build state names

+// v2. Another instance adopting a promoted reuse of the name v2 must not

+// replace that copy (executable and sibling): the restaging pin set holds

+// every build a settings pin, a last_executed or a resolved T names.

+func TestStaleCopyABuildStateOrPinNamesIsNotOverwritten(t *testing.T) {

+ manual := UpdateSettings{Updates: UpdatesManual, Channel: "canary/x"}

+ pinned := UpdateSettings{Updates: UpdatesAuto, Channel: "canary/x", Pin: "v2"}

+ for name, c := range map[string]struct {

+ u UpdateSettings

+ bs BuildStateCopy

+ }{

+ "last_executed": {manual, BuildStateCopy{Known: true, Replica: true, State: &BuildState{LastExecuted: "v2"}}},

+ "resolved T": {manual, BuildStateCopy{Known: true, Replica: true, State: &BuildState{LastExecuted: "v1", Resolved: &ResolvedRequest{ID: 3, Target: "v2"}}}},

+ "pin": {pinned, BuildStateCopy{Known: true, Replica: true, State: &BuildState{LastExecuted: "v1"}}},

+ } {

+ t.Run(name, func(t *testing.T) {

+ f := newReuseFixture(t)

+ sib, err := os.ReadFile(StagedManifestPath(f.opt, "bab", "v2", "myapp"))

+ if err != nil && !os.IsNotExist(err) {

+ t.Fatal(err)

+ }

+ f.sleeperRow(t, "h2.example.com", "myapp", c.u, c.bs)

+ f.assertRefused(t, f.pass(t), ErrVersionReused.Error())

+ if now, _ := os.ReadFile(StagedManifestPath(f.opt, "bab", "v2", "myapp")); !bytes.Equal(now, sib) {

+ t.Fatalf("the referenced copy's sibling was replaced: %q, was %q", now, sib)

+ }

+ })

+ }

+}

+

+// Finding 2's completeness half: a held row of the same app whose copy cannot

+// be read completely (never read, or its last refresh failed) refuses the

+// restage, since its last_executed may be the reused name. A row of another

+// app with the same problem does not block this one.

+func TestAnIncompleteBuildStateRefusesARestageOnlyForItsApp(t *testing.T) {

+ manual := UpdateSettings{Updates: UpdatesManual, Channel: "canary/x"}

+ for name, bs := range map[string]BuildStateCopy{

+ "never read": {},

+ "refresh failed": {Known: true, Replica: true, ReadErr: "s3: timeout", State: &BuildState{LastExecuted: "v1"}},

+ } {

+ t.Run(name, func(t *testing.T) {

+ f := newReuseFixture(t)

+ f.sleeperRow(t, "h2.example.com", "myapp", manual, bs)

+ f.assertRefused(t, f.pass(t), "what refers to it is unknown")

+ })

+ t.Run(name+", another app", func(t *testing.T) {

+ f := newReuseFixture(t)

+ f.sleeperRow(t, "h3.example.com", "otherapp", manual, bs)

+ f.pass(t)

+ if got := f.stagedV2(t); got != "new v2" {

+ t.Fatalf("staged v2 = %q: another app's unreadable build state blocked this app's restage", got)

+ }

+ })

+ }

+}

diff --git a/internal/release/updatetarget.go b/internal/release/updatetarget.go

index 0a14836b5..7c7cb4ce1 100644

--- a/internal/release/updatetarget.go

+++ b/internal/release/updatetarget.go

@@ -76,17 +76,19 @@ func Target(in TargetInput) TargetDecision {

// resolution waits, so clearing the pin later installs this T, not a

// head that moved on. A completed request never enters here again.

//

- // None of it happens while the build state is unread (!Build.Known): the

- // watermark then sees only the consumed file, so a request that already

- // completed would look pending on a fresh box (R-TARGET-UNKNOWN). The pin,

- // held and adoption rules decide as before, with no resolution and no

- // failure completion.

+ // None of it happens while the build state is unread (!Build.Read(): never

+ // read, or its latest refresh failed and the copy may be stale): the

+ // watermark then sees only the consumed file, or an old copy's, so a

+ // request that already completed would look pending (R-TARGET-UNKNOWN).

+ // The pin, held and adoption rules decide as before, with no resolution

+ // and no failure completion.

var (

resolve *ResolvedRequest

failedReq uint64

reqTarget string // a resolved T, not failed, above the watermark

)

- if in.Build.Known {

+ read := in.Build.Read()

+ if read {

if u.InstallRequest > wm {

if res := resolvedFor(in.Consumed, state, u.InstallRequest); res != nil {

if failed(res.Target) {

@@ -149,7 +151,7 @@ func Target(in TargetInput) TargetDecision {

}

var d TargetDecision

switch {

- case !in.Build.Known:

+ case !read:

d = TargetDecision{Reason: ReasonUnknown}

case state != nil && state.LastExecuted != "":

d = TargetDecision{Version: state.LastExecuted, Reason: reason, Evidence: EvidenceRecord, Detail: state.LastExecuted}

@@ -227,8 +229,10 @@ func tagInMajor(tag string, major int) bool {

// - Baking names the head a pending, unresolved request waits on, beside

// the decision's reason, never in its place (Astra I1 #8): Reason is what

// the held wake check reads. Only with the build state read

-// (in.Build.Known), as Target does no request bookkeeping without it.

-// - Desired, Reason and Detail are the decision; At is the writer's.

+// (in.Build.Read()), as Target does no request bookkeeping without it.

+// - Desired, Reason and Detail are the decision; Settings names the

+// update_settings column it was made for (in.Route.UpdateSettings,

+// ruling R-HELD-PURE); At is the writer's.

func NextConsumed(in TargetInput, dec TargetDecision) Consumed {

next := Consumed{Host: in.Route.Host}

if in.Consumed != nil {

@@ -250,15 +254,18 @@ func NextConsumed(in TargetInput, dec TargetDecision) Consumed {

// build, takes the build state's T (ruling R-RESOLUTION-STATE-WINS), so

// the box's own record stops disagreeing with what Target followed. Only

// with the build state read: an unread copy says nothing.

- if in.Build.Known && state != nil && state.Resolved != nil && next.Resolved != nil &&

+ if in.Build.Read() && state != nil && state.Resolved != nil && next.Resolved != nil &&

next.Resolved.ID == state.Resolved.ID && next.Resolved.Target != state.Resolved.Target {

next.Resolved = copyResolved(state.Resolved)

}

next.Baking = ""

- if in.Build.Known && in.Pointer != nil && in.HeadHeld != "" && !u.InstallFastTrack && dec.Resolve == nil &&

+ if in.Build.Read() && in.Pointer != nil && in.HeadHeld != "" && !u.InstallFastTrack && dec.Resolve == nil &&

u.InstallRequest > Watermark(&next, state) && resolvedFor(&next, state, u.InstallRequest) == nil {

next.Baking = in.Pointer.Version

}

next.Desired, next.Reason, next.Detail, next.At = dec.Version, string(dec.Reason), dec.Detail, time.Time{}

+ // The column this decision was made for (ruling R-HELD-PURE): the held

+ // wake check reads Reason only while the column is still this one.

+ next.Settings = SettingsDigest(in.Route.UpdateSettings)

return next

}

carlos / platform Public

settings-4: Astra round 1 fixes