amadan
settings-4 task 12a: the final-wave fixes queued by the task reviews
- claimwake: a pinned claimed start with no generation number hands the sidecar the file it ran (s.Staged) with the pin's demand. - claimwake: the claimed exec asks its attempt gate again after the record, as startSidecar does; a fence raised during the PUT execs nothing. - adopt: the up-to-date refusal clear keeps a refusal that still blocks either the selected build or the head. - wakecheck: the wait's re-ask is bounded by the wait, so a slow answer never holds the wake past wakeWaitBound. - movesinstall: a landing whose seal carries no state removes every version of a leftover build-state.json beside the lease. - buildstate (R-HELD-SCOPE): heldTarget and wakeTarget hold only Kind instance rows with no Unit, as root applies settings. - Doc and log fixes: SnapshotFingerprint's four churn exceptions, the stage-marker journal line, the bsErr operator line.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
13 files changed,
+315
−34
internal/activator/claimrecord_linux_test.go+64 −3internal/activator/claimwake.go+15 −1internal/activator/wakecheck.go+23 −3internal/activator/wakecheck_test.go+34 −0internal/host/agent.go+11 −3internal/host/buildstate.go+16 −5internal/host/buildstate_moves_test.go+47 −0internal/host/buildstate_test.go+49 −0internal/host/movesfinal_test.go+8 −1internal/host/movesinstall.go+19 −3internal/registry/registry.go+11 −8internal/release/adopt.go+4 −3internal/release/adoptsettings_test.go+14 −4
diff --git a/internal/activator/claimrecord_linux_test.go b/internal/activator/claimrecord_linux_test.go| index bab6806ac..7c56af25a 100644 |
| --- a/internal/activator/claimrecord_linux_test.go |
| +++ b/internal/activator/claimrecord_linux_test.go |
| @@ -279,6 +279,20 @@ func TestAClaimedProbeRecords(t *testing.T) { |
| // claim's Holder.Location (internal/host buildStates.locate, ruling |
| // R-CLAIMPREFIX-2), which is the claimed lease's. |
| func TestAClaimedSidecarRestartRecordsTheBuildItsPrimaryRan(t *testing.T) { |
| + claimedSidecarRestart(t, false) |
| +} |
| + |
| +// The same for a pinned claimed start (Task 12a): every claimed wake is |
| +// pinned once a planner is on, and with no generation number the pin's |
| +// start must still hand the sidecar the file it ran, by path, with the |
| +// pin's demand. Prepare's start was checked against the pin (s.Version is |
| +// Pin.Build), and s.Staged is what actually ran. |
| +func TestAPinnedClaimedSidecarRestartRunsTheClaimedFile(t *testing.T) { |
| + claimedSidecarRestart(t, true) |
| +} |
| + |
| +func claimedSidecarRestart(t *testing.T, pinned bool) { |
| + t.Helper() |
| f := newRootLaunchFixture(t) |
| f.root.grant = launchedChild(t) |
| host := f.launch.Target.Host |
| @@ -316,6 +330,15 @@ func TestAClaimedSidecarRestartRecordsTheBuildItsPrimaryRan(t *testing.T) { |
| }}}, b, reg) |
| l := &execLog{} |
| a.BeforeExec = l.hook |
| + id := claimedBuildOf(t, f, "v1") |
| + wantDemand := "" |
| + if pinned { |
| + boot := id |
| + boot.Demand, wantDemand = "hold", "hold" |
| + a.Planner = func(context.Context, registry.Route) WakePlan { |
| + return WakePlan{Boot: boot, Mode: ModeHold} |
| + } |
| + } |
| rt := registry.Route{Host: host, Kind: "instance", Hibernate: true, Socket: filepath.Join(f.runDir, "app.sock"), DBPath: filepath.Join(f.launchDir, "app.db")} |
| done, err := a.beginRequest(t.Context(), rt) |
| if err != nil { |
| @@ -323,9 +346,8 @@ func TestAClaimedSidecarRestartRecordsTheBuildItsPrimaryRan(t *testing.T) { |
| } |
| done() |
| g, ok := a.entryGen(host) |
| - id := claimedBuildOf(t, f, "v1") |
| - if !ok || g.Build != "v1" || g.StagedPath != id.StagedPath || g.Digest != id.Digest { |
| - t.Fatalf("serving %+v (%v), want the hashed v1 by path and digest", g, ok) |
| + if !ok || g.Build != "v1" || g.StagedPath != id.StagedPath || g.Digest != id.Digest || g.Demand != wantDemand { |
| + t.Fatalf("serving %+v (%v), want the hashed v1 by path and digest, demand %q", g, ok, wantDemand) |
| } |
| a.sweepOnce() |
| waitFor(t, "the sidecar start", func() bool { return len(b.sidecarsOn()) == 1 && a.sidecarSettled(host) }) |
| @@ -340,3 +362,42 @@ func TestAClaimedSidecarRestartRecordsTheBuildItsPrimaryRan(t *testing.T) { |
| t.Fatalf("locations %+v: want the primary's at the claimed lease, the sidecar's left to the claim read", locs) |
| } |
| } |
| + |
| +// The claimed exec asks its attempt gate again after the record, as |
| +// startSidecar does (Task 12a): a fence raised or a lease lost while the |
| +// build state was being written (the S3 PUT) execs nothing. |
| +func TestAClaimedStartExecsOnlyIfStillGatedAfterItsRecord(t *testing.T) { |
| + c := newClaimedRecord(t, nil) |
| + var mu sync.Mutex |
| + fenced, gates := false, 0 |
| + record := c.log.hook |
| + c.a.BeforeExec = func(ctx context.Context, host, build string) error { |
| + err := record(ctx, host, build) |
| + mu.Lock() |
| + fenced = true // the fence lands during the PUT |
| + mu.Unlock() |
| + return err |
| + } |
| + gate := func() error { |
| + mu.Lock() |
| + defer mu.Unlock() |
| + gates++ |
| + if fenced { |
| + return ErrFenced |
| + } |
| + return nil |
| + } |
| + err := c.a.startAdmitted(t.Context(), &actEntry{host: c.obs.Host}, c.obs, startPolicy{MaxStarts: 1, Window: 30 * time.Second, Gate: gate}) |
| + if !errors.Is(err, ErrFenced) { |
| + t.Fatalf("startAdmitted = %v, want the fence raised during the record", err) |
| + } |
| + if calls, _, _ := c.log.seen(); len(calls) != 1 { |
| + t.Fatalf("records %v, want the one record before the fence", calls) |
| + } |
| + mu.Lock() |
| + asked := gates |
| + mu.Unlock() |
| + if _, _, _, launches := c.counts(); launches != 0 || asked != 2 { |
| + t.Fatalf("launches=%d gates=%d, want the gate asked before and after the record and nothing execed", launches, asked) |
| + } |
| +} |
diff --git a/internal/activator/claimwake.go b/internal/activator/claimwake.go| index 107ae829e..8389e1eae 100644 |
| --- a/internal/activator/claimwake.go |
| +++ b/internal/activator/claimwake.go |
| @@ -526,6 +526,14 @@ func (a *Service) startAdmitted(ctx context.Context, e *actEntry, admitted hostc |
| unrecorded = true |
| return err |
| } |
| + if policy.Gate != nil { |
| + // Asked again after the record, as startSidecar does: a |
| + // fence raised or a lease lost while the build state was |
| + // written (the PUT) execs nothing (Task 12a). |
| + if err := policy.Gate(); err != nil { |
| + return err |
| + } |
| + } |
| if err := budget.start(ctx, s.OpenProbe); err != nil { |
| return err |
| } |
| @@ -612,13 +620,19 @@ func (a *Service) startAdmitted(ctx context.Context, e *actEntry, admitted hostc |
| g.StagedPath, g.Digest, g.Demand = policy.Pin.StagedPath, policy.Pin.Digest, policy.Pin.Demand |
| } |
| } |
| - if policy.Pin == nil && s.Staged.StagedPath != "" && s.Staged.Digest != "" && s.Staged.Version == s.Version { |
| + if (policy.Pin == nil || g.N == 0) && s.Staged.StagedPath != "" && s.Staged.Digest != "" && s.Staged.Version == s.Version { |
| // The file this start ran (the hashed resolution of the |
| // live link), so a legacy sidecar restart runs that file |
| // by path, never whatever the link names by then (Task |
| // 5d; ruling R-EXEC-SIDECAR). With no number, wake() |
| // keeps it on the canonical handle it gives this start. |
| + // A pinned start with no number takes the same file: its |
| + // version was checked against Pin.Build above, and |
| + // s.Staged is what ran; the demand is the pin's (Task 12a). |
| g.Build, g.StagedPath, g.Digest = s.Version, s.Staged.StagedPath, s.Staged.Digest |
| + if policy.Pin != nil { |
| + g.Demand = policy.Pin.Demand |
| + } |
| } |
| e.claimResult = &wokenInstance{proc: p, rep: rep, spawnVersion: s.Version, gen: g, stopOutcome: &stopObservation{}} |
| return nil |
diff --git a/internal/activator/wakecheck.go b/internal/activator/wakecheck.go| index dca5e029d..265a7ee11 100644 |
| --- a/internal/activator/wakecheck.go |
| +++ b/internal/activator/wakecheck.go |
| @@ -114,13 +114,19 @@ func (a *Service) wakeCheck(ctx context.Context, host string, direct bool) (stri |
| if a.liveVersion(host) == h.Target { |
| return h.Target, nil, nil |
| } |
| - if n%heldReask == 0 { |
| + if n%heldReask == 0 && wait.Err() == nil { |
| // The answer is asked again while the wake waits: a member's |
| // change (a pin, Update now, auto) or root's verdict (failed, |
| // unknown) ends this wait now, rather than at its bound onto a |
| // target that is no longer the instance's. The next wake checks |
| - // afresh. |
| - if next := a.HeldTarget(host); !sameHeld(next, h) { |
| + // afresh. The re-ask is bounded by the wait (Task 12a): it may |
| + // read the bucket for up to its own budget, and an answer that |
| + // comes back after the wait has ended is not waited for. |
| + next, ok := a.reaskHeld(wait, host) |
| + if !ok { |
| + return "", nil, fmt.Errorf("%w: %s is on %q, wants %s", errStaleLink, host, installed, h.Target) |
| + } |
| + if !sameHeld(next, h) { |
| switch { |
| case next.Held && (next.Unknown || next.Target == ""): |
| return "", nil, errBuildUnknown |
| @@ -142,6 +148,20 @@ func (a *Service) wakeCheck(ctx context.Context, host string, direct bool) (stri |
| // HeldTarget again (which may read the bucket). |
| const heldReask = 8 |
| +// reaskHeld is HeldTarget(host) bounded by wait: false when wait ends first. |
| +// The call itself is not cancelled (HeldTarget takes no context); it finishes |
| +// within its own budget into a buffered channel nobody reads. |
| +func (a *Service) reaskHeld(wait context.Context, host string) (HeldWake, bool) { |
| + got := make(chan HeldWake, 1) |
| + go func() { got <- a.HeldTarget(host) }() |
| + select { |
| + case h := <-got: |
| + return h, true |
| + case <-wait.Done(): |
| + return HeldWake{}, false |
| + } |
| +} |
| + |
| // sameHeld reports whether two answers send the wake to the same place. |
| func sameHeld(x, y HeldWake) bool { |
| return x.Held == y.Held && x.Target == y.Target && x.Unknown == y.Unknown && x.Failed == y.Failed |
diff --git a/internal/activator/wakecheck_test.go b/internal/activator/wakecheck_test.go| index e7d41f464..837a8b4ff 100644 |
| --- a/internal/activator/wakecheck_test.go |
| +++ b/internal/activator/wakecheck_test.go |
| @@ -542,3 +542,37 @@ func TestAHeldUnitRowRefusesALinkThatMovedAfterTheWait(t *testing.T) { |
| } |
| waitFor(t, "the lease to go back", func() bool { return unit.releaseCount() == 1 }) |
| } |
| + |
| +// The wait's re-ask is bounded by the wait (Task 12a): an answer that is slow |
| +// to come back (the agent's refresh reads the bucket, up to its own 5 s |
| +// budget) never holds the wake past wakeWaitBound. |
| +func TestAHeldWakesReaskNeverOutlastsTheWait(t *testing.T) { |
| + shrinkWakeWait(t, 1900*time.Millisecond) |
| + a, b := identityActivator(t, "v3") |
| + var asked atomic.Int32 |
| + a.HeldTarget = func(string) HeldWake { |
| + if asked.Add(1) > 1 { |
| + time.Sleep(5 * time.Second) // the refresh's own budget |
| + } |
| + return HeldWake{Target: "v1", Held: true} |
| + } |
| + a.RequestStage = func(string, string) {} // root never installs v1 |
| + // The check itself, timed: the request's own waiters give up at the same |
| + // bound, so through beginRequest an overrun shows only as their timeout |
| + // while the wake is still stuck in the re-ask. |
| + began := time.Now() |
| + _, _, err := a.wakeCheck(context.Background(), "one.example.com", true) |
| + took := time.Since(began) |
| + if !errors.Is(err, errStaleLink) { |
| + t.Fatalf("err = %v, want errStaleLink at the bound", err) |
| + } |
| + if asked.Load() < 2 { |
| + t.Fatalf("asked %d times: the wait never asked again", asked.Load()) |
| + } |
| + if took >= 3*time.Second { |
| + t.Fatalf("the wait ran %v, past its 1.9 s bound: the re-ask outlasted it", took) |
| + } |
| + if b.startCount() != 0 { |
| + t.Fatal("a wake that never saw its target ran something") |
| + } |
| +} |
diff --git a/internal/host/agent.go b/internal/host/agent.go| index f2f58a1ae..fa95c708a 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -297,13 +297,17 @@ func runAgent(args []string, buildClaims ClaimFactory) { |
| // wiring so it locates claimed hosts through cfg.Claims. nil, nil on a box |
| // with no replica bucket. The activator's BeforeExec hook comes from bs |
| // (buildStateHook, below): a failure here is one operator line, the |
| - // agent keeps running, and every spawn is refused until it is fixed. |
| + // agent keeps running, and every spawn that records its build is refused |
| + // until it is fixed: every exec-backed wake, worker, sidecar, swap and |
| + // claimed start (Tasks 5a-5d). A unit-backed row is started by systemd |
| + // on its live link and records nothing (activator identityFor is nil |
| + // for it), so it still starts. |
| bs, bsErr := newBuildStates(cfg, reg, release.OptDir()) |
| // The status report says this box enforces update settings only while |
| // the build-state store is up (settings spec §4.2; Astra I3 #1). |
| noteBuildStatesInit(bsErr) |
| if bsErr != nil { |
| - log.Printf("agent: build state store unavailable: %v. every instance and worker spawn is refused until it is fixed and the agent restarted", bsErr) |
| + log.Printf("agent: build state store unavailable: %v. every exec-backed instance, worker and sidecar spawn is refused until it is fixed and the agent restarted (unit-backed rows still start)", bsErr) |
| } |
| // Fleet box identity (customer-fleets plan, Task 6 fix round — controller |
| @@ -1509,7 +1513,11 @@ func stageMarkerRequester(dd string) func(host, ver string, err error) { |
| log.Printf("agent: cannot request root stage for %s %s: %v", host, ver, err) |
| return |
| } |
| - log.Printf("agent: %s needs %s staged and this process cannot write the version store — asked for a root pass", host, ver) |
| + // One line for every cause the marker relays (Task 12a): a build |
| + // this process cannot stage, an install request only root |
| + // resolves (ErrInstallRequestNeedsRoot), and a held wake's target |
| + // only root installs. The throttle does not carry the cause. |
| + log.Printf("agent: %s needs a root pass for %s (a build only root can stage or install, or an install request only root resolves); asked for one", host, ver) |
| }) |
| } |
diff --git a/internal/host/buildstate.go b/internal/host/buildstate.go| index f5de81452..b68d89c72 100644 |
| --- a/internal/host/buildstate.go |
| +++ b/internal/host/buildstate.go |
| @@ -524,9 +524,13 @@ func buildStateHook(bs *buildStates, initErr error) func(ctx context.Context, ho |
| // decides whether a major hold is in force (the agent does not read tags). |
| // A pending request's T is followed only with the build state read (Known): |
| // with it unread root does no request bookkeeping and will not install T. |
| +// |
| +// Settings hold only the rows root applies them to (settingsApply; ruling |
| +// R-HELD-SCOPE): every other row, a named unit or an exec worker under an |
| +// app-policy setting, is not held. |
| func (b *buildStates) heldTarget(host string) activator.HeldWake { |
| rt, err := b.reg.Get(host) |
| - if err != nil || rt == nil { |
| + if err != nil || rt == nil || !settingsApply(*rt) { |
| return activator.HeldWake{} |
| } |
| c, _ := release.ReadConsumed(b.optDir, host) |
| @@ -601,6 +605,12 @@ func (b *buildStates) heldTarget(host string) activator.HeldWake { |
| return activator.HeldWake{} // no build state: whatever root installed is the instance's |
| } |
| +// settingsApply reports a row root applies update settings to: Kind |
| +// instance with no Unit, release's adopt condition (ruling R-HELD-SCOPE). |
| +// Root and the wake must agree on it, or a wake waits for an install root |
| +// never makes. |
| +func settingsApply(rt registry.Route) bool { return rt.Kind == "instance" && rt.Unit == "" } |
| + |
| // retained is v's staged executable on this box when its bytes match its |
| // staged manifest's sha256 for this platform (release.RetainedStaged, the |
| // check stageAuthorized's recovery uses), else nil (Astra I3 #5). |
| @@ -614,12 +624,13 @@ const wakeHeldBudget = 5 * time.Second |
| // wakeTarget is heldTarget with the copy refreshed first (§4.2: "refreshed |
| // at every reconcile and before every wake"), for a route whose settings |
| -// could hold it. A route with default settings is today's wake: nothing is |
| -// read for it. A refresh that fails keeps the last copy (refresh records the |
| -// failure), and the answer is read from that, as root reads it. |
| +// could hold it. A route with default settings, or one root applies no |
| +// settings to (settingsApply), is today's wake: nothing is read for it. A |
| +// refresh that fails keeps the last copy (refresh records the failure), and |
| +// the answer is read from that, as root reads it. |
| func (b *buildStates) wakeTarget(ctx context.Context, host string) activator.HeldWake { |
| rt, err := b.reg.Get(host) |
| - if err != nil || rt == nil { |
| + if err != nil || rt == nil || !settingsApply(*rt) { |
| return activator.HeldWake{} |
| } |
| if u, err := release.ParseUpdateSettingsLenient(rt.UpdateSettings); err == nil && u.Default() { |
diff --git a/internal/host/buildstate_moves_test.go b/internal/host/buildstate_moves_test.go| index 8bd455b6f..ad2498776 100644 |
| --- a/internal/host/buildstate_moves_test.go |
| +++ b/internal/host/buildstate_moves_test.go |
| @@ -4,12 +4,14 @@ import ( |
| "context" |
| "strings" |
| "testing" |
| + "time" |
| "github.com/carlosframework/platform/internal/activator" |
| "github.com/carlosframework/platform/internal/hostclaim" |
| "github.com/carlosframework/platform/internal/hostwrite" |
| "github.com/carlosframework/platform/internal/lease" |
| "github.com/carlosframework/platform/internal/moves" |
| + "github.com/carlosframework/platform/internal/park" |
| "github.com/carlosframework/platform/internal/provision" |
| "github.com/carlosframework/platform/internal/release" |
| ) |
| @@ -315,6 +317,51 @@ func TestALandingWithNoBuildStatePlacesNone(t *testing.T) { |
| } |
| } |
| +// A landing whose seal carries no state removes what an earlier stay left |
| +// beside the target's lease (Task 12a): otherwise that stay's build state |
| +// would read as this instance's. The removal runs under the location's |
| +// lease, like the placement. |
| +func TestALandingWithNoBuildStateRemovesALeftoverOne(t *testing.T) { |
| + h := newLandingHarness(t) |
| + key := cutRoot + "/build-state.json" |
| + storeState(h.phase1Harness, "data-uk", key, "v2") // an earlier stay's |
| + h.flip(t) |
| + if st := h.seal(t).State; len(st) != 0 { |
| + t.Fatalf("seal state %+v, want none", st) |
| + } |
| + h.land(t) |
| + h.installed(t) |
| + if o, ok := h.f.buckets["data-uk"][key]; ok { |
| + t.Fatalf("an earlier stay's build state %q is still beside the lease", o.body) |
| + } |
| + if calls := h.vs.callsOn("delete-version data-uk/" + key); len(calls) == 0 { |
| + t.Fatal("the leftover was not removed by version") |
| + } |
| +} |
| + |
| +// A committed move's source location park acknowledges with a build state |
| +// beside the source's lease (the 8b reviewer's regression): the state is the |
| +// marker's twin, never an object of no generation that would hold the |
| +// park-replica open for ever. |
| +func TestTheSourceLocationParkAcknowledgesWithABuildState(t *testing.T) { |
| + lh := newLandingHarness(t) |
| + storeState(lh.phase1Harness, "data-ie", cutRoot+"/build-state.json", "v6") |
| + h := newFinalHarnessFrom(t, lh) |
| + // One admission on the holder: with none the engine names no parked |
| + // generations at all, and the build state is never classified. |
| + rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) { |
| + c.Holder.Admissions = append(c.Holder.Admissions, hostclaim.AdmissionRecord{Region: "ie", Session: 1, Box: "ie-box", Generation: "cur", Mode: "cold"}) |
| + }) |
| + h.commit(t) |
| + h.clock.Advance(park.SignatureAge + cutRequestLimit + time.Minute) |
| + h.sourcePass(t) |
| + for _, o := range h.claim(t).Obligations { |
| + if o.Kind == "park-replica" && o.Region == "ie" { |
| + t.Fatalf("the ie park-replica is still owed past its horizon: %+v", o) |
| + } |
| + } |
| +} |
| + |
| // returnSealState is the return seal's state class, checked against the |
| // lane object it names. |
| func returnSealState(t *testing.T, h *rollbackHarness, rbID string) (moves.SealedObject, string) { |
diff --git a/internal/host/buildstate_test.go b/internal/host/buildstate_test.go| index 98a799aa3..f4017014a 100644 |
| --- a/internal/host/buildstate_test.go |
| +++ b/internal/host/buildstate_test.go |
| @@ -981,6 +981,55 @@ func TestTheWakeTargetRefreshesOnlyAHeldRoute(t *testing.T) { |
| } |
| } |
| +// Ruling R-HELD-SCOPE (Task 12a): settings hold only the rows root applies |
| +// them to (Kind instance, no Unit; adopt.go's settings condition). A named |
| +// unit row and an exec worker row under an app-policy manual wake as today: |
| +// not held, and nothing is read for them. Before, the unit row read |
| +// "build unknown" on every wake with nothing to clear it, and the worker was |
| +// held on a build root never targets. |
| +func TestOnlyRowsRootHoldsAreHeldAtTheWake(t *testing.T) { |
| + b, s, reg := testBuildStates(t) |
| + ctx := context.Background() |
| + manual := release.UpdateSettings{Updates: release.UpdatesManual, Channel: "stable"} |
| + for _, rt := range []registry.Route{ |
| + {Host: "unit.example.com", Kind: "instance", App: "km", Hibernate: true, Socket: "/run/u.sock", Unit: "km-unit"}, |
| + {Host: "w.example.com", Kind: "sidecar", App: "km", Hibernate: true}, |
| + } { |
| + if err := reg.Upsert(rt); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := reg.SetUpdateSettings(rt.Host, manual.Column()); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + // The worker keeps a build state (its lease's), read and copied. |
| + if err := put(t, b, ctx, "w.example.com", release.BuildState{LastExecuted: "v2"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := reg.SetBuildState("w.example.com", release.BuildStateCopy{Known: true, Replica: true, State: &release.BuildState{LastExecuted: "v2"}}.Column()); err != nil { |
| + t.Fatal(err) |
| + } |
| + gets := &getCounter{Store: s} |
| + b.storeFor = storesOf(map[string]release.Store{"hib": gets}) |
| + for _, host := range []string{"unit.example.com", "w.example.com"} { |
| + if h := b.heldTarget(host); h.Held || h.Unknown || h.Target != "" { |
| + t.Errorf("%s: heldTarget = %+v; want today's wake", host, h) |
| + } |
| + if h := b.wakeTarget(ctx, host); h.Held || h.Unknown || h.Target != "" { |
| + t.Errorf("%s: wakeTarget = %+v; want today's wake", host, h) |
| + } |
| + } |
| + if n := gets.n(); n != 0 { |
| + t.Errorf("%d bucket reads for rows root never holds; want none", n) |
| + } |
| + // The instance row under the same settings is still held. |
| + setSettings(t, reg, manual) |
| + setCopy(t, reg, release.BuildStateCopy{Known: true, Replica: true, State: &release.BuildState{LastExecuted: "v3"}}) |
| + if h := b.heldTarget("a.example.com"); !h.Held || h.Target != "v3" { |
| + t.Fatalf("the instance: heldTarget = %+v; want held on v3", h) |
| + } |
| +} |
| + |
| type getCounter struct { |
| release.Store |
| mu sync.Mutex |
diff --git a/internal/host/movesfinal_test.go b/internal/host/movesfinal_test.go| index c4cf9d286..58abdb359 100644 |
| --- a/internal/host/movesfinal_test.go |
| +++ b/internal/host/movesfinal_test.go |
| @@ -47,7 +47,14 @@ type finalHarness struct { |
| func newFinalHarness(t *testing.T) *finalHarness { |
| t.Helper() |
| - h := &finalHarness{landingHarness: newLandingHarness(t)} |
| + return newFinalHarnessFrom(t, newLandingHarness(t)) |
| +} |
| + |
| +// newFinalHarnessFrom is newFinalHarness over a landing harness the caller |
| +// prepared before the flip. |
| +func newFinalHarnessFrom(t *testing.T, lh *landingHarness) *finalHarness { |
| + t.Helper() |
| + h := &finalHarness{landingHarness: lh} |
| h.flip(t) |
| h.land(t) |
| h.installed(t) |
diff --git a/internal/host/movesinstall.go b/internal/host/movesinstall.go| index 7c9b6afa9..04b372f81 100644 |
| --- a/internal/host/movesinstall.go |
| +++ b/internal/host/movesinstall.go |
| @@ -619,17 +619,33 @@ func (ir *installRun) building(ctx context.Context) (bool, error) { |
| // request a stale round signed can land a newer version in the lane until |
| // the seal's bound, and that version is never placed. A seal with no state |
| // (the source had none, or a seal from before the class existed) places |
| -// nothing. A replay copies again. |
| +// nothing, and removes every version of a build state an earlier stay left |
| +// at the location (Task 12a): it is the marker's twin, so the landing's park |
| +// passes it over, and otherwise that stay's state would read as this |
| +// instance's. A replay copies, or removes, again. |
| func (ir *installRun) placeBuildState(ctx context.Context, vs lease.VersionedStore, seal moves.Seal, holder hostclaim.Holder) error { |
| + bucket := ir.cfg().OwnBucket |
| + dst := release.BuildStateKey(strings.TrimSuffix(holder.Location, "/")) |
| if len(seal.State) == 0 { |
| + vers, err := vs.ListVersions(ctx, bucket, dst) |
| + if err != nil { |
| + return fmt.Errorf("list a leftover build state at %s: %w", dst, err) |
| + } |
| + for _, v := range vers { |
| + if v.Key != dst { |
| + continue |
| + } |
| + if err := vs.DeleteVersion(ctx, bucket, dst, v.VersionID); err != nil { |
| + return fmt.Errorf("remove a leftover build state at %s (version %s): %w", dst, v.VersionID, err) |
| + } |
| + log.Printf("moves: %s: the %s carries no build state; removed version %s of an earlier stay's at %s", ir.rec.Host, ir.src.Name, v.VersionID, dst) |
| + } |
| return nil |
| } |
| o := seal.State[0] |
| if o.Key != ir.src.Lane+moves.BuildStateSuffix { |
| return fmt.Errorf("%w: the sealed build state %s is not the %s's lane key", moves.ErrBadSeal, o.Key, ir.src.Name) |
| } |
| - bucket := ir.cfg().OwnBucket |
| - dst := release.BuildStateKey(strings.TrimSuffix(holder.Location, "/")) |
| if _, _, err := vs.CopyVersion(ctx, bucket, o.Key, o.VersionID, bucket, dst, false); err != nil { |
| return fmt.Errorf("place the build state at %s: %w", dst, err) |
| } |
diff --git a/internal/registry/registry.go b/internal/registry/registry.go| index 0ac7f9d62..384d5c1ad 100644 |
| --- a/internal/registry/registry.go |
| +++ b/internal/registry/registry.go |
| @@ -3139,14 +3139,17 @@ FROM route_paths ORDER BY host, prefix`) |
| // values are scanned generically and written in column order, which is why |
| // this needs no per-column typing the way Fingerprint does. |
| // |
| -// "Every column" has exactly three exceptions, snapshotChurnCols: the |
| -// scheduling stamps (and the serving_version rollout stamp) the box rewrites |
| -// on its own clock with no operator change behind them. Hashing those made the gate never hold (#207 — both |
| -// production boxes wrote a generation on essentially every 5-minute tick, |
| -// so the 8-generation history was ~40 minutes deep, not 8 changes deep). |
| -// They are still IN the snapshot; only the decision to take one ignores |
| -// them. A restore therefore loses up to one tick of those stamps, which the |
| -// activator and DNS sweeps re-derive on their first pass anyway. |
| +// "Every column" has exactly four exceptions, snapshotChurnCols: the |
| +// scheduling stamps (next_due, dns_checked_at), the serving_version rollout |
| +// stamp, and build_state, the agent's copy of each instance's build state |
| +// (its ReadAt moves on every refresh). The box rewrites them on its own |
| +// clock with no operator change behind them. Hashing those made the gate |
| +// never hold (#207: both production boxes wrote a generation on |
| +// essentially every 5-minute tick, so the 8-generation history was ~40 |
| +// minutes deep, not 8 changes deep). They are still IN the snapshot; only |
| +// the decision to take one ignores them. A restore therefore loses up to |
| +// one tick of those stamps, which the activator and DNS sweeps and the |
| +// build-state refresh re-derive on their first pass anyway. |
| // |
| // route_aliases and wildcard_domains are not hashed either: both are |
| // pass-owned materializations of domain claims (domains.Reconcile rebuilds |
diff --git a/internal/release/adopt.go b/internal/release/adopt.go| index 688a779bf..37e54ae71 100644 |
| --- a/internal/release/adopt.go |
| +++ b/internal/release/adopt.go |
| @@ -1237,10 +1237,11 @@ func adoptHost(ctx context.Context, work hostwrite.Work, reg *registry.Registry, |
| // next root pass retries. A record that still blocks stays: the link |
| // is on the refused build only when its rollback failed, and that is |
| // worth showing. The record is judged against the build this route |
| - // selected (sel): under a pin or a hold, a refusal of the head is |
| - // not the live build's. |
| + // selected (sel) and against the head (ptr): under a pin or a hold |
| + // a refusal of the head that is still current stays, so an unpin |
| + // does not retry a promotion this box already refused (Task 12a). |
| if cfg.bakeDir != "" { |
| - if r, err := ReadRefusal(cfg.bakeDir, rt.Host); err == nil && r != nil && !r.Blocks(sel) { |
| + if r, err := ReadRefusal(cfg.bakeDir, rt.Host); err == nil && r != nil && !r.Blocks(sel) && !r.Blocks(ptr) { |
| _ = ClearRefusal(cfg.bakeDir, rt.Host) |
| } |
| } |
diff --git a/internal/release/adoptsettings_test.go b/internal/release/adoptsettings_test.go| index aad34b03d..3da350580 100644 |
| --- a/internal/release/adoptsettings_test.go |
| +++ b/internal/release/adoptsettings_test.go |
| @@ -803,9 +803,11 @@ func TestAnAgentPassRelaysAPendingRequestToRoot(t *testing.T) { |
| } |
| } |
| -// R-T6-MISC fix round 1: the up-to-date refusal clear judges the selected |
| -// build. Pinned and live on v1: a refusal of the head v2 is stale for this |
| -// route and cleared; a refusal of v1 itself (its rollback failed) stays. |
| +// R-T6-MISC fix round 1, amended by Task 12a: the up-to-date refusal clear |
| +// judges both the selected build and the head. Pinned and live on v1: a |
| +// refusal of the head v2 that is still current is kept (unpinning would |
| +// otherwise retry a refused promotion once); a refusal of v1 itself (its |
| +// rollback failed) stays too. |
| func TestTheLiveBuildsRefusalIsJudgedOnTheSelectedBuild(t *testing.T) { |
| s := NewDirStore(t.TempDir()) |
| ctx := context.Background() |
| @@ -835,8 +837,16 @@ func TestTheLiveBuildsRefusalIsJudgedOnTheSelectedBuild(t *testing.T) { |
| t.Fatal(err) |
| } |
| pass() |
| + if r, err := ReadRefusal(bake, "a.example.com"); err != nil || r == nil || r.Version != "v2" { |
| + t.Fatalf("refusal = %+v, %v; the pin cleared a refusal of the head that is still current", r, err) |
| + } |
| + // A refusal of an earlier promotion of v2 blocks neither build: cleared. |
| + if err := WriteRefusal(bake, Refusal{Host: "a.example.com", App: "km", Version: "v2", PointerPromotedAt: head.PromotedAt.Add(-time.Hour), Reason: "did not come up"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + pass() |
| if r, err := ReadRefusal(bake, "a.example.com"); err != nil || r != nil { |
| - t.Fatalf("refusal = %+v, %v; the head's refusal was kept as though it were the live build's", r, err) |
| + t.Fatalf("refusal = %+v, %v; a refusal that blocks nothing was kept", r, err) |
| } |
| if err := WriteRefusal(bake, Refusal{Host: "a.example.com", App: "km", Version: "v1", Reason: "did not come up"}); err != nil { |
| t.Fatal(err) |