amadan

carlos / platform Public

zdd-5 task 9: a first wake with too little health window left boots the good build untried

B's health deadline is fixed from the wake's start, so a slow lease,
restore or seeding could leave B an expired or near-expired window at its
exec, and the cut-short check would record B as a failed build. Below
firstWakeMinVerdict (20 s) of window at the exec, B is not exec'd: the
good build boots through the same gated startPinnedAt, nothing is recorded
against B, and the wake reports as a healthy non-first wake of the good
build. With no good build, or in hold mode, nothing changes.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude b09bccf73b4060faa95779df4044caa833038e8f parent 7614d62
3 files changed, +191 −5
  • internal/activator/activator.go +5 −1
  • internal/activator/wakeplan.go +66 −4
  • internal/activator/wakeplan_test.go +120 −0
diff --git a/internal/activator/activator.go b/internal/activator/activator.go
index e10f87baf..ad620c19a 100644
--- a/internal/activator/activator.go
+++ b/internal/activator/activator.go
@@ -1505,11 +1505,15 @@ type wokenInstance struct {
// planned is the generation a planned wake left serving; failed and
// failReason name a first wake's build that failed; fellBack says the
- // good build replaced it (zero-downtime spec §8.2).
+ // good build replaced it (zero-downtime spec §8.2). untried says a
+ // first wake's B was never exec'd because too little of its health
+ // window was left (firstWakeMinVerdict), so the good build was booted
+ // in its place as an ordinary wake.
planned Gen
failed *release.StagedIdentity
failReason string
fellBack bool
+ untried bool
}
// doWake runs the side effects of one wake. It takes the ENTRY, not the bare
diff --git a/internal/activator/wakeplan.go b/internal/activator/wakeplan.go
index 5b5fc943b..49c2a79b8 100644
--- a/internal/activator/wakeplan.go
+++ b/internal/activator/wakeplan.go
@@ -9,6 +9,7 @@ import (
"context"
"errors"
"fmt"
+ "log"
"time"
"github.com/carlosframework/platform/internal/registry"
@@ -181,6 +182,12 @@ var (
firstWakeKillReserve = 25 * time.Second
firstWakeKillBound = 2 * time.Second
firstWakeSettle = 10 * time.Second
+ // firstWakeMinVerdict is the least health window B is exec'd with on a
+ // first wake that has a good build. B's window is fixed from the wake's
+ // start, so a slow lease, restore or seeding eats it; below this floor
+ // a failure would be the clock's, not the build's, and the good build
+ // boots instead with nothing recorded against B.
+ firstWakeMinVerdict = 20 * time.Second
)
// genKiller (plan 3 Task 2, opset.go) is plan 2's generation kill
@@ -285,19 +292,41 @@ func (a *Service) killFailed(rt registry.Route, g Gen, p Proc, worker bool) erro
// For a first wake in overlap mode with a good build, B's health window
// ends at the wake's start plus 55 s minus 25 s, a failed B is killed
// outright, and the good build gets whatever time remains. When B's end is
-// not proven, nothing more starts.
+// not proven, nothing more starts. When less than firstWakeMinVerdict of
+// B's window is left at its exec (a slow lease, restore or seeding), B is
+// not exec'd at all: the good build boots instead and nothing is recorded,
+// since a health check cut that short would fail B for the clock's sake.
func (a *Service) spawnPlanned(ctx context.Context, e *actEntry, rt registry.Route, plan WakePlan, w *wokenInstance, lease Lease) (Proc, string, Gen, error) {
plan, err := a.checkPlan(ctx, e, rt, plan)
if err != nil {
return nil, "", Gen{}, err
}
canFall := plan.FirstWake && plan.Fallback != nil && plan.Mode == ModeOverlap
+ boot := plan.Boot
+ gate := a.planGate(ctx, e, lease, plan, boot.Version)
var healthBy time.Time
+ var left time.Duration
if canFall {
healthBy = e.wakeStart.Add(firstWakeBudget - firstWakeKillReserve)
+ gate = verdictFloorGate(gate, healthBy, &left)
+ }
+ proc, ver, g, err := a.startPinnedAt(ctx, rt, genOf(boot), gate, healthBy)
+ if proc == nil && errors.Is(err, errNoTimeForVerdict) {
+ // B never ran: too little of its window was left for a verdict at
+ // the exec. The good build boots in its place through the same
+ // gate, and the wake is reported as the good build's ordinary wake
+ // (w.untried): no failure of B, no fallback, no observation.
+ // Operator log only; nothing happened to the member's build.
+ good := *plan.Fallback
+ log.Printf("activator: %s: first wake of %s had %v of its health window left at the exec, under the %v floor; booting the good build %s without trying it",
+ rt.Host, boot.Version, left.Round(time.Millisecond), firstWakeMinVerdict, good.Version)
+ w.untried = true
+ gproc, gver, gg, gerr := a.startPinnedAt(ctx, rt, genOf(good), a.planGate(ctx, e, lease, plan, good.Version), time.Time{})
+ if gerr != nil {
+ return gproc, gver, gg, fmt.Errorf("the good build %s, booted without trying %s: %w", good.Version, boot.Version, gerr)
+ }
+ return gproc, gver, withIdentity(gg, good), nil
}
- boot := plan.Boot
- proc, ver, g, err := a.startPinnedAt(ctx, rt, genOf(boot), a.planGate(ctx, e, lease, plan, boot.Version), healthBy)
if err == nil {
return proc, ver, withIdentity(g, boot), nil
}
@@ -335,6 +364,33 @@ func (a *Service) spawnPlanned(ctx context.Context, e *actEntry, rt registry.Rou
return fproc, fver, withIdentity(fg, good), nil
}
+// errNoTimeForVerdict: at B's exec, less than firstWakeMinVerdict of its
+// health window was left. B is not exec'd; it is never a failure of B.
+var errNoTimeForVerdict = errors.New("too little of the first wake's health window is left for a verdict")
+
+// verdictFloorGate is gate with the first-wake floor asked after it, at
+// the exec: B's window is the earlier of exec + spawnHealthTimeout and
+// healthBy, less now. Below firstWakeMinVerdict it refuses with
+// errNoTimeForVerdict and stores the window in *left for the log. The
+// ownership, lease and plan checks go first, so their refusal (abandonment)
+// is never mistaken for a lack of time.
+func verdictFloorGate(gate func() error, healthBy time.Time, left *time.Duration) func() error {
+ return func() error {
+ if err := gate(); err != nil {
+ return err
+ }
+ now := time.Now()
+ end := now.Add(spawnHealthTimeout)
+ if healthBy.Before(end) {
+ end = healthBy
+ }
+ if *left = end.Sub(now); *left < firstWakeMinVerdict {
+ return errNoTimeForVerdict
+ }
+ return nil
+ }
+}
+
// spawnPlannedWorker is spawnPlanned for a socketless child (spec §6.5).
// Each spawn uses startWorker's contract exactly as doWake's worker branch
// does (PF8): the build is published before its exec, planGate is asked at
@@ -451,6 +507,12 @@ func (a *Service) afterWake(e *actEntry, w *wokenInstance, err error) {
}
case w.fellBack:
a.emitGen(GenEvent{Kind: GenFellBack, Route: e.route, Gen: w.planned, Failed: *w.failed, Mode: plan.Mode, FirstWake: true, Reason: w.failReason, At: now, FailedUnder: plan.FailedUnder})
+ case w.untried:
+ // B was never exec'd (too little time for a verdict) and the good
+ // build serves: reported exactly as a healthy wake of the good build
+ // that is not a first wake, so the scheduler records it as served
+ // and B stays a candidate for the next wake.
+ a.emitGen(GenEvent{Kind: GenHealthy, Route: e.route, Gen: w.planned, Mode: plan.Mode, At: now})
default:
a.emitGen(GenEvent{Kind: GenHealthy, Route: e.route, Gen: w.planned, Mode: plan.Mode, FirstWake: plan.FirstWake, At: now})
switch {
@@ -474,7 +536,7 @@ func (a *Service) afterWake(e *actEntry, w *wokenInstance, err error) {
// checks there, so a stop they begin cancels the observation. The caller
// holds a.mu.
func (a *Service) armFirstWatchLocked(e *actEntry, w *wokenInstance) {
- if a.Planner == nil || a.OnGen == nil || e.worker || !e.plan.planned() || !e.plan.FirstWake || w.fellBack || w.failed != nil {
+ if a.Planner == nil || a.OnGen == nil || e.worker || !e.plan.planned() || !e.plan.FirstWake || w.fellBack || w.failed != nil || w.untried {
return
}
ctx, cancel := context.WithCancel(context.Background())
diff --git a/internal/activator/wakeplan_test.go b/internal/activator/wakeplan_test.go
index 495514f98..d3895b955 100644
--- a/internal/activator/wakeplan_test.go
+++ b/internal/activator/wakeplan_test.go
@@ -678,6 +678,9 @@ func TestAPlannedFirstWakeOnTheRealExecBackendFallsBackInTime(t *testing.T) {
wakeWaitBound = time.Second
t.Cleanup(func() { wakeWaitBound = oldBound })
shrinkFirstWake(t, 4*time.Second, 2*time.Second, time.Millisecond) // B's health window: 2 s
+ // The floor shrinks with the window, or B (2 s) would never be
+ // exec'd under the spec's 20 s floor.
+ shrinkVerdictFloor(t, time.Second)
eb := testExecBackend(t)
eb.RunDirOverride = t.TempDir()
host := "real.example.com"
@@ -1095,3 +1098,120 @@ func TestBounceForNamesTheCause(t *testing.T) {
t.Fatalf("events = %v", sink.msgs)
}
}
+
+// shrinkVerdictFloor sets the least health window a first wake's B is
+// exec'd with, beside shrinkFirstWake's budget, so a test drives the floor
+// in milliseconds rather than the spec's seconds.
+func shrinkVerdictFloor(t *testing.T, floor time.Duration) {
+ t.Helper()
+ old := firstWakeMinVerdict
+ firstWakeMinVerdict = floor
+ t.Cleanup(func() { firstWakeMinVerdict = old })
+}
+
+// slowFirstCheck is a plan's selection gate whose first call, checkPlan's
+// before the exec, takes d: it stands in for a slow lease, restore or
+// seeding, all of which run before B's exec on the wake's own clock.
+func slowFirstCheck(d time.Duration) func(string) error {
+ var once sync.Once
+ return func(string) error {
+ once.Do(func() { time.Sleep(d) })
+ return nil
+ }
+}
+
+// Review fix round 1 (risk c): B's health window is fixed from the wake's
+// start, so a slow before-exec phase leaves B too little time for a verdict.
+// Below firstWakeMinVerdict B is never exec'd: the good build boots through
+// the same gate, nothing is recorded against B, and the wake reports as a
+// healthy wake of the good build that is not a first wake.
+func TestAFirstWakeWithNoTimeForAVerdictBootsTheGoodBuild(t *testing.T) {
+ // B's window ends 2 s after the wake began; the floor is 1 s; the work
+ // before the exec takes 1.5 s, leaving B about 0.5 s.
+ shrinkFirstWake(t, 3*time.Second, time.Second, firstWakeSettle)
+ shrinkVerdictFloor(t, time.Second)
+ dir := t.TempDir()
+ good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2")
+ a, b, evs := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good, Check: slowFirstCheck(1500 * time.Millisecond)})
+ done, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
+ if err != nil {
+ t.Fatalf("the waiting request was not served by the good build: %v", err)
+ }
+ done()
+ if gens := b.started(); len(gens) != 1 || gens[0].Build != "v1" {
+ t.Fatalf("spawned %+v, want the good v1 alone: B must not exec without time for a verdict", gens)
+ }
+ if k := b.kills(); len(k) != 0 {
+ t.Fatalf("killed %v, want nothing", k)
+ }
+ ev := evs.wait(t, GenHealthy)
+ if ev.FirstWake || ev.Gen.Build != "v1" || ev.Gen.StagedPath != good.StagedPath {
+ t.Fatalf("healthy event = %+v, want the good build, not a first wake", ev)
+ }
+ time.Sleep(50 * time.Millisecond) // anything afterWake would emit after GenHealthy
+ if k := evs.kinds(); len(k) != 1 {
+ t.Fatalf("events %v, want GenHealthy alone: nothing recorded against B, no observation", k)
+ }
+ if g, ok := a.Current("one.example.com"); !ok || g.Build != "v1" {
+ t.Fatalf("Current = %+v %v, want the good build serving and not under observation", g, ok)
+ }
+}
+
+// The floor needs a good build to boot in B's place. With none, B is exec'd
+// as a wake does today: spawnHealthTimeout from the exec, with no cap.
+func TestAFirstWakeWithNoGoodBuildExecsBWhateverTimeIsLeft(t *testing.T) {
+ shrinkFirstWake(t, 3*time.Second, time.Second, 50*time.Millisecond)
+ shrinkVerdictFloor(t, time.Second)
+ dir := t.TempDir()
+ next := stagedID(t, dir, "v2")
+ a, b, evs := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Check: slowFirstCheck(1500 * time.Millisecond)})
+ var capped atomic.Bool
+ a.genHealth = func(ctx context.Context, g Gen) error {
+ if dl, ok := ctx.Deadline(); ok && time.Until(dl) < time.Second {
+ capped.Store(true)
+ }
+ return ctx.Err()
+ }
+ done, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
+ if err != nil {
+ t.Fatalf("the wake: %v", err)
+ }
+ done()
+ if gens := b.started(); len(gens) != 1 || gens[0].Build != "v2" {
+ t.Fatalf("spawned %+v, want B: there is no good build to boot instead", gens)
+ }
+ if capped.Load() {
+ t.Fatal("B's health check was capped: with no good build it keeps today's full window")
+ }
+ if ev := evs.wait(t, GenHealthy); !ev.FirstWake || ev.Gen.Build != "v2" {
+ t.Fatalf("healthy event = %+v, want B's first wake", ev)
+ }
+}
+
+// With at least the floor left, B is exec'd as before, and a failing B is
+// still a failure that falls back (the real-budget case is
+// TestFirstWakeFallsBackToTheGoodBuildWithinTheBudget).
+func TestAFirstWakeWithTimeForAVerdictStillExecsB(t *testing.T) {
+ shrinkFirstWake(t, 3*time.Second, time.Second, firstWakeSettle)
+ shrinkVerdictFloor(t, time.Second)
+ dir := t.TempDir()
+ good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2")
+ a, b, evs := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good, Check: slowFirstCheck(0)})
+ a.genHealth = func(_ context.Context, g Gen) error {
+ if g.Build == "v2" {
+ return errors.New("GET / answered 500")
+ }
+ return nil
+ }
+ done, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
+ if err != nil {
+ t.Fatalf("the waiting request was not served by the good build: %v", err)
+ }
+ done()
+ if gens := b.started(); len(gens) != 2 || gens[0].Build != "v2" || gens[1].Build != "v1" {
+ t.Fatalf("spawned %+v, want v2 then the good v1", gens)
+ }
+ if ev := evs.wait(t, GenFellBack); ev.Failed.Version != "v2" || ev.Gen.Build != "v1" {
+ t.Fatalf("fallback event = %+v", ev)
+ }
+}