amadan

carlos / platform Public

zdd-6 task 14b: refusal texts and the unpublished-build wake reason, drafts for copy review

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 17cd0e4c09b4aa8be6b9cda54977de0854dd293e parent cf0b8eb
9 files changed, +85 −23
  • internal/activator/activator.go +1 −0
  • internal/activator/logsink.go +10 −0
  • internal/activator/wakeplan_test.go +12 −7
  • internal/host/rollout.go +9 −4
  • internal/host/rolloutdeclined.go +3 −1
  • internal/host/rolloutdeclined_test.go +2 −2
  • internal/host/rolloutrefusaldrafts_test.go +21 −0
  • internal/rollout/wakeplan.go +15 −4
  • internal/rollout/wakeplan_test.go +12 −5
diff --git a/internal/activator/activator.go b/internal/activator/activator.go
index 088c398aa..1de2ee04b 100644
--- a/internal/activator/activator.go
+++ b/internal/activator/activator.go
@@ -2931,6 +2931,7 @@ func (a *Service) respawnWorkerLocked(e *actEntry, stopOld Proc, why string, ont
// on its own terms, moves the worker onto another build.
a.mu.Unlock()
log.Printf("activator: WORKER %s failed to respawn (%s): %v — the sweep's crash restart takes it from here", host, why, err)
+ // TODO-COPY zdd.sidecar.restart-retrying (main's text, em dash and all: flag-off bytes stay as they are)
emitEvent(a.Logs, rt, "error", "sidecar could not be restarted — retrying")
default:
e.proc = proc
diff --git a/internal/activator/logsink.go b/internal/activator/logsink.go
index 9caf80813..49c84827f 100644
--- a/internal/activator/logsink.go
+++ b/internal/activator/logsink.go
@@ -71,6 +71,16 @@ func memberSafeWakeReason(err error) string {
// N1).
err = ErrFenced
}
+ if errors.Is(err, ErrBuildNotPublished) {
+ // A worker wake refused because its build could not be published
+ // before the exec (plan 4's ErrBuildNotPublished, flag on or off).
+ // The member gets a reason in their terms; the write error is the
+ // operator's (the sweep retries the write). errStagedBuildNotPublished
+ // is a distinct sentinel and was mapped to the fence above.
+ //
+ // TODO-COPY zdd.wake.not-published
+ return "wake failed: the box could not record which build this instance runs, so it did not start it. the next request tries again"
+ }
s := err.Error()
if errors.Is(err, errClaimsAdmission) {
// claimAdmission's wrapper is the classifier's, not the member's
diff --git a/internal/activator/wakeplan_test.go b/internal/activator/wakeplan_test.go
index ead4ea146..954e57512 100644
--- a/internal/activator/wakeplan_test.go
+++ b/internal/activator/wakeplan_test.go
@@ -1001,18 +1001,23 @@ func TestErrBuildNotPublishedIsAbandonment(t *testing.T) {
}
}
-// Final review F9: the worker path keeps main's sentinel with main's text
-// (origin/main a3bf7d6c), so a worker wake whose publication fails with the
-// flag off reads to the member exactly as it did. The staged instance path
-// has a sentinel of its own: abandonment too, and its member event reads as
-// the maintenance fence's existing wording, never the raw text.
+// Final review F9: the worker path keeps main's sentinel text (origin/main
+// a3bf7d6c). Zero-downtime plan 6 (Task 14b) gives a worker wake whose
+// publication fails a member-safe event of its own, whatever the flag: the
+// draft below (zdd.wake.not-published), never the wrapped detail. The staged
+// instance path has a sentinel of its own: abandonment too, and its member
+// event reads as the maintenance fence's existing wording, never the raw text.
func TestTheWorkerPublicationSentinelKeepsMainsText(t *testing.T) {
if got, want := ErrBuildNotPublished.Error(), "worker build not published before its exec"; got != want {
t.Fatalf("ErrBuildNotPublished = %q, want main's %q", got, want)
}
worker := fmt.Errorf("start bot.x.internal: %w", fmt.Errorf("%w: disk full", ErrBuildNotPublished))
- if got, want := memberSafeWakeReason(worker), "wake failed: start bot.x.internal: worker build not published before its exec: disk full"; got != want {
- t.Fatalf("worker member text = %q, want main's passthrough %q", got, want)
+ const draft = "wake failed: the box could not record which build this instance runs, so it did not start it. the next request tries again"
+ if got := memberSafeWakeReason(worker); got != draft {
+ t.Fatalf("worker member text = %q, want the draft %q", got, draft)
+ }
+ if got := memberSafeWakeReason(fmt.Errorf("%w: disk full", ErrBuildNotPublished)); got != draft || strings.Contains(got, "disk full") {
+ t.Fatalf("wrapped member text = %q, want the draft alone", got)
}
inst := fmt.Errorf("one.example.com: refusing to wake: %w", errStagedBuildNotPublished)
if errors.Is(inst, ErrBuildNotPublished) || errors.Is(worker, errStagedBuildNotPublished) {
diff --git a/internal/host/rollout.go b/internal/host/rollout.go
index 72ab0b3b7..27648f441 100644
--- a/internal/host/rollout.go
+++ b/internal/host/rollout.go
@@ -40,9 +40,12 @@ const (
// a member through the activator's "wake failed:" event, so it names the
// missing fact and never the raw error, which goes to the agent's log.
var refuseTexts = map[string]string{
- rolloutNoStore: "overlap swaps are on but the deploy store is unavailable",
- rolloutNoReleaseKey: "overlap swaps are on but the release key is unavailable",
- rolloutBookUnreadable: "overlap swaps are on but the failure book is unreadable",
+ // TODO-COPY zdd.refuse.no-store
+ rolloutNoStore: "this box cannot start instances right now: the deploy store is unavailable",
+ // TODO-COPY zdd.refuse.no-release-key
+ rolloutNoReleaseKey: "this box cannot start instances right now: the release key is unavailable",
+ // TODO-COPY zdd.refuse.book-unreadable
+ rolloutBookUnreadable: "this box cannot start instances right now: its record of failed builds cannot be read",
}
var rolloutMgr atomic.Pointer[rollout.Manager]
@@ -500,7 +503,9 @@ func rolloutStatusProbe() *status.RolloutReport {
// rolloutRestartRefused is the RestartError an awake managed host's config
// ack carries while managed starts are refused. It is fixed text: the cause
// goes to the agent's log, never to the member (plan 6 words it).
-const rolloutRestartRefused = "restart refused: rolling restarts are unavailable on this box"
+//
+// TODO-COPY zdd.refuse.restart
+const rolloutRestartRefused = "restart refused: this box cannot restart instances right now"
// managedAwake reports whether host is a managed exec instance the activator
// has a live entry for (waking, awake or stopping, as Owns counts them), so
diff --git a/internal/host/rolloutdeclined.go b/internal/host/rolloutdeclined.go
index d63a6659c..498174206 100644
--- a/internal/host/rolloutdeclined.go
+++ b/internal/host/rolloutdeclined.go
@@ -22,7 +22,9 @@ const rolloutBookHonoured = "honoured"
// refuseNoSwaps is the wake refusal for a managed instance whose build needs
// the scheduler this box cannot run: refuseManaged's text for a reason it has
// no words for, and a declined box's for a build a failure record holds.
-const refuseNoSwaps = "overlap swaps are on but this box cannot run them"
+//
+// TODO-COPY zdd.refuse.no-swaps
+const refuseNoSwaps = "this box cannot run two builds side by side, which this instance's build needs"
// failureHolds is the failure book a box that declined CARLOS_SWAP=overlap
// still honours (final review F6, which replaced PF15's "today's paths"). An
diff --git a/internal/host/rolloutdeclined_test.go b/internal/host/rolloutdeclined_test.go
index b1bcf4d1e..5cda71769 100644
--- a/internal/host/rolloutdeclined_test.go
+++ b/internal/host/rolloutdeclined_test.go
@@ -196,8 +196,8 @@ func TestDeclinedBoxRefusesAWakeOntoAHeldBuild(t *testing.T) {
{blocked, refuseNoSwaps},
{noGood, ""},
{repointedNoGood, ""},
- {downNoGood, "build v2 failed in hold mode on this instance; recovery is explicit"},
- {downGood, "build v2 failed in hold mode on this instance; recovery is explicit"},
+ {downNoGood, "build v2 failed in hold mode on this instance, so it stays down until you roll back or retry the build"},
+ {downGood, "build v2 failed in hold mode on this instance, so it stays down until you roll back or retry the build"},
{repointed, refuseNoSwaps},
{good, ""},
{managedRoute("retried.example.com", "v2"), ""},
diff --git a/internal/host/rolloutrefusaldrafts_test.go b/internal/host/rolloutrefusaldrafts_test.go
new file mode 100644
index 000000000..c864b6d47
--- /dev/null
+++ b/internal/host/rolloutrefusaldrafts_test.go
@@ -0,0 +1,21 @@
+package host
+
+import "testing"
+
+// The member-facing refusal texts are drafts pending copy review (zdd.refuse.*
+// and zdd.refuse.restart in the plan 6 copy list). The other tests compare
+// against the constants, so this one pins the words themselves: a change here
+// is a copy change and goes through review.
+func TestRolloutRefusalDrafts(t *testing.T) {
+ for name, tc := range map[string]struct{ got, want string }{
+ "no-store": {refuseTexts[rolloutNoStore], "this box cannot start instances right now: the deploy store is unavailable"},
+ "no-release-key": {refuseTexts[rolloutNoReleaseKey], "this box cannot start instances right now: the release key is unavailable"},
+ "book-unreadable": {refuseTexts[rolloutBookUnreadable], "this box cannot start instances right now: its record of failed builds cannot be read"},
+ "no-swaps": {refuseNoSwaps, "this box cannot run two builds side by side, which this instance's build needs"},
+ "restart": {rolloutRestartRefused, "restart refused: this box cannot restart instances right now"},
+ } {
+ if tc.got != tc.want {
+ t.Errorf("%s: %q, want the draft %q", name, tc.got, tc.want)
+ }
+ }
+}
diff --git a/internal/rollout/wakeplan.go b/internal/rollout/wakeplan.go
index 1bcc8ecf2..94c3007a8 100644
--- a/internal/rollout/wakeplan.go
+++ b/internal/rollout/wakeplan.go
@@ -44,7 +44,8 @@ type wakeInputs struct {
// so it names the build only. A box that declined CARLOS_SWAP=overlap but
// honours the failure records refuses with the same words (internal/host).
func HeldDownRefusal(build string) string {
- return "build " + build + " failed in hold mode on this instance; recovery is explicit"
+ // TODO-COPY zdd.refuse.held-down
+ return "build " + build + " failed in hold mode on this instance, so it stays down until you roll back or retry the build"
}
// decideWake is spec §8.2's "when the wake picks a build", as a pure
@@ -64,6 +65,7 @@ func decideWake(in wakeInputs) (activator.WakePlan, error) {
if installed == "" {
// Plan installs a cold host first; a plan for a host with nothing
// installed refuses rather than let the live link run unplanned.
+ // TODO-COPY zdd.refuse.not-installed
return activator.WakePlan{Refuse: "no build is installed for this instance yet"}, nil
}
hasGood := rt.GoodVersion != "" && rt.GoodDigest != ""
@@ -96,6 +98,7 @@ func decideWake(in wakeInputs) (activator.WakePlan, error) {
if hasGood {
return recovery, err
}
+ // TODO-COPY zdd.refuse.not-staged
return activator.WakePlan{Refuse: "build " + installed + " is not staged correctly on this box"}, err
}
mode := AttemptMode(in.Policy, demand)
@@ -141,6 +144,7 @@ func decideHoldRan(in wakeInputs, mark HoldRan, hasGood bool, recovery activator
if err != nil {
// A staging refusal cannot boot the good build instead here, as it
// does for an instance with no marker: that would be the fallback.
+ // TODO-COPY zdd.refuse.not-staged
return activator.WakePlan{Refuse: "build " + installed + " is not staged correctly on this box"}, err
}
mode := AttemptMode(in.Policy, demand)
@@ -197,7 +201,8 @@ func (m *Manager) Plan(ctx context.Context, rt registry.Route) (plan activator.W
// hold that went down, or a swap whose entry stopped under it, is
// recorded by done() before anything wakes the instance, so no wake
// can boot the good build ahead of a hold-down record.
- return activator.WakePlan{Refuse: "an attempt is still running for this instance"}
+ // TODO-COPY zdd.refuse.attempt-running
+ return activator.WakePlan{Refuse: "this instance is still moving to another build"}
}
if rt.Version == "" {
// A cold managed host (spec §6.4, §8.4): its build is installed and
@@ -206,10 +211,12 @@ func (m *Manager) Plan(ctx context.Context, rt registry.Route) (plan activator.W
// Nothing managed ever runs through an empty plan: the live-link
// wake would materialize the channel head and exec it unplanned.
if m.d.Materialize == nil {
+ // TODO-COPY zdd.refuse.not-installed
return activator.WakePlan{Refuse: "no build is installed for this instance yet"}
}
if err := m.d.Materialize(ctx, rt.Host); err != nil {
m.d.Logf("rollout: %s: install before its wake: %v", rt.Host, err)
+ // TODO-COPY zdd.refuse.not-installable
return activator.WakePlan{Refuse: "its build could not be installed"}
}
fresh, err := m.d.Routes.Get(rt.Host)
@@ -217,6 +224,7 @@ func (m *Manager) Plan(ctx context.Context, rt registry.Route) (plan activator.W
if err != nil {
m.d.Logf("rollout: %s: read the row after its install: %v", rt.Host, err)
}
+ // TODO-COPY zdd.refuse.not-installed
return activator.WakePlan{Refuse: "no build is installed for this instance yet"}
}
rt = *fresh
@@ -231,6 +239,7 @@ func (m *Manager) Plan(ctx context.Context, rt registry.Route) (plan activator.W
if err != nil {
m.d.Logf("rollout: %s: read the row after its plan: %v", rt.Host, err)
}
+ // TODO-COPY zdd.refuse.not-installed
return activator.WakePlan{Refuse: "no build is installed for this instance yet"}
}
if fresh.Version == rt.Version {
@@ -238,6 +247,7 @@ func (m *Manager) Plan(ctx context.Context, rt registry.Route) (plan activator.W
}
if again {
m.d.Logf("rollout: %s: its installed build moved twice while its wake was planned (%s, now %s); refusing the wake", rt.Host, rt.Version, fresh.Version)
+ // TODO-COPY zdd.refuse.not-installed
return activator.WakePlan{Refuse: "no build is installed for this instance yet"}
}
m.d.Logf("rollout: %s: its installed build moved from %s to %s while its wake was planned; planning again", rt.Host, rt.Version, fresh.Version)
@@ -302,9 +312,10 @@ func (m *Manager) planCheck(rt registry.Route, plan activator.WakePlan) func(bui
boot := plan.Boot.Version
hasGood := rt.GoodVersion != "" && rt.GoodDigest != ""
// The texts reach the member's "wake failed" event (memberSafeWakeReason
- // passes a planned wake's error through), so they are existing words.
+ // passes a planned wake's error through).
held := func(build string) error {
- return fmt.Errorf("build %s is held by a failure recorded since the wake was planned", build)
+ // TODO-COPY zdd.refuse.held-since-plan
+ return fmt.Errorf("build %s failed on this box while this instance was starting, so it was not started", build)
}
return func(build string) error {
if mark, ok := holdRanFor(m.d.Book, acct, app, host); ok {
diff --git a/internal/rollout/wakeplan_test.go b/internal/rollout/wakeplan_test.go
index be1a6577f..9d1fbc3d1 100644
--- a/internal/rollout/wakeplan_test.go
+++ b/internal/rollout/wakeplan_test.go
@@ -32,7 +32,7 @@ func TestDecideWake(t *testing.T) {
down := &FailureBook{}
down.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "a", HeldDown: []string{"a"}, At: t0})
- if p, _ := decideWake(inputs(wakeRoute("", ""), &FailureBook{}, overlap)); p.Boot.StagedPath != "" || p.Refuse == "" {
+ if p, _ := decideWake(inputs(wakeRoute("", ""), &FailureBook{}, overlap)); p.Boot.StagedPath != "" || p.Refuse != "no build is installed for this instance yet" {
t.Fatal("nothing stamped: today's wake, unplanned")
}
p, _ := decideWake(inputs(wakeRoute("v1", "v1"), &FailureBook{}, overlap))
@@ -451,9 +451,14 @@ func TestAPlansCheckRefusesABuildHeldSinceThePlan(t *testing.T) {
if _, err := h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "b", At: t0}); err != nil {
t.Fatal(err)
}
- if err := p.Check("v2"); err == nil {
+ err := p.Check("v2")
+ if err == nil {
t.Fatal("v2 failed on a sibling after the plan, and the check still let it boot")
}
+ // Draft wording (zdd.refuse.held-since-plan), pending copy review.
+ if want := "build v2 failed on this box while this instance was starting, so it was not started"; err.Error() != want {
+ t.Fatalf("refusal %q, want the draft %q", err, want)
+ }
if err := p.Check("v1"); err != nil {
t.Fatalf("the good build was refused: %v", err)
}
@@ -503,13 +508,15 @@ func TestAForeignRowsFailedHoldIsHeldDown(t *testing.T) {
h.install("v2", "a")
h.pass()
waitFor(t, "the attempt to start", func() bool { _, started, _ := h.act.snapshot(); return len(started) == 1 })
- if p := h.m.Plan(context.Background(), h.rows.get("a")); p.Refuse == "" {
+ // Draft wording (zdd.refuse.attempt-running, zdd.refuse.held-down), pending copy review.
+ if p := h.m.Plan(context.Background(), h.rows.get("a")); p.Refuse != "this instance is still moving to another build" {
t.Fatalf("a wake was planned while the instance's attempt ran: %+v", p)
}
close(gate)
waitFor(t, "the hold-down record", func() bool { return h.book.HeldDown("bab", "blog", "v2", "a") })
- if p := h.m.Plan(context.Background(), h.rows.get("a")); p.Refuse == "" || p.Boot.StagedPath != "" {
- t.Fatalf("after the hold went down the wake planned %+v, want refused (held down)", p)
+ wantDown := "build v2 failed in hold mode on this instance, so it stays down until you roll back or retry the build"
+ if p := h.m.Plan(context.Background(), h.rows.get("a")); p.Refuse != wantDown || p.Boot.StagedPath != "" {
+ t.Fatalf("after the hold went down the wake planned %+v, want refused %q (held down)", p, wantDown)
}
}