amadan
release: a wrong-program refusal is its own kind, and the live binary always joins the history
Refusal.Kind (wrong-program) drives the gate line, the alert and status --live so none of them hint at --retry, which would loop. The live binary's program is recorded whenever it is missing from the history, not only when the history is empty.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
11 files changed,
+160
−14
cli.go+4 −0internal/host/alertsink.go+15 −1internal/host/refusalwatch.go+1 −1internal/host/refusalwatch_test.go+29 −0internal/release/adopt.go+18 −7internal/release/programhistory_test.go+55 −1internal/release/refusal.go+13 −1internal/status/contract_test.go+2 −2internal/status/dryrun.go+1 −1internal/status/snapshot.go+3 −0statuslive_test.go+19 −0
CI passed — run details
diff --git a/cli.go b/cli.go| index d0e329fa1..3aa6cb5a7 100644 |
| --- a/cli.go |
| +++ b/cli.go |
| @@ -3446,6 +3446,10 @@ func printLiveBoxes(w io.Writer, boxes []status.Box, now time.Time) { |
| humanBytes(m.MemAvailBytes), humanBytes(m.MemTotalBytes), |
| humanFloat(m.Load1), humanBytes(m.DiskFreeBytes), alertsWord(m.Alerts)) |
| for _, r := range m.ReleaseRefusals { |
| + if r.Kind == release.RefusalWrongProgram { |
| + fmt.Fprintf(w, " refused %s %s since %s: wrong program (%s)\n", r.Host, r.Version, r.RefusedAt.UTC().Format(time.RFC3339), strings.TrimPrefix(r.Reason, "wrong program: ")) |
| + continue |
| + } |
| failed := "" |
| if r.RecoveryError != "" { |
| failed = " (rollback failed: " + r.RecoveryError + ")" |
diff --git a/internal/host/alertsink.go b/internal/host/alertsink.go| index 4bc9f112f..14c02f95e 100644 |
| --- a/internal/host/alertsink.go |
| +++ b/internal/host/alertsink.go |
| @@ -135,8 +135,11 @@ func (a *alertSinks) deliverRelease(ctx context.Context, r release.Refusal) { |
| if a.webhookURL != "" && a.post != nil { |
| body, err := json.Marshal(struct { |
| Kind string `json:"kind"` |
| + // The record's own Kind is shadowed by the envelope's "kind" |
| + // above, so it travels as refusal_kind. |
| + RefusalKind string `json:"refusal_kind,omitempty"` |
| release.Refusal |
| - }{"release_refused", r}) |
| + }{"release_refused", r.Kind, r}) |
| if err == nil { |
| if err := a.post(a.webhookURL, body); err != nil { |
| a.counts.webhook.Add(1) |
| @@ -159,6 +162,17 @@ func (a *alertSinks) deliverRelease(ctx context.Context, r release.Refusal) { |
| // failed reads as one in the subject (spec §2.6): "rolled back" over a box |
| // whose restored build never came up would send the operator back to bed. |
| func formatReleaseAlert(r release.Refusal) (subject, msg string) { |
| + if r.Kind == release.RefusalWrongProgram { |
| + // Nothing started and nothing was rolled back: the box never left its |
| + // old build, and "--retry" would be refused the same way. |
| + subject = fmt.Sprintf("%s: %s refused, wrong program (still on %s)", r.Host, r.Version, r.RestoredVersion) |
| + if len(subject) > 100 { // SNS subject limit |
| + subject = subject[:100] |
| + } |
| + msg = fmt.Sprintf("%s refused %s at %s: %s\n\nShip it with --program-change if the change is intended, or promote a different build.", |
| + r.Host, r.Version, r.RefusedAt.Format(time.RFC3339), r.Reason) |
| + return subject, msg |
| + } |
| if r.Recovered { |
| subject = fmt.Sprintf("%s: %s did not start, rolled back to %s", r.Host, r.Version, r.RestoredVersion) |
| } else { |
diff --git a/internal/host/refusalwatch.go b/internal/host/refusalwatch.go| index 32667a6d2..30a77ac69 100644 |
| --- a/internal/host/refusalwatch.go |
| +++ b/internal/host/refusalwatch.go |
| @@ -29,7 +29,7 @@ func (w *refusalWatch) tick(ctx context.Context, bakeDir string, sinks *alertSin |
| } |
| var rows []status.ReleaseRefusal |
| for _, r := range recs { |
| - row := status.ReleaseRefusal{Host: r.Host, Version: r.Version, Reason: r.Reason, RefusedAt: r.RefusedAt} |
| + row := status.ReleaseRefusal{Host: r.Host, Version: r.Version, Reason: r.Reason, RefusedAt: r.RefusedAt, Kind: r.Kind} |
| if !r.Recovered { |
| row.RecoveryError = recoveryError(r) |
| } |
diff --git a/internal/host/refusalwatch_test.go b/internal/host/refusalwatch_test.go| index 8aa5ce3a6..3df1d8c97 100644 |
| --- a/internal/host/refusalwatch_test.go |
| +++ b/internal/host/refusalwatch_test.go |
| @@ -92,3 +92,32 @@ func TestRefusalWatchReportsTheRecoveryOutcome(t *testing.T) { |
| }) |
| } |
| } |
| + |
| +// D2's refusal is not a rollback: the subject says so and the message points |
| +// at --program-change, never --retry. The webhook keeps its envelope kind and |
| +// carries the record's as refusal_kind; the status row carries it too. |
| +func TestRefusalWatchReportsAWrongProgram(t *testing.T) { |
| + bake := t.TempDir() |
| + now := time.Date(2026, 10, 6, 9, 31, 0, 0, time.UTC) |
| + rec := release.Refusal{Host: "console.example.com", App: "console", Version: "v2", RefusedAt: now.Add(-time.Minute), |
| + Reason: "wrong program: example.com/carlos", Kind: release.RefusalWrongProgram, Recovered: true, RestoredVersion: "v1"} |
| + if err := release.WriteRefusal(bake, rec); err != nil { |
| + t.Fatal(err) |
| + } |
| + subject, msg := formatReleaseAlert(rec) |
| + if subject != "console.example.com: v2 refused, wrong program (still on v1)" { |
| + t.Fatalf("subject = %q", subject) |
| + } |
| + if !strings.Contains(msg, "--program-change") || strings.Contains(msg, "--retry") { |
| + t.Fatalf("message = %q", msg) |
| + } |
| + var posts []string |
| + sinks := newAlertSinks("https://hook.example", func(_ string, b []byte) error { posts = append(posts, string(b)); return nil }, "", nil, &alertCounters{}) |
| + rows := (&refusalWatch{}).tick(context.Background(), bake, sinks, now) |
| + if len(posts) != 1 || !strings.Contains(posts[0], `"kind":"release_refused"`) || !strings.Contains(posts[0], `"refusal_kind":"wrong-program"`) { |
| + t.Fatalf("posts = %v", posts) |
| + } |
| + if len(rows) != 1 || rows[0].Kind != release.RefusalWrongProgram { |
| + t.Fatalf("rows = %+v", rows) |
| + } |
| +} |
diff --git a/internal/release/adopt.go b/internal/release/adopt.go| index 27401c497..2564ab52c 100644 |
| --- a/internal/release/adopt.go |
| +++ b/internal/release/adopt.go |
| @@ -994,8 +994,14 @@ func adoptHost(ctx context.Context, reg *registry.Registry, s Store, optDir stri |
| if r, err := ReadRefusal(cfg.bakeDir, rt.Host); err != nil { |
| fmt.Printf(" %s: reading refusal record: %v\n", rt.Host, err) |
| } else if r.Blocks(ptr) { |
| - fmt.Printf(" %s: refused: %s did not come up on this box at %s (%s); retry with: carlos promote --retry %s %s\n", |
| - rt.Host, ver, r.RefusedAt.Format(time.RFC3339), r.Reason, ver, channel) |
| + if r.Kind == RefusalWrongProgram { |
| + // --retry would re-stage the same bytes and be refused again. |
| + fmt.Printf(" %s: refused: %s is the wrong program for this route (%s); still on %s; ship it with --program-change if the change is intended\n", |
| + rt.Host, ver, r.Reason, r.RestoredVersion) |
| + } else { |
| + fmt.Printf(" %s: refused: %s did not come up on this box at %s (%s); retry with: carlos promote --retry %s %s\n", |
| + rt.Host, ver, r.RefusedAt.Format(time.RFC3339), r.Reason, ver, channel) |
| + } |
| return |
| } |
| } |
| @@ -1016,10 +1022,14 @@ func adoptHost(ctx context.Context, reg *registry.Registry, s Store, optDir stri |
| newProg := FileProgram(VersionPath(optDir, RouteAccount(rt), ver, rt.App)) |
| if cfg.bakeDir != "" && newProg != "" { |
| hist := ReadPrograms(cfg.bakeDir, rt.Host) |
| - if len(hist) == 0 && cur != "" { |
| - if seed := FileProgram(VersionPath(optDir, RouteAccount(rt), cur, rt.App)); seed != "" { |
| - _ = RecordProgram(cfg.bakeDir, rt.Host, seed) |
| - hist = []string{seed} |
| + // The live binary is what the route is running, so its program belongs |
| + // in the history whatever the file already says: a history of [A] on a |
| + // route that is live on B (a change made before this check existed, or |
| + // a record lost) must not refuse the next B build. |
| + if cur != "" { |
| + if live := FileProgram(VersionPath(optDir, RouteAccount(rt), cur, rt.App)); live != "" && !slices.Contains(hist, live) { |
| + _ = RecordProgram(cfg.bakeDir, rt.Host, live) |
| + hist = append(slices.Clone(hist), live) |
| } |
| } |
| if len(hist) > 0 && !slices.Contains(hist, newProg) && !(m != nil && m.ProgramChange) { |
| @@ -1032,7 +1042,8 @@ func adoptHost(ctx context.Context, reg *registry.Registry, s Store, optDir stri |
| // Nothing was restarted, so the box is still on its old build. |
| if err := WriteRefusal(cfg.bakeDir, Refusal{Host: rt.Host, App: rt.App, Version: ver, SHA256: sha, |
| Platform: adoptPlatform, PointerPromotedAt: ptr.PromotedAt, RefusedAt: now.UTC(), |
| - Reason: "wrong program: " + newProg, Recovered: true}); err != nil { |
| + Reason: "wrong program: " + newProg, |
| + Kind: RefusalWrongProgram, Recovered: true, RestoredVersion: cur}); err != nil { |
| fmt.Printf(" %s: writing refusal record: %v\n", rt.Host, err) |
| } |
| return |
diff --git a/internal/release/programhistory_test.go b/internal/release/programhistory_test.go| index 24aff87e7..519eebe93 100644 |
| --- a/internal/release/programhistory_test.go |
| +++ b/internal/release/programhistory_test.go |
| @@ -3,6 +3,7 @@ package release |
| import ( |
| "bytes" |
| "context" |
| + "slices" |
| "strings" |
| "testing" |
| ) |
| @@ -43,9 +44,62 @@ func TestBoxRefusesWrongProgramWithNoManifestField(t *testing.T) { |
| if !strings.Contains(out.String(), "example.com/carlos") || !strings.Contains(out.String(), "not pointed live") { |
| t.Fatalf("printed %q, want the program refusal", out.String()) |
| } |
| - if r, _ := ReadRefusal(bake, host); r == nil || r.Version != "new" { |
| + r, _ := ReadRefusal(bake, host) |
| + if r == nil || r.Version != "new" { |
| t.Fatalf("refusal = %+v, want one for new", r) |
| } |
| + if r.Kind != RefusalWrongProgram || !r.Recovered || r.RestoredVersion != "old" || r.RecoveryError != "" { |
| + t.Fatalf("refusal = %+v, want a recovered wrong-program record still on old", r) |
| + } |
| + // The next pass reads the record at the gate: no --retry (it would loop), |
| + // and the way out is named. |
| + out.Reset() |
| + withCapturedStdout(t, &out, func() { |
| + if err := Pass(ctx, reg, s, pub, opt, restart, BakeDir(bake)); err != nil { |
| + t.Fatalf("second Pass: %v", err) |
| + } |
| + }) |
| + if !strings.Contains(out.String(), "--program-change") || strings.Contains(out.String(), "--retry") || !strings.Contains(out.String(), "still on old") { |
| + t.Fatalf("gate line = %q, want --program-change, still on old, and no --retry", out.String()) |
| + } |
| +} |
| + |
| +// The history is [A] but the route is live on a B binary (a change made before |
| +// the check existed, or a lost record): a new B build is the same program the |
| +// route is running and must not be refused. |
| +func TestBoxRecordsTheLiveProgramEvenWithAHistory(t *testing.T) { |
| + host := "console.example.com" |
| + s := NewDirStore(t.TempDir()) |
| + ctx := context.Background() |
| + priv, pub := testKeys(t) |
| + opt, bake := t.TempDir(), t.TempDir() |
| + reg := testRegistry(t) |
| + b1, b2 := tinyProgram(t, "example.com/b"), tinyProgram(t, "example.com/b") |
| + b2 = append(b2[:len(b2):len(b2)], 0) // distinct bytes, same program |
| + shipRelease(t, s, "bab", "console", "vb1", adoptPlatform, b1) |
| + shipRelease(t, s, "bab", "console", "vb2", adoptPlatform, b2) |
| + if err := StageArtifact(opt, "bab", "vb1", "console", b1); err != nil { |
| + t.Fatal(err) |
| + } |
| + addRoute(t, reg, host, "console", "canary/x", false) |
| + if err := PointLive(opt, host, "bab", "vb1", "console"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := RecordProgram(bake, host, "example.com/a"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := Promote(ctx, s, priv, "bab", "console", "canary/x", "vb2"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }, BakeDir(bake)); err != nil { |
| + t.Fatalf("Pass: %v", err) |
| + } |
| + if v, _ := LiveVersion(opt, host); v != "vb2" { |
| + t.Fatalf("live = %q, want vb2", v) |
| + } |
| + if got := ReadPrograms(bake, host); !slices.Contains(got, "example.com/b") { |
| + t.Fatalf("history = %v, want example.com/b recorded", got) |
| + } |
| } |
| func TestBoxSeedsHistoryFromTheLiveBinary(t *testing.T) { |
diff --git a/internal/release/refusal.go b/internal/release/refusal.go| index 43b18fdb4..9a5cbce56 100644 |
| --- a/internal/release/refusal.go |
| +++ b/internal/release/refusal.go |
| @@ -31,8 +31,20 @@ import ( |
| const refusalSchema = 1 |
| +// What a refusal is for. RefusalNotUp (the zero value, so every record written |
| +// before Kind existed reads as one) is D1: the unit did not come up and was |
| +// rolled back. RefusalWrongProgram is D2: the staged binary is a different Go |
| +// program from any this route has run, so it was never pointed live and |
| +// nothing restarted. A "--retry" cures the first and loops on the second. |
| +const ( |
| + RefusalNotUp = "" |
| + RefusalWrongProgram = "wrong-program" |
| +) |
| + |
| type Refusal struct { |
| - Schema int `json:"schema"` |
| + Schema int `json:"schema"` |
| + // Kind is RefusalNotUp or RefusalWrongProgram. Omitted when not-up. |
| + Kind string `json:"kind,omitempty"` |
| Host string `json:"host"` |
| App string `json:"app"` |
| Version string `json:"version"` |
diff --git a/internal/status/contract_test.go b/internal/status/contract_test.go| index 6c7cc254e..0acd2099f 100644 |
| --- a/internal/status/contract_test.go |
| +++ b/internal/status/contract_test.go |
| @@ -85,7 +85,7 @@ var allowedBoxKeys = map[string][]string{ |
| "metrics.config_refusals[]": {"host", "reason", "since", "last", "count"}, |
| // release_refusals (D1) names the host and build the box rolled back, in |
| // the private block only, pinned to the field level for the same reason. |
| - "metrics.release_refusals[]": {"host", "version", "reason", "refused_at", "recovery_error"}, |
| + "metrics.release_refusals[]": {"host", "version", "reason", "refused_at", "kind", "recovery_error"}, |
| } |
| // allowedFleetKeys are THE schema for Fleet's JSON output. Same friction as |
| @@ -309,7 +309,7 @@ func fixtureProbes() status.Probes { |
| // by the allowlist walk too. |
| ReleaseRefusals: []status.ReleaseRefusal{ |
| {Host: "contract-rolled-back.internal", Version: "v2", Reason: "exited: status=2", RefusedAt: time.Now(), |
| - RecoveryError: "rollback did not come up either"}, |
| + Kind: "wrong-program", RecoveryError: "rollback did not come up either"}, |
| }, |
| ConfigGuard: "bypassed (CARLOS_NO_SEALED_CONFIG)", |
| MemFrac: &memFrac, |
diff --git a/internal/status/dryrun.go b/internal/status/dryrun.go| index fd9026ae5..5ccc610d6 100644 |
| --- a/internal/status/dryrun.go |
| +++ b/internal/status/dryrun.go |
| @@ -213,7 +213,7 @@ func dryRunFixtures() ([]registry.Route, Probes) { |
| ConfigRefusalsTotal: 1, |
| // One rolled-back build (D1), for the same scope check. |
| ReleaseRefusals: []ReleaseRefusal{ |
| - {Host: dryRunReleaseRefusalHost, Version: "v-dryrun", Reason: "exited: status=2", RefusedAt: time.Now()}, |
| + {Host: dryRunReleaseRefusalHost, Version: "v-dryrun", Reason: "exited: status=2", RefusedAt: time.Now(), Kind: "wrong-program"}, |
| }, |
| // The probe table, keyed by a route host and carrying a backend |
| // address — see dryRunUpstreamNeedles. One up and one down upstream, |
diff --git a/internal/status/snapshot.go b/internal/status/snapshot.go| index eb4ea92f9..0b95e9efb 100644 |
| --- a/internal/status/snapshot.go |
| +++ b/internal/status/snapshot.go |
| @@ -177,6 +177,9 @@ type ReleaseRefusal struct { |
| Version string `json:"version"` |
| Reason string `json:"reason"` |
| RefusedAt time.Time `json:"refused_at"` |
| + // Kind is release.Refusal's: "" for a unit that did not come up, |
| + // "wrong-program" for a build refused before it was ever pointed live. |
| + Kind string `json:"kind,omitempty"` |
| // RecoveryError is set when the rollback itself did not come up (the |
| // record's Recovered is false): why. "" is a recovered rollback. |
| RecoveryError string `json:"recovery_error,omitempty"` |
diff --git a/statuslive_test.go b/statuslive_test.go| index 320eb894e..21a4c0e3f 100644 |
| --- a/statuslive_test.go |
| +++ b/statuslive_test.go |
| @@ -185,6 +185,25 @@ func TestPrintLiveBoxesShowsAFailedRollback(t *testing.T) { |
| } |
| } |
| +// A wrong-program refusal (D2) is not a failed start. |
| +func TestPrintLiveBoxesShowsAWrongProgramRefusal(t *testing.T) { |
| + now := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) |
| + var buf bytes.Buffer |
| + printLiveBoxes(&buf, []status.Box{ |
| + {GeneratedAt: now, Label: "ada-stargazing", Metrics: &status.BoxMetrics{ |
| + Tunnel: "connected", |
| + ReleaseRefusals: []status.ReleaseRefusal{{ |
| + Host: "console.example.com", Version: "v2", Kind: "wrong-program", |
| + RefusedAt: time.Date(2026, 10, 6, 9, 30, 40, 0, time.UTC), Reason: "wrong program: example.com/carlos", |
| + }}, |
| + }}, |
| + }, now) |
| + want := "refused console.example.com v2 since 2026-10-06T09:30:40Z: wrong program (example.com/carlos)\n" |
| + if !strings.Contains(buf.String(), want) { |
| + t.Fatalf("missing %q in:\n%s", want, buf.String()) |
| + } |
| +} |
| + |
| func TestHumanBytesAndFloat(t *testing.T) { |
| for _, c := range []struct { |
| in *int64 |