amadan

carlos / platform Public

release: refusal records carry the rollback's real outcome

Final review F6 (Plan 1 D1), with the ledger's Task 3, 6 and 8 minors.

adoptHost writes the group's refusal records AFTER the rollback restart. release.Refusal gains restored_version, recovered and recovery_error: recovered only when the bound link was rolled back and the restart on it came up. rollbackGroup is passed restored = cur only when the bound rollback happened, so siblings are never repointed onto a build their process is not running, and a repointed sibling is stamped with the bound route's PrevVersion rather than its own (which names restored by then).

The watcher's SNS subject reads '<host>: <ver> did not start, rolled back to <cur>' or '<host>: <ver> did not start and the rollback did not come up either', the message carries the error, and the webhook body carries the record's new fields. status.ReleaseRefusal gains recovery_error (private metrics block; contract allowlist updated) and status --live appends '(rollback failed: <err>)'.

Tests: recovered records, rollback restart failure (records and alerts), bound rollback failure leaves siblings alone, sibling stamped with bound's history, SNS/webhook/status row for both outcomes, the live line.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 27922317236e9b80f709c3f90832372e272195fd parent ce103d1
11 files changed, +314 −23
  • cli.go +5 −1
  • internal/host/alertsink.go +30 −6
  • internal/host/refusalwatch.go +5 −1
  • internal/host/refusalwatch_test.go +49 −0
  • internal/release/adopt.go +37 −5
  • internal/release/refusal.go +9 −0
  • internal/release/rollback.go +27 −7
  • internal/release/rollback_test.go +125 −0
  • internal/status/contract_test.go +3 −2
  • internal/status/snapshot.go +3 −0
  • statuslive_test.go +21 −1
diff --git a/cli.go b/cli.go
index 119842e51..f843ef76c 100644
--- a/cli.go
+++ b/cli.go
@@ -3435,7 +3435,11 @@ func printLiveBoxes(w io.Writer, boxes []status.Box, now time.Time) {
humanBytes(m.MemAvailBytes), humanBytes(m.MemTotalBytes),
humanFloat(m.Load1), humanBytes(m.DiskFreeBytes), alertsWord(m.Alerts))
for _, r := range m.ReleaseRefusals {
- fmt.Fprintf(w, " refused %s %s since %s: %s\n", r.Host, r.Version, r.RefusedAt.UTC().Format(time.RFC3339), r.Reason)
+ failed := ""
+ if r.RecoveryError != "" {
+ failed = " (rollback failed: " + r.RecoveryError + ")"
+ }
+ fmt.Fprintf(w, " refused %s %s since %s: %s%s\n", r.Host, r.Version, r.RefusedAt.UTC().Format(time.RFC3339), r.Reason, failed)
}
}
}
diff --git a/internal/host/alertsink.go b/internal/host/alertsink.go
index e3adc9f90..4bc9f112f 100644
--- a/internal/host/alertsink.go
+++ b/internal/host/alertsink.go
@@ -145,12 +145,7 @@ func (a *alertSinks) deliverRelease(ctx context.Context, r release.Refusal) {
}
}
if a.topicARN != "" && a.pub != nil {
- subject := fmt.Sprintf("%s: %s did not start, rolled back", r.Host, r.Version)
- if len(subject) > 100 { // SNS subject limit
- subject = subject[:100]
- }
- msg := fmt.Sprintf("%s refused %s at %s: %s\n\nRetry with: carlos promote --retry %s <channel>",
- r.Host, r.Version, r.RefusedAt.Format(time.RFC3339), r.Reason, r.Version)
+ subject, msg := formatReleaseAlert(r)
pctx, cancel := context.WithTimeout(ctx, alertPublishTimeout)
defer cancel()
if _, err := a.pub.Publish(pctx, &sns.PublishInput{TopicArn: &a.topicARN, Subject: &subject, Message: &msg}); err != nil {
@@ -160,6 +155,35 @@ func (a *alertSinks) deliverRelease(ctx context.Context, r release.Refusal) {
}
}
+// formatReleaseAlert is a refusal's SNS subject and message. A recovery that
+// failed reads as one in the subject (spec §2.6): "rolled back" over a box
+// whose restored build never came up would send the operator back to bed.
+func formatReleaseAlert(r release.Refusal) (subject, msg string) {
+ if r.Recovered {
+ subject = fmt.Sprintf("%s: %s did not start, rolled back to %s", r.Host, r.Version, r.RestoredVersion)
+ } else {
+ subject = fmt.Sprintf("%s: %s did not start and the rollback did not come up either", r.Host, r.Version)
+ }
+ if len(subject) > 100 { // SNS subject limit
+ subject = subject[:100]
+ }
+ msg = fmt.Sprintf("%s refused %s at %s: %s", r.Host, r.Version, r.RefusedAt.Format(time.RFC3339), r.Reason)
+ if !r.Recovered {
+ msg += "\n\nThe rollback did not come up either: " + recoveryError(r)
+ }
+ msg += fmt.Sprintf("\n\nRetry with: carlos promote --retry %s <channel>", r.Version)
+ return subject, msg
+}
+
+// recoveryError is why a refusal's recovery failed, never empty for one that
+// did: a record written before the outcome was recorded has none to give.
+func recoveryError(r release.Refusal) string {
+ if r.RecoveryError != "" {
+ return r.RecoveryError
+ }
+ return "outcome not recorded"
+}
+
// topicARNPattern is the whole shape of a standard SNS topic ARN: partition,
// region, a 12 digit account, and a topic name that is only letters, digits,
// hyphen and underscore. The name has no colon, so a subscription ARN
diff --git a/internal/host/refusalwatch.go b/internal/host/refusalwatch.go
index 3ea5b9556..32667a6d2 100644
--- a/internal/host/refusalwatch.go
+++ b/internal/host/refusalwatch.go
@@ -29,7 +29,11 @@ func (w *refusalWatch) tick(ctx context.Context, bakeDir string, sinks *alertSin
}
var rows []status.ReleaseRefusal
for _, r := range recs {
- rows = append(rows, status.ReleaseRefusal{Host: r.Host, Version: r.Version, Reason: r.Reason, RefusedAt: r.RefusedAt})
+ row := status.ReleaseRefusal{Host: r.Host, Version: r.Version, Reason: r.Reason, RefusedAt: r.RefusedAt}
+ if !r.Recovered {
+ row.RecoveryError = recoveryError(r)
+ }
+ rows = append(rows, row)
if last, ok := w.seen[r.Host]; ok && !r.RefusedAt.After(last) {
continue
}
diff --git a/internal/host/refusalwatch_test.go b/internal/host/refusalwatch_test.go
index 0a2d72baa..8aa5ce3a6 100644
--- a/internal/host/refusalwatch_test.go
+++ b/internal/host/refusalwatch_test.go
@@ -43,3 +43,52 @@ func TestRefusalWatchSkipsOldRecordsAtStartup(t *testing.T) {
t.Fatalf("an agent restart re-alerted a refusal from two hours ago")
}
}
+
+// Final review F6 and ledger Task 8: SNS and the webhook tell a recovered
+// rollback from one whose restored build did not come up either, and the
+// status row carries the failure for `status --live`.
+func TestRefusalWatchReportsTheRecoveryOutcome(t *testing.T) {
+ now := time.Date(2026, 10, 6, 9, 31, 0, 0, time.UTC)
+ for _, c := range []struct {
+ name string
+ rec release.Refusal
+ subject string
+ inMessage string
+ inBody string
+ rowErr string
+ }{
+ {"recovered", release.Refusal{RestoredVersion: "v1", Recovered: true},
+ "console.example.com: v2 did not start, rolled back to v1", "Retry with: carlos promote --retry v2",
+ `"recovered":true`, ""},
+ {"failed recovery", release.Refusal{RestoredVersion: "v1", RecoveryError: "start console.service: exit status 1"},
+ "console.example.com: v2 did not start and the rollback did not come up either",
+ "The rollback did not come up either: start console.service: exit status 1",
+ `"recovery_error":"start console.service: exit status 1"`, "start console.service: exit status 1"},
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ bake := t.TempDir()
+ r := c.rec
+ r.Host, r.App, r.Version, r.RefusedAt, r.Reason = "console.example.com", "console", "v2", now.Add(-time.Minute), "exited: status=2"
+ if err := release.WriteRefusal(bake, r); err != nil {
+ t.Fatal(err)
+ }
+ var posts []string
+ pub := &fakeSNS{}
+ sinks := newAlertSinks("https://hook.example", func(_ string, b []byte) error { posts = append(posts, string(b)); return nil },
+ testTopicARN, pub, &alertCounters{})
+ rows := (&refusalWatch{}).tick(context.Background(), bake, sinks, now)
+ if len(pub.in) != 1 || *pub.in[0].Subject != c.subject {
+ t.Fatalf("sns = %+v, want one publish with subject %q", pub.in, c.subject)
+ }
+ if !strings.Contains(*pub.in[0].Message, c.inMessage) {
+ t.Fatalf("sns message = %q, want %q in it", *pub.in[0].Message, c.inMessage)
+ }
+ if len(posts) != 1 || !strings.Contains(posts[0], c.inBody) {
+ t.Fatalf("webhook = %v, want %s in the body", posts, c.inBody)
+ }
+ if len(rows) != 1 || rows[0].RecoveryError != c.rowErr {
+ t.Fatalf("status rows = %+v, want recovery_error %q", rows, c.rowErr)
+ }
+ })
+ }
+}
diff --git a/internal/release/adopt.go b/internal/release/adopt.go
index 0362a84ca..4eac8ea96 100644
--- a/internal/release/adopt.go
+++ b/internal/release/adopt.go
@@ -1141,9 +1141,11 @@ func adoptHost(ctx context.Context, reg *registry.Registry, s Store, optDir stri
var notUp *UnitNotUpError
isNotUp := errors.As(rerr, &notUp)
rolledBack := false
+ var rollbackErr error
if cur != "" {
if perr := PointLive(optDir, rt.Host, RouteAccount(rt), cur, rt.App); perr != nil {
fmt.Printf(" %s: could not roll the symlink back to %s: %v\n", rt.Host, cur, perr)
+ rollbackErr = perr
} else {
rolledBack = true
}
@@ -1220,22 +1222,52 @@ func adoptHost(ctx context.Context, reg *registry.Registry, s Store, optDir stri
if isNotUp && cfg.rollbackRestart != nil {
start = cfg.rollbackRestart
}
+ // D1's group (rollback.go): siblings follow bound onto the
+ // restored build only when bound's own rollback happened.
+ var group []string
if isNotUp {
- sha := ""
- if m != nil {
- sha = m.Artifacts[adoptPlatform].SHA256
+ restored := ""
+ if rolledBack {
+ restored = cur
}
- rollbackGroup(reg, optDir, cfg, rt, ptr, ver, cur, sha, adoptPlatform, notUp, now)
+ group = rollbackGroup(reg, optDir, rt, ver, restored)
if notUp.Journal != "" {
fmt.Printf(" %s: %s's last lines:\n%s\n", rt.Host, notUp.Unit, indent(notUp.Journal, " "))
}
}
+ started := false
+ var serr error
if start != nil && rolledBack && !daemonRestart {
- if serr := start(rt.Host); serr != nil {
+ started = true
+ if serr = start(rt.Host); serr != nil {
fmt.Printf(" %s: rolled back to %s but could not start it: %v — this host may be serving nothing\n",
rt.Host, cur, serr)
}
}
+ if isNotUp {
+ // The refusals go down AFTER the restart, carrying what it
+ // did: a record written before it would tell the watcher
+ // "rolled back" about a box whose restored build never came
+ // up (final review F6).
+ rec := recovery{recovered: true}
+ switch {
+ case cur == "":
+ rec = recovery{err: "there is no previous version to roll back to"}
+ case !rolledBack:
+ rec = recovery{err: fmt.Sprintf("could not roll the symlink back to %s: %v", cur, rollbackErr)}
+ case !started:
+ rec = recovery{restored: cur, err: "nothing on this pass restarts the unit on " + cur}
+ case serr != nil:
+ rec = recovery{restored: cur, err: serr.Error()}
+ default:
+ rec.restored = cur
+ }
+ sha := ""
+ if m != nil {
+ sha = m.Artifacts[adoptPlatform].SHA256
+ }
+ writeGroupRefusals(cfg, group, rt.App, ptr, ver, sha, adoptPlatform, notUp, rec, now)
+ }
return
}
}
diff --git a/internal/release/refusal.go b/internal/release/refusal.go
index d9804aab3..43b18fdb4 100644
--- a/internal/release/refusal.go
+++ b/internal/release/refusal.go
@@ -41,6 +41,15 @@ type Refusal struct {
PointerPromotedAt time.Time `json:"pointer_promoted_at"`
RefusedAt time.Time `json:"refused_at"`
Reason string `json:"reason"`
+ // The rollback's outcome, recorded after its restart (spec §2.6: a
+ // recovery that failed must not be published as a rollback).
+ // RestoredVersion is what the bound host's link was put back on, "" when
+ // that repoint itself failed or there was nothing to put back. Recovered
+ // is true only when the link was rolled back AND the restart on it came
+ // up; RecoveryError says why not otherwise.
+ RestoredVersion string `json:"restored_version,omitempty"`
+ Recovered bool `json:"recovered"`
+ RecoveryError string `json:"recovery_error,omitempty"`
}
func RefusalDir(bakeDir string) string { return filepath.Join(bakeDir, "refusals") }
diff --git a/internal/release/rollback.go b/internal/release/rollback.go
index 57f1e3f50..27485f73a 100644
--- a/internal/release/rollback.go
+++ b/internal/release/rollback.go
@@ -65,10 +65,15 @@ func processGroup(reg *registry.Registry, bound registry.Route) []registry.Route
}
// rollbackGroup puts every group route whose link names the rejected version
-// onto restored (what bound now runs), stamps it to match, and writes a
-// refusal for bound and every group host.
-func rollbackGroup(reg *registry.Registry, optDir string, cfg passConfig, bound registry.Route,
- ptr *ChannelPointer, rejected, restored, sha, platform string, nu *UnitNotUpError, now time.Time) {
+// onto restored (what bound now runs), stamps it to match, and returns bound
+// and every group host: the hosts writeGroupRefusals refuses. restored ""
+// (bound's own rollback failed, or it had nothing to go back to) repoints
+// nothing: a sibling must not claim a recovery the bound route never made.
+//
+// A sibling is stamped with bound's PrevVersion, not its own. Earlier in this
+// pass it was stamped rejected over restored, so its own PrevVersion now
+// names restored; bound, never stamped, still holds the history both share.
+func rollbackGroup(reg *registry.Registry, optDir string, bound registry.Route, rejected, restored string) []string {
hosts := []string{bound.Host}
for _, g := range processGroup(reg, bound) {
hosts = append(hosts, g.Host)
@@ -80,10 +85,24 @@ func rollbackGroup(reg *registry.Registry, optDir string, cfg passConfig, bound
fmt.Printf(" %s: could not roll the symlink back to %s with its process: %v\n", g.Host, restored, err)
continue
}
- if err := reg.SetVersions(g.Host, restored, g.PrevVersion); err != nil {
+ if err := reg.SetVersions(g.Host, restored, bound.PrevVersion); err != nil {
fmt.Printf(" %s: rolled back to %s but recording the version failed: %v\n", g.Host, restored, err)
}
}
+ return hosts
+}
+
+// recovery is how a D1 rollback ended, recorded on every refusal it writes.
+type recovery struct {
+ restored string // the version bound's link was put back on; "" if none
+ recovered bool
+ err string // why not, when !recovered
+}
+
+// writeGroupRefusals writes one refusal per host, AFTER the rollback restart,
+// so each record carries the recovery's real outcome (spec §2.6).
+func writeGroupRefusals(cfg passConfig, hosts []string, app string, ptr *ChannelPointer,
+ rejected, sha, platform string, nu *UnitNotUpError, rec recovery, now time.Time) {
if cfg.bakeDir == "" {
return
}
@@ -93,8 +112,9 @@ func rollbackGroup(reg *registry.Registry, optDir string, cfg passConfig, bound
reason += " — " + lines[len(lines)-1]
}
for _, h := range hosts {
- r := Refusal{Host: h, App: bound.App, Version: rejected, SHA256: sha, Platform: platform,
- PointerPromotedAt: ptr.PromotedAt, RefusedAt: now.UTC(), Reason: reason}
+ r := Refusal{Host: h, App: app, Version: rejected, SHA256: sha, Platform: platform,
+ PointerPromotedAt: ptr.PromotedAt, RefusedAt: now.UTC(), Reason: reason,
+ RestoredVersion: rec.restored, Recovered: rec.recovered, RecoveryError: rec.err}
if err := WriteRefusal(cfg.bakeDir, r); err != nil {
fmt.Printf(" %s: writing refusal record: %v\n", h, err)
}
diff --git a/internal/release/rollback_test.go b/internal/release/rollback_test.go
index 5687b96e1..f1bd8d8e1 100644
--- a/internal/release/rollback_test.go
+++ b/internal/release/rollback_test.go
@@ -2,6 +2,12 @@ package release
import (
"bytes"
+ "context"
+ "crypto/ed25519"
+ "errors"
+ "os"
+ "path/filepath"
+ "strings"
"testing"
"github.com/carlosframework/platform/internal/registry"
@@ -83,6 +89,125 @@ func TestNotUpRollsTheWholeGroupBackAndRefusesEveryHost(t *testing.T) {
if rolledBackHost != "platform.example.com" {
t.Fatalf("rollback restart ran for %q, want the bound host", rolledBackHost)
}
+ for _, h := range []string{"console.example.com", "platform.example.com"} {
+ if r, _ := ReadRefusal(bake, h); !r.Recovered || r.RestoredVersion != "old" || r.RecoveryError != "" {
+ t.Fatalf("%s refusal = %+v, want recovered onto old", h, r)
+ }
+ }
+}
+
+// notUpGroupFixture is the flagship shape again: console.example.com is
+// served by platform.example.com's console.service, the bound route has a
+// history (old over older), and the canary pointer names the rejected new.
+func notUpGroupFixture(t *testing.T) (context.Context, *registry.Registry, Store, ed25519.PublicKey, string) {
+ t.Helper()
+ ctx, reg, s, pub, opt := startUnitFixture(t, "console.example.com")
+ if err := reg.SetUnit("console.example.com", ""); err != nil {
+ t.Fatal(err)
+ }
+ groupFixtureRoute(t, reg, "platform.example.com", "", "127.0.0.1:8090", "console.service")
+ if err := reg.SetUpstreams("console.example.com", "127.0.0.1:8090", "", ""); err != nil {
+ t.Fatal(err)
+ }
+ if err := PointLive(opt, "platform.example.com", "bab", "old", "console"); err != nil {
+ t.Fatal(err)
+ }
+ if err := reg.SetVersions("platform.example.com", "old", "older"); err != nil {
+ t.Fatal(err)
+ }
+ return ctx, reg, s, pub, opt
+}
+
+// Final review F6 (and the ledger's Task 3 minor): a sibling rolled back with
+// its process is stamped with the BOUND route's previous version. Its own
+// PrevVersion names restored by then (this pass stamped it new over old).
+func TestRollbackGroupStampsTheBoundRoutesHistory(t *testing.T) {
+ ctx, reg, s, pub, opt := notUpGroupFixture(t)
+ restart := func(h string) error {
+ if h == "platform.example.com" {
+ return &UnitNotUpError{Unit: "console.service", Reason: "exited: status=2"}
+ }
+ return nil
+ }
+ var out bytes.Buffer
+ withCapturedStdout(t, &out, func() {
+ if err := Pass(ctx, reg, s, pub, opt, restart, BakeDir(t.TempDir()), WithRollbackRestart(func(string) error { return nil })); err != nil {
+ t.Fatalf("Pass: %v", err)
+ }
+ })
+ if rt := mustRoute(t, reg, "console.example.com"); rt.Version != "old" || rt.PrevVersion != "older" {
+ t.Fatalf("sibling stamp = %q over %q, want old over older (bound's history)", rt.Version, rt.PrevVersion)
+ }
+}
+
+// Final review F6: the rollback restart failed. Every record says so, with
+// the error, so the watcher does not publish a recovery that never happened.
+func TestRollbackRestartFailureIsRecordedAsNotRecovered(t *testing.T) {
+ ctx, reg, s, pub, opt := notUpGroupFixture(t)
+ restart := func(h string) error {
+ if h == "platform.example.com" {
+ return &UnitNotUpError{Unit: "console.service", Reason: "exited: status=2"}
+ }
+ return nil
+ }
+ rb := func(string) error {
+ return errors.New("rollback did not come up either: console.service did not come up: unit entered failed state")
+ }
+ bake := t.TempDir()
+ var out bytes.Buffer
+ withCapturedStdout(t, &out, func() {
+ if err := Pass(ctx, reg, s, pub, opt, restart, BakeDir(bake), WithRollbackRestart(rb)); err != nil {
+ t.Fatalf("Pass: %v", err)
+ }
+ })
+ for _, h := range []string{"console.example.com", "platform.example.com"} {
+ r, _ := ReadRefusal(bake, h)
+ if r == nil || r.Recovered || r.RestoredVersion != "old" ||
+ !strings.Contains(r.RecoveryError, "rollback did not come up either") {
+ t.Fatalf("%s refusal = %+v, want not recovered, restored old, carrying the restart's error", h, r)
+ }
+ }
+}
+
+// Final review F6: the BOUND route's own symlink rollback failed, so the
+// process still runs (or will run) the rejected build. Siblings are not
+// repointed onto a version their process is not running, and the records
+// say there was no recovery.
+func TestBoundRollbackFailureLeavesSiblingsAlone(t *testing.T) {
+ ctx, reg, s, pub, opt := notUpGroupFixture(t)
+ restart := func(h string) error {
+ if h == "platform.example.com" {
+ // A non-empty directory where PointLive's temp link goes: the
+ // repoint back to old fails for this host only.
+ if err := os.MkdirAll(filepath.Join(opt, "live", "platform.example.com.tmp", "x"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ return &UnitNotUpError{Unit: "console.service", Reason: "exited: status=2"}
+ }
+ return nil
+ }
+ rbCalled := false
+ bake := t.TempDir()
+ var out bytes.Buffer
+ withCapturedStdout(t, &out, func() {
+ if err := Pass(ctx, reg, s, pub, opt, restart, BakeDir(bake),
+ WithRollbackRestart(func(string) error { rbCalled = true; return nil })); err != nil {
+ t.Fatalf("Pass: %v", err)
+ }
+ })
+ if v, _ := LiveVersion(opt, "platform.example.com"); v != "new" {
+ t.Fatalf("bound live = %q; the fixture meant its rollback to fail", v)
+ }
+ if v, _ := LiveVersion(opt, "console.example.com"); v != "new" {
+ t.Fatalf("sibling live = %q, want it left on new: its process never went back to old", v)
+ }
+ if rbCalled {
+ t.Fatal("the rollback restart ran with the link still on the rejected build")
+ }
+ r, _ := ReadRefusal(bake, "platform.example.com")
+ if r == nil || r.Recovered || r.RestoredVersion != "" || !strings.Contains(r.RecoveryError, "could not roll the symlink back to old") {
+ t.Fatalf("bound refusal = %+v, want not recovered with the repoint's error", r)
+ }
}
// Final review F4: a same-upstream route following ANOTHER channel is a
diff --git a/internal/status/contract_test.go b/internal/status/contract_test.go
index 4b7f0fe04..6c7cc254e 100644
--- a/internal/status/contract_test.go
+++ b/internal/status/contract_test.go
@@ -85,7 +85,7 @@ var allowedBoxKeys = map[string][]string{
"metrics.config_refusals[]": {"host", "reason", "since", "last", "count"},
// release_refusals (D1) names the host and build the box rolled back, in
// the private block only, pinned to the field level for the same reason.
- "metrics.release_refusals[]": {"host", "version", "reason", "refused_at"},
+ "metrics.release_refusals[]": {"host", "version", "reason", "refused_at", "recovery_error"},
}
// allowedFleetKeys are THE schema for Fleet's JSON output. Same friction as
@@ -308,7 +308,8 @@ func fixtureProbes() status.Probes {
// One rolled-back build, so metrics.release_refusals is exercised
// by the allowlist walk too.
ReleaseRefusals: []status.ReleaseRefusal{
- {Host: "contract-rolled-back.internal", Version: "v2", Reason: "exited: status=2", RefusedAt: time.Now()},
+ {Host: "contract-rolled-back.internal", Version: "v2", Reason: "exited: status=2", RefusedAt: time.Now(),
+ RecoveryError: "rollback did not come up either"},
},
ConfigGuard: "bypassed (CARLOS_NO_SEALED_CONFIG)",
MemFrac: &memFrac,
diff --git a/internal/status/snapshot.go b/internal/status/snapshot.go
index 5301e5316..eb4ea92f9 100644
--- a/internal/status/snapshot.go
+++ b/internal/status/snapshot.go
@@ -177,6 +177,9 @@ type ReleaseRefusal struct {
Version string `json:"version"`
Reason string `json:"reason"`
RefusedAt time.Time `json:"refused_at"`
+ // RecoveryError is set when the rollback itself did not come up (the
+ // record's Recovered is false): why. "" is a recovered rollback.
+ RecoveryError string `json:"recovery_error,omitempty"`
}
// UpstreamState is one upstream's health as the edge's prober currently sees
diff --git a/statuslive_test.go b/statuslive_test.go
index 57a0291f8..320eb894e 100644
--- a/statuslive_test.go
+++ b/statuslive_test.go
@@ -159,7 +159,27 @@ func TestPrintLiveBoxesShowsReleaseRefusals(t *testing.T) {
}},
}},
}, now)
- want := "refused console.example.com v2 since 2026-10-06T09:30:40Z: exited: status=2"
+ want := "refused console.example.com v2 since 2026-10-06T09:30:40Z: exited: status=2\n"
+ if !strings.Contains(buf.String(), want) {
+ t.Fatalf("missing %q in:\n%s", want, buf.String())
+ }
+}
+
+// Final review F6: a rollback that did not come up says so on the same line.
+func TestPrintLiveBoxesShowsAFailedRollback(t *testing.T) {
+ now := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
+ var buf bytes.Buffer
+ printLiveBoxes(&buf, []status.Box{
+ {GeneratedAt: now, Label: "ada-stargazing", Metrics: &status.BoxMetrics{
+ Tunnel: "connected",
+ ReleaseRefusals: []status.ReleaseRefusal{{
+ Host: "console.example.com", Version: "v2",
+ RefusedAt: time.Date(2026, 10, 6, 9, 30, 40, 0, time.UTC), Reason: "exited: status=2",
+ RecoveryError: "start console.service: exit status 1",
+ }},
+ }},
+ }, now)
+ want := "refused console.example.com v2 since 2026-10-06T09:30:40Z: exited: status=2 (rollback failed: start console.service: exit status 1)"
if !strings.Contains(buf.String(), want) {
t.Fatalf("missing %q in:\n%s", want, buf.String())
}