carlos / platform Public

host: alert on a refused build and show it in status --live

A release refusal record (D1) becomes one alert per record through the agent's webhook and SNS sinks, and a ReleaseRefusals row in the private metrics block that carlos status --live prints under the box. Records older than 30 minutes when first seen are not alerted, so an agent restart does not re-send old news.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

claude 2 weeks ago 6abd12d52d273723075a1639bfdcf7ef615bf2a5 parent b6a26ab

8 files changed, +188 −1

cli.go +3 −0
internal/host/agent.go +3 −0
internal/host/alertsink.go +35 −0
internal/host/metrics_loop.go +7 −0
internal/host/refusalwatch.go +55 −0
internal/host/refusalwatch_test.go +45 −0
internal/status/snapshot.go +19 −1
statuslive_test.go +21 −0

diff --git a/cli.go b/cli.go

index 6c92cf1a9..119842e51 100644

--- a/cli.go

+++ b/cli.go

@@ -3434,6 +3434,9 @@ func printLiveBoxes(w io.Writer, boxes []status.Box, now time.Time) {

b.Label, orDash(b.Region), orDash(m.Release), orDash(m.Tunnel),

humanBytes(m.MemAvailBytes), humanBytes(m.MemTotalBytes),

humanFloat(m.Load1), humanBytes(m.DiskFreeBytes), alertsWord(m.Alerts))

+ for _, r := range m.ReleaseRefusals {

+ fmt.Fprintf(w, " refused %s %s since %s: %s\n", r.Host, r.Version, r.RefusedAt.UTC().Format(time.RFC3339), r.Reason)

+ }

}

diff --git a/internal/host/agent.go b/internal/host/agent.go

index b124c4d13..131cb5d0c 100644

--- a/internal/host/agent.go

+++ b/internal/host/agent.go

@@ -2942,6 +2942,9 @@ func statusLoop(reg *registry.Registry) {

// config (amadan #213): private Metrics only, nothing public.

p.ConfigRefusals, p.ConfigRefusalsTotal = configRefusalProbe()

p.ConfigGuard = configGuardProbe()

+ // Builds the root pass rolled back (D1), as the metrics loop's

+ // refusal watch last read them. Private Metrics only: hosts.

+ p.ReleaseRefusals = releaseRefusalProbe()

// A fleet box (CARLOS_FLEET set) never gets here as a contender: it

// self-reports only, never attempting the election at all — no

diff --git a/internal/host/alertsink.go b/internal/host/alertsink.go

index c9aca34d8..e3adc9f90 100644

--- a/internal/host/alertsink.go

+++ b/internal/host/alertsink.go

@@ -17,6 +17,7 @@ import (

"github.com/aws/aws-sdk-go-v2/service/sns"

"github.com/carlosframework/platform/internal/metrics"

+ "github.com/carlosframework/platform/internal/release"

"github.com/carlosframework/platform/internal/status"

)

@@ -125,6 +126,40 @@ func (a *alertSinks) deliver(ctx context.Context, rules metrics.Rules, at time.T

}

+// deliverRelease sends a D1 refusal to every configured sink. The webhook

+// body is the record with kind "release_refused"; SNS gets a sentence.

+func (a *alertSinks) deliverRelease(ctx context.Context, r release.Refusal) {

+ if a == nil {

+ return

+ }

+ if a.webhookURL != "" && a.post != nil {

+ body, err := json.Marshal(struct {

+ Kind string `json:"kind"`

+ release.Refusal

+ }{"release_refused", r})

+ if err == nil {

+ if err := a.post(a.webhookURL, body); err != nil {

+ a.counts.webhook.Add(1)

+ log.Printf("host: alert webhook post (release refusal %s): %v", r.Host, err)

+ }

+ if a.topicARN != "" && a.pub != nil {

+ subject := fmt.Sprintf("%s: %s did not start, rolled back", r.Host, r.Version)

+ if len(subject) > 100 { // SNS subject limit

+ subject = subject[:100]

+ }

+ msg := fmt.Sprintf("%s refused %s at %s: %s\n\nRetry with: carlos promote --retry %s <channel>",

+ r.Host, r.Version, r.RefusedAt.Format(time.RFC3339), r.Reason, r.Version)

+ pctx, cancel := context.WithTimeout(ctx, alertPublishTimeout)

+ defer cancel()

+ if _, err := a.pub.Publish(pctx, &sns.PublishInput{TopicArn: &a.topicARN, Subject: &subject, Message: &msg}); err != nil {

+ a.counts.sns.Add(1)

+ log.Printf("host: alert sns publish (release refusal %s): %v", r.Host, err)

+ }

// topicARNPattern is the whole shape of a standard SNS topic ARN: partition,

// region, a 12 digit account, and a topic name that is only letters, digits,

// hyphen and underscore. The name has no colon, so a subscription ARN

diff --git a/internal/host/metrics_loop.go b/internal/host/metrics_loop.go

index cbd2cc382..285765629 100644

--- a/internal/host/metrics_loop.go

+++ b/internal/host/metrics_loop.go

@@ -808,10 +808,17 @@ func metricsLoop(execRoot string, stampedUnits func() []string) {

sm := &sampler{}

probes := realSampleProbes(execRoot, stampedUnits)

tick := 0

+ // One watch for the life of the loop: its memory of what it has already

+ // alerted is what keeps a standing refusal from paging every minute.

+ // dataDir() is the bake dir the agent hands release.BakeDir.

+ var refusals refusalWatch

for {

time.Sleep(60 * time.Second)

tick++

+ rows := refusals.tick(ctx, dataDir(), sinks, time.Now())

+ releaseRefusals.Store(&rows)

sm.tick(probes)

latest := sm.buf[len(sm.buf)-1]

if err := alertTick(ctx, s, box, rules, &st, latest, statePath, sinks); err != nil {

diff --git a/internal/host/refusalwatch.go b/internal/host/refusalwatch.go

new file mode 100644

index 000000000..3ea5b9556

--- /dev/null

+++ b/internal/host/refusalwatch.go

@@ -0,0 +1,55 @@

+// refusalWatch turns D1's refusal records (release.Refusal, written by the

+// root pass) into one alert each and a status row each. It runs on the

+// agent's metrics tick. A record older than refusalAlertWindow when first

+// seen is not alerted, so an agent restart does not re-send old news.

+package host

+import (

+ "context"

+ "log"

+ "sync/atomic"

+ "time"

+ "github.com/carlosframework/platform/internal/release"

+ "github.com/carlosframework/platform/internal/status"

+const refusalAlertWindow = 30 * time.Minute

+type refusalWatch struct{ seen map[string]time.Time }

+func (w *refusalWatch) tick(ctx context.Context, bakeDir string, sinks *alertSinks, now time.Time) []status.ReleaseRefusal {

+ if w.seen == nil {

+ w.seen = map[string]time.Time{}

+ }

+ recs, err := release.ListRefusals(bakeDir)

+ if err != nil {

+ log.Printf("agent: refusals: %v", err)

+ return nil

+ }

+ var rows []status.ReleaseRefusal

+ for _, r := range recs {

+ rows = append(rows, status.ReleaseRefusal{Host: r.Host, Version: r.Version, Reason: r.Reason, RefusedAt: r.RefusedAt})

+ if last, ok := w.seen[r.Host]; ok && !r.RefusedAt.After(last) {

+ continue

+ }

+ w.seen[r.Host] = r.RefusedAt

+ if now.Sub(r.RefusedAt) <= refusalAlertWindow {

+ sinks.deliverRelease(ctx, r)

+ }

+ return rows

+// releaseRefusals is the metrics loop's latest answer, read by the status

+// loop's probe. They are separate goroutines, so this is the seam between

+// them (the same shape as processAlertCounters). Nil until the first tick.

+var releaseRefusals atomic.Pointer[[]status.ReleaseRefusal]

+// releaseRefusalProbe feeds status.Probes.ReleaseRefusals.

+func releaseRefusalProbe() []status.ReleaseRefusal {

+ if p := releaseRefusals.Load(); p != nil {

+ return append([]status.ReleaseRefusal(nil), (*p)...)

+ }

+ return nil

diff --git a/internal/host/refusalwatch_test.go b/internal/host/refusalwatch_test.go

new file mode 100644

index 000000000..0a2d72baa

--- /dev/null

+++ b/internal/host/refusalwatch_test.go

@@ -0,0 +1,45 @@

+package host

+import (

+ "context"

+ "strings"

+ "testing"

+ "time"

+ "github.com/carlosframework/platform/internal/release"

+func TestRefusalWatchAlertsOncePerRecord(t *testing.T) {

+ bake := t.TempDir()

+ now := time.Date(2026, 10, 6, 9, 31, 0, 0, time.UTC)

+ if err := release.WriteRefusal(bake, release.Refusal{Host: "console.example.com", App: "console",

+ Version: "v2", RefusedAt: now.Add(-time.Minute), Reason: "exited: status=2"}); err != nil {

+ t.Fatal(err)

+ }

+ var posts []string

+ sinks := newAlertSinks("https://hook.example", func(_ string, b []byte) error { posts = append(posts, string(b)); return nil }, "", nil, &alertCounters{})

+ w := &refusalWatch{}

+ got := w.tick(context.Background(), bake, sinks, now)

+ w.tick(context.Background(), bake, sinks, now.Add(time.Minute))

+ if len(posts) != 1 || !strings.Contains(posts[0], `"release_refused"`) || !strings.Contains(posts[0], "v2") {

+ t.Fatalf("posts = %v, want exactly one release_refused alert", posts)

+ }

+ if len(got) != 1 || got[0].Host != "console.example.com" {

+ t.Fatalf("status rows = %+v", got)

+ }

+func TestRefusalWatchSkipsOldRecordsAtStartup(t *testing.T) {

+ bake := t.TempDir()

+ now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)

+ if err := release.WriteRefusal(bake, release.Refusal{Host: "a.example.com", Version: "v2",

+ RefusedAt: now.Add(-2 * time.Hour)}); err != nil {

+ t.Fatal(err)

+ }

+ var posts int

+ sinks := newAlertSinks("https://hook.example", func(string, []byte) error { posts++; return nil }, "", nil, &alertCounters{})

+ (&refusalWatch{}).tick(context.Background(), bake, sinks, now)

+ if posts != 0 {

+ t.Fatalf("an agent restart re-alerted a refusal from two hours ago")

+ }

diff --git a/internal/status/snapshot.go b/internal/status/snapshot.go

index e1b76eaeb..5301e5316 100644

--- a/internal/status/snapshot.go

+++ b/internal/status/snapshot.go

@@ -141,6 +141,9 @@ type Probes struct {

// how many are refused in all when the guard capped the list.

ConfigRefusals []ConfigRefusal

ConfigRefusalsTotal int

+ // ReleaseRefusals is the builds this box rolled back and is now refusing

+ // to re-adopt (D1); nil when there are none. Private Metrics only.

+ ReleaseRefusals []ReleaseRefusal

// ConfigGuard is "" in the normal mode and a words-only note when the

// config guard's explicit bypass is in effect on this box (amadan #213).

ConfigGuard string

@@ -166,6 +169,16 @@ type ConfigRefusal struct {

Count int `json:"count"`

}

+// ReleaseRefusal is a build this box rolled back because its unit did not

+// come up (D1, release.Refusal). PRIVATE like the rest of Box.Metrics: Host

+// is a real route name.

+type ReleaseRefusal struct {

+ Host string `json:"host"`

+ Version string `json:"version"`

+ Reason string `json:"reason"`

+ RefusedAt time.Time `json:"refused_at"`

// UpstreamState is one upstream's health as the edge's prober currently sees

// it: what it believes, when that belief last changed, and why.

@@ -299,6 +312,10 @@ type BoxMetrics struct {

// ConfigRefusalsTotal is the count of refused hosts when ConfigRefusals

// was capped, so a long list reads as long. Omitted when zero.

ConfigRefusalsTotal int `json:"config_refusals_total,omitempty"`

+ // ReleaseRefusals is the builds this box rolled back because their unit

+ // did not come up (D1), one per host. Hostname-bearing, so private like

+ // ConfigRefusals. Omitted when empty.

+ ReleaseRefusals []ReleaseRefusal `json:"release_refusals,omitempty"`

// ConfigGuard says the box's admission guard is bypassed, e.g.

// "bypassed (CARLOS_NO_SEALED_CONFIG)": it admits placed exec instances

// without checking their sealed config. Omitted in the normal mode, so an

@@ -697,7 +714,7 @@ func CollectBox(routes []registry.Route, p Probes, now time.Time) Box {

p.DiskFreeBytes != nil || p.Release != "" || p.Tunnel != "" ||

p.InstanceID != "" || p.Platform != "" || p.SystemChannel != "" || p.System != nil ||

len(routeVersions) > 0 || len(aliasReadings) > 0 || len(p.Upstreams) > 0 ||

- len(routeDBBytes) > 0 || len(p.ConfigRefusals) > 0 || p.ConfigGuard != "" ||

+ len(routeDBBytes) > 0 || len(p.ConfigRefusals) > 0 || len(p.ReleaseRefusals) > 0 || p.ConfigGuard != "" ||

p.Alerts != nil {

box.Metrics = &BoxMetrics{

MemTotalBytes: p.MemTotalBytes, MemAvailBytes: p.MemAvailBytes,

@@ -718,6 +735,7 @@ func CollectBox(routes []registry.Route, p Probes, now time.Time) Box {

// A fresh copy: the guard's own slice is its to mutate.

ConfigRefusals: append([]ConfigRefusal(nil), p.ConfigRefusals...),

ConfigRefusalsTotal: p.ConfigRefusalsTotal,

+ ReleaseRefusals: append([]ReleaseRefusal(nil), p.ReleaseRefusals...),

ConfigGuard: p.ConfigGuard,

}

diff --git a/statuslive_test.go b/statuslive_test.go

index 93677c05e..57a0291f8 100644

--- a/statuslive_test.go

+++ b/statuslive_test.go

@@ -144,6 +144,27 @@ func TestPrintLiveBoxesSaysWhetherAlertsAreDelivered(t *testing.T) {

}

+// TestPrintLiveBoxesShowsReleaseRefusals: a build the box rolled back (D1)

+// reads as its own line under the box, naming the host, the version, since

+// when and why, so the operator does not have to read the journal.

+func TestPrintLiveBoxesShowsReleaseRefusals(t *testing.T) {

+ now := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)

+ var buf bytes.Buffer

+ printLiveBoxes(&buf, []status.Box{

+ {GeneratedAt: now, Label: "ada-stargazing", Metrics: &status.BoxMetrics{

+ Tunnel: "connected",

+ ReleaseRefusals: []status.ReleaseRefusal{{

+ Host: "console.example.com", Version: "v2",

+ RefusedAt: time.Date(2026, 10, 6, 9, 30, 40, 0, time.UTC), Reason: "exited: status=2",

+ }},

+ }, now)

+ want := "refused console.example.com v2 since 2026-10-06T09:30:40Z: exited: status=2"

+ if !strings.Contains(buf.String(), want) {

+ t.Fatalf("missing %q in:\n%s", want, buf.String())

+ }

func TestHumanBytesAndFloat(t *testing.T) {

for _, c := range []struct {

in *int64