amadan

carlos / platform Public

console: alert watcher — keymail on fire/clear via operators' own grants

The box judges itself and publishes alerts/boxes/<box>.json, but it has no
keymail identity: the only send path that exists is the console's per-user
OAuth grant. So the console polls that prefix every minute, diffs each
rule's firing state against what it last told operators (the new alert_seen
table), and mails the difference — each operator with that operator's OWN
grant token.

Only transitions are news. A rule the console has never seen that is firing
is a transition; one it has never seen that is quiet is recorded silently,
because "the disk is fine" is not a page. A send failure for any operator
leaves alert_seen unrecorded, so the next tick retries the whole transition:
a duplicate page is cheap, a missed one is what this exists to prevent.
Operators with no grant are skipped and named in the log, once per pass and
only when a pass has something to say. An unreadable or unparsable alert
object is logged and stepped over — a watcher that dies on one bad file is a
watcher that stops paging.

StartAlertWatcher is called from serve(), never NewServer: constructing a
Server, which every console test does dozens of times, must not start a
goroutine or send mail. Delivery goes through a sendApp field so alertPass
can be asserted on with no keymail server in play. Nothing here logs a grant
token; keymail's own errors carry method and path only.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Paul Campbell aa7ef0899d1866836b31fa776e5faff18209003f parent 3ada7ce
6 files changed, +584 −0
  • cmd/console/main.go +5 −0
  • internal/console/alertwatch.go +215 −0
  • internal/console/alertwatch_test.go +231 −0
  • internal/console/server.go +9 −0
  • internal/console/store.go +63 −0
  • internal/console/store_test.go +61 −0
diff --git a/cmd/console/main.go b/cmd/console/main.go
index 3f771b83c..659433cec 100644
--- a/cmd/console/main.go
+++ b/cmd/console/main.go
@@ -218,6 +218,11 @@ func serve() {
if err != nil {
log.Fatal(err)
}
+ // The alert watcher polls alerts/boxes/*.json every minute and keymails
+ // operators on each fire/clear (internal/console/alertwatch.go). Started
+ // here rather than in NewServer so constructing a Server — which every
+ // console test does — never starts a goroutine or sends mail.
+ go srv.StartAlertWatcher(context.Background())
addr := envOr("CONSOLE_ADDR", "127.0.0.1:8090")
log.Printf("console: %s listening on %s", origin, addr)
log.Fatal(http.ListenAndServe(addr, srv))
diff --git a/internal/console/alertwatch.go b/internal/console/alertwatch.go
new file mode 100644
index 000000000..70df565d8
--- /dev/null
+++ b/internal/console/alertwatch.go
@@ -0,0 +1,215 @@
+package console
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "log"
+ "sort"
+ "strings"
+ "time"
+
+ "github.com/carlosframework/platform/internal/metrics"
+)
+
+// The keymail half of alert delivery lives here rather than on the box,
+// because the only keymail send path that exists is the console's per-user
+// OAuth grant (Keymail.SendApp) and the agent has no keymail identity. The
+// box judges itself and publishes alerts/boxes/<box>.json; this watcher
+// polls that prefix, diffs each rule's firing state against what it last
+// told operators (the alert_seen table), and mails the difference — each
+// operator with that operator's OWN grant token, so nobody is sent mail on
+// a credential they did not personally issue.
+//
+// The failure posture throughout is "never go silent": an unreadable or
+// unparsable alert object is logged and skipped, a send failure leaves the
+// transition unrecorded so the next pass retries the whole thing, and no
+// path here can panic a console that is otherwise serving fine.
+
+// alertWatchInterval is the poll cadence from the spec: a minute is fast
+// enough for a disk filling up and slow enough to be free.
+const alertWatchInterval = time.Minute
+
+// alertPrefix is the bucket prefix the agent publishes AlertState under
+// (metrics.AlertKey).
+const alertPrefix = "alerts/boxes/"
+
+// StartAlertWatcher runs alertPass immediately and then every minute until
+// ctx is done. Started from cmd/console's serve(), deliberately NOT from
+// NewServer: tests construct servers by the dozen and none of them should
+// grow a background goroutine or a keymail send.
+func (s *Server) StartAlertWatcher(ctx context.Context) {
+ t := time.NewTicker(alertWatchInterval)
+ defer t.Stop()
+ for {
+ if err := s.alertPass(ctx); err != nil {
+ log.Printf("console: alert watch: %v", err)
+ }
+ select {
+ case <-ctx.Done():
+ return
+ case <-t.C:
+ }
+ }
+}
+
+// operatorGrant is one recipient and the token the console sends AS them
+// with. The token is never logged.
+type operatorGrant struct {
+ address string
+ token string
+}
+
+// alertRun is one pass's working state: who to mail, and the operators with
+// no grant to mail through — reported at most once per pass, and only if
+// the pass had something to say, so a fleet at rest logs nothing.
+type alertRun struct {
+ to []operatorGrant
+ missing []string
+ noted bool
+}
+
+// recipients returns the mailable operators, logging the token-less ones the
+// first time a pass actually needs to send.
+func (r *alertRun) recipients() []operatorGrant {
+ if !r.noted && len(r.missing) > 0 {
+ r.noted = true
+ log.Printf("console: alert watch: no keymail grant for %s — not notified (they must sign in to the console once)",
+ strings.Join(r.missing, ", "))
+ }
+ return r.to
+}
+
+// alertPass reads every alerts/boxes/*.json, diffs each rule's firing state
+// against alert_seen, and for each transition mails every operator holding a
+// grant and records the new state. It returns an error only for a failure
+// that makes the whole pass meaningless (listing the bucket, or reading the
+// operator list); per-box and per-rule trouble is logged and stepped over.
+func (s *Server) alertPass(ctx context.Context) error {
+ if s.cfg.Releases == nil || s.sendApp == nil {
+ return nil
+ }
+ keys, err := s.cfg.Releases.List(ctx, alertPrefix)
+ if err != nil {
+ return fmt.Errorf("list %s: %w", alertPrefix, err)
+ }
+ if len(keys) == 0 {
+ return nil // no box has ever published an alert: nothing to do
+ }
+ run, err := s.alertRecipients()
+ if err != nil {
+ return err
+ }
+ for _, key := range keys {
+ if !strings.HasSuffix(key, ".json") {
+ continue
+ }
+ body, _, err := s.cfg.Releases.Get(ctx, key)
+ if err != nil {
+ log.Printf("console: alert watch: read %s: %v", key, err)
+ continue
+ }
+ var st metrics.AlertState
+ if err := json.Unmarshal(body, &st); err != nil {
+ log.Printf("console: alert watch: unparsable %s: %v", key, err)
+ continue
+ }
+ box := st.Box
+ if box == "" {
+ // Trust the key over an empty body field: the key is what the
+ // publisher's own AlertKey produced.
+ box = strings.TrimSuffix(strings.TrimPrefix(key, alertPrefix), ".json")
+ }
+ names := make([]string, 0, len(st.Rules))
+ for name := range st.Rules {
+ names = append(names, name)
+ }
+ sort.Strings(names)
+ for _, name := range names {
+ s.alertRule(ctx, run, box, name, st.Rules[name], st.UpdatedAt)
+ }
+ }
+ return nil
+}
+
+// alertRecipients resolves the operator list into recipients, splitting off
+// the operators the console cannot write as.
+func (s *Server) alertRecipients() (*alertRun, error) {
+ addrs, err := s.store.Operators()
+ if err != nil {
+ return nil, fmt.Errorf("operators: %w", err)
+ }
+ run := &alertRun{}
+ for _, addr := range addrs {
+ tok, err := s.store.GrantToken(addr)
+ if err != nil {
+ return nil, fmt.Errorf("grant token for %s: %w", addr, err)
+ }
+ if tok == "" {
+ run.missing = append(run.missing, addr)
+ continue
+ }
+ run.to = append(run.to, operatorGrant{address: addr, token: tok})
+ }
+ return run, nil
+}
+
+// alertRule handles one rule of one box: nothing to do unless its firing
+// state differs from what operators were last told.
+func (s *Server) alertRule(ctx context.Context, run *alertRun, box, rule string, rs metrics.RuleState, updated time.Time) {
+ seen, known, err := s.store.AlertSeen(box, rule)
+ if err != nil {
+ log.Printf("console: alert watch: alert_seen %s/%s: %v", box, rule, err)
+ return
+ }
+ if known && seen == rs.Firing {
+ return
+ }
+ if !known && !rs.Firing {
+ // First sighting of a quiet rule. Record it so a later fire is a
+ // transition; say nothing — "everything is fine" is not a page.
+ if err := s.store.SetAlertSeen(box, rule, false); err != nil {
+ log.Printf("console: alert watch: record %s/%s: %v", box, rule, err)
+ }
+ return
+ }
+ subject, body := alertMessage(box, rule, rs, updated)
+ delivered := true
+ for _, op := range run.recipients() {
+ if _, err := s.sendApp(ctx, op.token, op.address, subject, body); err != nil {
+ // The address is safe to log; the token never is.
+ log.Printf("console: alert watch: keymail %s/%s to %s: %v", box, rule, op.address, err)
+ delivered = false
+ }
+ }
+ if !delivered {
+ // Leave the transition unrecorded: the next pass retries the whole
+ // thing, which may re-mail an operator who did get it. A duplicate
+ // page is cheap; a missed one is what this system exists to prevent.
+ return
+ }
+ if err := s.store.SetAlertSeen(box, rule, rs.Firing); err != nil {
+ log.Printf("console: alert watch: record %s/%s: %v", box, rule, err)
+ }
+}
+
+// alertMessage is the operator-facing copy. Plain sentences, and no needle
+// beyond the value that triggered the transition — this mail lands in an
+// inbox, not on the operator dashboard.
+func alertMessage(box, rule string, rs metrics.RuleState, updated time.Time) (subject, body string) {
+ at := updated
+ if rs.Firing && !rs.FiredAt.IsZero() {
+ at = rs.FiredAt
+ }
+ stamp := at.UTC().Format(time.RFC3339)
+ if rs.Firing {
+ subject = fmt.Sprintf("[carlos] %s: %s alert", box, rule)
+ body = fmt.Sprintf("The %s alert is firing on %s.\n\nValue at the transition: %g\nTime (UTC): %s\n\n"+
+ "You are getting this because you hold the CARLOS fleet-operator role.\n", rule, box, rs.Value, stamp)
+ return subject, body
+ }
+ subject = fmt.Sprintf("[carlos] %s: %s cleared", box, rule)
+ body = fmt.Sprintf("The %s alert has cleared on %s.\n\nValue at the transition: %g\nTime (UTC): %s\n\n"+
+ "You are getting this because you hold the CARLOS fleet-operator role.\n", rule, box, rs.Value, stamp)
+ return subject, body
+}
diff --git a/internal/console/alertwatch_test.go b/internal/console/alertwatch_test.go
new file mode 100644
index 000000000..02ad99d71
--- /dev/null
+++ b/internal/console/alertwatch_test.go
@@ -0,0 +1,231 @@
+package console
+
+import (
+ "context"
+ "encoding/json"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/metrics"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// sentMail is one captured sendApp call. The token is captured so a test can
+// assert each operator is mailed with THEIR OWN grant, never a shared one.
+type sentMail struct {
+ token, to, subject, body string
+}
+
+// alertFixture is a console fixture with two operators — one holding a
+// keymail grant, one not — and a captured sendApp.
+type alertFixture struct {
+ *consoleFixture
+ sent []sentMail
+ err error // what the captured sendApp returns
+}
+
+func newAlertFixture(t *testing.T) *alertFixture {
+ t.Helper()
+ fx := newConsoleFixture(t)
+ af := &alertFixture{consoleFixture: fx}
+ for _, addr := range []string{"ops-a@keymail.dev", "ops-b@keymail.dev"} {
+ if err := fx.st.SetOperator(addr, true); err != nil {
+ t.Fatal(err)
+ }
+ }
+ // Only ops-a ever granted the console send access.
+ if err := fx.st.SetGrantToken("ops-a@keymail.dev", "grant-a", time.Now()); err != nil {
+ t.Fatal(err)
+ }
+ fx.srv.sendApp = func(_ context.Context, token, to, subject, body string) (string, error) {
+ af.sent = append(af.sent, sentMail{token, to, subject, body})
+ if af.err != nil {
+ return "", af.err
+ }
+ return "msg-1", nil
+ }
+ return af
+}
+
+func (af *alertFixture) writeAlert(t *testing.T, st metrics.AlertState) {
+ t.Helper()
+ b, err := json.Marshal(st)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // Publishing is a CAS write (create-only on ""), so re-publishing a box's
+ // state — which is what a fire→clear transition is — needs the etag.
+ key := metrics.AlertKey(st.Box)
+ etag := ""
+ if _, cur, err := af.rel.Get(context.Background(), key); err == nil {
+ etag = cur
+ }
+ if _, err := af.rel.Put(context.Background(), key, b, etag); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func (af *alertFixture) pass(t *testing.T) {
+ t.Helper()
+ af.sent = nil
+ if err := af.srv.alertPass(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func firingDisk(firing bool) metrics.AlertState {
+ rs := metrics.RuleState{Firing: firing, Value: 0.95}
+ if firing {
+ rs.FiredAt = time.Date(2026, 8, 2, 10, 0, 0, 0, time.UTC)
+ rs.Since = rs.FiredAt
+ } else {
+ rs.Value = 0.80
+ }
+ return metrics.AlertState{
+ V: 1,
+ Box: "box-a",
+ UpdatedAt: time.Date(2026, 8, 2, 10, 5, 0, 0, time.UTC),
+ Rules: map[string]metrics.RuleState{"disk": rs},
+ }
+}
+
+// TestAlertPassFiresOnceThenGoesQuiet is the spec's core promise: a keymail
+// on the TRANSITION, to the operators who hold a grant, and silence while
+// the alert stays in the state the console already reported.
+func TestAlertPassFiresOnceThenGoesQuiet(t *testing.T) {
+ af := newAlertFixture(t)
+ af.writeAlert(t, firingDisk(true))
+
+ af.pass(t)
+ if len(af.sent) != 1 {
+ t.Fatalf("first pass sent %d keymails, want 1: %+v", len(af.sent), af.sent)
+ }
+ got := af.sent[0]
+ if got.to != "ops-a@keymail.dev" || got.token != "grant-a" {
+ t.Fatalf("sent to %q with token %q, want ops-a@keymail.dev/grant-a", got.to, got.token)
+ }
+ if got.subject != "[carlos] box-a: disk alert" {
+ t.Fatalf("subject = %q", got.subject)
+ }
+ for _, want := range []string{"disk", "box-a", "0.95", "2026-08-02T10:00:00Z"} {
+ if !strings.Contains(got.body, want) {
+ t.Fatalf("body is missing %q:\n%s", want, got.body)
+ }
+ }
+ if strings.Contains(got.body, "grant-a") {
+ t.Fatal("body leaked the grant token")
+ }
+
+ // Recorded: the same state on the next pass says nothing.
+ firing, known, err := af.st.AlertSeen("box-a", "disk")
+ if err != nil || !known || !firing {
+ t.Fatalf("AlertSeen = %v,%v,%v — want true,true,nil", firing, known, err)
+ }
+ af.pass(t)
+ if len(af.sent) != 0 {
+ t.Fatalf("second pass sent %d keymails, want 0: %+v", len(af.sent), af.sent)
+ }
+}
+
+// TestAlertPassClears covers the other half of the transition: the alert
+// going away is news too.
+func TestAlertPassClears(t *testing.T) {
+ af := newAlertFixture(t)
+ af.writeAlert(t, firingDisk(true))
+ af.pass(t)
+
+ af.writeAlert(t, firingDisk(false))
+ af.pass(t)
+ if len(af.sent) != 1 {
+ t.Fatalf("clear pass sent %d keymails, want 1: %+v", len(af.sent), af.sent)
+ }
+ if af.sent[0].subject != "[carlos] box-a: disk cleared" {
+ t.Fatalf("subject = %q", af.sent[0].subject)
+ }
+ af.pass(t)
+ if len(af.sent) != 0 {
+ t.Fatalf("pass after clear sent %d keymails, want 0", len(af.sent))
+ }
+}
+
+// TestAlertPassRetriesAfterSendFailure: a failed send leaves the transition
+// unrecorded, so the next tick delivers it. A missed page is worse than a
+// duplicate one.
+func TestAlertPassRetriesAfterSendFailure(t *testing.T) {
+ af := newAlertFixture(t)
+ af.writeAlert(t, firingDisk(true))
+ af.err = context.DeadlineExceeded
+
+ af.pass(t)
+ if len(af.sent) != 1 {
+ t.Fatalf("failing pass attempted %d sends, want 1", len(af.sent))
+ }
+ if _, known, err := af.st.AlertSeen("box-a", "disk"); err != nil || known {
+ t.Fatalf("AlertSeen recorded despite a send failure (known=%v, err=%v)", known, err)
+ }
+
+ af.err = nil
+ af.pass(t)
+ if len(af.sent) != 1 {
+ t.Fatalf("retry pass sent %d keymails, want 1", len(af.sent))
+ }
+ if firing, known, err := af.st.AlertSeen("box-a", "disk"); err != nil || !known || !firing {
+ t.Fatalf("AlertSeen = %v,%v,%v after a successful retry", firing, known, err)
+ }
+}
+
+// TestAlertPassQuietFirstSighting: a rule the console has never seen that is
+// NOT firing is recorded silently. Nobody wants a keymail saying the disk is
+// fine.
+func TestAlertPassQuietFirstSighting(t *testing.T) {
+ af := newAlertFixture(t)
+ af.writeAlert(t, firingDisk(false))
+
+ af.pass(t)
+ if len(af.sent) != 0 {
+ t.Fatalf("quiet first sighting sent %d keymails, want 0: %+v", len(af.sent), af.sent)
+ }
+ if firing, known, err := af.st.AlertSeen("box-a", "disk"); err != nil || !known || firing {
+ t.Fatalf("AlertSeen = %v,%v,%v — want false,true,nil", firing, known, err)
+ }
+}
+
+// TestAlertPassSurvivesJunk: an unparsable object under alerts/ is skipped,
+// and the boxes either side of it are still processed. A watcher that dies
+// on one bad file is a watcher that stops paging.
+func TestAlertPassSurvivesJunk(t *testing.T) {
+ af := newAlertFixture(t)
+ if _, err := af.rel.Put(context.Background(), "alerts/boxes/box-junk.json", []byte("{not json"), ""); err != nil {
+ t.Fatal(err)
+ }
+ af.writeAlert(t, firingDisk(true))
+
+ af.pass(t)
+ if len(af.sent) != 1 {
+ t.Fatalf("pass sent %d keymails, want 1: %+v", len(af.sent), af.sent)
+ }
+}
+
+// TestAlertPassEmptyBucket: no alerts/ prefix at all is a no-op, not an
+// error — a fleet that has never published an alert is the normal case.
+func TestAlertPassEmptyBucket(t *testing.T) {
+ fx := newConsoleFixture(t)
+ fx.srv.sendApp = func(context.Context, string, string, string, string) (string, error) {
+ t.Fatal("sent a keymail with no alerts published")
+ return "", nil
+ }
+ if err := fx.srv.alertPass(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// TestNewServerDefaultsSendApp: production wiring goes through the console's
+// own keymail client; nothing else in NewServer starts a goroutine.
+func TestNewServerDefaultsSendApp(t *testing.T) {
+ fx := newConsoleFixture(t)
+ if fx.srv.sendApp == nil {
+ t.Fatal("NewServer left sendApp nil — the watcher would never deliver")
+ }
+ var _ release.Store = fx.rel
+}
diff --git a/internal/console/server.go b/internal/console/server.go
index 61807d3f7..8f7391309 100644
--- a/internal/console/server.go
+++ b/internal/console/server.go
@@ -120,6 +120,10 @@ type Server struct {
// notification attempt. Tests only — the send is off the request
// goroutine, and a test that asserted on it without this would be racing.
cliNotified chan struct{}
+ // sendApp is how the alert watcher (alertwatch.go) delivers keymail:
+ // km.SendApp in production, a capture in tests. A field rather than a
+ // direct call so an alertPass assertion needs no keymail server.
+ sendApp func(ctx context.Context, token, to, subject, body string) (string, error)
}
// NewServer wires a server over an opened Store, loading (or minting) the
@@ -141,6 +145,11 @@ func NewServer(cfg Config, st *Store) (*Server, error) {
secure: strings.HasPrefix(cfg.Origin, "https://"),
cli: newCLILimits(),
}
+ if s.km != nil {
+ s.sendApp = s.km.SendApp
+ }
+ // No goroutine is started here: StartAlertWatcher is serve()'s job, so a
+ // test that constructs a Server gets a Server and nothing else.
s.mux.HandleFunc("GET /{$}", s.handleHome)
s.mux.HandleFunc("POST /auth/continue", s.handleContinue)
s.mux.HandleFunc("GET /auth/signin", s.handleSignin)
diff --git a/internal/console/store.go b/internal/console/store.go
index 1271001eb..07e471c65 100644
--- a/internal/console/store.go
+++ b/internal/console/store.go
@@ -160,6 +160,17 @@ func (s *Store) migrate() error {
created_at INTEGER NOT NULL,
expires_at INTEGER NOT NULL,
accepted_at INTEGER NOT NULL DEFAULT 0
+);`,
+ // The alert watcher's memory (alertwatch.go): the firing state the
+ // console last told operators about, per (box, rule). A row's absence
+ // means "never seen", which is why AlertSeen reports known separately
+ // — a first sighting of a firing rule is news, a first sighting of a
+ // quiet one is not.
+ `CREATE TABLE IF NOT EXISTS alert_seen (
+ box TEXT NOT NULL,
+ rule TEXT NOT NULL,
+ firing INTEGER NOT NULL,
+ PRIMARY KEY (box, rule)
);`,
} {
if _, err := s.db.Exec(stmt); err != nil {
@@ -1064,6 +1075,58 @@ func (s *Store) IsOperator(address string) (bool, error) {
return v != 0, nil
}
+// Operators lists every address holding the fleet-operator bit, sorted —
+// the alert watcher's recipient list. Sorted so a pass mails people in a
+// stable order and a test can assert on it.
+func (s *Store) Operators() ([]string, error) {
+ rows, err := s.db.Query(`SELECT address FROM users WHERE operator != 0 ORDER BY address`)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+ var out []string
+ for rows.Next() {
+ var a string
+ if err := rows.Scan(&a); err != nil {
+ return nil, err
+ }
+ out = append(out, a)
+ }
+ return out, rows.Err()
+}
+
+// ---- alert watcher state (alertwatch.go) ------------------------------------
+
+// AlertSeen reports the firing state the console last NOTIFIED operators
+// about for (box, rule), and whether it has ever seen that rule at all.
+// known=false is not the same as firing=false: a rule the console has never
+// seen fires as news, while one it has seen quiet stays quiet.
+func (s *Store) AlertSeen(box, rule string) (firing bool, known bool, err error) {
+ var v int
+ err = s.db.QueryRow(`SELECT firing FROM alert_seen WHERE box = ? AND rule = ?`, box, rule).Scan(&v)
+ if err == sql.ErrNoRows {
+ return false, false, nil
+ }
+ if err != nil {
+ return false, false, err
+ }
+ return v != 0, true, nil
+}
+
+// SetAlertSeen records that operators have been told (box, rule) is in this
+// state. Written only AFTER every send for a transition succeeded, so a
+// delivery failure leaves the transition pending and the next pass retries
+// it — a duplicate page beats a missed one.
+func (s *Store) SetAlertSeen(box, rule string, firing bool) error {
+ v := 0
+ if firing {
+ v = 1
+ }
+ _, err := s.db.Exec(`INSERT INTO alert_seen (box, rule, firing) VALUES (?, ?, ?)
+ON CONFLICT(box, rule) DO UPDATE SET firing = excluded.firing`, box, rule, v)
+ return err
+}
+
// ---- grants (the console's Keymail access tokens) ---------------------------
// SetGrantToken stores the Keymail access token the user's OAuth approval
diff --git a/internal/console/store_test.go b/internal/console/store_test.go
index 54b788bd4..c59908ebe 100644
--- a/internal/console/store_test.go
+++ b/internal/console/store_test.go
@@ -330,6 +330,67 @@ func TestOperatorMigrationIdempotent(t *testing.T) {
}
}
+// TestOperatorsList: the alert watcher's recipient list is exactly the
+// addresses holding the operator bit, sorted, and revoking removes you.
+func TestOperatorsList(t *testing.T) {
+ s := openTestStore(t)
+ if ops, err := s.Operators(); err != nil || len(ops) != 0 {
+ t.Fatalf("Operators on an empty store = %v %v, want none", ops, err)
+ }
+ for _, a := range []string{"zoe@keymail.dev", "amy@keymail.dev"} {
+ if err := s.SetOperator(a, true); err != nil {
+ t.Fatal(err)
+ }
+ }
+ // A plain user (never an operator) is not on the list.
+ if _, err := s.EnsureUser("bob@keymail.dev", time.Now()); err != nil {
+ t.Fatal(err)
+ }
+ ops, err := s.Operators()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !reflect.DeepEqual(ops, []string{"amy@keymail.dev", "zoe@keymail.dev"}) {
+ t.Fatalf("Operators = %v", ops)
+ }
+ if err := s.SetOperator("zoe@keymail.dev", false); err != nil {
+ t.Fatal(err)
+ }
+ if ops, _ := s.Operators(); !reflect.DeepEqual(ops, []string{"amy@keymail.dev"}) {
+ t.Fatalf("Operators after revoke = %v", ops)
+ }
+}
+
+// TestAlertSeenRoundTrip: the watcher's memory. "Never seen" and "seen, not
+// firing" are different answers — the first is news on a fire, the second is
+// what makes the second pass quiet.
+func TestAlertSeenRoundTrip(t *testing.T) {
+ s := openTestStore(t)
+ if firing, known, err := s.AlertSeen("box-a", "disk"); err != nil || known || firing {
+ t.Fatalf("AlertSeen before anything = %v %v %v, want false false nil", firing, known, err)
+ }
+ if err := s.SetAlertSeen("box-a", "disk", true); err != nil {
+ t.Fatal(err)
+ }
+ if firing, known, err := s.AlertSeen("box-a", "disk"); err != nil || !known || !firing {
+ t.Fatalf("AlertSeen after fire = %v %v %v, want true true nil", firing, known, err)
+ }
+ // Same (box, rule) upserts rather than duplicating.
+ if err := s.SetAlertSeen("box-a", "disk", false); err != nil {
+ t.Fatal(err)
+ }
+ if firing, known, err := s.AlertSeen("box-a", "disk"); err != nil || !known || firing {
+ t.Fatalf("AlertSeen after clear = %v %v %v, want false true nil", firing, known, err)
+ }
+ // Rules and boxes are independent keys.
+ if _, known, _ := s.AlertSeen("box-a", "mem"); known {
+ t.Fatal("AlertSeen(box-a, mem) is known — the rule key is not being used")
+ }
+ if _, known, _ := s.AlertSeen("box-b", "disk"); known {
+ t.Fatal("AlertSeen(box-b, disk) is known — the box key is not being used")
+ }
+}
+
func TestCreateAppClaimOnce(t *testing.T) {
s := openTestStore(t)
now := time.Now()