amadan
console: key alert_seen per recipient so one dead grant pages one operator
All-or-nothing recording meant a single revoked keymail grant re-sent the whole transition to every healthy operator, every minute, for as long as the alert stayed up. The state is now keyed (box, rule, address): each mailable operator is diffed on their own, a success records that operator's row, and a failure leaves only theirs unrecorded — so the next pass retries exactly the operator who missed out. The old two-column table exists only on unreleased dev consoles, so the migration is replaced in place rather than chained. When no operator holds a grant at all there is nothing to send, and the transition is recorded against the synthetic empty address so it stops replaying and re-logging forever. That drops the alert: an operator who gains a grant afterwards is not caught up on it, because an unknown address falls back to the synthetic row. That is a real hole, so it is named in the comments and asserted in a test rather than left to be discovered.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
4 files changed,
+257
−93
internal/console/alertwatch.go+92 −41internal/console/alertwatch_test.go+105 −19internal/console/store.go+33 −21internal/console/store_test.go+27 −12
diff --git a/internal/console/alertwatch.go b/internal/console/alertwatch.go| index 70df565d8..e0023ebad 100644 |
| --- a/internal/console/alertwatch.go |
| +++ b/internal/console/alertwatch.go |
| @@ -17,14 +17,20 @@ import ( |
| // OAuth grant (Keymail.SendApp) and the agent has no keymail identity. The |
| // box judges itself and publishes alerts/boxes/<box>.json; this watcher |
| // polls that prefix, diffs each rule's firing state against what it last |
| -// told operators (the alert_seen table), and mails the difference — each |
| -// operator with that operator's OWN grant token, so nobody is sent mail on |
| -// a credential they did not personally issue. |
| +// told each operator (the alert_seen table), and mails the difference — |
| +// each operator with that operator's OWN grant token, so nobody is sent |
| +// mail on a credential they did not personally issue. |
| +// |
| +// The diff is per RECIPIENT, not per rule, because delivery fails per |
| +// recipient: a success records that operator's row and a failure leaves |
| +// only theirs unrecorded, so the next pass retries exactly the operator |
| +// who missed out. One operator's revoked grant can no longer re-page every |
| +// healthy operator on the fleet, tick after tick. |
| // |
| // The failure posture throughout is "never go silent": an unreadable or |
| -// unparsable alert object is logged and skipped, a send failure leaves the |
| -// transition unrecorded so the next pass retries the whole thing, and no |
| -// path here can panic a console that is otherwise serving fine. |
| +// unparsable alert object is logged and skipped, an undelivered transition |
| +// stays pending for that operator until it lands, and no path here can |
| +// panic a console that is otherwise serving fine. |
| // alertWatchInterval is the poll cadence from the spec: a minute is fast |
| // enough for a disk filling up and slow enough to be free. |
| @@ -69,20 +75,27 @@ type alertRun struct { |
| noted bool |
| } |
| -// recipients returns the mailable operators, logging the token-less ones the |
| -// first time a pass actually needs to send. |
| -func (r *alertRun) recipients() []operatorGrant { |
| - if !r.noted && len(r.missing) > 0 { |
| - r.noted = true |
| - log.Printf("console: alert watch: no keymail grant for %s — not notified (they must sign in to the console once)", |
| - strings.Join(r.missing, ", ")) |
| +// noteMissing names the operators the console cannot write as, at most once |
| +// per pass, called only when the pass actually has something to deliver. |
| +func (r *alertRun) noteMissing() { |
| + if r.noted || len(r.missing) == 0 { |
| + return |
| } |
| - return r.to |
| + r.noted = true |
| + log.Printf("console: alert watch: no keymail grant for %s — not notified (they must sign in to the console once)", |
| + strings.Join(r.missing, ", ")) |
| } |
| -// alertPass reads every alerts/boxes/*.json, diffs each rule's firing state |
| -// against alert_seen, and for each transition mails every operator holding a |
| -// grant and records the new state. It returns an error only for a failure |
| +// noMailboxAddress is the synthetic alert_seen recipient standing for "no |
| +// operator was mailable when this transition happened". Recording it stops |
| +// the transition replaying — and being re-logged — every minute forever, |
| +// at the honest cost that the alert is simply lost: an operator who gains a |
| +// grant later is not caught up on it (see alertSeenFor). |
| +const noMailboxAddress = "" |
| + |
| +// alertPass reads every alerts/boxes/*.json and, for each rule, diffs its |
| +// firing state against what alert_seen says each operator was last told, |
| +// mailing and recording per recipient. It returns an error only for a failure |
| // that makes the whole pass meaningless (listing the bucket, or reading the |
| // operator list); per-box and per-rule trouble is logged and stepped over. |
| func (s *Server) alertPass(ctx context.Context) error { |
| @@ -154,42 +167,80 @@ func (s *Server) alertRecipients() (*alertRun, error) { |
| return run, nil |
| } |
| -// alertRule handles one rule of one box: nothing to do unless its firing |
| -// state differs from what operators were last told. |
| +// alertRule handles one rule of one box: every mailable operator whose |
| +// recorded state differs from the published one gets a keymail, and each |
| +// send is recorded on its own. |
| func (s *Server) alertRule(ctx context.Context, run *alertRun, box, rule string, rs metrics.RuleState, updated time.Time) { |
| - seen, known, err := s.store.AlertSeen(box, rule) |
| - if err != nil { |
| - log.Printf("console: alert watch: alert_seen %s/%s: %v", box, rule, err) |
| - return |
| - } |
| - if known && seen == rs.Firing { |
| + if len(run.to) == 0 { |
| + // Nobody the console can write as. Record the transition against the |
| + // synthetic recipient so it stops replaying, and say so out loud — |
| + // this is the one path where an alert is genuinely dropped. |
| + if s.alertDue(box, rule, noMailboxAddress, rs.Firing) { |
| + run.noteMissing() |
| + log.Printf("console: alert watch: %s/%s transition dropped — no operator holds a keymail grant", box, rule) |
| + s.recordAlertSeen(box, rule, noMailboxAddress, rs.Firing) |
| + } |
| return |
| } |
| - if !known && !rs.Firing { |
| - // First sighting of a quiet rule. Record it so a later fire is a |
| - // transition; say nothing — "everything is fine" is not a page. |
| - if err := s.store.SetAlertSeen(box, rule, false); err != nil { |
| - log.Printf("console: alert watch: record %s/%s: %v", box, rule, err) |
| + var due []operatorGrant |
| + for _, op := range run.to { |
| + if s.alertDue(box, rule, op.address, rs.Firing) { |
| + due = append(due, op) |
| } |
| + } |
| + if len(due) == 0 { |
| return |
| } |
| + run.noteMissing() |
| subject, body := alertMessage(box, rule, rs, updated) |
| - delivered := true |
| - for _, op := range run.recipients() { |
| + for _, op := range due { |
| if _, err := s.sendApp(ctx, op.token, op.address, subject, body); err != nil { |
| - // The address is safe to log; the token never is. |
| + // The address is safe to log; the token never is. Leaving this |
| + // operator's row unrecorded is the retry: the next pass mails |
| + // them and nobody else. |
| log.Printf("console: alert watch: keymail %s/%s to %s: %v", box, rule, op.address, err) |
| - delivered = false |
| + continue |
| } |
| + s.recordAlertSeen(box, rule, op.address, rs.Firing) |
| } |
| - if !delivered { |
| - // Leave the transition unrecorded: the next pass retries the whole |
| - // thing, which may re-mail an operator who did get it. A duplicate |
| - // page is cheap; a missed one is what this system exists to prevent. |
| - return |
| +} |
| + |
| +// alertDue reports whether address must be told that (box, rule) is now in |
| +// this firing state, recording a quiet first sighting as it goes — nobody |
| +// wants mail saying the disk is fine. |
| +// |
| +// An address the console has never written to about this rule falls back to |
| +// the synthetic no-mailbox row: if the transition was already dropped for |
| +// want of any grant at all, an operator who gains a grant afterwards is not |
| +// retroactively paged about it. |
| +func (s *Server) alertDue(box, rule, address string, firing bool) bool { |
| + seen, known, err := s.store.AlertSeen(box, rule, address) |
| + if err != nil { |
| + log.Printf("console: alert watch: alert_seen %s/%s for %s: %v", box, rule, address, err) |
| + return false |
| + } |
| + if !known && address != noMailboxAddress { |
| + seen, known, err = s.store.AlertSeen(box, rule, noMailboxAddress) |
| + if err != nil { |
| + log.Printf("console: alert watch: alert_seen %s/%s: %v", box, rule, err) |
| + return false |
| + } |
| } |
| - if err := s.store.SetAlertSeen(box, rule, rs.Firing); err != nil { |
| - log.Printf("console: alert watch: record %s/%s: %v", box, rule, err) |
| + if known && seen == firing { |
| + return false |
| + } |
| + if !known && !firing { |
| + // First sighting of a quiet rule. Record it so a later fire is a |
| + // transition; say nothing. |
| + s.recordAlertSeen(box, rule, address, false) |
| + return false |
| + } |
| + return true |
| +} |
| + |
| +func (s *Server) recordAlertSeen(box, rule, address string, firing bool) { |
| + if err := s.store.SetAlertSeen(box, rule, address, firing); err != nil { |
| + log.Printf("console: alert watch: record %s/%s for %s: %v", box, rule, address, err) |
| } |
| } |
diff --git a/internal/console/alertwatch_test.go b/internal/console/alertwatch_test.go| index 02ad99d71..75ac99fa2 100644 |
| --- a/internal/console/alertwatch_test.go |
| +++ b/internal/console/alertwatch_test.go |
| @@ -17,30 +17,25 @@ type sentMail struct { |
| token, to, subject, body string |
| } |
| -// alertFixture is a console fixture with two operators — one holding a |
| -// keymail grant, one not — and a captured sendApp. |
| +// alertFixture is a console fixture with a captured sendApp. err makes |
| +// sends fail; failFor narrows the failure to one recipient, which is how |
| +// the per-recipient retry is pinned. |
| type alertFixture struct { |
| *consoleFixture |
| - sent []sentMail |
| - err error // what the captured sendApp returns |
| + sent []sentMail |
| + err error |
| + failFor string |
| } |
| -func newAlertFixture(t *testing.T) *alertFixture { |
| +// newCapturedFixture wires the capture and nothing else — no operators, so |
| +// a test can build its own recipient shape. |
| +func newCapturedFixture(t *testing.T) *alertFixture { |
| t.Helper() |
| fx := newConsoleFixture(t) |
| af := &alertFixture{consoleFixture: fx} |
| - for _, addr := range []string{"ops-a@keymail.dev", "ops-b@keymail.dev"} { |
| - if err := fx.st.SetOperator(addr, true); err != nil { |
| - t.Fatal(err) |
| - } |
| - } |
| - // Only ops-a ever granted the console send access. |
| - if err := fx.st.SetGrantToken("ops-a@keymail.dev", "grant-a", time.Now()); err != nil { |
| - t.Fatal(err) |
| - } |
| fx.srv.sendApp = func(_ context.Context, token, to, subject, body string) (string, error) { |
| af.sent = append(af.sent, sentMail{token, to, subject, body}) |
| - if af.err != nil { |
| + if af.err != nil && (af.failFor == "" || af.failFor == to) { |
| return "", af.err |
| } |
| return "msg-1", nil |
| @@ -48,6 +43,22 @@ func newAlertFixture(t *testing.T) *alertFixture { |
| return af |
| } |
| +// newAlertFixture is the common shape: two operators, only one of whom ever |
| +// granted the console send access. |
| +func newAlertFixture(t *testing.T) *alertFixture { |
| + t.Helper() |
| + af := newCapturedFixture(t) |
| + for _, addr := range []string{"ops-a@keymail.dev", "ops-b@keymail.dev"} { |
| + if err := af.st.SetOperator(addr, true); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + if err := af.st.SetGrantToken("ops-a@keymail.dev", "grant-a", time.Now()); err != nil { |
| + t.Fatal(err) |
| + } |
| + return af |
| +} |
| + |
| func (af *alertFixture) writeAlert(t *testing.T, st metrics.AlertState) { |
| t.Helper() |
| b, err := json.Marshal(st) |
| @@ -118,7 +129,7 @@ func TestAlertPassFiresOnceThenGoesQuiet(t *testing.T) { |
| } |
| // Recorded: the same state on the next pass says nothing. |
| - firing, known, err := af.st.AlertSeen("box-a", "disk") |
| + firing, known, err := af.st.AlertSeen("box-a", "disk", "ops-a@keymail.dev") |
| if err != nil || !known || !firing { |
| t.Fatalf("AlertSeen = %v,%v,%v — want true,true,nil", firing, known, err) |
| } |
| @@ -161,7 +172,7 @@ func TestAlertPassRetriesAfterSendFailure(t *testing.T) { |
| if len(af.sent) != 1 { |
| t.Fatalf("failing pass attempted %d sends, want 1", len(af.sent)) |
| } |
| - if _, known, err := af.st.AlertSeen("box-a", "disk"); err != nil || known { |
| + if _, known, err := af.st.AlertSeen("box-a", "disk", "ops-a@keymail.dev"); err != nil || known { |
| t.Fatalf("AlertSeen recorded despite a send failure (known=%v, err=%v)", known, err) |
| } |
| @@ -170,11 +181,86 @@ func TestAlertPassRetriesAfterSendFailure(t *testing.T) { |
| if len(af.sent) != 1 { |
| t.Fatalf("retry pass sent %d keymails, want 1", len(af.sent)) |
| } |
| - if firing, known, err := af.st.AlertSeen("box-a", "disk"); err != nil || !known || !firing { |
| + if firing, known, err := af.st.AlertSeen("box-a", "disk", "ops-a@keymail.dev"); err != nil || !known || !firing { |
| t.Fatalf("AlertSeen = %v,%v,%v after a successful retry", firing, known, err) |
| } |
| } |
| +// TestAlertPassRetriesOnlyTheFailedOperator is why alert_seen is keyed per |
| +// recipient: one operator's dead grant must not re-page every healthy |
| +// operator on the fleet, tick after tick. |
| +func TestAlertPassRetriesOnlyTheFailedOperator(t *testing.T) { |
| + af := newAlertFixture(t) |
| + if err := af.st.SetGrantToken("ops-b@keymail.dev", "grant-b", time.Now()); err != nil { |
| + t.Fatal(err) |
| + } |
| + af.writeAlert(t, firingDisk(true)) |
| + af.err = context.DeadlineExceeded |
| + af.failFor = "ops-b@keymail.dev" |
| + |
| + af.pass(t) |
| + if len(af.sent) != 2 { |
| + t.Fatalf("first pass attempted %d sends, want 2: %+v", len(af.sent), af.sent) |
| + } |
| + if firing, known, err := af.st.AlertSeen("box-a", "disk", "ops-a@keymail.dev"); err != nil || !known || !firing { |
| + t.Fatalf("ops-a AlertSeen = %v %v %v — a healthy delivery was not recorded", firing, known, err) |
| + } |
| + if _, known, err := af.st.AlertSeen("box-a", "disk", "ops-b@keymail.dev"); err != nil || known { |
| + t.Fatalf("ops-b AlertSeen recorded despite a failed send (known=%v, err=%v)", known, err) |
| + } |
| + |
| + af.err = nil |
| + af.pass(t) |
| + if len(af.sent) != 1 || af.sent[0].to != "ops-b@keymail.dev" { |
| + t.Fatalf("retry pass sent %+v, want exactly one mail to ops-b", af.sent) |
| + } |
| + if af.sent[0].token != "grant-b" { |
| + t.Fatalf("retry used token %q, want ops-b's own grant", af.sent[0].token) |
| + } |
| + af.pass(t) |
| + if len(af.sent) != 0 { |
| + t.Fatalf("pass after the retry sent %d keymails, want 0", len(af.sent)) |
| + } |
| +} |
| + |
| +// TestAlertPassNoMailableOperator pins the one path where an alert is |
| +// genuinely dropped: nobody holds a grant, so there is nothing to send and |
| +// the transition is recorded against the synthetic recipient so it does not |
| +// replay every minute forever. The honest cost — documented, and asserted |
| +// here rather than left to be discovered — is that an operator who gains a |
| +// grant afterwards is never told about that transition. |
| +func TestAlertPassNoMailableOperator(t *testing.T) { |
| + af := newCapturedFixture(t) |
| + if err := af.st.SetOperator("ops-b@keymail.dev", true); err != nil { |
| + t.Fatal(err) |
| + } |
| + af.writeAlert(t, firingDisk(true)) |
| + |
| + af.pass(t) |
| + if len(af.sent) != 0 { |
| + t.Fatalf("sent %d keymails with no grant anywhere: %+v", len(af.sent), af.sent) |
| + } |
| + if firing, known, err := af.st.AlertSeen("box-a", "disk", ""); err != nil || !known || !firing { |
| + t.Fatalf("synthetic AlertSeen = %v %v %v, want true true nil — the drop was not recorded", firing, known, err) |
| + } |
| + |
| + // The grant arrives after the fact. The alert is still firing, but the |
| + // console has already recorded the transition as handled: no catch-up. |
| + if err := af.st.SetGrantToken("ops-b@keymail.dev", "grant-b", time.Now()); err != nil { |
| + t.Fatal(err) |
| + } |
| + af.pass(t) |
| + if len(af.sent) != 0 { |
| + t.Fatalf("a newly-granted operator was retroactively paged: %+v", af.sent) |
| + } |
| + // But the NEXT transition reaches them normally. |
| + af.writeAlert(t, firingDisk(false)) |
| + af.pass(t) |
| + if len(af.sent) != 1 || af.sent[0].to != "ops-b@keymail.dev" { |
| + t.Fatalf("clear after the grant arrived sent %+v, want one mail to ops-b", af.sent) |
| + } |
| +} |
| + |
| // TestAlertPassQuietFirstSighting: a rule the console has never seen that is |
| // NOT firing is recorded silently. Nobody wants a keymail saying the disk is |
| // fine. |
| @@ -186,7 +272,7 @@ func TestAlertPassQuietFirstSighting(t *testing.T) { |
| if len(af.sent) != 0 { |
| t.Fatalf("quiet first sighting sent %d keymails, want 0: %+v", len(af.sent), af.sent) |
| } |
| - if firing, known, err := af.st.AlertSeen("box-a", "disk"); err != nil || !known || firing { |
| + if firing, known, err := af.st.AlertSeen("box-a", "disk", "ops-a@keymail.dev"); err != nil || !known || firing { |
| t.Fatalf("AlertSeen = %v,%v,%v — want false,true,nil", firing, known, err) |
| } |
| } |
diff --git a/internal/console/store.go b/internal/console/store.go| index 07e471c65..301e154e0 100644 |
| --- a/internal/console/store.go |
| +++ b/internal/console/store.go |
| @@ -162,15 +162,22 @@ func (s *Store) migrate() error { |
| accepted_at INTEGER NOT NULL DEFAULT 0 |
| );`, |
| // The alert watcher's memory (alertwatch.go): the firing state the |
| - // console last told operators about, per (box, rule). A row's absence |
| - // means "never seen", which is why AlertSeen reports known separately |
| - // — a first sighting of a firing rule is news, a first sighting of a |
| - // quiet one is not. |
| + // console last told ONE operator about, per (box, rule, address). A |
| + // row's absence means "never told them", which is why AlertSeen |
| + // reports known separately — a first sighting of a firing rule is |
| + // news, a first sighting of a quiet one is not. |
| + // |
| + // Per-recipient rather than per-rule because delivery fails per |
| + // recipient: one operator's revoked keymail grant must not re-page |
| + // every healthy operator on the next pass. The empty address is a |
| + // synthetic row meaning "nobody was mailable when this transition |
| + // happened" (see alertwatch.go). |
| `CREATE TABLE IF NOT EXISTS alert_seen ( |
| - box TEXT NOT NULL, |
| - rule TEXT NOT NULL, |
| - firing INTEGER NOT NULL, |
| - PRIMARY KEY (box, rule) |
| + box TEXT NOT NULL, |
| + rule TEXT NOT NULL, |
| + address TEXT NOT NULL, |
| + firing INTEGER NOT NULL, |
| + PRIMARY KEY (box, rule, address) |
| );`, |
| } { |
| if _, err := s.db.Exec(stmt); err != nil { |
| @@ -1097,13 +1104,18 @@ func (s *Store) Operators() ([]string, error) { |
| // ---- alert watcher state (alertwatch.go) ------------------------------------ |
| -// AlertSeen reports the firing state the console last NOTIFIED operators |
| -// about for (box, rule), and whether it has ever seen that rule at all. |
| -// known=false is not the same as firing=false: a rule the console has never |
| -// seen fires as news, while one it has seen quiet stays quiet. |
| -func (s *Store) AlertSeen(box, rule string) (firing bool, known bool, err error) { |
| +// AlertSeen reports the firing state the console last NOTIFIED address |
| +// about for (box, rule), and whether it has ever told them anything about |
| +// that rule. known=false is not the same as firing=false: a rule this |
| +// operator has never been told about fires as news, while one they were |
| +// last told was quiet stays quiet. |
| +// |
| +// The empty address is the watcher's synthetic "nobody was mailable" |
| +// recipient (alertwatch.go), not a real user. |
| +func (s *Store) AlertSeen(box, rule, address string) (firing bool, known bool, err error) { |
| var v int |
| - err = s.db.QueryRow(`SELECT firing FROM alert_seen WHERE box = ? AND rule = ?`, box, rule).Scan(&v) |
| + err = s.db.QueryRow(`SELECT firing FROM alert_seen WHERE box = ? AND rule = ? AND address = ?`, |
| + box, rule, address).Scan(&v) |
| if err == sql.ErrNoRows { |
| return false, false, nil |
| } |
| @@ -1113,17 +1125,17 @@ func (s *Store) AlertSeen(box, rule string) (firing bool, known bool, err error) |
| return v != 0, true, nil |
| } |
| -// SetAlertSeen records that operators have been told (box, rule) is in this |
| -// state. Written only AFTER every send for a transition succeeded, so a |
| -// delivery failure leaves the transition pending and the next pass retries |
| -// it — a duplicate page beats a missed one. |
| -func (s *Store) SetAlertSeen(box, rule string, firing bool) error { |
| +// SetAlertSeen records that address has been told (box, rule) is in this |
| +// state. Written per recipient, only after THAT recipient's send succeeded, |
| +// so one dead grant leaves one row unrecorded and the next pass retries |
| +// that operator alone. |
| +func (s *Store) SetAlertSeen(box, rule, address string, firing bool) error { |
| v := 0 |
| if firing { |
| v = 1 |
| } |
| - _, err := s.db.Exec(`INSERT INTO alert_seen (box, rule, firing) VALUES (?, ?, ?) |
| -ON CONFLICT(box, rule) DO UPDATE SET firing = excluded.firing`, box, rule, v) |
| + _, err := s.db.Exec(`INSERT INTO alert_seen (box, rule, address, firing) VALUES (?, ?, ?, ?) |
| +ON CONFLICT(box, rule, address) DO UPDATE SET firing = excluded.firing`, box, rule, address, v) |
| return err |
| } |
diff --git a/internal/console/store_test.go b/internal/console/store_test.go| index c59908ebe..4becf8bfd 100644 |
| --- a/internal/console/store_test.go |
| +++ b/internal/console/store_test.go |
| @@ -361,34 +361,49 @@ func TestOperatorsList(t *testing.T) { |
| } |
| } |
| -// TestAlertSeenRoundTrip: the watcher's memory. "Never seen" and "seen, not |
| -// firing" are different answers — the first is news on a fire, the second is |
| -// what makes the second pass quiet. |
| +// TestAlertSeenRoundTrip: the watcher's memory, one row per recipient. |
| +// "Never told them" and "told them it was quiet" are different answers — |
| +// the first is news on a fire, the second is what makes the second pass |
| +// quiet — and one operator's row must never answer for another's. |
| func TestAlertSeenRoundTrip(t *testing.T) { |
| s := openTestStore(t) |
| - if firing, known, err := s.AlertSeen("box-a", "disk"); err != nil || known || firing { |
| + const amy, zoe = "amy@keymail.dev", "zoe@keymail.dev" |
| + if firing, known, err := s.AlertSeen("box-a", "disk", amy); err != nil || known || firing { |
| t.Fatalf("AlertSeen before anything = %v %v %v, want false false nil", firing, known, err) |
| } |
| - if err := s.SetAlertSeen("box-a", "disk", true); err != nil { |
| + if err := s.SetAlertSeen("box-a", "disk", amy, true); err != nil { |
| t.Fatal(err) |
| } |
| - if firing, known, err := s.AlertSeen("box-a", "disk"); err != nil || !known || !firing { |
| + if firing, known, err := s.AlertSeen("box-a", "disk", amy); err != nil || !known || !firing { |
| t.Fatalf("AlertSeen after fire = %v %v %v, want true true nil", firing, known, err) |
| } |
| - // Same (box, rule) upserts rather than duplicating. |
| - if err := s.SetAlertSeen("box-a", "disk", false); err != nil { |
| + // Same (box, rule, address) upserts rather than duplicating. |
| + if err := s.SetAlertSeen("box-a", "disk", amy, false); err != nil { |
| t.Fatal(err) |
| } |
| - if firing, known, err := s.AlertSeen("box-a", "disk"); err != nil || !known || firing { |
| + if firing, known, err := s.AlertSeen("box-a", "disk", amy); err != nil || !known || firing { |
| t.Fatalf("AlertSeen after clear = %v %v %v, want false true nil", firing, known, err) |
| } |
| - // Rules and boxes are independent keys. |
| - if _, known, _ := s.AlertSeen("box-a", "mem"); known { |
| + // Rules, boxes and addresses are all independent parts of the key. |
| + if _, known, _ := s.AlertSeen("box-a", "mem", amy); known { |
| t.Fatal("AlertSeen(box-a, mem) is known — the rule key is not being used") |
| } |
| - if _, known, _ := s.AlertSeen("box-b", "disk"); known { |
| + if _, known, _ := s.AlertSeen("box-b", "disk", amy); known { |
| t.Fatal("AlertSeen(box-b, disk) is known — the box key is not being used") |
| } |
| + if _, known, _ := s.AlertSeen("box-a", "disk", zoe); known { |
| + t.Fatal("AlertSeen(box-a, disk, zoe) is known — amy's row is answering for zoe") |
| + } |
| + // The synthetic "nobody was mailable" recipient is its own row. |
| + if err := s.SetAlertSeen("box-a", "disk", "", true); err != nil { |
| + t.Fatal(err) |
| + } |
| + if firing, known, err := s.AlertSeen("box-a", "disk", ""); err != nil || !known || !firing { |
| + t.Fatalf("AlertSeen for the empty address = %v %v %v, want true true nil", firing, known, err) |
| + } |
| + if firing, _, _ := s.AlertSeen("box-a", "disk", amy); firing { |
| + t.Fatal("the synthetic row overwrote amy's") |
| + } |
| } |
| func TestCreateAppClaimOnce(t *testing.T) { |