amadan

carlos / platform Public

console: mail_sender_counts, its one-shot backfill and the legacy drop (plan 2, task 8)

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Paul Campbell pushed by claude 8099af1002e5e9b6ce6e60091624773b38216d15 parent 5e8e319
3 files changed, +137 −0
  • cmd/console/main.go +3 −0
  • internal/console/mailcounts_test.go +66 −0
  • internal/console/store.go +68 −0
diff --git a/cmd/console/main.go b/cmd/console/main.go
index 566f06565..f8716ab43 100644
--- a/cmd/console/main.go
+++ b/cmd/console/main.go
@@ -350,6 +350,9 @@ func serve() {
return
}
log.Printf("console: mail legacy sweep -> %d email.json objects removed", n)
+ if err := st.DropLegacyMailCounts(true); err != nil {
+ log.Printf("console: mail legacy sweep: dropping mail_counts -> %v", err)
+ }
}()
}
// The economics pass writes the daily usage/census/storage/statement
diff --git a/internal/console/mailcounts_test.go b/internal/console/mailcounts_test.go
index f514b4647..cf4d11970 100644
--- a/internal/console/mailcounts_test.go
+++ b/internal/console/mailcounts_test.go
@@ -2,6 +2,7 @@ package console
import (
"context"
+ "path/filepath"
"strings"
"testing"
"time"
@@ -456,3 +457,68 @@ func TestResumeStaleCapPauseNeverTouchesAKindlessPause(t *testing.T) {
t.Fatal("a kind-less pause from yesterday was lifted")
}
}
+
+// SQLite cannot widen a primary key, so per-sender counts are a new table,
+// backfilled once from mail_counts with sender = 'app'. The backfill runs at
+// most once for the life of the database: a second migrate must not
+// resurrect rows a purge or a rename has since removed.
+func TestMailSenderCountsAreBackfilledExactlyOnce(t *testing.T) {
+ dir := t.TempDir()
+ path := filepath.Join(dir, "console.db")
+ st, err := Open(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := st.db.Exec(`INSERT INTO mail_counts (account, app, day, sends, bounces, complaints, rejects, deliveries)
+VALUES ('bag', 'harbourmail', '2026-09-14', 7, 1, 0, 0, 6)`); err != nil {
+ t.Fatal(err)
+ }
+ // Force the backfill to run again by winding the marker back, which is
+ // what a console rolled back and forward would look like.
+ if _, err := st.db.Exec(`PRAGMA user_version = 0`); err != nil {
+ t.Fatal(err)
+ }
+ if err := st.migrate(); err != nil {
+ t.Fatal(err)
+ }
+ var sends, deliveries int
+ if err := st.db.QueryRow(`SELECT sends, deliveries FROM mail_sender_counts
+WHERE account = 'bag' AND app = 'harbourmail' AND sender = 'app' AND day = '2026-09-14'`).Scan(&sends, &deliveries); err != nil {
+ t.Fatalf("the backfill did not land: %v", err)
+ }
+ if sends != 7 || deliveries != 6 {
+ t.Fatalf("backfilled %d sends and %d deliveries, want 7 and 6", sends, deliveries)
+ }
+
+ // A row removed after the backfill stays removed across a restart.
+ if _, err := st.db.Exec(`DELETE FROM mail_sender_counts WHERE account = 'bag'`); err != nil {
+ t.Fatal(err)
+ }
+ if err := st.migrate(); err != nil {
+ t.Fatal(err)
+ }
+ var n int
+ if err := st.db.QueryRow(`SELECT count(*) FROM mail_sender_counts WHERE account = 'bag'`).Scan(&n); err != nil {
+ t.Fatal(err)
+ }
+ if n != 0 {
+ t.Fatalf("the backfill resurrected %d deleted rows", n)
+ }
+
+ // The legacy table is dropped only when an operator asks, a release later.
+ if err := st.DropLegacyMailCounts(false); err != nil {
+ t.Fatal(err)
+ }
+ if err := st.db.QueryRow(`SELECT count(*) FROM mail_counts`).Scan(&n); err != nil {
+ t.Fatalf("mail_counts was dropped while the flag was off: %v", err)
+ }
+ if err := st.DropLegacyMailCounts(true); err != nil {
+ t.Fatal(err)
+ }
+ if err := st.db.QueryRow(`SELECT count(*) FROM mail_counts`).Scan(&n); err == nil {
+ t.Fatal("mail_counts survived its own drop")
+ }
+ if err := st.Close(); err != nil {
+ t.Fatal(err)
+ }
+}
diff --git a/internal/console/store.go b/internal/console/store.go
index eca0917ce..0b4f3f3c3 100644
--- a/internal/console/store.go
+++ b/internal/console/store.go
@@ -206,6 +206,27 @@ func (s *Store) migrate() error {
complaints INTEGER NOT NULL DEFAULT 0,
rejects INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (account, app, day)
+);`,
+ // Per-SENDER counts (senders design §8). A new table rather than a
+ // widened key, because SQLite cannot ALTER a primary key. Every count
+ // is written here from now on; mail_counts stays, unwritten, and is
+ // dropped a release later by DropLegacyMailCounts.
+ //
+ // day is a UTC date string, not a timestamp, exactly as mail_counts'
+ // is — which is why this table cannot join mailAppTables blindly: the
+ // rename, purge and reap sweeps key on mailAppTimeColumn and there is
+ // no such column here.
+ `CREATE TABLE IF NOT EXISTS mail_sender_counts (
+ account TEXT NOT NULL,
+ app TEXT NOT NULL,
+ sender TEXT NOT NULL,
+ day TEXT NOT NULL, -- UTC YYYY-MM-DD
+ sends INTEGER NOT NULL DEFAULT 0,
+ bounces INTEGER NOT NULL DEFAULT 0,
+ complaints INTEGER NOT NULL DEFAULT 0,
+ rejects INTEGER NOT NULL DEFAULT 0,
+ deliveries INTEGER NOT NULL DEFAULT 0,
+ PRIMARY KEY (account, app, sender, day)
);`,
// What happened to each message, per (account, app) — one row per
// recipient per SES notification (mailevents.go). The counts above
@@ -368,6 +389,16 @@ func (s *Store) migrate() error {
return err
}
}
+ // The one-shot backfill of mail_sender_counts from mail_counts (senders
+ // design §8). It runs AFTER the ALTER list because mail_counts.deliveries
+ // is added there, and it runs at most once for the life of the database:
+ // a backfill repeated at every start would resurrect rows a purge or a
+ // rename had since removed, since mail_counts is not swept by anything.
+ // PRAGMA user_version is the marker because the console has no other
+ // place to keep one and this is the first thing that has needed it.
+ if err := s.backfillMailSenderCounts(); err != nil {
+ return err
+ }
if err := s.migrateAppsKey(); err != nil {
return err
}
@@ -384,6 +415,43 @@ func (s *Store) migrate() error {
return s.migratePendingChanges()
}
+// mailCountsBackfillVersion is the user_version this migration stamps. The
+// next migration that needs a one-shot marker takes 2 and reads this
+// constant rather than the literal.
+const mailCountsBackfillVersion = 1
+
+func (s *Store) backfillMailSenderCounts() error {
+ var version int
+ if err := s.db.QueryRow(`PRAGMA user_version`).Scan(&version); err != nil {
+ return err
+ }
+ if version >= mailCountsBackfillVersion {
+ return nil
+ }
+ if _, err := s.db.Exec(`INSERT OR IGNORE INTO mail_sender_counts
+ (account, app, sender, day, sends, bounces, complaints, rejects, deliveries)
+SELECT account, app, 'app', day, sends, bounces, complaints, rejects, deliveries FROM mail_counts`); err != nil {
+ return err
+ }
+ // PRAGMA takes no bound parameter, and the value is a constant in this
+ // file rather than anything a caller supplies.
+ _, err := s.db.Exec(fmt.Sprintf(`PRAGMA user_version = %d`, mailCountsBackfillVersion))
+ return err
+}
+
+// DropLegacyMailCounts removes the pre-senders counts table. Guarded by a
+// flag that defaults off and driven by an operator a release after the
+// backfill, for the same reason mail.SweepLegacyRecords is: the rollback
+// window has to close before the old shape goes, and dropping it early would
+// leave a rolled-back console counting into a table that is not there.
+func (s *Store) DropLegacyMailCounts(enabled bool) error {
+ if !enabled {
+ return nil
+ }
+ _, err := s.db.Exec(`DROP TABLE IF EXISTS mail_counts`)
+ return err
+}
+
// migrateAppsKey moves the apps table from PRIMARY KEY(app) — a GLOBAL name
// land-grab — to PRIMARY KEY(account_id, app), which is what "app names are
// unique within an account, never globally" actually means. SQLite cannot