amadan

carlos / platform Public

console: a stalled mail ensure no longer holds sends past their deadline

The mailer's lock is a one-slot channel acquired with the caller's
context, so ready() and Ensure() give up with ErrConsoleMailNotReady
(wrapping ctx.Err()) instead of queueing behind a hung AWS call. The
startup ensure runs under a two-minute deadline.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Paul Campbell pushed by claude f03b9438bd6cfc69eabb61eecafc030c2d39a884 parent f2c3a31
3 files changed, +77 −9
  • cmd/console/main.go +9 −2
  • internal/console/consolemail.go +27 −7
  • internal/console/consolemail_test.go +41 −0
diff --git a/cmd/console/main.go b/cmd/console/main.go
index 53f119a68..f4335a374 100644
--- a/cmd/console/main.go
+++ b/cmd/console/main.go
@@ -388,8 +388,15 @@ func serve() {
// The console mailer's identity ensure (consoleMailer above) — makes or
// re-reads the sending domain's SES/DKIM identity. Started here, not in
// NewServer, same reasoning as StartAlertWatcher: constructing a Server
- // never starts a goroutine or sends mail.
- go ensureConsoleMail(context.Background())
+ // never starts a goroutine or sends mail. Bounded: the AWS SDK has no
+ // overall timeout, and an ensure that never returns would hold the
+ // mailer for good (a send gives up at its own deadline, but every one
+ // would).
+ go func() {
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+ defer cancel()
+ ensureConsoleMail(ctx)
+ }()
// The senders migration's tail (tenant-email senders design §5): once a
// release has passed with every console writing senders/app.json, an
// operator sets CARLOS_MAIL_LEGACY_SWEEP=1 and the next start deletes the
diff --git a/internal/console/consolemail.go b/internal/console/consolemail.go
index b0c309d3b..08984128b 100644
--- a/internal/console/consolemail.go
+++ b/internal/console/consolemail.go
@@ -11,7 +11,6 @@ import (
"fmt"
"log"
"strings"
- "sync"
"time"
"github.com/carlosframework/platform/internal/dnsrec"
@@ -56,7 +55,10 @@ type sesConsoleMailer struct {
ses mail.EmailSender
now func() time.Time
- mu sync.Mutex
+ // lock is a one-slot mutex a waiter can give up on: an ensure stalled
+ // inside an AWS call must not hold a Send past its own deadline.
+ // Everything below it is guarded by it.
+ lock chan struct{}
ensured bool
verified bool
configSet string
@@ -65,7 +67,8 @@ type sesConsoleMailer struct {
func newSESConsoleMailer(from string, s release.Store, p mail.Provisioner, w dnsrec.Writer, d mail.Deployment, ses mail.EmailSender) *sesConsoleMailer {
_, domain, _ := strings.Cut(from, "@")
- return &sesConsoleMailer{from: from, domain: domain, store: s, prov: p, dns: w, dep: d, ses: ses, now: time.Now}
+ return &sesConsoleMailer{from: from, domain: domain, store: s, prov: p, dns: w, dep: d, ses: ses, now: time.Now,
+ lock: make(chan struct{}, 1)}
}
// NewConsoleMailer builds the console's mailer, or nil when this deployment
@@ -86,11 +89,26 @@ func NewConsoleMailer(from string, s release.Store, p mail.Provisioner, w dnsrec
// Ensure makes (or re-reads) the identity. Idempotent. Pending DNS records
// for a custom domain are logged for the operator to publish.
func (m *sesConsoleMailer) Ensure(ctx context.Context) error {
- m.mu.Lock()
- defer m.mu.Unlock()
+ if err := m.acquire(ctx); err != nil {
+ return err
+ }
+ defer m.unlock()
return m.ensureLocked(ctx)
}
+// acquire takes m.lock, or gives up with ErrConsoleMailNotReady when ctx
+// ends first.
+func (m *sesConsoleMailer) acquire(ctx context.Context) error {
+ select {
+ case m.lock <- struct{}{}:
+ return nil
+ case <-ctx.Done():
+ return fmt.Errorf("%w: %w", ErrConsoleMailNotReady, ctx.Err())
+ }
+}
+
+func (m *sesConsoleMailer) unlock() { <-m.lock }
+
func (m *sesConsoleMailer) ensureLocked(ctx context.Context) error {
if m.ensured {
return nil
@@ -108,8 +126,10 @@ func (m *sesConsoleMailer) ensureLocked(ctx context.Context) error {
}
func (m *sesConsoleMailer) ready(ctx context.Context) (string, error) {
- m.mu.Lock()
- defer m.mu.Unlock()
+ if err := m.acquire(ctx); err != nil {
+ return "", err
+ }
+ defer m.unlock()
if m.verified {
return m.configSet, nil
}
diff --git a/internal/console/consolemail_test.go b/internal/console/consolemail_test.go
index 777e1a4f9..d9903aec5 100644
--- a/internal/console/consolemail_test.go
+++ b/internal/console/consolemail_test.go
@@ -235,3 +235,44 @@ func TestReservedAppName(t *testing.T) {
}
}
}
+
+// stallingProvisioner never answers EnsureTenant until released, whatever
+// the context says: the AWS SDK call that hangs.
+type stallingProvisioner struct {
+ *mail.MemProvisioner
+ entered chan struct{}
+ release chan struct{}
+}
+
+func (p *stallingProvisioner) EnsureTenant(context.Context, string) error {
+ close(p.entered)
+ <-p.release
+ return nil
+}
+
+// Final review Important #3: an ensure stalled inside the provider holds the
+// mailer; a Send must still give up at its own deadline, not the ensure's.
+func TestConsoleMailerSendHonoursItsDeadlineWhileEnsureHolds(t *testing.T) {
+ p := &stallingProvisioner{MemProvisioner: mail.NewMemProvisioner("1"),
+ entered: make(chan struct{}), release: make(chan struct{})}
+ m := newSESConsoleMailer("noreply@console-mail.bab.oncarlos.test", release.NewDirStore(t.TempDir()), p,
+ dnsrec.NewMemWriter(), testMailDeployment(), &fakeSES{})
+ ensured := make(chan struct{})
+ go func() { defer close(ensured); m.Ensure(context.Background()) }()
+ // Let the ensure finish before TempDir's cleanup removes its store.
+ defer func() { close(p.release); <-ensured }()
+ <-p.entered
+
+ ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+ defer cancel()
+ done := make(chan error, 1)
+ go func() { done <- m.Send(ctx, "bill@gmail.com", "s", "b") }()
+ select {
+ case err := <-done:
+ if !errors.Is(err, ErrConsoleMailNotReady) || !errors.Is(err, context.DeadlineExceeded) {
+ t.Fatalf("Send while the ensure holds = %v, want ErrConsoleMailNotReady wrapping DeadlineExceeded", err)
+ }
+ case <-time.After(time.Second):
+ t.Fatal("Send blocked behind a stalled ensure past its own deadline")
+ }
+}