amadan
edge: housekeeper-gated issuance, certs+tokens published to bucket
One elected issuer per deployment mints; every other edge serves what it publishes (multi-region ingress spec, Part 2). publishingCache wraps the autocert.Cache the whole issuance path already funnels through, so per-host certs, the ACME account keys, HTTP-01 tokens AND wildcard.go's DNS-01 material all land under control/certs/ with their cache names unchanged — one prefix for Task 5's mirror to rebuild a cache from, and one for the fleet door's ACL to keep denying. Local cache first; a bucket failure is loud and never fails the local operation. Writes are last-writer-wins (putCert replaces rather than create-only, or every renewal would lose to its own previous cert). Nothing bumps control/serial. The gate arrives as edge.WithIssuerGate(housekeeperGate) — host hands it DOWN, never an import back — and it is the Task-3 gate, not the bare flag: HostPolicy answers errNotIssuer on a non-issuer, and the wildcard manager's single order site stands down after the cache load, before the CA, with the order itself bound to the holding's context. No gate configured (carlos edge without the agent, every single-box deployment) is ungated: mint as always. Serving is never gated.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
6 files changed,
+753
−46
internal/edge/certcache.go+8 −0internal/edge/certsync.go+250 −0internal/edge/certsync_test.go+321 −0internal/edge/edge.go+44 −7internal/edge/wildcard.go+121 −38internal/host/agent.go+9 −1
diff --git a/internal/edge/certcache.go b/internal/edge/certcache.go| index c534d429a..04400859d 100644 |
| --- a/internal/edge/certcache.go |
| +++ b/internal/edge/certcache.go |
| @@ -22,6 +22,14 @@ import ( |
| // restart with no migration. "s3" is reserved for a pooled-edge backend; |
| // until it exists it refuses to start, because failing loudly beats silently |
| // minting certs into a local dir the rest of a pool can't see. |
| +// |
| +// Sharing arrived as a WRAPPER around what this returns, not as a backend |
| +// selected here: certsync.go's publishingCache keeps the local cache as the |
| +// durable copy and publishes every entry to the control bucket under |
| +// control/certs/, which is what the multi-region spec's "one issuer, every |
| +// edge serves" needs — a pure remote backend would put a network round trip |
| +// on the issuance path and leave a box with nothing local when the bucket |
| +// blips. "s3" therefore stays reserved and still refuses. |
| func newCertCache(dir string) (autocert.Cache, error) { |
| switch backend := strings.TrimSpace(os.Getenv("CARLOS_CERT_CACHE")); backend { |
| case "", "dir": |
diff --git a/internal/edge/certsync.go b/internal/edge/certsync.go| new file mode 100644 |
| index 000000000..58c226ee6 |
| --- /dev/null |
| +++ b/internal/edge/certsync.go |
| @@ -0,0 +1,250 @@ |
| +package edge |
| + |
| +// Issuer-side certificate publishing (spec: |
| +// docs/superpowers/specs/2026-08-20-multi-region-ingress-design.md, Part 2). |
| +// |
| +// With more than one edge in a deployment, exactly ONE of them talks to the |
| +// CA — the housekeeper (internal/host/housekeeper.go) — and every artifact |
| +// that issuance produces is published to the control bucket under |
| +// control/certs/ so every OTHER edge can serve it. Two halves live here: |
| +// |
| +// - publishingCache wraps the autocert.Cache the whole issuance path |
| +// already funnels through (certcache.go's seam, reserved for exactly |
| +// this). autocert writes its ACME account key, every per-host cert+key |
| +// PEM, and every HTTP-01 token response through its Cache — cluster |
| +// support is that interface's documented purpose — and wildcard.go's |
| +// DNS-01 manager shares the same handle. Wrapping it therefore captures |
| +// per-host certs, wildcards, tokens and account keys with no autocert |
| +// fork and no second abstraction. |
| +// |
| +// - the issuance gate: HostPolicy (per-host HTTP-01) and the wildcard |
| +// manager's order path (DNS-01) both refuse to START work on a box that |
| +// is not the housekeeper. Serving is never gated: an edge serves from |
| +// its local cache and from the mirror of this prefix regardless. |
| +// |
| +// Cert writes never bump control/serial — the serial is the console's |
| +// (writer map, Part 1); the mirror side polls this prefix on its own |
| +// cadence. Renewals run ~30 days before expiry, so a poll cadence measured |
| +// in seconds is never on a deadline. |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "fmt" |
| + "log" |
| + "os" |
| + "strings" |
| + |
| + "golang.org/x/crypto/acme/autocert" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// certPrefix is where every issuance artifact lands in the control bucket. |
| +// One prefix for the lot, named by the cache key autocert (or wildcard.go) |
| +// chose, so the mirror can rebuild a whole cache from a single list — and so |
| +// the fleet door's ACL has exactly one prefix to keep denying. |
| +const certPrefix = "control/certs/" |
| + |
| +// certKey is the bucket key for cache entry name. The cache name goes |
| +// through verbatim: "example.com", "example.com+rsa", "acme_account+key", |
| +// "<token>+http-01", "wildcard-<parent>.pem", "wildcard-account.key". |
| +func certKey(name string) string { return certPrefix + name } |
| + |
| +// IssuerGate answers "may this process start certificate issuance, and under |
| +// what authority does that work run?" — the housekeeper's gate |
| +// (internal/host), handed DOWN at edge construction because internal/host |
| +// imports internal/edge and never the reverse. The context is cancelled the |
| +// instant this box stops being the housekeeper, so an order in flight |
| +// unwinds with the authority it started under. |
| +// |
| +// A nil gate means "ungated": `carlos edge` run without the agent, and every |
| +// test, has no election in the process at all, and refusing issuance there |
| +// would leave the box with no certificates rather than with today's |
| +// behaviour. |
| +type IssuerGate func() (context.Context, bool) |
| + |
| +// issuerGate is the Edge's read of its gate, with the ungated default |
| +// applied. See the type's doc for why nil means "yes". |
| +func (e *Edge) issuerGate() (context.Context, bool) { |
| + if e.issuer == nil { |
| + return context.Background(), true |
| + } |
| + return e.issuer() |
| +} |
| + |
| +// errNotIssuer is what HostPolicy answers on a box that is not the |
| +// deployment's certificate issuer. Sentinel (errors.Is-able) so the gate is |
| +// distinguishable from "that host isn't mine", which is a different fault |
| +// with a different fix. |
| +var errNotIssuer = errors.New("acme: this box is not the deployment's certificate issuer") |
| + |
| +// issuerHostPolicy is the autocert HostPolicy: the registry check this edge |
| +// has always applied (Allowed), plus the issuance gate. |
| +// |
| +// Two autocert call sites reach this, and both are deliberate: |
| +// GetCertificate consults the policy BEFORE its cache lookup |
| +// (autocert.go:292 in x/crypto v0.52.0), and HTTPHandler consults it before |
| +// serving an HTTP-01 token (autocert.go:405). On a non-issuer both therefore |
| +// decline — which is correct only because the mirror layer sits behind them |
| +// and serves both certs and challenges from control/certs/. Nothing here may |
| +// be relaxed into "allow if we have it cached": autocert treats an EXPIRED |
| +// cached cert as a miss and would go straight to an order, so a policy that |
| +// says yes for a cached name is a policy that lets a non-issuer order. |
| +// |
| +// The flag is not a fence. Under a hung lease renewal two boxes can |
| +// transiently both believe they hold — the housekeeper's own stand-down is |
| +// bounded by its 30s poll, and a lost renewal is only noticed on that poll. |
| +// Brief double-issuance against the shared ACME account is inside the |
| +// spec's accepted envelope (one account, one rate-limit bucket, renewals ~30 |
| +// days early), so there is deliberately NO fencing token here and no |
| +// readiness predicate on the election: a fence would trade a rare duplicate |
| +// order for a deployment that can stop issuing altogether. |
| +func (e *Edge) issuerHostPolicy() autocert.HostPolicy { |
| + return func(_ context.Context, host string) error { |
| + if !e.Allowed(host) { |
| + return fmt.Errorf("acme: host %q not in registry", host) |
| + } |
| + if _, ok := e.issuerGate(); !ok { |
| + return errNotIssuer |
| + } |
| + return nil |
| + } |
| +} |
| + |
| +// publishingCache is the autocert.Cache every issuance artifact goes |
| +// through: the local cache first (durability, and autocert's own read path |
| +// keeps working byte-for-byte), then a best-effort copy into the control |
| +// bucket. |
| +// |
| +// Bucket failures are LOUD and never returned: a returned error aborts an |
| +// issuance that has already succeeded locally, trading a cert this box could |
| +// have served for a bucket blip. The mirror's next poll (and, failing that, |
| +// the next renewal) re-publishes. |
| +type publishingCache struct { |
| + inner autocert.Cache |
| + s release.Store |
| + // issuer gates the BUCKET half only — the local cache is written |
| + // unconditionally, because a non-issuer must still be able to serve what |
| + // it was handed. nil = ungated (see IssuerGate). |
| + // |
| + // It is the gate's boolean that is used and not its context: publishing |
| + // is not an order, and cancelling a publish because the holding ended |
| + // mid-write would strand a freshly minted cert on one box's disk with |
| + // nothing scheduled to re-publish it. The caller's context (autocert's, |
| + // already deadline-bounded) carries the write. |
| + issuer IssuerGate |
| +} |
| + |
| +func (p *publishingCache) Get(ctx context.Context, key string) ([]byte, error) { |
| + return p.inner.Get(ctx, key) |
| +} |
| + |
| +// Put writes locally, then publishes. Local first: if the disk write fails |
| +// there is no cert, and publishing one the box itself cannot serve would be |
| +// a lie in the bucket. |
| +func (p *publishingCache) Put(ctx context.Context, key string, data []byte) error { |
| + if err := p.inner.Put(ctx, key, data); err != nil { |
| + return err |
| + } |
| + if !p.publishes(key) { |
| + return nil |
| + } |
| + if err := putCert(ctx, p.s, key, data); err != nil { |
| + log.Printf("certsync: bucket publish failed for %q: %v — the cert is live on this box; the next renewal republishes", key, err) |
| + } |
| + return nil |
| +} |
| + |
| +// Delete removes both copies. The bucket copy goes even when the local |
| +// delete fails: the entry autocert deletes is a spent HTTP-01 token, and a |
| +// token left in a prefix every edge mirrors is litter every edge would keep |
| +// serving. |
| +// |
| +// It is gated like Put, and that is a deliberate trade: an issuer that |
| +// stands down mid-order leaves its token behind in the bucket, because the |
| +// writer map gives control/certs/** to the housekeeper alone and a box that |
| +// is no longer the housekeeper does not get to delete there. The spec's |
| +// answer to that litter is a TTL stamp swept from the mirror side, not a |
| +// second writer. |
| +func (p *publishingCache) Delete(ctx context.Context, key string) error { |
| + err := p.inner.Delete(ctx, key) |
| + if !p.publishes(key) { |
| + return err |
| + } |
| + if derr := p.s.Delete(ctx, certKey(key)); derr != nil { |
| + log.Printf("certsync: bucket delete failed for %q: %v — a spent challenge token may linger until it is overwritten", key, derr) |
| + } |
| + return err |
| +} |
| + |
| +// publishes reports whether this box should write key to the bucket: only |
| +// the issuer publishes, and only names that survive the round trip through a |
| +// bucket key do. |
| +func (p *publishingCache) publishes(key string) bool { |
| + if p.s == nil { |
| + return false |
| + } |
| + if p.issuer != nil { |
| + if _, ok := p.issuer(); !ok { |
| + // Not the issuer. Silent, like every other not-held path in this |
| + // deployment: it is the permanent normal state of N-1 boxes. |
| + return false |
| + } |
| + } |
| + if !publishableCertName(key) { |
| + log.Printf("certsync: refusing to publish cache entry %q — the name does not round-trip as a bucket key", key) |
| + return false |
| + } |
| + return true |
| +} |
| + |
| +// publishableCertName rejects cache names that would not survive as |
| +// "control/certs/<name>": a "/" would put the entry outside the flat prefix |
| +// the mirror maps back onto cache names, and a traversal segment has no |
| +// business in a key derived from a hostname. Nothing autocert or wildcard.go |
| +// generates fails this (autocert's HTTP-01 key is path.Base + "+http-01", |
| +// and domains are punycode-validated before they reach a cache key); it is |
| +// here so that a future key shape fails visibly rather than scattering |
| +// objects through the control prefix. |
| +func publishableCertName(name string) bool { |
| + if name == "" || strings.Contains(name, "/") { |
| + return false |
| + } |
| + for _, seg := range []string{"..", "\\"} { |
| + if strings.Contains(name, seg) { |
| + return false |
| + } |
| + } |
| + return true |
| +} |
| + |
| +// putCert writes key's body regardless of prior state: create-only when |
| +// absent, etag-replace when present, re-reading on a lost race — the |
| +// putReplacing idiom from internal/host's snapshot loop, needed here for the |
| +// same reason (release.Store.Put with an empty ifMatch is create-only, so a |
| +// renewal would otherwise lose to its own previous cert). |
| +// |
| +// Last-writer-wins is the DESIGN here, not a compromise: one gated issuer |
| +// means there is normally no second writer at all, and the transient |
| +// double-holder the housekeeper's poll window allows writes an equally valid |
| +// cert for the same name. Whichever lands last is servable, and the next |
| +// renewal converges the pair. |
| +func putCert(ctx context.Context, s release.Store, name string, body []byte) error { |
| + key := certKey(name) |
| + for attempt := 0; attempt < 3; attempt++ { |
| + _, etag, err := s.Get(ctx, key) |
| + switch { |
| + case errors.Is(err, os.ErrNotExist): |
| + etag = "" |
| + case err != nil: |
| + return fmt.Errorf("get %s: %w", key, err) |
| + } |
| + if _, err := s.Put(ctx, key, body, etag); err == nil { |
| + return nil |
| + } else if !errors.Is(err, release.ErrCASConflict) { |
| + return fmt.Errorf("put %s: %w", key, err) |
| + } |
| + } |
| + return fmt.Errorf("%s: lost CAS race three times", key) |
| +} |
diff --git a/internal/edge/certsync_test.go b/internal/edge/certsync_test.go| new file mode 100644 |
| index 000000000..225c14462 |
| --- /dev/null |
| +++ b/internal/edge/certsync_test.go |
| @@ -0,0 +1,321 @@ |
| +package edge |
| + |
| +import ( |
| + "bytes" |
| + "context" |
| + "errors" |
| + "log" |
| + "net/http/httptest" |
| + "strings" |
| + "sync" |
| + "testing" |
| + "time" |
| + |
| + "golang.org/x/crypto/acme/autocert" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// certStoreGet is the two-value read the tests want: the Store's third |
| +// return (the etag) is never interesting here. |
| +func certStoreGet(t *testing.T, s release.Store, key string) ([]byte, error) { |
| + t.Helper() |
| + b, _, err := s.Get(context.Background(), key) |
| + return b, err |
| +} |
| + |
| +func TestCertKeyShape(t *testing.T) { |
| + // Pinned: Task 5's mirror lists this exact prefix and Task 6's door ACL |
| + // test names these exact keys. autocert's cache names go through |
| + // UNCHANGED — per-host PEM, the account key, HTTP-01 tokens — and so do |
| + // the wildcard manager's (wildcard.go), which is what lets one mirror of |
| + // one prefix rebuild a whole cache. |
| + for name, want := range map[string]string{ |
| + "example.com": "control/certs/example.com", |
| + "example.com+rsa": "control/certs/example.com+rsa", |
| + "acme_account+key": "control/certs/acme_account+key", |
| + "tok+http-01": "control/certs/tok+http-01", |
| + "wildcard-go.tito.io.pem": "control/certs/wildcard-go.tito.io.pem", |
| + "wildcard-account.key": "control/certs/wildcard-account.key", |
| + } { |
| + if got := certKey(name); got != want { |
| + t.Fatalf("certKey(%q) = %q, want %q", name, got, want) |
| + } |
| + } |
| +} |
| + |
| +func TestPublishingCacheMirrorsPutsToBucket(t *testing.T) { |
| + s := release.NewDirStore(t.TempDir()) |
| + pc := &publishingCache{inner: autocert.DirCache(t.TempDir()), s: s} |
| + if err := pc.Put(context.Background(), "example.com", []byte("PEM")); err != nil { |
| + t.Fatal(err) |
| + } |
| + got, err := certStoreGet(t, s, "control/certs/example.com") |
| + if err != nil || string(got) != "PEM" { |
| + t.Fatalf("bucket copy = %q, %v", got, err) |
| + } |
| + // Local cache also holds it (autocert's own read path keeps working). |
| + if b, err := pc.Get(context.Background(), "example.com"); err != nil || string(b) != "PEM" { |
| + t.Fatalf("local copy = %q, %v", b, err) |
| + } |
| +} |
| + |
| +func TestPublishingCacheDeleteRemovesBucketCopy(t *testing.T) { |
| + s := release.NewDirStore(t.TempDir()) |
| + pc := &publishingCache{inner: autocert.DirCache(t.TempDir()), s: s} |
| + _ = pc.Put(context.Background(), "tok+http-01", []byte("challenge")) |
| + if err := pc.Delete(context.Background(), "tok+http-01"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := certStoreGet(t, s, "control/certs/tok+http-01"); err == nil { |
| + t.Fatal("token must be deleted from bucket after validation") |
| + } |
| + if _, err := pc.Get(context.Background(), "tok+http-01"); !errors.Is(err, autocert.ErrCacheMiss) { |
| + t.Fatalf("local copy after Delete: want ErrCacheMiss, got %v", err) |
| + } |
| +} |
| + |
| +// A renewal writes the SAME key again. A plain create-only Put would lose |
| +// every renewal to a CAS conflict, so the publish is last-writer-wins. |
| +func TestPublishingCacheReplacesAnExistingBucketCopy(t *testing.T) { |
| + s := release.NewDirStore(t.TempDir()) |
| + pc := &publishingCache{inner: autocert.DirCache(t.TempDir()), s: s} |
| + if err := pc.Put(context.Background(), "example.com", []byte("OLD")); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := pc.Put(context.Background(), "example.com", []byte("NEW")); err != nil { |
| + t.Fatal(err) |
| + } |
| + got, err := certStoreGet(t, s, "control/certs/example.com") |
| + if err != nil || string(got) != "NEW" { |
| + t.Fatalf("bucket copy after renewal = %q, %v — the publish must replace, not create-only", got, err) |
| + } |
| +} |
| + |
| +// failingStore fails every write; reads are the real store's. |
| +type failingStore struct { |
| + release.Store |
| + mu sync.Mutex |
| + puts int |
| + dels int |
| + putErr error |
| +} |
| + |
| +func (f *failingStore) Put(context.Context, string, []byte, string) (string, error) { |
| + f.mu.Lock() |
| + defer f.mu.Unlock() |
| + f.puts++ |
| + return "", f.putErr |
| +} |
| + |
| +func (f *failingStore) Delete(context.Context, string) error { |
| + f.mu.Lock() |
| + defer f.mu.Unlock() |
| + f.dels++ |
| + return errors.New("bucket down") |
| +} |
| + |
| +// Local issuance must survive a bucket blip: the cert is on disk, the |
| +// failure is loud, and autocert never sees an error (a returned error would |
| +// abort an issuance that actually succeeded). |
| +func TestPublishingCacheBucketFailureNeverFailsTheLocalOperation(t *testing.T) { |
| + var buf bytes.Buffer |
| + prev := log.Writer() |
| + log.SetOutput(&buf) |
| + t.Cleanup(func() { log.SetOutput(prev) }) |
| + |
| + s := &failingStore{Store: release.NewDirStore(t.TempDir()), putErr: errors.New("bucket down")} |
| + dir := t.TempDir() |
| + pc := &publishingCache{inner: autocert.DirCache(dir), s: s} |
| + |
| + if err := pc.Put(context.Background(), "example.com", []byte("PEM")); err != nil { |
| + t.Fatalf("Put must not fail on a bucket error: %v", err) |
| + } |
| + if b, err := pc.Get(context.Background(), "example.com"); err != nil || string(b) != "PEM" { |
| + t.Fatalf("local copy = %q, %v — local durability comes first", b, err) |
| + } |
| + if err := pc.Delete(context.Background(), "example.com"); err != nil { |
| + t.Fatalf("Delete must not fail on a bucket error: %v", err) |
| + } |
| + if out := buf.String(); !strings.Contains(out, "certsync: bucket publish failed") || |
| + !strings.Contains(out, "certsync: bucket delete failed") { |
| + t.Fatalf("a bucket failure must be loud; log was:\n%s", out) |
| + } |
| +} |
| + |
| +// The gate is the whole point: only the housekeeper publishes. A non-issuer |
| +// keeps its local cache (it must still serve) and writes nothing. |
| +func TestPublishingCacheNonIssuerPublishesNothing(t *testing.T) { |
| + s := &failingStore{Store: release.NewDirStore(t.TempDir())} |
| + pc := &publishingCache{ |
| + inner: autocert.DirCache(t.TempDir()), |
| + s: s, |
| + issuer: func() (context.Context, bool) { return nil, false }, |
| + } |
| + if err := pc.Put(context.Background(), "example.com", []byte("PEM")); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := pc.Delete(context.Background(), "example.com"); err != nil { |
| + t.Fatal(err) |
| + } |
| + s.mu.Lock() |
| + defer s.mu.Unlock() |
| + if s.puts != 0 || s.dels != 0 { |
| + t.Fatalf("non-issuer touched the bucket: %d put(s), %d delete(s)", s.puts, s.dels) |
| + } |
| +} |
| + |
| +// ---- the issuance gate ------------------------------------------------------ |
| + |
| +func testIssuerEdge(hosts ...string) *Edge { |
| + e := &Edge{} |
| + m := map[string]upstream{} |
| + for _, h := range hosts { |
| + m[h] = upstream{} |
| + } |
| + e.upstreams.Store(&m) |
| + return e |
| +} |
| + |
| +func TestHostPolicyUngatedIsTodaysBehaviour(t *testing.T) { |
| + e := testIssuerEdge("app.bab.oncarlos.com") |
| + pol := e.issuerHostPolicy() |
| + if err := pol(context.Background(), "app.bab.oncarlos.com"); err != nil { |
| + t.Fatalf("no gate configured (carlos edge, single box): want issuance allowed, got %v", err) |
| + } |
| + if err := pol(context.Background(), "stranger.example"); err == nil { |
| + t.Fatal("unknown host must still be refused") |
| + } else if errors.Is(err, errNotIssuer) { |
| + t.Fatalf("unknown host must be refused as unknown, not as not-issuer: %v", err) |
| + } |
| +} |
| + |
| +func TestHostPolicyRefusesOrdersOnANonIssuer(t *testing.T) { |
| + e := testIssuerEdge("app.bab.oncarlos.com") |
| + e.issuer = func() (context.Context, bool) { return nil, false } |
| + err := e.issuerHostPolicy()(context.Background(), "app.bab.oncarlos.com") |
| + if !errors.Is(err, errNotIssuer) { |
| + t.Fatalf("non-issuer HostPolicy = %v, want errNotIssuer", err) |
| + } |
| +} |
| + |
| +func TestHostPolicyAllowsOrdersWhileHolding(t *testing.T) { |
| + e := testIssuerEdge("app.bab.oncarlos.com") |
| + e.issuer = func() (context.Context, bool) { return context.Background(), true } |
| + if err := e.issuerHostPolicy()(context.Background(), "app.bab.oncarlos.com"); err != nil { |
| + t.Fatalf("issuer HostPolicy = %v, want nil", err) |
| + } |
| +} |
| + |
| +// ---- the wildcard (DNS-01) gate -------------------------------------------- |
| + |
| +// countingCache counts Gets per key — the wildcard order path's first cache |
| +// read is its ACME account key, so "did an order start?" is observable |
| +// without a CA. |
| +type countingCache struct { |
| + autocert.Cache |
| + mu sync.Mutex |
| + gets map[string]int |
| +} |
| + |
| +func newCountingCache(t *testing.T) *countingCache { |
| + return &countingCache{Cache: autocert.DirCache(t.TempDir()), gets: map[string]int{}} |
| +} |
| + |
| +func (c *countingCache) Get(ctx context.Context, key string) ([]byte, error) { |
| + c.mu.Lock() |
| + c.gets[key]++ |
| + c.mu.Unlock() |
| + return c.Cache.Get(ctx, key) |
| +} |
| + |
| +func (c *countingCache) count(key string) int { |
| + c.mu.Lock() |
| + defer c.mu.Unlock() |
| + return c.gets[key] |
| +} |
| + |
| +// deadDirectory is an ACME directory URL nothing is listening on, so an |
| +// order that DOES start fails immediately and offline. |
| +func deadDirectory(t *testing.T) string { |
| + srv := httptest.NewServer(nil) |
| + url := srv.URL + "/directory" |
| + srv.Close() |
| + return url |
| +} |
| + |
| +func wildcardGateManager(t *testing.T, cache autocert.Cache) *wildcardManager { |
| + w := newWildcardManager([]string{"go.tito.io"}, "", func() []string { return nil }, nil, "", cache, &fakeDNS{}) |
| + w.directory = deadDirectory(t) |
| + w.refresh() |
| + return w |
| +} |
| + |
| +func TestWildcardPassDoesNotOrderOnANonIssuer(t *testing.T) { |
| + cache := newCountingCache(t) |
| + w := wildcardGateManager(t, cache) |
| + w.issuer = func() (context.Context, bool) { return nil, false } |
| + |
| + // A stood-down pass comes back within the hour: the next one is what |
| + // loads the issuer's wildcard out of this box's cache and into memory. |
| + if delay := w.pass(); delay > time.Hour { |
| + t.Fatalf("non-issuer pass delay = %s, want <= 1h", delay) |
| + } |
| + |
| + if n := cache.count(wildcardAccountKeyName); n != 0 { |
| + t.Fatalf("a non-issuer started an ACME order (%d account-key read(s))", n) |
| + } |
| + if w.backingOff("go.tito.io", time.Now()) { |
| + t.Fatal("standing down is not a failure: it must not arm the per-parent backoff") |
| + } |
| +} |
| + |
| +func TestWildcardPassOrdersWhileHolding(t *testing.T) { |
| + cache := newCountingCache(t) |
| + w := wildcardGateManager(t, cache) |
| + w.issuer = func() (context.Context, bool) { return context.Background(), true } |
| + |
| + w.pass() |
| + |
| + if n := cache.count(wildcardAccountKeyName); n == 0 { |
| + t.Fatal("the issuer must attempt the order (no account-key read happened)") |
| + } |
| + if !w.backingOff("go.tito.io", time.Now()) { |
| + t.Fatal("a failed order must arm the per-parent backoff") |
| + } |
| +} |
| + |
| +// The order runs under the HOLDING's context, so a stand-down mid-order is |
| +// observable inside the order rather than at the next 12h tick. |
| +func TestWildcardOrderRunsUnderTheHoldingContext(t *testing.T) { |
| + cache := newCountingCache(t) |
| + w := wildcardGateManager(t, cache) |
| + held, standDown := context.WithCancel(context.Background()) |
| + w.issuer = func() (context.Context, bool) { return held, true } |
| + standDown() |
| + |
| + w.pass() |
| + |
| + // The account key read is the first thing obtain does; with the holding |
| + // already cancelled the order must not have reached the CA. The cache is |
| + // context-free, so what this pins is that the cancelled context is the |
| + // one the order carries: obtain fails before any network call. |
| + if err := held.Err(); err == nil { |
| + t.Fatal("test bug: the holding should be cancelled") |
| + } |
| + w.mu.Lock() |
| + got := w.certs["go.tito.io"] |
| + w.mu.Unlock() |
| + if got != nil { |
| + t.Fatal("an order under a cancelled holding must not produce a cert") |
| + } |
| +} |
| + |
| +func TestWildcardUngatedOrdersAsToday(t *testing.T) { |
| + cache := newCountingCache(t) |
| + w := wildcardGateManager(t, cache) // no issuer gate at all |
| + w.pass() |
| + if n := cache.count(wildcardAccountKeyName); n == 0 { |
| + t.Fatal("with no gate configured (carlos edge, single box) issuance must run exactly as today") |
| + } |
| +} |
diff --git a/internal/edge/edge.go b/internal/edge/edge.go| index 12c148917..ea01f594a 100644 |
| --- a/internal/edge/edge.go |
| +++ b/internal/edge/edge.go |
| @@ -161,8 +161,26 @@ type Edge struct { |
| // false, ServeHTTP never constructs a gzipWriter and the writer stack is |
| // byte-for-byte what it was before compression existed (compress.go). |
| gzipOn bool |
| + // issuer, when set, gates certificate ISSUANCE on this box holding the |
| + // deployment's housekeeper lease (certsync.go). It arrives through |
| + // WithIssuerGate because internal/host imports this package and never the |
| + // reverse. nil = ungated, which is `carlos edge` without the agent and |
| + // every single-box deployment's behaviour to date: mint as always. |
| + // Serving is never gated by it. |
| + issuer IssuerGate |
| } |
| +// RunOption configures the Edge that Run/RunWithActivator* builds, before it |
| +// serves anything. Variadic rather than another positional parameter: the |
| +// composing caller (internal/host's agent role) hands host-owned state DOWN |
| +// into this package, and every existing call site keeps compiling unchanged. |
| +type RunOption func(*Edge) |
| + |
| +// WithIssuerGate hands the edge the deployment's certificate-issuer gate — |
| +// the housekeeper's, in the agent role. Without it the edge issues exactly |
| +// as it always has. See IssuerGate (certsync.go). |
| +func WithIssuerGate(g IssuerGate) RunOption { return func(e *Edge) { e.issuer = g } } |
| + |
| // upstream is one served host: the handler for its backend, plus — when the |
| // activator manages it — the route itself, so a request can trigger the wake. |
| type upstream struct { |
| @@ -1676,7 +1694,13 @@ func RunWithActivator(args []string, act Activator, cache *Cache) { |
| // Edge (e.g. to read tunnel state for status reporting) without racing the |
| // configuration it needs. It runs on the calling goroutine, so a hook that |
| // blocks holds the front door shut: hand off to a goroutine and return. |
| -func RunWithActivatorReady(args []string, act Activator, cache *Cache, ready func(*Edge)) { |
| +// |
| +// opts configure the Edge the moment it exists — before any EnableXxx, and |
| +// long before the TLS listener — so a knob the composing caller owns |
| +// (WithIssuerGate) is in place for the construction decisions that read it. |
| +// The ready hook is the wrong home for those: it fires once, at the end, for |
| +// state a caller wants to REACH. |
| +func RunWithActivatorReady(args []string, act Activator, cache *Cache, ready func(*Edge), opts ...RunOption) { |
| dataDir := envOr("CARLOS_DATA_DIR", "/data") |
| fs := flag.NewFlagSet("edge", flag.ExitOnError) |
| registryPath := fs.String("registry", filepath.Join(dataDir, "registry.db"), "registry SQLite path (default $CARLOS_DATA_DIR/registry.db)") |
| @@ -1698,6 +1722,9 @@ func RunWithActivatorReady(args []string, act Activator, cache *Cache, ready fun |
| if err != nil { |
| log.Fatalf("edge: load routes: %v", err) |
| } |
| + for _, opt := range opts { |
| + opt(e) |
| + } |
| e.fallbackURL = *fallbackURL |
| e.EnableCache(cache) |
| @@ -1761,16 +1788,26 @@ func RunWithActivatorReady(args []string, act Activator, cache *Cache, ready fun |
| if err != nil { |
| log.Fatalf("edge: %v", err) |
| } |
| + // Publish every issuance artifact to the control bucket (certsync.go): |
| + // per-host certs, the ACME account keys, HTTP-01 tokens, and — through |
| + // the same handle below — the DNS-01 wildcards. Only the deployment's |
| + // elected issuer writes; every other edge serves from the mirror of that |
| + // prefix. No deployment store configured means no bucket to publish to: |
| + // a single box keeping its certs on its own disk, exactly as before. |
| + if e.staticStore != nil { |
| + certCache = &publishingCache{inner: certCache, s: e.staticStore, issuer: e.issuerGate} |
| + log.Printf("edge: publishing issued certs to %s* in the deployment bucket", certPrefix) |
| + } else { |
| + log.Printf("edge: cert publishing off (no deployment store configured) — certs stay on this box") |
| + } |
| m := &autocert.Manager{ |
| Prompt: autocert.AcceptTOS, |
| Cache: certCache, |
| - HostPolicy: func(_ context.Context, host string) error { |
| - if e.Allowed(host) { |
| - return nil |
| - } |
| - return fmt.Errorf("acme: host %q not in registry", host) |
| - }, |
| + // Registry membership AND the issuance gate — see issuerHostPolicy |
| + // (certsync.go) for why the gate cannot be relaxed for cached names, |
| + // and for why a transient double-holder is deliberately not fenced. |
| + HostPolicy: e.issuerHostPolicy(), |
| } |
| tlsCfg := m.TLSConfig() |
| // Wildcard certs (wildcard.go, ACME DNS-01): covered names — one label |
diff --git a/internal/edge/wildcard.go b/internal/edge/wildcard.go| index 41d65a9f9..b8c21463c 100644 |
| --- a/internal/edge/wildcard.go |
| +++ b/internal/edge/wildcard.go |
| @@ -97,6 +97,18 @@ type wildcardManager struct { |
| // with no usable cert (a new account's first route). Capacity 1: a wake |
| // is a level, not a count. |
| wake chan struct{} |
| + |
| + // issuer gates ORDERING (certsync.go): in a multi-edge deployment only |
| + // the housekeeper talks to the CA, and its context ends the moment the |
| + // holding does, so a stand-down mid-order is observable inside the order. |
| + // nil = ungated, which is every single-box deployment and `carlos edge` |
| + // without the agent. Loading and serving a cert some other box obtained |
| + // is never gated — that is the whole point of the shared cache. |
| + issuer IssuerGate |
| + |
| + // lastNotIssuerLog rate-limits the one line a non-issuer prints when it |
| + // stands down from a pass. Read/written under mu. |
| + lastNotIssuerLog time.Time |
| } |
| func newWildcardManager(staticDomains []string, appsDomain string, hosts func() []string, delegated func() []string, delegationZone string, cache autocert.Cache, dns dnsProvider) *wildcardManager { |
| @@ -386,43 +398,7 @@ func parseWildcardPEM(raw []byte) (*tls.Certificate, error) { |
| // with no usable cert (a new account). |
| func (w *wildcardManager) run() { |
| for { |
| - delay := 12 * time.Hour |
| - w.mu.Lock() |
| - parents := append([]string(nil), w.parents...) |
| - w.mu.Unlock() |
| - for _, p := range parents { |
| - if !w.needsObtain(p, time.Now()) { |
| - continue |
| - } |
| - if w.backingOff(p, time.Now()) { |
| - // A wake (fired because some OTHER parent needs a cert) does |
| - // not shorten this parent's own hourly retry. |
| - continue |
| - } |
| - // Another process (or pool sibling) may have obtained it already. |
| - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) |
| - w.loadCached(ctx, p) |
| - cancel() |
| - if !w.needsObtain(p, time.Now()) { |
| - continue |
| - } |
| - ctx, cancel = context.WithTimeout(context.Background(), 10*time.Minute) |
| - err := w.obtain(ctx, p) |
| - cancel() |
| - if err != nil { |
| - log.Printf("edge: wildcard *.%s: %v — per-host issuance keeps covering; retrying in 1h", p, err) |
| - delay = time.Hour |
| - w.mu.Lock() |
| - w.nextAttempt[p] = time.Now().Add(time.Hour) |
| - w.mu.Unlock() |
| - continue |
| - } |
| - w.mu.Lock() |
| - delete(w.nextAttempt, p) |
| - cert := w.certs[p] |
| - w.mu.Unlock() |
| - log.Printf("edge: wildcard *.%s obtained (valid until %s)", p, cert.Leaf.NotAfter.Format(time.RFC3339)) |
| - } |
| + delay := w.pass() |
| select { |
| case <-time.After(delay): |
| case <-w.wake: |
| @@ -430,6 +406,107 @@ func (w *wildcardManager) run() { |
| } |
| } |
| +// pass is one sweep over the current parent set — load what the cache |
| +// already has, order what is still missing — and returns how long run() |
| +// should wait before the next one. Split out of run() so the gating and |
| +// backoff decisions are testable without spinning the loop. |
| +func (w *wildcardManager) pass() time.Duration { |
| + delay := 12 * time.Hour |
| + w.mu.Lock() |
| + parents := append([]string(nil), w.parents...) |
| + w.mu.Unlock() |
| + standDowns := 0 |
| + for _, p := range parents { |
| + if !w.needsObtain(p, time.Now()) { |
| + continue |
| + } |
| + if w.backingOff(p, time.Now()) { |
| + // A wake (fired because some OTHER parent needs a cert) does |
| + // not shorten this parent's own hourly retry. |
| + continue |
| + } |
| + // Another process (or pool sibling) may have obtained it already. |
| + // Ungated on purpose: reading a cert someone else issued — including |
| + // one this box's mirror of control/certs/ has just landed — is |
| + // SERVING, and every edge serves. |
| + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) |
| + w.loadCached(ctx, p) |
| + cancel() |
| + if !w.needsObtain(p, time.Now()) { |
| + continue |
| + } |
| + // The order gate (certsync.go): one elected issuer talks to the CA. |
| + // A non-issuer stands down HERE, after the cache load and before the |
| + // CA — so it still picks up the issuer's wildcard, and never arms the |
| + // per-parent backoff (standing down is not a failure; the moment this |
| + // box wins the lease it must be able to order immediately). |
| + held, ok := w.issuerGate() |
| + if !ok { |
| + standDowns++ |
| + continue |
| + } |
| + // The order runs under the HOLDING's context, not a bare background |
| + // one: an ACME order takes seconds to minutes, and work started as |
| + // the issuer must not outlive the authority it started under. |
| + ctx, cancel = context.WithTimeout(held, 10*time.Minute) |
| + err := w.obtain(ctx, p) |
| + cancel() |
| + if err != nil { |
| + log.Printf("edge: wildcard *.%s: %v — per-host issuance keeps covering; retrying in 1h", p, err) |
| + delay = time.Hour |
| + w.mu.Lock() |
| + w.nextAttempt[p] = time.Now().Add(time.Hour) |
| + w.mu.Unlock() |
| + continue |
| + } |
| + w.mu.Lock() |
| + delete(w.nextAttempt, p) |
| + cert := w.certs[p] |
| + w.mu.Unlock() |
| + log.Printf("edge: wildcard *.%s obtained (valid until %s)", p, cert.Leaf.NotAfter.Format(time.RFC3339)) |
| + } |
| + if standDowns > 0 { |
| + w.logStandDown(standDowns) |
| + // Come back in an hour, not twelve. A non-issuer's next pass is what |
| + // LOADS the wildcard the issuer has meanwhile minted into this box's |
| + // cache — certFor serves from the in-memory map loadCached fills — and |
| + // refresh() only wakes the loop on a route swap, which a quiet box may |
| + // not see for a day. No ACME traffic is involved in the retry. |
| + if delay > time.Hour { |
| + delay = time.Hour |
| + } |
| + } |
| + return delay |
| +} |
| + |
| +// issuerGate is this manager's read of the deployment's issuer gate, with |
| +// the ungated default applied (see IssuerGate, certsync.go). |
| +func (w *wildcardManager) issuerGate() (context.Context, bool) { |
| + if w.issuer == nil { |
| + return context.Background(), true |
| + } |
| + return w.issuer() |
| +} |
| + |
| +// logStandDown says once an hour, at most, that this box wanted wildcards it |
| +// is not allowed to order. Not silent — a box with no wildcard and no reason |
| +// in the journal is the hard kind of production puzzle — and not per-pass |
| +// either: refresh() can wake a pass on every route swap, and on a |
| +// non-issuer every one of those would repeat the same line. |
| +func (w *wildcardManager) logStandDown(parents int) { |
| + now := time.Now() |
| + w.mu.Lock() |
| + quiet := now.Sub(w.lastNotIssuerLog) < time.Hour |
| + if !quiet { |
| + w.lastNotIssuerLog = now |
| + } |
| + w.mu.Unlock() |
| + if quiet { |
| + return |
| + } |
| + log.Printf("edge: wildcard: %d parent(s) need issuance but this box is not the deployment's certificate issuer — serving what the issuer publishes", parents) |
| +} |
| + |
| // obtain runs one full ACME DNS-01 issuance for *.parent and swaps the |
| // result in. |
| func (w *wildcardManager) obtain(ctx context.Context, parent string) error { |
| @@ -656,7 +733,13 @@ func newWildcardFromEnv(e *Edge, cache autocert.Cache) *wildcardManager { |
| mu.Unlock() |
| return out |
| } |
| - return newWildcardManager(static, appsDomain, e.currentHosts, delegated, zone, cache, dns) |
| + w := newWildcardManager(static, appsDomain, e.currentHosts, delegated, zone, cache, dns) |
| + // DNS-01 orders are gated on the same election per-host HTTP-01 orders |
| + // are (certsync.go): one issuer per deployment, whichever protocol the |
| + // name needs. The cache handle is already the publishing one, so the |
| + // wildcard PEM and its ACME account key distribute with everything else. |
| + w.issuer = e.issuerGate |
| + return w |
| } |
| // wrapGetCertificate serves the wildcard for covered names and hands |
diff --git a/internal/host/agent.go b/internal/host/agent.go| index 24ee06d0b..59ac7428a 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -345,7 +345,15 @@ func RunAgent(args []string) { |
| m := edge.NewUsageMeter() |
| e.EnableUsage(m) |
| go usageLoop(m.Drain) |
| - }) |
| + }, |
| + // Certificate issuance is a deployment-global duty, so it runs where |
| + // the housekeeper runs (housekeeper.go). The gate goes DOWN into the |
| + // edge as a func — this package imports internal/edge and never the |
| + // reverse — and it is the GATE, not the bare flag: an ACME order runs |
| + // for seconds to minutes, and the context ends the instant this box |
| + // stops being the housekeeper. Armed above, so this closure is live |
| + // and correct by the time the edge reads it. |
| + edge.WithIssuerGate(housekeeperGate)) |
| // The edge has drained; take the children down too (they're this process's |
| // children — orphaning them would strand their leases until TTL expiry). |