carlos / platform Public

Clone
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package domains
2
3import (
4 "context"
5 "crypto/ed25519"
6 "crypto/x509"
7 "encoding/json"
8 "encoding/pem"
9 "errors"
10 "fmt"
11 "log"
12 "net"
13 "os"
14 "path/filepath"
15 "slices"
16 "strings"
17 "time"
18
19 "github.com/carlosframework/platform/internal/registry"
20 "github.com/carlosframework/platform/internal/release"
21)
22
23// Config is everything one reconcile pass reads its world through. Every
24// field is a seam: the tests inject fixtures, the agent injects production
25// values (internal/host wires it).
26type Config struct {
27 HostClaim bool
28 // AppsDomain is the deployment's platform apps domain (flagship:
29 // "oncarlos.com"). "" makes the WHOLE pass inert — no minting, no
30 // manifests read, zero registry writes — the litestream convention:
31 // unconfigured means the feature does not exist.
32 AppsDomain string
33 // EdgeIPs answers this deployment's current edge addresses; a host
34 // resolving to any of them is "pointing". A FUNC, not a slice: the set
35 // moves at runtime as edges arm and retire, while the agent's config is
36 // built once at loop arm — a boot-frozen list is exactly the bug this
37 // replaced (a correctly-pointed domain steered to another region's edge
38 // read as pointing elsewhere). nil, or a func returning nothing, leaves
39 // DNS stamping standing down for that pass — see section 4.
40 EdgeIPs func() []string
41 // Resolve is the DNS seam (net resolver in production, a fake in
42 // tests); nil skips DNS stamping entirely.
43 Resolve func(host string) ([]string, error)
44 // AcmeDir is the edge's autocert DirCache directory; "" skips TLS
45 // stamping entirely.
46 AcmeDir string
47 // Now is the clock seam; nil = time.Now.
48 Now func() time.Time
49 // ReleasePub verifies the production flag: before a trash marker is
50 // allowed to drop an app's routes (§1c) and again before the auto-purge
51 // pass destroys anything (§1d). nil disables purging entirely — a box
52 // that cannot verify must not destroy — and leaves the route-drop
53 // unguarded, which §1c explains and accepts.
54 ReleasePub ed25519.PublicKey
55}
56
57// Reconcile runs the spec §3 steps in order, idempotently: bucket account
58// manifests → registry accounts + route linkage, platform-domain minting,
59// canary-host minting/removal, then DNS and TLS health stamps. Per-item
60// failures are logged and skipped — one bad manifest or one unresolvable
61// host must never take the pass down; only setup-level failures (listing the
62// bucket or the registry) return an error.
63//
64// Removing a route happens on exactly two grounds. INFERRED: a route with
65// Platform set whose canary channel pointer is gone from the bucket — and
66// inference stops there, because custom domains are owner property.
67// INSTRUCTED: every route of an app carrying a valid trash marker, custom
68// domains included. The marker is not something this pass inferred; it is the
69// owner saying stop serving this app, and a custom domain still serving a
70// "deleted" app would be the worse betrayal (§1c) — unless the app is flagged
71// production, which outranks the marker on any box that can verify the flag.
72func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg Config) error {
73 if cfg.AppsDomain == "" {
74 return nil // inert until configured
75 }
76 if cfg.Now == nil {
77 cfg.Now = time.Now
78 }
79
80 // One bucket listing feeds the app→account key-path scan and the
81 // canary-pointer scan below; accounts/ is listed separately because it
82 // must be read FIRST — the account set is what makes a key path
83 // unambiguous.
84 acctKeys, err := s.List(ctx, "accounts/")
85 if err != nil {
86 return fmt.Errorf("domains: list accounts: %w", err)
87 }
88 keys, err := s.List(ctx, "apps/")
89 if err != nil {
90 return fmt.Errorf("domains: list bucket: %w", err)
91 }
92 routes, err := reg.List()
93 if err != nil {
94 return fmt.Errorf("domains: list routes: %w", err)
95 }
96
97 // --- 1. account materialization ------------------------------------------
98 known := materializeAccounts(ctx, s, acctKeys, reg)
99
100 // --- 1c. trashed apps ----------------------------------------------------
101 // A valid trash marker (apps/<sqid>/<app>/deleted.json) is the owner's
102 // explicit instruction to stop serving the app. This extends the pass's
103 // destructive-move charter deliberately: inference never destroys owner
104 // property, but a marker is not inference — so ALL the app's routes go,
105 // custom domains included. A custom domain that kept serving a "deleted"
106 // app would be the worse betrayal. A marker that fails validation
107 // (ReadAppTombstone) changes nothing: an instruction we cannot fully
108 // believe must not destroy routes, so the app stays live and the log
109 // says why, every pass, until someone fixes or removes the object.
110 //
111 // And a valid marker does NOT outrank the production flag. The marker is
112 // unsigned by design — anything holding PutObject on the bucket can write
113 // one — so without this check a writer with no signing key could park a
114 // marker on a production app and take its routes down fleet-wide, on
115 // every box, at the next tick. Dropping every route of a production app
116 // is an outage, which is precisely what the flag exists to prevent, so
117 // the same contradiction the purge path refuses (§1d) is refused here:
118 // flag ON, or flag UNREADABLE, and the marker is not acted on at all.
119 //
120 // Collected before the linkage below purely so the set is complete when
121 // the removal pass (right after 1b's stamping) reads it. The linkage
122 // itself does NOT skip trashed apps — see the stamping loop for why.
123 trashed := map[appKey]*release.AppTombstone{}
124 for _, key := range keys {
125 acct, app, prefix, ok := appKeyOwner(key, known)
126 if !ok || key != prefix+"deleted.json" {
127 continue
128 }
129 tomb, err := release.ReadAppTombstone(ctx, s, acct, app)
130 if err != nil {
131 log.Printf("domains: ignoring trash marker for %s/%s: %v", acct, app, err)
132 continue
133 }
134 if tomb == nil {
135 continue
136 }
137 // A KEYLESS box acts on the marker, deliberately. It cannot verify a
138 // flag, and the alternative — hold every trash forever wherever there
139 // is no pubkey — would break hiding entirely on legitimately keyless
140 // deployments, whose owners have no way to make the guard pass. The
141 // guard is where it matters: every SERVING box carries the pubkey
142 // already (adoption needs it), so production traffic is covered, and
143 // the destructive half (§1d) refuses outright without a key.
144 if cfg.ReleasePub != nil {
145 flag, err := release.GetProduction(ctx, s, cfg.ReleasePub, acct, app)
146 if err != nil {
147 log.Printf("domains: cannot read the production flag for %s/%s — refusing to act on the trash marker: %v", acct, app, err)
148 continue
149 }
150 if flag != nil && flag.Production {
151 log.Printf("domains: trash marker on production app %s/%s — refusing to act on it; turn production off first", acct, app)
152 continue
153 }
154 }
155 trashed[appKey{acct, app}] = tomb
156 }
157
158 // --- 1b. app→account linkage, from the KEY PATH --------------------------
159 // apps/<sqid>/<app>/… says the app belongs to that account. A legacy
160 // unqualified apps/<app>/… key says nothing at all since the read-fallback
161 // was retired — the runtime cannot read those objects, so they must not
162 // mint a route or claim an app (appKeyOwner). The shapes are told apart by
163 // whether the first segment is a KNOWN account sqid, which is why the
164 // account set had to be read first.
165 // owners is app name -> the accounts that hold objects for it. A LIST, not
166 // one account: the same app name legitimately exists in several accounts,
167 // and collapsing them to one would re-home every tenant's app onto
168 // whichever account the bucket listing happened to reach last.
169 owners := map[string][]string{}
170 for _, key := range keys {
171 acct, app, _, ok := appKeyOwner(key, known)
172 if !ok {
173 continue
174 }
175 if !slices.Contains(owners[app], acct) {
176 owners[app] = append(owners[app], acct)
177 }
178 }
179 // Trashed apps are stamped here like any other, deliberately: the stamp is
180 // what makes the removal lookup below match. A route with an empty account
181 // column resolves to ops until 1b writes the account the BUCKET asserts,
182 // so skipping trashed apps here would leave such a row reading "ops" while
183 // its marker sits under the tenant's sqid — and the app would go on
184 // serving from its custom domain forever. Stamping a row that is about to
185 // be deleted costs one write and buys "ALL of them".
186 for i := range routes {
187 if routes[i].App == "" {
188 continue // an app-less route (status page, bare proxy) has no account
189 }
190 acct := resolveRouteAccount(routes[i], owners[routes[i].App], cfg.AppsDomain)
191 if acct == "" || routes[i].Account == acct {
192 continue
193 }
194 if err := reg.SetAccount(routes[i].Host, acct); err != nil {
195 log.Printf("domains: stamp account on %s: %v", routes[i].Host, err)
196 continue
197 }
198 routes[i].Account = acct
199 }
200
201 // Trashed apps stop routing NOW: every route of a trashed (account, app)
202 // is removed, and the shrunken slice is what minting sees, so nothing
203 // below re-mints an apex or canary host for an app in the trash.
204 //
205 // Except a host in the import journal (seeded-instances spec §5.7): its
206 // row is an uncommitted import's, and the journal owns it until the
207 // rollback puts the original route back and retires the journal row.
208 // Deleting it here would lose the import's gate and leave the rollback
209 // no row to restore over. The provision pass skips these hosts for the
210 // same reason. The row goes on a later pass, once the journal lets go.
211 // A journal that cannot be read could hide any host, so then no trashed
212 // route is removed this pass.
213 //
214 // The list is from the top of the pass, and this loop runs alongside
215 // the provision pass. A rollback can put another owner's original row
216 // back at a listed host and retire its journal row in between, so the
217 // delete is conditional on the row still being the one listed
218 // (DeleteIfUnchanged); a row that changed is left for the next pass.
219 if len(trashed) > 0 {
220 journaled, jerr := importJournalHosts(reg)
221 if jerr != nil {
222 log.Printf("domains: the import journal could not be read: %v (no route of a trashed app is removed this pass)", jerr)
223 }
224 keptLive := routes[:0]
225 for _, rt := range routes {
226 if rt.App != "" && trashed[appKey{release.RouteAccount(rt), rt.App}] != nil {
227 if jerr != nil || journaled[rt.Host] {
228 if jerr == nil {
229 log.Printf("domains: not removing route %s of trashed app %s yet: an import on it is still journaled", rt.Host, rt.App)
230 }
231 keptLive = append(keptLive, rt)
232 continue
233 }
234 deleted, err := reg.DeleteIfUnchanged(rt)
235 if err != nil {
236 log.Printf("domains: remove route %s of trashed app %s: %v", rt.Host, rt.App, err)
237 keptLive = append(keptLive, rt) // still there; try next pass
238 continue
239 }
240 if !deleted {
241 log.Printf("domains: not removing route %s of trashed app %s: the row changed since this pass listed it", rt.Host, rt.App)
242 keptLive = append(keptLive, rt) // judged again next pass, from a fresh list
243 continue
244 }
245 log.Printf("domains: removed route %s — app %s is in the trash", rt.Host, rt.App)
246 continue
247 }
248 keptLive = append(keptLive, rt)
249 }
250 routes = keptLive
251 }
252
253 // --- 1d. auto-purge ------------------------------------------------------
254 // Trashed longer than TrashCooloff → destroy. The marker's own deleted_at
255 // is the clock (bucket-side; restore deletes the marker, so there is
256 // nothing to replay). PurgeApp RE-READS the marker immediately before
257 // destroying — this listing is only a hint — and re-checks the production
258 // flag; both guards fail toward keeping data. Idempotent and race-proof:
259 // concurrent boxes converge because every step tolerates already-gone and
260 // the marker goes last.
261 for k, tomb := range trashed {
262 if cfg.Now().Sub(tomb.DeletedAt) < release.TrashCooloff {
263 continue
264 }
265 if cfg.HostClaim {
266 if err := release.RequestClaimAutoPurge(ctx, s, k.acct, k.app, cfg.Now()); err != nil {
267 log.Printf("domains: request auto-purge %s/%s: %v", k.acct, k.app, err)
268 }
269 continue
270 }
271 if err := release.PurgeApp(ctx, s, cfg.ReleasePub, k.acct, k.app, cfg.Now()); err != nil {
272 log.Printf("domains: auto-purge %s/%s: %v", k.acct, k.app, err)
273 continue
274 }
275 log.Printf("domains: purged %s/%s — trashed %s, past the %s cooloff",
276 k.acct, k.app, tomb.DeletedAt.Format(time.RFC3339), release.TrashCooloff)
277 }
278
279 // --- 2. platform-domain minting ------------------------------------------
280 byHost := map[string]bool{}
281 // hasApex tracks the (ACCOUNT, app) pairs that already own a platform
282 // apex. The account is part of the key because app names are unique per
283 // account, never globally: two tenants can both run a "blog", and each
284 // deserves its own apex.
285 //
286 // Within one account the guard is deliberately "already has one" rather
287 // than "already has the account-correct one": minting is additive (nothing
288 // here ever removes an apex), so keying on the expected host would mint a
289 // SECOND apex the moment an app's account changed, leaving both live
290 // forever with no rule for which is canonical. One app per account, one
291 // apex; moving an app between accounts is a deliberate operator migration,
292 // not a silent side effect.
293 hasApex := map[appKey]bool{}
294 for _, rt := range routes {
295 byHost[rt.Host] = true
296 if rt.Platform && !rt.Canary {
297 hasApex[appKey{release.RouteAccount(rt), rt.App}] = true
298 }
299 }
300 for _, k := range candidateApps(routes) {
301 if hasApex[k] {
302 continue
303 }
304 host := fmt.Sprintf("%s.%s.%s", k.app, k.acct, cfg.AppsDomain)
305 if byHost[host] {
306 continue // already minted — idempotent
307 }
308 src := firstReadyUpstream(routes, k)
309 if src == nil {
310 continue // nothing serving to copy an upstream from; next pass
311 }
312 nr := mintedRoute(host, k.app, k.acct, *src)
313 if err := reg.Upsert(nr); err != nil {
314 log.Printf("domains: mint %s: %v", host, err)
315 continue
316 }
317 log.Printf("domains: minted platform domain %s (upstream %s)", host, src.Host)
318 routes = append(routes, nr)
319 byHost[host] = true
320 hasApex[k] = true
321 }
322
323 // --- 3. canary hosts: mint for hostless pointers, remove for retired ----
324 canaryChannels := map[appKey]map[string]bool{} // (account, app) -> live canary/<name> channels
325 for _, key := range keys {
326 acctOfKey, app, _, ok := appKeyOwner(key, known)
327 if !ok {
328 continue
329 }
330 app, name, ok := canaryPointer(key, known)
331 if !ok {
332 continue
333 }
334 k := appKey{acctOfKey, app}
335 if trashed[k] != nil {
336 continue // no canary host for an app in the trash
337 }
338 if canaryChannels[k] == nil {
339 canaryChannels[k] = map[string]bool{}
340 }
341 channel := "canary/" + name
342 canaryChannels[k][channel] = true
343 followed := false
344 for _, rt := range routes {
345 if rt.App == app && release.RouteAccount(rt) == k.acct && rt.Channel == channel {
346 followed = true
347 break
348 }
349 }
350 if followed {
351 continue
352 }
353 acct := k.acct
354 src := firstReadyUpstream(routes, k)
355 if src == nil {
356 continue // app has nothing serving yet; the pointer waits
357 }
358 // Static apps ONLY. A minted route can only copy an upstream that
359 // already exists, and for an instance/service app the only one
360 // available is the app's stable process — so a "canary" host here
361 // would proxy straight to stable while the console labeled it with
362 // the canary's version: a URL that lies about what it serves.
363 // Spawning a real per-host canary instance is the not-yet-built
364 // `carlos canary`'s job, not something reconciliation can fake. For
365 // static apps the copied Channel genuinely IS the backend (the edge
366 // resolves the pointer to find that channel's tar), so those mint
367 // honestly.
368 if src.Kind != "static" {
369 log.Printf("domains: canary channel %s for app %s has no host: minting one needs a per-host canary instance (carlos canary), not an alias to stable — skipping", channel, app)
370 continue
371 }
372 host := fmt.Sprintf("%s.%s.%s.%s", name, app, acct, cfg.AppsDomain)
373 nr := mintedRoute(host, app, acct, *src)
374 nr.Channel = channel
375 nr.Canary = true
376 if err := reg.Upsert(nr); err != nil {
377 log.Printf("domains: mint canary %s: %v", host, err)
378 continue
379 }
380 log.Printf("domains: minted canary domain %s (channel %s)", host, channel)
381 routes = append(routes, nr)
382 byHost[host] = true
383 }
384 // Removal: ONLY canary hosts this pass minted, whose pointer is now gone
385 // from the bucket. Three conditions, each load-bearing: Platform (a
386 // custom route on the same channel is owner property — the flag, not the
387 // channel, grants removal authority), Canary (a STATIC app's apex mint
388 // legitimately carries a canary channel too, since that's how it finds
389 // its tar — removing it on retirement would delete the app's platform
390 // domain and re-mint it next pass, forever), and the channel prefix.
391 kept := routes[:0]
392 for _, rt := range routes {
393 if rt.Platform && rt.Canary && strings.HasPrefix(rt.Channel, "canary/") &&
394 !canaryChannels[appKey{release.RouteAccount(rt), rt.App}][rt.Channel] {
395 if err := reg.Delete(rt.Host); err != nil {
396 log.Printf("domains: remove retired canary route %s: %v", rt.Host, err)
397 kept = append(kept, rt) // still there; stamp it honestly below
398 continue
399 }
400 log.Printf("domains: removed retired canary route %s (channel %s)", rt.Host, rt.Channel)
401 continue
402 }
403 kept = append(kept, rt)
404 }
405 routes = kept
406
407 // Sections 4 and 5 — the per-box health stamps — live in StampHealth
408 // (#303): they write THIS box's registry and must run on every box, not
409 // only on the housekeeper this pass is gated to. The agent runs it once
410 // per tick after this pass, on every box; this pass does not, so the
411 // housekeeper stamps once like everyone else.
412 return nil
413}
414
415// StampHealth is the per-box half of the domains pass: DNS health (section
416// 4, "pointing"/"elsewhere"/"" on routes and aliases) and TLS health
417// (section 5, cert_expires_at). Both stamp the LOCAL registry from a local
418// view — this box's resolver, this box's autocert cache — so they are
419// every box's job on every tick, with no lease involved: gating them to
420// the housekeeper (#289) froze every other box's stamps (#303), and the
421// edges-file publisher's "pointing" bound (#287) needs the HOME box's own
422// row stamped.
423//
424// Housekeeper or not, the stamps are what the console shows: a home box's
425// status snapshot carries them, and a non-housekeeper home box would show
426// every attached hostname as un-pointing forever (#303) — and the edges-file
427// publisher's "pointing" bound (#287) reads the home box's own row.
428//
429// routes may be nil, in which case the registry is listed here. Inert with
430// no AppsDomain, per Config's contract.
431func StampHealth(ctx context.Context, reg *registry.Registry, cfg Config, routes []registry.Route) error {
432 if cfg.AppsDomain == "" {
433 return nil
434 }
435 if cfg.Now == nil {
436 cfg.Now = time.Now
437 }
438 if routes == nil {
439 var err error
440 if routes, err = reg.List(); err != nil {
441 return fmt.Errorf("domains: list routes: %w", err)
442 }
443 }
444 // aliases are the extra hostnames route_aliases holds (Task 2/#151),
445 // health-stamped by sections 4 and 5 below exactly like the routes they
446 // point at. Listed ONCE, above both sections, and guarded the same way
447 // each section already guards itself — a box with neither Resolve nor
448 // AcmeDir configured pays no aliases query at all.
449 var aliases []registry.RouteAlias
450 if cfg.Resolve != nil || cfg.AcmeDir != "" {
451 var err error
452 aliases, err = reg.ListAliases()
453 if err != nil {
454 return fmt.Errorf("domains: list aliases: %w", err)
455 }
456 }
457
458 // --- 4. DNS health --------------------------------------------------------
459 // The edge set is asked for ONCE PER PASS, not once per box start: under
460 // latency steering a customer's correctly-pointed domain resolves to
461 // whichever region's edge the resolver is nearest, so the comparison set
462 // has to be the deployment's live one.
463 if cfg.Resolve != nil && cfg.EdgeIPs != nil {
464 edge := map[string]bool{}
465 for _, ip := range cfg.EdgeIPs() {
466 edge[ip] = true
467 }
468 // Load-bearing: stamping against an empty set would mark every host
469 // in the deployment `elsewhere` at once — a fleet-wide lying red
470 // raised by a bucket read that failed, not by anyone's DNS. Unknown
471 // is not a verdict this pass is allowed to guess at, so it stands
472 // down and leaves the previous stamps alone until it knows again.
473 //
474 // This is the SET-level case of the same rule the per-host loop
475 // below applies: what we do not know, we do not stamp.
476 if len(edge) == 0 {
477 log.Printf("domains: no edge addresses known this pass — DNS stamping stands down rather than stamping every host elsewhere")
478 } else {
479 now := cfg.Now().Unix()
480 // OUR failure never overwrites a verdict. "unresolved" means the
481 // lookup itself failed (SERVFAIL, timeout, no resolver) — one bad
482 // DNS tick must not flip a working domain, because the edges-file
483 // publisher gates on "pointing" (#287), so that flip would
484 // unpublish every alias fleet-wide for a cadence. The stamp time
485 // is left alone too, so staleness stays visible (dns_checked_at
486 // stops advancing) rather than being laundered into a fresh
487 // unknown.
488 //
489 // An authoritative NXDOMAIN ("") is NOT that case and DOES stamp:
490 // the resolver answered, and the answer is that the record is
491 // gone. A customer who deletes their record must stop reading
492 // "pointing" — refusing that news would be the same lie in the
493 // opposite direction.
494 for _, rt := range routes {
495 status := dnsStatus(cfg.Resolve, rt.Host, edge)
496 if status == unresolvedDNS && rt.DNSStatus != "" {
497 continue
498 }
499 if err := reg.SetDNSStatus(rt.Host, status, now); err != nil {
500 log.Printf("domains: stamp dns on %s: %v", rt.Host, err)
501 }
502 }
503 for _, a := range aliases {
504 status := dnsStatus(cfg.Resolve, a.Alias, edge)
505 if status == unresolvedDNS && a.DNSStatus != "" {
506 continue
507 }
508 if err := reg.SetAliasDNSStatus(a.Alias, status, now); err != nil {
509 log.Printf("domains: stamp dns on alias %s: %v", a.Alias, err)
510 }
511 }
512 }
513 }
514
515 // --- 5. TLS health --------------------------------------------------------
516 if cfg.AcmeDir != "" {
517 for _, rt := range routes {
518 at := hostCertExpiry(cfg.AcmeDir, rt.Host, cfg.AppsDomain)
519 if err := reg.SetCertExpiresAt(rt.Host, at); err != nil {
520 log.Printf("domains: stamp cert expiry on %s: %v", rt.Host, err)
521 }
522 }
523 for _, a := range aliases {
524 at := hostCertExpiry(cfg.AcmeDir, a.Alias, cfg.AppsDomain)
525 if err := reg.SetAliasCertExpiresAt(a.Alias, at); err != nil {
526 log.Printf("domains: stamp cert expiry on alias %s: %v", a.Alias, err)
527 }
528 }
529 }
530 return nil
531}
532
533// unresolvedDNS is the stamp for "we could not look this up" as distinct
534// from "the resolver says there is no such record". The distinction is the
535// difference between blaming the customer and owning the outage.
536const unresolvedDNS = "unresolved"
537
538// dnsStatus resolves host and classifies the answer against edge. Three
539// distinct unknowns, deliberately:
540//
541// - "" — the resolver answered authoritatively that the name does not
542// exist. The customer has not created the record yet; guidance is right.
543// - unresolvedDNS — the lookup FAILED (SERVFAIL, timeout, no resolver).
544// That is OUR problem, and telling the customer their record is missing
545// would blame them for our outage. Section 4 never lets it overwrite an
546// existing verdict.
547// - "elsewhere"/"pointing" — a real answer, judged against the set.
548//
549// Shared by the route and alias loops in section 4 so the two verdicts can
550// never drift apart.
551func dnsStatus(resolve func(host string) ([]string, error), host string, edge map[string]bool) string {
552 ips, err := resolve(host)
553 if err != nil {
554 var dnsErr *net.DNSError
555 if errors.As(err, &dnsErr) && dnsErr.IsNotFound {
556 return ""
557 }
558 return unresolvedDNS
559 }
560 for _, ip := range ips {
561 if edge[ip] {
562 return "pointing"
563 }
564 }
565 return "elsewhere"
566}
567
568// OpsSqid is the built-in ops account's frozen sqid (registry id 1). Every
569// deployment has it, console or not.
570var OpsSqid = Sqid(1)
571
572// AccountManifestSqid reports whether key is an account manifest
573// (accounts/<sqid>/account.json, exactly one path level deep) and which sqid
574// its PATH claims.
575//
576// Exported because the console materializes accounts from the SAME objects
577// this reconcile pass does. One parser, not two: if the two ever disagreed
578// about what counts as a manifest, the console and the registry would hold
579// different account sets on the same box — and the account set is what tells a
580// qualified bucket key from a legacy one.
581func AccountManifestSqid(key string) (string, bool) {
582 rest, ok := strings.CutPrefix(key, "accounts/")
583 if !ok {
584 return "", false
585 }
586 sqid, ok := strings.CutSuffix(rest, "/account.json")
587 if !ok || sqid == "" || strings.Contains(sqid, "/") {
588 return "", false
589 }
590 return sqid, true
591}
592
593// materializeAccounts turns the deployment's account manifests into this
594// box's registry account rows, and returns the set of sqids that exist.
595//
596// The return value is the point: known is what makes an apps/<sqid>/<app>/…
597// key path unambiguous (appKeyOwner), so it drives the trash-marker scan,
598// the app->account linkage and the canary-pointer scan — every part of the
599// pass that reads a bucket key. It is a function rather than an inline loop
600// so that invariant is reachable by a test; there is one vantage, and reg is
601// never nil.
602func materializeAccounts(ctx context.Context, s release.Store, acctKeys []string, reg *registry.Registry) map[string]bool {
603 // The console originates account truth as accounts/<sqid>/account.json in
604 // the bucket; this is the only place it becomes registry state, and the id
605 // in the manifest is TAKEN AS GIVEN — never assigned here. That is the
606 // whole durability property: a rebuilt registry, or a second box, ends up
607 // with byte-identical account identity, so platform domains never re-home.
608 // A deployment with no console has no manifests: everything stays under
609 // the built-in ops account.
610 known := map[string]bool{OpsSqid: true} // sqid -> materialized
611 for _, key := range acctKeys {
612 sqid, ok := AccountManifestSqid(key)
613 if !ok {
614 continue
615 }
616 body, _, err := s.Get(ctx, key)
617 if err != nil {
618 log.Printf("domains: skipping %s: read failed: %v", key, err)
619 continue
620 }
621 var m AccountManifest
622 if err := json.Unmarshal(body, &m); err != nil || m.ID <= 0 || m.Sqid == "" {
623 log.Printf("domains: skipping %s: malformed account manifest: %v", key, err)
624 continue
625 }
626 // Three things must agree: the key path, the body's sqid, and the
627 // frozen encoding of the body's id. A manifest that disagrees with
628 // itself is a lie about identity — materializing it would put an
629 // account at a domain that stops resolving the moment anything
630 // recomputes the sqid — so it is skipped, loudly, never guessed at.
631 if m.Sqid != sqid || Sqid(m.ID) != sqid {
632 log.Printf("domains: skipping %s: sqid %q/%q is not Sqid(%d)=%q", key, sqid, m.Sqid, m.ID, Sqid(m.ID))
633 continue
634 }
635 // A tombstoned manifest is materialized nowhere and, if this registry
636 // still carries the account from before it was retired, un-materialized
637 // here. Leaving the row would keep the account listed by `carlos
638 // accounts list` on every box forever — a deletion that only ever took
639 // effect in the console that performed it.
640 if m.Deleted() {
641 if err := reg.DeleteAccount(m.Sqid); err != nil {
642 log.Printf("domains: retire account %s: %v", sqid, err)
643 }
644 continue
645 }
646 // KNOWN FIRST, then the write. The manifest is the authority on
647 // which accounts EXIST; the registry row is only this box's local
648 // copy of that fact, and the two answer different questions. known
649 // answers the first — it is what tells a qualified bucket key
650 // (apps/<sqid>/<app>/…) from a legacy one, and nothing else.
651 //
652 // So a failed row-write must not un-know the account. If it did,
653 // every one of that account's keys would read as legacy for the rest
654 // of the pass (appKeyOwner returns !ok), and this box would silently
655 // skip the account's trash markers, route linkage and canary
656 // pointers — a transient SQLite error turning into "this tenant's
657 // apps stop reconciling" until some later pass happens to write the
658 // row. Staying known costs only a stale `carlos accounts list` row on
659 // this one box, which the next pass fixes.
660 known[sqid] = true
661 if err := reg.PutAccount(m.ID, m.Sqid, m.Name); err != nil {
662 log.Printf("domains: materialize account %s: %v", sqid, err)
663 continue
664 }
665 }
666 return known
667}
668
669// appKeyOwner reads the app→account linkage straight out of a bucket key:
670// apps/<sqid>/<app>/… belongs to that account. known is the set of
671// materialized account sqids, which is what makes a key path unambiguous —
672// hence accounts being read first.
673//
674// A LEGACY unqualified apps/<app>/… key is not linkage. It used to be (it
675// meant "ops, until this app is migrated"), but the read-fallback's
676// retirement on 2026-08-03 made those objects unreadable to the runtime, and
677// bytes nobody can serve say nothing about who owns an app. Acting on them
678// here would be worse than useless: reconciliation would mint routes in front
679// of builds no box can install, and a leftover copy under a migrated app
680// would claim it back for ops. Nothing removes legacy objects, so this stays
681// true for as long as they sit there.
682//
683// Keys with nothing after the app segment (apps/<sqid>/<app>) are not linkage
684// either: an object sitting directly at the app prefix says nothing about
685// ownership. It also returns the app's key prefix, so callers can parse what
686// comes after it without rebuilding it.
687//
688// Requiring a KNOWN sqid also closes a latent bug the old legacy branch hid:
689// a genuinely qualified key whose account had not materialized yet fell
690// through to it and came back as (ops, "<sqid>") — the account's own sqid
691// read as an app NAME, on any box whose account manifests reconcile had not
692// reached. That minted apps out of account ids. Now it returns nothing and
693// the next pass, with the account materialized, reads the key correctly.
694func appKeyOwner(key string, known map[string]bool) (acct, app, prefix string, ok bool) {
695 rest, ok := strings.CutPrefix(key, "apps/")
696 if !ok {
697 return "", "", "", false
698 }
699 acct, rest, ok = strings.Cut(rest, "/")
700 if !ok || !known[acct] { // legacy apps/<app>/…, or an account we know nothing about
701 return "", "", "", false
702 }
703 app, _, hasThird := strings.Cut(rest, "/")
704 if !hasThird || app == "" {
705 return "", "", "", false
706 }
707 // The one place the two namespaces can genuinely collide: an app whose
708 // NAME happens to equal a materialized account sqid — a legacy key for an
709 // app called "bab" is apps/bab/channels/…, which parses as ops's app
710 // "channels". A qualified key's third segment is the app name; a legacy
711 // key's second is always one of the release plane's own words, so those
712 // two words are never an app here.
713 if app == "channels" || app == "releases" {
714 return "", "", "", false
715 }
716 return acct, app, "apps/" + acct + "/" + app + "/", true
717}
718
719// canaryPointer reports whether key is an account-qualified canary channel
720// pointer (apps/<sqid>/<app>/channels/canary/<name>.json), and its app +
721// canary name. A legacy unqualified pointer is not one: see appKeyOwner.
722func canaryPointer(key string, known map[string]bool) (app, name string, ok bool) {
723 _, app, prefix, ok := appKeyOwner(key, known)
724 if !ok {
725 return "", "", false
726 }
727 rest, ok := strings.CutPrefix(key, prefix+"channels/canary/")
728 if !ok {
729 return "", "", false
730 }
731 name, ok = strings.CutSuffix(rest, ".json")
732 if !ok || name == "" || strings.Contains(name, "/") {
733 return "", "", false
734 }
735 return app, name, true
736}
737
738// appKey is an app's identity: the account it belongs to plus its name. App
739// names are unique WITHIN an account, never globally, so nothing about an app
740// may be keyed on the name alone.
741type appKey struct{ acct, app string }
742
743// resolveRouteAccount decides which account a route's app belongs to, given
744// the accounts the BUCKET says hold objects under that app name. "" means
745// leave the row alone.
746//
747// The rules, in order:
748// - the row's own account already holds objects: it is right, keep it;
749// - exactly one account holds them: adopt it (this is what makes a
750// migration land);
751// - several do, and the row names none of them: leave it alone. Picking one
752// would re-home somebody's app onto another tenant's bytes, which is far
753// worse than a row that reconciliation declines to touch.
754func resolveRouteAccount(rt registry.Route, owners []string, appsDomain string) string {
755 if len(owners) == 0 {
756 if rt.Account == "" {
757 return OpsSqid // nothing shipped yet: ops, like a console-less box
758 }
759 return "" // the row says something; the bucket says nothing. Believe the row.
760 }
761 // A row that names one of the owners is right; keep it.
762 if release.ValidAccount(rt.Account) && slices.Contains(owners, rt.Account) {
763 return rt.Account
764 }
765 // The row names nothing (or something the bucket doesn't hold). Before
766 // any fallback: a platform host literally names its owner in its own
767 // labels — <name>.<sqid>.<apps domain>. #344 was the ops fallback
768 // winning this tie (a stray pipeline.json under ops vs the tenant's
769 // whole app), and the config sync then deleting the tenant's delivered
770 // config for the mis-stamped account.
771 if sq := release.HostAccountLabel(rt.Host, appsDomain); sq != "" && slices.Contains(owners, sq) {
772 return sq
773 }
774 if len(owners) == 1 {
775 return owners[0]
776 }
777 // Several owners and nothing that decides between them. NOT ops-if-
778 // present: an unstamped row already reads as ops at runtime, so
779 // stamping buys nothing — and a wrong stamp is what #344 cost.
780 log.Printf("domains: %s (app %q) names account %q, but objects exist under %v — leaving it alone",
781 rt.Host, rt.App, rt.Account, owners)
782 return ""
783}
784
785// candidateApps returns the (account, app) pairs eligible for a platform
786// domain, in first-seen (host) order: every app — static sites included, a
787// shipped static tree deserves an addressable URL as much as an instance
788// does — outside the platform-reserved svc/* namespace, with at least one
789// route that is NOT itself platform-minted. An app whose only routes are
790// minted ones has been removed by its operator, and re-minting from a mint
791// would keep a ghost alive forever.
792//
793// The account is the one already stamped on the route by the linkage step
794// above, so an app whose objects have moved mints under the account that now
795// owns them.
796func candidateApps(routes []registry.Route) []appKey {
797 seen := map[appKey]bool{}
798 real := map[appKey]bool{} // has a non-platform route
799 var order []appKey
800 for _, rt := range routes {
801 if rt.App == "" || strings.HasPrefix(rt.App, "svc/") {
802 continue
803 }
804 k := appKey{release.RouteAccount(rt), rt.App}
805 if !seen[k] {
806 seen[k] = true
807 order = append(order, k)
808 }
809 if !rt.Platform {
810 real[k] = true
811 }
812 }
813 var out []appKey
814 for _, k := range order {
815 if real[k] {
816 out = append(out, k)
817 }
818 }
819 return out
820}
821
822// firstReadyUpstream picks the deterministic upstream a minted domain
823// serves from: the app's first ready route in host order (reg.List's order)
824// that has something to copy and is not itself a minted route — minting
825// from a mint would be self-reference. "Something to copy" is a proxy
826// upstream (Socket or Addr) for instances and services, or — for static
827// routes — the release channel, since a static route's backend IS its
828// channel pointer, not a socket.
829func firstReadyUpstream(routes []registry.Route, k appKey) *registry.Route {
830 for i := range routes {
831 rt := &routes[i]
832 if rt.App != k.app || release.RouteAccount(*rt) != k.acct || rt.Platform || rt.Status != "ready" {
833 continue
834 }
835 if rt.Socket != "" || rt.Addr != "" || (rt.Kind == "static" && rt.Channel != "") {
836 return rt
837 }
838 }
839 return nil
840}
841
842// mintedRoute is the platform route shape: same upstream trio
843// (Socket/Addr/Backing) and Kind as the source, ready (the upstream is —
844// that's how it was chosen), platform-flagged, account-stamped.
845//
846// Channel is copied for static routes and NOT for anything else, because
847// the two serving classes reach their backend by different mechanisms. A
848// proxied route (instance/service) reaches its backend through Socket/Addr,
849// already copied above — the channel would be decoration, and a copied
850// canary channel would make the apex route look like a canary follower, so
851// retiring that canary would delete the app's platform domain. A static
852// route has no socket at all: the edge's static handler resolves the app's
853// CHANNEL POINTER to find the shipped tar, so a channel-less static mint
854// would serve nothing. Hence the asymmetry — same field, opposite verdicts,
855// for reasons that live one layer down in internal/edge.
856//
857// Environment is never set here, by design (2026-08-02-plan-environments.md
858// Task 2): reconcile does not derive an environment binding for a minted
859// route, so `carlos route -environment` is allowed on a plain Platform apex.
860// It is NOT allowed on a Platform+Canary route (platformCanaryOwnedRow,
861// cli.go) — not because this function derives Environment, but because if
862// such a route's `followed` check (below) goes false again, the Upsert(nr)
863// that follows rebuilds the whole row from this function's values. That is
864// the same zero-value-on-re-mint caveat runRoute documents for
865// Channel/Version/Status/DNS/Cert. Since the 2026-08-02 review,
866// registry.Upsert preserves a non-empty existing environment through exactly
867// this path (IMPORTANT 3), so a binding is no longer silently lost here —
868// but a row this function rebuilds is still not one the route verb hands an
869// operator, which is why the CLI refusal exists on top of the preservation.
870//
871// Region is copied from src (customer-fleets follow-up): a fleet-placed
872// source's minted apex must carry the same "fleet/<name>" Region so the
873// platform edge dispatches it through the fleet's channel rather than
874// treating it as unconstrained. When src is fleet-placed
875// (registry.FleetPlacement(src.Region)), Socket is deliberately left empty
876// even though it was just copied above the src.Kind == "static" carve-out
877// would otherwise leave it — a fleet-placed source's Socket is a path on
878// the FLEET BOX, not this platform process, so copying it verbatim onto the
879// minted apex would give it a dead local socket instead of letting it ride
880// the fleet dispatch Region now enables.
881func mintedRoute(host, app, account string, src registry.Route) registry.Route {
882 rt := registry.Route{
883 Host: host,
884 Kind: src.Kind,
885 Socket: src.Socket,
886 Addr: src.Addr,
887 Backing: src.Backing,
888 App: app,
889 Status: "ready",
890 Platform: true,
891 Account: account,
892 Region: src.Region,
893 }
894 if src.Kind == "static" {
895 rt.Channel = src.Channel
896 }
897 if _, ok := registry.FleetPlacement(src.Region); ok {
898 rt.Socket = ""
899 }
900 return rt
901}
902
903// hostCertExpiry resolves host's cert expiry from the acme cache dir as the
904// LATER of the per-host autocert entry and the covering wildcard's PEM
905// (wildcard-<parent>.pem). Two reasons a bare per-host lookup isn't enough: a
906// wildcard-covered host never gets a per-host entry at all (without the
907// wildcard fallback status would call it certless forever), and a host
908// GRANDFATHERED onto a per-host cert from before its wildcard existed keeps
909// that file on disk forever — autocert stops renewing it once the wildcard
910// starts serving the host, so the stale file's expiry would shadow the
911// wildcard's and status would cry "expiring" over a cert the edge no longer
912// serves. The edge always serves whichever cert is actually valid, so the max
913// of the two is the only truthful answer. 0 = unknown, never a guess.
914//
915// The wildcard parent is host's generic one-label strip, not an
916// appsDomain-shaped derivation (appsDomain is otherwise unused here): one
917// label folds into "*.", so any host's covering wildcard is its one-label
918// parent's, whether that parent is an <acct>.<apps-domain> derivation, a
919// static domain (go.tito.io), or a delegated customer domain (woodstar.app).
920// Trying the read is cheaper than knowing which — a miss is a miss. An apex
921// host's own strip lands one label short of the parent that names ITS
922// covering wildcard file, so the apex never matches its own wildcard.
923func hostCertExpiry(acmeDir, host, appsDomain string) int64 {
924 perHost := certExpiry(filepath.Join(acmeDir, host))
925 rsa := certExpiry(filepath.Join(acmeDir, host+"+rsa"))
926 if rsa > perHost {
927 perHost = rsa
928 }
929 wildcard := int64(0)
930 if i := strings.IndexByte(host, '.'); i > 0 {
931 parent := strings.ToLower(host[i+1:])
932 wildcard = certExpiry(filepath.Join(acmeDir, "wildcard-"+parent+".pem"))
933 }
934 if wildcard > perHost {
935 return wildcard
936 }
937 return perHost
938}
939
940// certExpiry reads the autocert DirCache entry at path (the PEM bundle
941// cachePut writes: private key block first, then the cert chain) and returns
942// the leaf's NotAfter as unix seconds. Anything short of a parseable
943// certificate — no file, no CERTIFICATE block, a corrupt one — is 0:
944// unknown, never a guess.
945func certExpiry(path string) int64 {
946 body, err := os.ReadFile(path)
947 if err != nil {
948 return 0
949 }
950 for len(body) > 0 {
951 var block *pem.Block
952 block, body = pem.Decode(body)
953 if block == nil {
954 return 0
955 }
956 if block.Type != "CERTIFICATE" {
957 continue // the private-key block(s) come first in the bundle
958 }
959 cert, err := x509.ParseCertificate(block.Bytes)
960 if err != nil {
961 return 0
962 }
963 return cert.NotAfter.Unix()
964 }
965 return 0
966}
967
968// importJournalHosts is every host the registry's import journal holds.
969func importJournalHosts(reg *registry.Registry) (map[string]bool, error) {
970 js, err := reg.ImportJournals()
971 if err != nil {
972 return nil, err
973 }
974 hosts := make(map[string]bool, len(js))
975 for _, j := range js {
976 hosts[j.Host] = true
977 }
978 return hosts, nil
979}
980