amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package domains_test
2
3import (
4 "bytes"
5 "context"
6 "crypto/ecdsa"
7 "crypto/ed25519"
8 "crypto/elliptic"
9 "crypto/rand"
10 "crypto/x509"
11 "crypto/x509/pkix"
12 "encoding/pem"
13 "errors"
14 "fmt"
15 "log"
16 "math/big"
17 "net"
18 "os"
19 "path/filepath"
20 "reflect"
21 "slices"
22 "strings"
23 "testing"
24 "time"
25
26 "github.com/carlosframework/platform/internal/domains"
27 "github.com/carlosframework/platform/internal/registry"
28 "github.com/carlosframework/platform/internal/release"
29)
30
31var ctx = context.Background()
32
33func dirStore(t *testing.T) *release.DirStore {
34 t.Helper()
35 return release.NewDirStore(t.TempDir())
36}
37
38func regWith(t *testing.T, routes ...registry.Route) *registry.Registry {
39 t.Helper()
40 reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db"))
41 if err != nil {
42 t.Fatal(err)
43 }
44 for _, rt := range routes {
45 if err := reg.Upsert(rt); err != nil {
46 t.Fatal(err)
47 }
48 }
49 return reg
50}
51
52// writeAccountManifest writes the console-shaped account manifest at
53// accounts/<sqid>/account.json — the bucket IS where account identity lives,
54// and the id in it is taken as given. The sqid is the frozen encoding of the
55// id (never anything else), so the helper derives it and hands it back.
56func writeAccountManifest(t *testing.T, s release.Store, id int64, name string) string {
57 t.Helper()
58 sqid := domains.Sqid(id)
59 body := []byte(`{"id":` + fmt.Sprint(id) + `,"sqid":"` + sqid + `","name":"` + name +
60 `","created_at":"2026-08-02T09:00:00Z"}`)
61 if _, err := s.Put(ctx, "accounts/"+sqid+"/account.json", body, ""); err != nil {
62 t.Fatal(err)
63 }
64 return sqid
65}
66
67// putKey drops an object at key so the key-path linkage has something to read.
68func putKey(t *testing.T, s release.Store, key string) {
69 t.Helper()
70 if _, err := s.Put(ctx, key, []byte("{}"), ""); err != nil {
71 t.Fatal(err)
72 }
73}
74
75// cfg is the standard reconcile config: apps domain set, a fake resolver
76// saying every host points at the edge, DNS/TLS seams otherwise defaulted.
77//
78// EdgeIPs is a func because the live set moves under the pass (in production
79// it folds every box's published edges file each pass); a fixture that
80// answers the same one address every time keeps every existing assertion
81// here reading exactly as it did when the field was a slice.
82func cfg(t *testing.T) domains.Config {
83 t.Helper()
84 return domains.Config{
85 AppsDomain: "oncarlos.test",
86 EdgeIPs: func() []string { return []string{"203.0.113.7"} },
87 Resolve: func(host string) ([]string, error) { return []string{"203.0.113.7"}, nil },
88 Now: func() time.Time { return time.Unix(1754000000, 0) },
89 }
90}
91
92func mustGet(t *testing.T, reg *registry.Registry, host string) *registry.Route {
93 t.Helper()
94 rt, err := reg.Get(host)
95 if err != nil {
96 t.Fatal(err)
97 }
98 if rt == nil {
99 t.Fatalf("route %s missing", host)
100 }
101 return rt
102}
103
104func routeCount(t *testing.T, reg *registry.Registry) int {
105 t.Helper()
106 routes, err := reg.List()
107 if err != nil {
108 t.Fatal(err)
109 }
110 return len(routes)
111}
112
113func TestReconcileMintsPlatformDomainUnderOps(t *testing.T) {
114 s, reg := dirStore(t), regWith(t, registry.Route{Host: "kass.training", Kind: "instance",
115 App: "kass", Status: "ready", Addr: "127.0.0.1:9000"})
116 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
117 t.Fatal(err)
118 }
119 // No manifest anywhere: the app lives under the built-in ops account
120 // (id 1, Sqid "bab" — frozen).
121 got := mustGet(t, reg, "kass.bab.oncarlos.test")
122 if !got.Platform || got.Addr != "127.0.0.1:9000" || got.Account != "bab" || got.App != "kass" {
123 t.Errorf("minted route = %+v", got)
124 }
125 n := routeCount(t, reg)
126 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
127 t.Fatal(err)
128 }
129 if routeCount(t, reg) != n {
130 t.Errorf("second Reconcile changed the route count: %d -> %d", n, routeCount(t, reg))
131 }
132}
133
134// The durability property, at the reconcile layer: whatever id the bucket
135// says, the registry takes as given. It never assigns one of its own, so two
136// boxes — or one box rebuilt — materialize identical sqids.
137func TestReconcileMaterializesAccountsFromManifests(t *testing.T) {
138 s, reg := dirStore(t), regWith(t)
139 sqid := writeAccountManifest(t, s, 793481920157, "Kass")
140 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
141 t.Fatal(err)
142 }
143 id, name, _ := reg.AccountBySqid(sqid)
144 if id != 793481920157 || name != "Kass" {
145 t.Errorf("materialized id=%d name=%q — the bucket's id must be taken as given", id, name)
146 }
147 // A second pass over the same bucket is a no-op, and a REBUILT registry
148 // re-materializes the identical id — the durability property.
149 reg2 := regWith(t)
150 if err := domains.Reconcile(ctx, s, reg2, cfg(t)); err != nil {
151 t.Fatal(err)
152 }
153 if id2, _, _ := reg2.AccountBySqid(sqid); id2 != id {
154 t.Errorf("rebuilt registry materialized id %d, want %d", id2, id)
155 }
156}
157
158// A tombstoned manifest is materialized nowhere, and un-materializes an account
159// this registry already carried. Without the second half, a deletion would take
160// effect only in the console that performed it: every other box on the
161// deployment would keep listing the account forever.
162func TestReconcileRetiresTombstonedAccounts(t *testing.T) {
163 s, reg := dirStore(t), regWith(t)
164 sqid := writeAccountManifest(t, s, 793481920157, "Kass")
165 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
166 t.Fatal(err)
167 }
168 if id, _, _ := reg.AccountBySqid(sqid); id == 0 {
169 t.Fatal("the account did not materialize in the first place")
170 }
171 if err := domains.TombstoneAccount(ctx, s, sqid, time.Now()); err != nil {
172 t.Fatal(err)
173 }
174 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
175 t.Fatal(err)
176 }
177 if id, name, _ := reg.AccountBySqid(sqid); id != 0 {
178 t.Errorf("retired account still in the registry as id=%d name=%q", id, name)
179 }
180 // A registry built from scratch never learns about it at all.
181 reg2 := regWith(t)
182 if err := domains.Reconcile(ctx, s, reg2, cfg(t)); err != nil {
183 t.Fatal(err)
184 }
185 if id, _, _ := reg2.AccountBySqid(sqid); id != 0 {
186 t.Errorf("a rebuilt registry materialized the retired account as id=%d", id)
187 }
188 // Ops is never deletable, however the manifest is written.
189 if err := domains.TombstoneAccount(ctx, s, domains.Sqid(domains.ReservedAccountID), time.Now()); err == nil {
190 t.Error("tombstoning the ops account was allowed")
191 }
192}
193
194// The app→account linkage IS the bucket key path: an app under
195// apps/<sqid>/<app>/… belongs to that account. A legacy unqualified
196// apps/<app>/… key is not linkage at all any more — the runtime cannot read
197// it, so it says nothing about who owns what. oldapp here still lands under
198// ops, but from its ROUTE's default, not from the leftover object.
199func TestReconcileDerivesAccountFromKeyPath(t *testing.T) {
200 s, reg := dirStore(t), regWith(t,
201 registry.Route{Host: "kass.training", Kind: "instance", App: "kass", Status: "ready", Addr: "127.0.0.1:9000"},
202 registry.Route{Host: "old.example.com", Kind: "instance", App: "oldapp", Status: "ready", Addr: "127.0.0.1:9001"})
203 sqid := writeAccountManifest(t, s, 793481920157, "Kass")
204 putKey(t, s, "apps/"+sqid+"/kass/channels/stable.json")
205 putKey(t, s, "apps/oldapp/channels/stable.json")
206 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
207 t.Fatal(err)
208 }
209 // kass mints under its account; oldapp mints under bab (ops).
210 kass := mustGet(t, reg, "kass."+sqid+".oncarlos.test")
211 if !kass.Platform || kass.Account != sqid {
212 t.Errorf("kass minted route = %+v, want platform under %s", kass, sqid)
213 }
214 old := mustGet(t, reg, "oldapp.bab.oncarlos.test")
215 if !old.Platform || old.Account != "bab" {
216 t.Errorf("oldapp minted route = %+v, want platform under bab", old)
217 }
218 // The source route carries the sqid too, so adopt builds the right keys.
219 if src := mustGet(t, reg, "kass.training"); src.Account != sqid {
220 t.Errorf("source route account = %q, want %s", src.Account, sqid)
221 }
222}
223
224// A leftover legacy object must not claim an app for ops. This is the shape
225// every migrated app's bucket is in — the migration copies and deliberately
226// never deletes — and the route was added before any of it, so it carries no
227// account yet.
228//
229// It used to go the other way: the legacy leftover made OPS an owner of
230// kass's objects, an unstamped route reads as ops, ops was among the owners,
231// and the route got stamped ops — pointing adopt at a prefix holding nothing
232// but the copy nobody serves. With legacy keys ignored there is exactly one
233// owner, the account the app actually lives in.
234func TestReconcileIgnoresLeftoverLegacyKeys(t *testing.T) {
235 s, reg := dirStore(t), regWith(t,
236 registry.Route{Host: "kass.training", Kind: "static", App: "kass",
237 Status: "ready", Channel: "stable"})
238 sqid := writeAccountManifest(t, s, 793481920157, "Kass")
239 putKey(t, s, "apps/"+sqid+"/kass/channels/stable.json")
240 putKey(t, s, "apps/kass/channels/stable.json")
241 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
242 t.Fatal(err)
243 }
244 if src := mustGet(t, reg, "kass.training"); src.Account != sqid {
245 t.Errorf("source route account = %q, want %s — a legacy leftover claimed the app for ops",
246 src.Account, sqid)
247 }
248 mustGet(t, reg, "kass."+sqid+".oncarlos.test") // the apex mints under the real account
249}
250
251// A legacy canary pointer mints no canary host. The host would resolve a
252// channel the fleet cannot read, so it would serve a 404 under a URL the
253// console advertises as live.
254func TestReconcileMintsNothingForALegacyCanaryPointer(t *testing.T) {
255 s, reg := dirStore(t), regWith(t, registry.Route{Host: "kass.training", Kind: "static",
256 App: "kass", Status: "ready", Channel: "stable"})
257 putKey(t, s, "apps/kass/channels/canary/pricing.json")
258 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
259 t.Fatal(err)
260 }
261 if rt, err := reg.Get("pricing.kass.bab.oncarlos.test"); err != nil || rt != nil {
262 t.Errorf("a legacy canary pointer minted %+v (%v) — nothing can serve it", rt, err)
263 }
264}
265
266// PINNED, NOT ENDORSED. A platform-minted canary host stamped with one
267// account while its app's real canary pointer lives under another is removed
268// on the next pass. The stamp is what decides, and a single stray object
269// under the stamped account is enough to hold it there: resolveRouteAccount's
270// first rule is "the row's own account already holds objects: it is right,
271// keep it", so step 1b never re-stamps the row onto the account the pointer
272// is actually under, and the retirement loop then finds no pointer for the
273// (account, app) pair the row names.
274//
275// This shape predates the fallback's retirement — the retirement only changed
276// which pointers count — and a live pre-flight on 2026-08-03 found it does not
277// occur on either fleet: tito's reconcile is inert (no apps domain configured)
278// and the flagship registry holds zero Platform+Canary rows. It is pinned here
279// so the behaviour is known rather than discovered by a deleted host.
280//
281// If it ever does bite, the fix belongs in the STAMPING rule (a row whose
282// stamped account holds no pointer for its own channel is not "right"), never
283// in the retirement loop — softening the loop would strand every genuinely
284// retired canary host forever.
285func TestReconcileRemovesACanaryHostStampedWithTheWrongAccount(t *testing.T) {
286 s, reg := dirStore(t), regWith(t,
287 registry.Route{Host: "blog.kass.test", Kind: "static", App: "blog",
288 Status: "ready", Channel: "stable", Account: "bab"},
289 registry.Route{Host: "pricing.blog.bab.oncarlos.test", Kind: "static", App: "blog",
290 Status: "ready", Channel: "canary/pricing", Platform: true, Canary: true, Account: "bab"},
291 )
292 sqid := writeAccountManifest(t, s, 793481920157, "Kass")
293 // One stray object under ops pins both rows to bab...
294 putKey(t, s, "apps/bab/blog/releases/v1/manifest.json")
295 // ...while the canary pointer the fleet actually follows is the tenant's.
296 putKey(t, s, "apps/"+sqid+"/blog/channels/canary/pricing.json")
297 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
298 t.Fatal(err)
299 }
300 if rt, err := reg.Get("pricing.blog.bab.oncarlos.test"); err != nil || rt != nil {
301 t.Errorf("canary route survived: %+v %v — if this now passes, the stamping rule changed; "+
302 "re-read the comment above before deleting this test", rt, err)
303 }
304 // The row that pinned it keeps its stamp, which is the mechanism at work.
305 if src := mustGet(t, reg, "blog.kass.test"); src.Account != "bab" {
306 t.Errorf("source route account = %q, want bab", src.Account)
307 }
308}
309
310// A platform-minted canary host whose pointer only ever existed at the legacy
311// keys is retired like any other pointerless canary. It is the same removal
312// rule, reached because legacy keys stopped counting as live pointers: the
313// host resolved to a build the fleet cannot install, so keeping it alive
314// would advertise a URL that 404s.
315func TestReconcileRetiresCanaryHostLeftOnALegacyPointer(t *testing.T) {
316 s, reg := dirStore(t), regWith(t,
317 registry.Route{Host: "kass.training", Kind: "static", App: "kass",
318 Status: "ready", Channel: "stable"},
319 registry.Route{Host: "pricing.kass.bab.oncarlos.test", Kind: "static", App: "kass",
320 Status: "ready", Channel: "canary/pricing", Platform: true, Canary: true, Account: "bab"},
321 )
322 putKey(t, s, "apps/kass/channels/canary/pricing.json")
323 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
324 t.Fatal(err)
325 }
326 if rt, err := reg.Get("pricing.kass.bab.oncarlos.test"); err != nil || rt != nil {
327 t.Errorf("canary host on a legacy-only pointer survived: %+v %v", rt, err)
328 }
329}
330
331// A malformed account manifest is one skipped line, never a failed pass, and
332// never a guessed id.
333func TestReconcileSkipsMalformedAccountManifest(t *testing.T) {
334 s, reg := dirStore(t), regWith(t, registry.Route{Host: "kass.training", Kind: "instance",
335 App: "kass", Status: "ready", Addr: "127.0.0.1:9000"})
336 putKey(t, s, "accounts/bad/account.json") // "{}" — no id, no sqid
337 if _, err := s.Put(ctx, "accounts/worse/account.json", []byte("{not json"), ""); err != nil {
338 t.Fatal(err)
339 }
340 sqid := writeAccountManifest(t, s, 793481920157, "Kass")
341 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
342 t.Fatalf("one bad manifest must not fail the pass: %v", err)
343 }
344 if id, _, _ := reg.AccountBySqid(sqid); id != 793481920157 {
345 t.Error("good manifest not processed after the bad ones")
346 }
347 for _, sqid := range []string{"bad", "worse"} {
348 if id, _, _ := reg.AccountBySqid(sqid); id != 0 {
349 t.Errorf("materialized a malformed manifest as %s (id %d)", sqid, id)
350 }
351 }
352}
353
354// A manifest whose sqid is not the frozen encoding of its own id is a lie
355// about identity — materializing it would put an account at a domain that
356// stops resolving the moment anything recomputes the sqid.
357func TestReconcileRejectsMismatchedSqid(t *testing.T) {
358 s, reg := dirStore(t), regWith(t)
359 if _, err := s.Put(ctx, "accounts/kqmv/account.json",
360 []byte(`{"id":42,"sqid":"kqmv","name":"Liar"}`), ""); err != nil {
361 t.Fatal(err)
362 }
363 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
364 t.Fatal(err)
365 }
366 if id, _, _ := reg.AccountBySqid("kqmv"); id != 0 {
367 t.Errorf("materialized a manifest whose sqid isn't Sqid(id): id=%d", id)
368 }
369}
370
371// A static app deserves an addressable URL as much as an instance does
372// (platform-site is exactly that case). Its minted route must carry
373// Kind=static AND the source route's Channel — the static handler resolves
374// that channel pointer to find the tar, so a channel-less static route has
375// nothing to serve.
376func TestReconcileMintsStaticAppWithChannel(t *testing.T) {
377 s, reg := dirStore(t), regWith(t, registry.Route{Host: "carlosframework.com", Kind: "static",
378 App: "platform-site", Status: "ready", Channel: "stable"})
379 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
380 t.Fatal(err)
381 }
382 got := mustGet(t, reg, "platform-site.bab.oncarlos.test")
383 if !got.Platform || got.Kind != "static" || got.Channel != "stable" || got.Account != "bab" {
384 t.Errorf("minted static route = %+v, want platform static on channel stable", got)
385 }
386 // Idempotence with the channel copy in place: the minted route now
387 // follows "stable" itself, which must not confuse a second pass into
388 // minting again (nor into treating it as a canary follower).
389 n := routeCount(t, reg)
390 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
391 t.Fatal(err)
392 }
393 if routeCount(t, reg) != n {
394 t.Errorf("second Reconcile changed the route count: %d -> %d", n, routeCount(t, reg))
395 }
396 if got := mustGet(t, reg, "platform-site.bab.oncarlos.test"); got.Channel != "stable" {
397 t.Errorf("second pass lost the minted route's channel: %+v", got)
398 }
399}
400
401// A static app riding a canary channel mints an apex route ON that channel
402// (it must, to serve anything) — and that route must NOT be swept away as a
403// "retired canary follower" when the channel goes: only genuinely
404// canary-minted hosts are removal candidates.
405func TestReconcileKeepsStaticApexOnRetiredCanaryChannel(t *testing.T) {
406 s, reg := dirStore(t), regWith(t,
407 registry.Route{Host: "site.custom", Kind: "static", App: "site",
408 Status: "ready", Channel: "canary/x"},
409 registry.Route{Host: "site.bab.oncarlos.test", Kind: "static", App: "site",
410 Status: "ready", Channel: "canary/x", Platform: true},
411 )
412 // No canary/x pointer exists in the bucket at all (retired).
413 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
414 t.Fatal(err)
415 }
416 mustGet(t, reg, "site.bab.oncarlos.test") // the apex mint survives
417}
418
419// Canary hosts are minted for STATIC apps only, where the copied channel
420// genuinely is the backend (the static handler resolves the pointer to find
421// the tar). See TestReconcileSkipsCanaryMintForInstanceApps for the other
422// half of that rule and why it exists.
423func TestReconcileMintsCanaryHost(t *testing.T) {
424 s, reg := dirStore(t), regWith(t, registry.Route{Host: "kass.training", Kind: "static",
425 App: "kass", Status: "ready", Channel: "stable"})
426 // The pointer's existence is what reconcile reads — content is not its
427 // business (the edge verifies signatures when serving).
428 if _, err := s.Put(ctx, "apps/bab/kass/channels/canary/pricing.json", []byte("{}"), ""); err != nil {
429 t.Fatal(err)
430 }
431 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
432 t.Fatal(err)
433 }
434 got := mustGet(t, reg, "pricing.kass.bab.oncarlos.test")
435 if !got.Platform || got.Channel != "canary/pricing" || got.Kind != "static" || !got.Canary {
436 t.Errorf("minted canary route = %+v", got)
437 }
438 // A followed canary never doubles up: the minted route follows the
439 // channel now, so a second pass mints nothing new.
440 n := routeCount(t, reg)
441 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
442 t.Fatal(err)
443 }
444 if routeCount(t, reg) != n {
445 t.Errorf("second Reconcile changed the route count: %d -> %d", n, routeCount(t, reg))
446 }
447}
448
449// An instance app's canary host would be a LIE: a minted route can only
450// copy an existing upstream, and the only upstream available is the app's
451// stable process — so the "canary" URL would proxy straight to stable while
452// the console labeled it with the canary's version. Nothing here spawns a
453// per-host canary instance; that gap belongs to the not-yet-built `carlos
454// canary`. Minting nothing is the honest answer until then.
455func TestReconcileSkipsCanaryMintForInstanceApps(t *testing.T) {
456 s, reg := dirStore(t), regWith(t, registry.Route{Host: "kass.training", Kind: "instance",
457 App: "kass", Status: "ready", Addr: "127.0.0.1:9000"})
458 if _, err := s.Put(ctx, "apps/bab/kass/channels/canary/pricing.json", []byte("{}"), ""); err != nil {
459 t.Fatal(err)
460 }
461 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
462 t.Fatal(err)
463 }
464 if rt, err := reg.Get("pricing.kass.bab.oncarlos.test"); err != nil || rt != nil {
465 t.Errorf("minted a canary host for an instance app: %+v %v — it would serve stable, not the canary", rt, err)
466 }
467 // The app's ordinary apex still mints: only the canary host is withheld.
468 mustGet(t, reg, "kass.bab.oncarlos.test")
469}
470
471// An account change (or any hand-edited account column) must never leave an
472// app with two live apex domains: minting is additive — the pass never
473// removes an apex — so the guard is "already has one", not "has the
474// account-correct one".
475func TestReconcileNeverMintsASecondApex(t *testing.T) {
476 s, reg := dirStore(t), regWith(t,
477 registry.Route{Host: "kass.training", Kind: "instance", App: "kass",
478 Status: "ready", Addr: "127.0.0.1:9000"},
479 registry.Route{Host: "kass.bab.oncarlos.test", Kind: "instance", App: "kass",
480 Status: "ready", Addr: "127.0.0.1:9000", Platform: true, Account: "bab"},
481 )
482 // The app moves to another account: its objects are now under that sqid.
483 sqid := writeAccountManifest(t, s, 793481920157, "Kass")
484 putKey(t, s, "apps/"+sqid+"/kass/channels/stable.json")
485 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
486 t.Fatal(err)
487 }
488 if rt, err := reg.Get("kass." + sqid + ".oncarlos.test"); err != nil || rt != nil {
489 t.Errorf("minted a second apex %+v %v — one app, one platform apex", rt, err)
490 }
491 mustGet(t, reg, "kass.bab.oncarlos.test") // the existing apex stays live
492}
493
494func TestReconcileRemovesRetiredCanaryHostOnly(t *testing.T) {
495 // The canary/old pointer is GONE from the bucket; its platform-minted
496 // follower must be removed, but the custom route following the same
497 // channel is owner property — reconciliation never touches it.
498 s, reg := dirStore(t), regWith(t,
499 registry.Route{Host: "kass.training", Kind: "instance", App: "kass",
500 Status: "ready", Addr: "127.0.0.1:9000"},
501 registry.Route{Host: "old.kass.bab.oncarlos.test", Kind: "instance", App: "kass",
502 Status: "ready", Addr: "127.0.0.1:9000", Channel: "canary/old", Platform: true, Canary: true},
503 registry.Route{Host: "rc.kass.training", Kind: "instance", App: "kass",
504 Status: "ready", Addr: "127.0.0.1:9000", Channel: "canary/old"},
505 )
506 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
507 t.Fatal(err)
508 }
509 if rt, err := reg.Get("old.kass.bab.oncarlos.test"); err != nil || rt != nil {
510 t.Errorf("retired canary's platform route still present: %+v %v", rt, err)
511 }
512 mustGet(t, reg, "rc.kass.training") // the custom follower survives
513}
514
515func TestReconcileNeverTouchesCustomDomains(t *testing.T) {
516 // A custom route whose app is dead (nothing ready, nothing shipped):
517 // reconciliation must leave it exactly alone — and mint nothing, since
518 // there is no ready upstream to copy.
519 s, reg := dirStore(t), regWith(t, registry.Route{Host: "kass.training", Kind: "instance",
520 App: "kass", Status: "failed", Addr: "127.0.0.1:9000"})
521 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
522 t.Fatal(err)
523 }
524 mustGet(t, reg, "kass.training")
525 if rt, err := reg.Get("kass.bab.oncarlos.test"); err != nil || rt != nil {
526 t.Errorf("minted a platform route with no ready upstream: %+v %v", rt, err)
527 }
528}
529
530func TestReconcileStampsDNS(t *testing.T) {
531 s, reg := dirStore(t), regWith(t,
532 registry.Route{Host: "points.test", Kind: "static", Status: "ready"},
533 registry.Route{Host: "elsewhere.test", Kind: "static", Status: "ready"},
534 registry.Route{Host: "broken.test", Kind: "static", Status: "ready"},
535 )
536 c := cfg(t)
537 c.Resolve = func(host string) ([]string, error) {
538 switch host {
539 case "points.test":
540 return []string{"198.51.100.9", "203.0.113.7"}, nil // edge IP among others
541 case "elsewhere.test":
542 return []string{"198.51.100.9"}, nil
543 default:
544 return nil, errors.New("server misbehaving") // not a *net.DNSError: our lookup failed, not an authoritative answer
545 }
546 }
547 if err := domains.Reconcile(ctx, s, reg, c); err != nil {
548 t.Fatal(err)
549 }
550 if err := domains.StampHealth(ctx, reg, c, nil); err != nil { // as domainsLoop does after the pass
551 t.Fatal(err)
552 }
553 now := c.Now().Unix()
554 for host, want := range map[string]string{
555 "points.test": "pointing",
556 "elsewhere.test": "elsewhere",
557 // A non-DNSError resolver failure is OURS, not an authoritative
558 // "no such name" — it must not read as "the customer has no record".
559 "broken.test": "unresolved",
560 } {
561 got := mustGet(t, reg, host)
562 if got.DNSStatus != want {
563 t.Errorf("%s dns_status = %q, want %q", host, got.DNSStatus, want)
564 }
565 if got.DNSCheckedAt != now {
566 t.Errorf("%s dns_checked_at = %d, want %d", host, got.DNSCheckedAt, now)
567 }
568 }
569}
570
571// The edge set is now asked for once PER PASS, not fixed at box start: under
572// latency steering a correctly-pointed customer domain resolves to whichever
573// region's edge is nearest, so a set frozen at boot reads a healthy domain as
574// "elsewhere" the moment another region arms. This pins that the pass reads
575// the CURRENT answer — a second pass with a widened set restamps.
576func TestReconcileReadsTheEdgeSetEveryPass(t *testing.T) {
577 reg := regWith(t,
578 registry.Route{Host: "faraway.test", Kind: "static", Status: "ready"},
579 )
580 c := cfg(t)
581 c.Resolve = func(host string) ([]string, error) { return []string{"198.51.100.9"}, nil }
582 calls := 0
583 // A second region's edge arms between the two passes.
584 c.EdgeIPs = func() []string {
585 calls++
586 if calls > 1 {
587 return []string{"203.0.113.7", "198.51.100.9"}
588 }
589 return []string{"203.0.113.7"}
590 }
591 if err := domains.StampHealth(ctx, reg, c, nil); err != nil {
592 t.Fatal(err)
593 }
594 if got := mustGet(t, reg, "faraway.test"); got.DNSStatus != "elsewhere" {
595 t.Fatalf("pass 1: dns_status = %q, want elsewhere — that edge is not armed yet", got.DNSStatus)
596 }
597 if err := domains.StampHealth(ctx, reg, c, nil); err != nil {
598 t.Fatal(err)
599 }
600 if got := mustGet(t, reg, "faraway.test"); got.DNSStatus != "pointing" {
601 t.Fatalf("pass 2: dns_status = %q, want pointing — the set is read per pass, not frozen at arm", got.DNSStatus)
602 }
603}
604
605// A pass that knows NO edge addresses stands down rather than stamping.
606// Stamping against an empty set would mark every host in the deployment
607// "elsewhere" at once — a fleet-wide lying red raised by a failed bucket
608// read, not by anybody's DNS. Existing stamps are left exactly as they were:
609// the last thing this pass actually knew beats a verdict it had to guess.
610func TestReconcileStandsDownWithNoEdgeIPs(t *testing.T) {
611 reg := regWith(t,
612 registry.Route{Host: "points.test", Kind: "static", Status: "ready"},
613 )
614 if err := domains.StampHealth(ctx, reg, cfg(t), nil); err != nil {
615 t.Fatal(err)
616 }
617 if got := mustGet(t, reg, "points.test"); got.DNSStatus != "pointing" {
618 t.Fatalf("setup: dns_status = %q, want pointing", got.DNSStatus)
619 }
620
621 for name, edge := range map[string]func() []string{
622 "an empty fold": func() []string { return nil },
623 "no func at all": nil,
624 } {
625 c := cfg(t)
626 c.EdgeIPs = edge
627 if err := domains.StampHealth(ctx, reg, c, nil); err != nil {
628 t.Fatalf("%s: %v", name, err)
629 }
630 if got := mustGet(t, reg, "points.test"); got.DNSStatus != "pointing" {
631 t.Errorf("%s: dns_status = %q, want the previous stamp left standing", name, got.DNSStatus)
632 }
633 }
634}
635
636// writeCertBundle writes what autocert's cachePut writes under DirCache key
637// <host>: the private key PEM block first, then the CERTIFICATE chain — so
638// the parser is exercised on skipping the key block.
639func writeCertBundle(t *testing.T, dir, host string, notAfter time.Time) {
640 t.Helper()
641 key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
642 if err != nil {
643 t.Fatal(err)
644 }
645 tmpl := &x509.Certificate{
646 SerialNumber: big.NewInt(1),
647 Subject: pkix.Name{CommonName: host},
648 DNSNames: []string{host},
649 NotBefore: notAfter.Add(-24 * time.Hour),
650 NotAfter: notAfter,
651 }
652 der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
653 if err != nil {
654 t.Fatal(err)
655 }
656 kb, err := x509.MarshalECPrivateKey(key)
657 if err != nil {
658 t.Fatal(err)
659 }
660 var buf bytes.Buffer
661 if err := pem.Encode(&buf, &pem.Block{Type: "EC PRIVATE KEY", Bytes: kb}); err != nil {
662 t.Fatal(err)
663 }
664 if err := pem.Encode(&buf, &pem.Block{Type: "CERTIFICATE", Bytes: der}); err != nil {
665 t.Fatal(err)
666 }
667 if err := os.WriteFile(filepath.Join(dir, host), buf.Bytes(), 0o600); err != nil {
668 t.Fatal(err)
669 }
670}
671
672func TestReconcileStampsCertExpiry(t *testing.T) {
673 s, reg := dirStore(t), regWith(t,
674 registry.Route{Host: "kass.training", Kind: "instance", App: "kass",
675 Status: "ready", Addr: "127.0.0.1:9000"},
676 registry.Route{Host: "junk.test", Kind: "static", Status: "ready"},
677 )
678 acmeDir := t.TempDir()
679 notAfter := time.Date(2026, 10, 12, 0, 0, 0, 0, time.UTC)
680 writeCertBundle(t, acmeDir, "kass.training", notAfter)
681 // An unparseable cache entry is unknown, same as absent — never a crash,
682 // never a guess.
683 if err := os.WriteFile(filepath.Join(acmeDir, "junk.test"), []byte("not pem at all"), 0o600); err != nil {
684 t.Fatal(err)
685 }
686 c := cfg(t)
687 c.AcmeDir = acmeDir
688 if err := domains.Reconcile(ctx, s, reg, c); err != nil {
689 t.Fatal(err)
690 }
691 if err := domains.StampHealth(ctx, reg, c, nil); err != nil { // as domainsLoop does after the pass
692 t.Fatal(err)
693 }
694 if got := mustGet(t, reg, "kass.training"); got.CertExpiresAt != notAfter.Unix() {
695 t.Errorf("kass.training cert_expires_at = %d, want %d", got.CertExpiresAt, notAfter.Unix())
696 }
697 if got := mustGet(t, reg, "junk.test"); got.CertExpiresAt != 0 {
698 t.Errorf("junk.test cert_expires_at = %d, want 0 (unparseable = unknown)", got.CertExpiresAt)
699 }
700 // The freshly minted platform route has no cached cert yet: 0.
701 if got := mustGet(t, reg, "kass.bab.oncarlos.test"); got.CertExpiresAt != 0 {
702 t.Errorf("minted route cert_expires_at = %d, want 0 (absent = unknown)", got.CertExpiresAt)
703 }
704}
705
706// aliasByName finds an alias row by hostname among reg.ListAliases(), the
707// same read-back-from-the-registry shape mustGet uses for routes.
708func aliasByName(t *testing.T, reg *registry.Registry, alias string) registry.RouteAlias {
709 t.Helper()
710 aliases, err := reg.ListAliases()
711 if err != nil {
712 t.Fatal(err)
713 }
714 for _, a := range aliases {
715 if a.Alias == alias {
716 return a
717 }
718 }
719 t.Fatalf("alias %s missing", alias)
720 return registry.RouteAlias{}
721}
722
723func TestReconcileStampsAliasDNS(t *testing.T) {
724 s, reg := dirStore(t), regWith(t,
725 registry.Route{Host: "app.test", Kind: "static", Status: "ready"},
726 )
727 for _, a := range []string{"points-alias.test", "elsewhere-alias.test", "broken-alias.test"} {
728 if err := reg.UpsertAlias(registry.RouteAlias{Alias: a, Host: "app.test"}); err != nil {
729 t.Fatal(err)
730 }
731 }
732 c := cfg(t)
733 c.Resolve = func(host string) ([]string, error) {
734 switch host {
735 case "points-alias.test":
736 return []string{"198.51.100.9", "203.0.113.7"}, nil // edge IP among others
737 case "elsewhere-alias.test":
738 return []string{"198.51.100.9"}, nil
739 default:
740 return nil, errors.New("server misbehaving") // not a *net.DNSError: our lookup failed, not an authoritative answer
741 }
742 }
743 if err := domains.Reconcile(ctx, s, reg, c); err != nil {
744 t.Fatal(err)
745 }
746 if err := domains.StampHealth(ctx, reg, c, nil); err != nil { // as domainsLoop does after the pass
747 t.Fatal(err)
748 }
749 now := c.Now().Unix()
750 for alias, want := range map[string]string{
751 "points-alias.test": "pointing",
752 "elsewhere-alias.test": "elsewhere",
753 // A non-DNSError resolver failure is OURS, not an authoritative
754 // "no such name" — it must not read as "the customer has no record".
755 "broken-alias.test": "unresolved",
756 } {
757 got := aliasByName(t, reg, alias)
758 if got.DNSStatus != want {
759 t.Errorf("%s dns_status = %q, want %q", alias, got.DNSStatus, want)
760 }
761 if got.DNSCheckedAt != now {
762 t.Errorf("%s dns_checked_at = %d, want %d", alias, got.DNSCheckedAt, now)
763 }
764 }
765}
766
767func TestReconcileStampsAliasCert(t *testing.T) {
768 s, reg := dirStore(t), regWith(t,
769 registry.Route{Host: "kass.training", Kind: "instance", App: "kass",
770 Status: "ready", Addr: "127.0.0.1:9000"},
771 )
772 if err := reg.UpsertAlias(registry.RouteAlias{Alias: "kass-alias.training", Host: "kass.training"}); err != nil {
773 t.Fatal(err)
774 }
775 acmeDir := t.TempDir()
776 notAfter := time.Date(2026, 10, 12, 0, 0, 0, 0, time.UTC)
777 writeCertBundle(t, acmeDir, "kass-alias.training", notAfter)
778 c := cfg(t)
779 c.AcmeDir = acmeDir
780 if err := domains.Reconcile(ctx, s, reg, c); err != nil {
781 t.Fatal(err)
782 }
783 if err := domains.StampHealth(ctx, reg, c, nil); err != nil { // as domainsLoop does after the pass
784 t.Fatal(err)
785 }
786 if got := aliasByName(t, reg, "kass-alias.training"); got.CertExpiresAt != notAfter.Unix() {
787 t.Errorf("kass-alias.training cert_expires_at = %d, want %d", got.CertExpiresAt, notAfter.Unix())
788 }
789}
790
791func TestReconcileInertWithoutAppsDomain(t *testing.T) {
792 s, reg := dirStore(t), regWith(t, registry.Route{Host: "kass.training", Kind: "instance",
793 App: "kass", Status: "ready", Addr: "127.0.0.1:9000"})
794 writeAccountManifest(t, s, 793481920157, "Kass") // present but must not be read
795 before, err := reg.List()
796 if err != nil {
797 t.Fatal(err)
798 }
799 c := cfg(t)
800 c.AppsDomain = ""
801 if err := domains.Reconcile(ctx, s, reg, c); err != nil {
802 t.Fatal(err)
803 }
804 after, err := reg.List()
805 if err != nil {
806 t.Fatal(err)
807 }
808 if !reflect.DeepEqual(before, after) {
809 t.Errorf("unconfigured Reconcile changed the registry:\nbefore %+v\nafter %+v", before, after)
810 }
811 if accts, _ := reg.Accounts(); len(accts) != 1 {
812 t.Errorf("unconfigured Reconcile materialized accounts: %+v (want only ops)", accts)
813 }
814}
815
816// routesFor returns every route in the registry whose App is app — the shape
817// the trash tests assert on, since "stops serving" means EVERY route, not one
818// named host.
819func routesFor(t *testing.T, reg *registry.Registry, app string) []registry.Route {
820 t.Helper()
821 routes, err := reg.List()
822 if err != nil {
823 t.Fatal(err)
824 }
825 var out []registry.Route
826 for _, rt := range routes {
827 if rt.App == app {
828 out = append(out, rt)
829 }
830 }
831 return out
832}
833
834// A trash marker is the owner saying "stop serving this app", so reconcile
835// removes ALL of its routes — the custom domain included, which is the one
836// destructive move inference alone would never be allowed. A custom domain
837// still serving a "deleted" app would be the worse betrayal. And nothing
838// re-mints: no apex, no canary host, on this pass or any later one.
839func TestReconcileTrashedAppDropsAllRoutesAndMintsNothing(t *testing.T) {
840 s, reg := dirStore(t), regWith(t,
841 registry.Route{Host: "derp.custom", Kind: "static", App: "derp",
842 Status: "ready", Channel: "stable", Account: domains.Sqid(793481920157)},
843 registry.Route{Host: "derp." + domains.Sqid(793481920157) + ".oncarlos.test", Kind: "static",
844 App: "derp", Status: "ready", Channel: "stable", Platform: true,
845 Account: domains.Sqid(793481920157)},
846 registry.Route{Host: "keep.custom", Kind: "static", App: "keep",
847 Status: "ready", Channel: "stable", Account: domains.Sqid(793481920157)},
848 )
849 sqid := writeAccountManifest(t, s, 793481920157, "Kass")
850 putKey(t, s, "apps/"+sqid+"/derp/channels/canary/x.json")
851 putKey(t, s, "apps/"+sqid+"/keep/channels/stable.json")
852 if err := release.TrashApp(ctx, s, sqid, "derp", "paul@keymail.dev", time.Unix(1753900000, 0)); err != nil {
853 t.Fatal(err)
854 }
855
856 for pass := 1; pass <= 2; pass++ {
857 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
858 t.Fatalf("pass %d: %v", pass, err)
859 }
860 if got := routesFor(t, reg, "derp"); len(got) != 0 {
861 t.Errorf("pass %d: trashed app still routed: %+v", pass, got)
862 }
863 // The neighbour in the same account is untouched by every pass: its
864 // custom route stands and its apex mints as it always would.
865 mustGet(t, reg, "keep.custom")
866 if apex := mustGet(t, reg, "keep."+sqid+".oncarlos.test"); !apex.Platform || apex.Account != sqid {
867 t.Errorf("pass %d: neighbour apex = %+v", pass, apex)
868 }
869 }
870}
871
872// "ALL of them" includes a route that never got its account column stamped.
873// Such a row reads as ops until step 1b stamps it, so the removal lookup would
874// miss a marker written under the tenant's account — the app would keep
875// serving from its custom domain forever. The stamp has to land FIRST, from
876// the account the bucket actually asserts, and then the removal matches.
877func TestReconcileTrashedAppWithUnstampedRouteStillDrops(t *testing.T) {
878 s, reg := dirStore(t), regWith(t,
879 registry.Route{Host: "derp.custom", Kind: "static", App: "derp",
880 Status: "ready", Channel: "stable"}, // no Account column at all
881 )
882 sqid := writeAccountManifest(t, s, 793481920157, "Kass")
883 putKey(t, s, "apps/"+sqid+"/derp/channels/stable.json")
884 if err := release.TrashApp(ctx, s, sqid, "derp", "paul@keymail.dev", time.Unix(1753900000, 0)); err != nil {
885 t.Fatal(err)
886 }
887 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
888 t.Fatal(err)
889 }
890 if got := routesFor(t, reg, "derp"); len(got) != 0 {
891 t.Errorf("unstamped route of a trashed app survived: %+v", got)
892 }
893}
894
895// THE safety property of the whole trash mechanism: app names are unique
896// within an account and never globally, so two tenants legitimately both run a
897// "derp". The removal key is (account, app) — one `release.RouteAccount(rt)`
898// away from being a name-only match, and that one word is the difference
899// between deleting an app and deleting a stranger's. Account A trashing its
900// derp must leave B's derp exactly as it found it.
901func TestReconcileTrashMarkerNeverCrossesAccounts(t *testing.T) {
902 a, b := domains.Sqid(793481920157), domains.Sqid(555000111222)
903 s, reg := dirStore(t), regWith(t,
904 registry.Route{Host: "derp.a.custom", Kind: "static", App: "derp",
905 Status: "ready", Channel: "stable", Account: a},
906 registry.Route{Host: "derp." + a + ".oncarlos.test", Kind: "static", App: "derp",
907 Status: "ready", Channel: "stable", Platform: true, Account: a},
908 registry.Route{Host: "derp.b.custom", Kind: "static", App: "derp",
909 Status: "ready", Channel: "stable", Account: b},
910 registry.Route{Host: "derp." + b + ".oncarlos.test", Kind: "static", App: "derp",
911 Status: "ready", Channel: "stable", Platform: true, Account: b},
912 )
913 if got := writeAccountManifest(t, s, 793481920157, "Ayy"); got != a {
914 t.Fatalf("account A sqid = %q, want %q", got, a)
915 }
916 if got := writeAccountManifest(t, s, 555000111222, "Bee"); got != b {
917 t.Fatalf("account B sqid = %q, want %q", got, b)
918 }
919 putKey(t, s, "apps/"+a+"/derp/channels/stable.json")
920 putKey(t, s, "apps/"+b+"/derp/channels/stable.json")
921 // Only A trashes its derp.
922 if err := release.TrashApp(ctx, s, a, "derp", "paul@keymail.dev", time.Unix(1753900000, 0)); err != nil {
923 t.Fatal(err)
924 }
925
926 wantHosts := []string{"derp.b.custom", "derp." + b + ".oncarlos.test"}
927 slices.Sort(wantHosts) // reg.List returns hosts in sorted order
928 var first []registry.Route
929 for pass := 1; pass <= 2; pass++ {
930 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
931 t.Fatalf("pass %d: %v", pass, err)
932 }
933 got := routesFor(t, reg, "derp")
934 var hosts []string
935 for _, rt := range got {
936 hosts = append(hosts, rt.Host)
937 if rt.Account != b {
938 t.Errorf("pass %d: surviving route %s stamped %q, want B (%s)", pass, rt.Host, rt.Account, b)
939 }
940 }
941 if !reflect.DeepEqual(hosts, wantHosts) {
942 t.Fatalf("pass %d: derp routes = %v, want exactly B's %v — A's marker must not reach B",
943 pass, hosts, wantHosts)
944 }
945 if pass == 1 {
946 first = got
947 continue
948 }
949 // Byte-identical across the second pass too: no re-stamp, no re-mint,
950 // no field quietly rewritten under B.
951 if !reflect.DeepEqual(first, got) {
952 t.Errorf("second pass changed B's routes:\nfirst %+v\nsecond %+v", first, got)
953 }
954 }
955}
956
957// A marker we cannot fully believe must not destroy routes: ReadAppTombstone
958// errors, reconcile logs it and treats the app as live, every pass, until
959// someone fixes or removes the object.
960func TestReconcileMalformedTrashMarkerIsIgnoredLoudly(t *testing.T) {
961 s, reg := dirStore(t), regWith(t,
962 registry.Route{Host: "derp.custom", Kind: "static", App: "derp",
963 Status: "ready", Channel: "stable", Account: domains.Sqid(793481920157)},
964 )
965 sqid := writeAccountManifest(t, s, 793481920157, "Kass")
966 putKey(t, s, "apps/"+sqid+"/derp/channels/stable.json")
967 // The body claims a different app than the path it sits at.
968 if _, err := s.Put(ctx, release.AppTombstoneKey(sqid, "derp"),
969 []byte(`{"schema":1,"account":"`+sqid+`","app":"elsewhere","deleted_at":"2026-08-04T09:00:00Z"}`), ""); err != nil {
970 t.Fatal(err)
971 }
972 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
973 t.Fatal(err)
974 }
975 mustGet(t, reg, "derp.custom") // routes stay
976 // ...and minting proceeds exactly as if no marker existed.
977 if apex := mustGet(t, reg, "derp."+sqid+".oncarlos.test"); !apex.Platform || apex.Account != sqid {
978 t.Errorf("apex = %+v — an unbelievable marker must change nothing", apex)
979 }
980}
981
982// A retired apps/<app>/account.json (the domains spec's linkage manifest,
983// superseded by the key path) is just another object in the bucket now — it
984// must not be read, and above all must not re-home an app.
985func TestReconcileIgnoresRetiredAppAccountManifests(t *testing.T) {
986 s, reg := dirStore(t), regWith(t, registry.Route{Host: "kass.training", Kind: "instance",
987 App: "kass", Status: "ready", Addr: "127.0.0.1:9000"})
988 if _, err := s.Put(ctx, "apps/kass/account.json",
989 []byte(`{"account":"paul@keymail.dev","owner":"paul@keymail.dev"}`), ""); err != nil {
990 t.Fatal(err)
991 }
992 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
993 t.Fatal(err)
994 }
995 if src := mustGet(t, reg, "kass.training"); src.Account != "bab" {
996 t.Errorf("retired linkage manifest re-homed the app: %+v", src)
997 }
998 mustGet(t, reg, "kass.bab.oncarlos.test")
999}
1000
1001// releaseKeys mints a root release key pair the way the deployment does, so
1002// the purge pass has something real to verify a production flag against.
1003func releaseKeys(t *testing.T) (ed25519.PrivateKey, ed25519.PublicKey) {
1004 t.Helper()
1005 seedB64, pubB64, err := release.GenerateKey()
1006 if err != nil {
1007 t.Fatal(err)
1008 }
1009 priv, err := release.ParseSigningKey(seedB64, "test")
1010 if err != nil {
1011 t.Fatal(err)
1012 }
1013 pub, err := release.ParsePublicKey(pubB64, "test")
1014 if err != nil {
1015 t.Fatal(err)
1016 }
1017 return priv, pub
1018}
1019
1020// appKeysLeft is what is still in the bucket under an app — the purge's only
1021// observable effect.
1022func appKeysLeft(t *testing.T, s release.Store, acct, app string) []string {
1023 t.Helper()
1024 keys, err := s.List(ctx, "apps/"+acct+"/"+app+"/")
1025 if err != nil {
1026 t.Fatal(err)
1027 }
1028 slices.Sort(keys)
1029 return keys
1030}
1031
1032// Past the cooloff, the pass DESTROYS: the app's objects and, last, its
1033// marker. The neighbour app in the same account is untouched, and the pass
1034// converges — a second run over an empty prefix changes nothing and does not
1035// error.
1036func TestReconcilePurgesAppsPastTheCooloff(t *testing.T) {
1037 _, pub := releaseKeys(t)
1038 s, reg := dirStore(t), regWith(t,
1039 registry.Route{Host: "derp.custom", Kind: "static", App: "derp",
1040 Status: "ready", Channel: "stable", Account: domains.Sqid(793481920157)},
1041 )
1042 sqid := writeAccountManifest(t, s, 793481920157, "Kass")
1043 putKey(t, s, "apps/"+sqid+"/derp/channels/stable.json")
1044 putKey(t, s, "apps/"+sqid+"/derp/releases/1.0.0/manifest.json")
1045 putKey(t, s, "apps/"+sqid+"/keep/channels/stable.json")
1046 // Trashed 31 days before the config's clock.
1047 deleted := time.Unix(1754000000, 0).Add(-31 * 24 * time.Hour)
1048 if err := release.TrashApp(ctx, s, sqid, "derp", "paul@keymail.dev", deleted); err != nil {
1049 t.Fatal(err)
1050 }
1051
1052 c := cfg(t)
1053 c.ReleasePub = pub
1054 for pass := 1; pass <= 2; pass++ {
1055 if err := domains.Reconcile(ctx, s, reg, c); err != nil {
1056 t.Fatalf("pass %d: %v", pass, err)
1057 }
1058 if got := appKeysLeft(t, s, sqid, "derp"); len(got) != 0 {
1059 t.Fatalf("pass %d: purge left objects behind: %v", pass, got)
1060 }
1061 if got := routesFor(t, reg, "derp"); len(got) != 0 {
1062 t.Errorf("pass %d: purged app still routed: %+v", pass, got)
1063 }
1064 if got := appKeysLeft(t, s, sqid, "keep"); len(got) != 1 {
1065 t.Fatalf("pass %d: the purge reached the neighbour app: %v", pass, got)
1066 }
1067 }
1068}
1069
1070// Inside the cooloff the app is hidden but INTACT — that window is the whole
1071// point of a trash, and restoring it must give the owner their bytes back.
1072func TestReconcileDoesNotPurgeInsideTheCooloff(t *testing.T) {
1073 _, pub := releaseKeys(t)
1074 s, reg := dirStore(t), regWith(t,
1075 registry.Route{Host: "derp.custom", Kind: "static", App: "derp",
1076 Status: "ready", Channel: "stable", Account: domains.Sqid(793481920157)},
1077 )
1078 sqid := writeAccountManifest(t, s, 793481920157, "Kass")
1079 putKey(t, s, "apps/"+sqid+"/derp/channels/stable.json")
1080 deleted := time.Unix(1754000000, 0).Add(-29 * 24 * time.Hour)
1081 if err := release.TrashApp(ctx, s, sqid, "derp", "paul@keymail.dev", deleted); err != nil {
1082 t.Fatal(err)
1083 }
1084
1085 c := cfg(t)
1086 c.ReleasePub = pub
1087 if err := domains.Reconcile(ctx, s, reg, c); err != nil {
1088 t.Fatal(err)
1089 }
1090 // Routes gone (the app is hidden), objects and marker still there.
1091 if got := routesFor(t, reg, "derp"); len(got) != 0 {
1092 t.Errorf("trashed app still routed: %+v", got)
1093 }
1094 want := []string{
1095 "apps/" + sqid + "/derp/channels/stable.json",
1096 release.AppTombstoneKey(sqid, "derp"),
1097 }
1098 slices.Sort(want)
1099 if got := appKeysLeft(t, s, sqid, "derp"); !reflect.DeepEqual(got, want) {
1100 t.Fatalf("inside the cooloff the bucket = %v, want %v — nothing may be destroyed yet", got, want)
1101 }
1102 if tomb, err := release.ReadAppTombstone(ctx, s, sqid, "derp"); err != nil || tomb == nil {
1103 t.Fatalf("marker gone inside the cooloff: %v, %v", tomb, err)
1104 }
1105}
1106
1107// A box with no release public key cannot verify a production flag, so it
1108// cannot prove an app is safe to destroy: purging is OFF. Hiding still
1109// happens — that is reversible, destruction is not.
1110func TestReconcileWithoutAReleaseKeyHidesButNeverDestroys(t *testing.T) {
1111 s, reg := dirStore(t), regWith(t,
1112 registry.Route{Host: "derp.custom", Kind: "static", App: "derp",
1113 Status: "ready", Channel: "stable", Account: domains.Sqid(793481920157)},
1114 )
1115 sqid := writeAccountManifest(t, s, 793481920157, "Kass")
1116 putKey(t, s, "apps/"+sqid+"/derp/channels/stable.json")
1117 deleted := time.Unix(1754000000, 0).Add(-31 * 24 * time.Hour)
1118 if err := release.TrashApp(ctx, s, sqid, "derp", "paul@keymail.dev", deleted); err != nil {
1119 t.Fatal(err)
1120 }
1121
1122 c := cfg(t) // no ReleasePub
1123 if err := domains.Reconcile(ctx, s, reg, c); err != nil {
1124 t.Fatal(err)
1125 }
1126 if got := routesFor(t, reg, "derp"); len(got) != 0 {
1127 t.Errorf("trashed app still routed: %+v", got)
1128 }
1129 if got := appKeysLeft(t, s, sqid, "derp"); len(got) != 2 {
1130 t.Fatalf("a keyless box destroyed objects: %v", got)
1131 }
1132}
1133
1134// A trashed app that is ALSO flagged production is a contradiction the
1135// console refuses to create — but if one ever exists in a bucket, the flag
1136// wins and the bytes stay. Same for a flag that will not verify.
1137func TestReconcileNeverPurgesAProductionFlaggedApp(t *testing.T) {
1138 priv, pub := releaseKeys(t)
1139 s, reg := dirStore(t), regWith(t,
1140 registry.Route{Host: "derp.custom", Kind: "static", App: "derp",
1141 Status: "ready", Channel: "stable", Account: domains.Sqid(793481920157)},
1142 )
1143 sqid := writeAccountManifest(t, s, 793481920157, "Kass")
1144 putKey(t, s, "apps/"+sqid+"/derp/channels/stable.json")
1145 // The flag goes on FIRST: SetProduction refuses a trashed app.
1146 if err := release.SetProduction(ctx, s, priv, sqid, "derp", true, "paul@keymail.dev"); err != nil {
1147 t.Fatal(err)
1148 }
1149 deleted := time.Unix(1754000000, 0).Add(-31 * 24 * time.Hour)
1150 if err := release.TrashApp(ctx, s, sqid, "derp", "paul@keymail.dev", deleted); err != nil {
1151 t.Fatal(err)
1152 }
1153
1154 c := cfg(t)
1155 c.ReleasePub = pub
1156 if err := domains.Reconcile(ctx, s, reg, c); err != nil {
1157 t.Fatal(err)
1158 }
1159 if got := appKeysLeft(t, s, sqid, "derp"); len(got) != 3 {
1160 t.Fatalf("a production-flagged app was purged: %v", got)
1161 }
1162 if tomb, err := release.ReadAppTombstone(ctx, s, sqid, "derp"); err != nil || tomb == nil {
1163 t.Fatalf("marker gone: %v, %v", tomb, err)
1164 }
1165}
1166
1167// The route-drop is as dangerous as the purge, and cheaper to reach: a trash
1168// marker is UNSIGNED, so anyone holding PutObject can write one. If a marker
1169// alone could take a production app's routes down, a bucket writer with no
1170// signing key would have a fleet-wide outage button. So on a box that can
1171// verify, the flag outranks the marker: nothing is dropped, nothing is
1172// destroyed, and the contradiction is said out loud every pass.
1173func TestReconcileNeverDropsRoutesOfAProductionFlaggedApp(t *testing.T) {
1174 priv, pub := releaseKeys(t)
1175 s, reg := dirStore(t), regWith(t,
1176 registry.Route{Host: "derp.custom", Kind: "static", App: "derp",
1177 Status: "ready", Channel: "stable", Account: domains.Sqid(793481920157)},
1178 )
1179 sqid := writeAccountManifest(t, s, 793481920157, "Kass")
1180 putKey(t, s, "apps/"+sqid+"/derp/channels/stable.json")
1181 // The flag goes on FIRST: SetProduction refuses a trashed app.
1182 if err := release.SetProduction(ctx, s, priv, sqid, "derp", true, "paul@keymail.dev"); err != nil {
1183 t.Fatal(err)
1184 }
1185 // Past the cooloff too, so a guard that only covered hiding would show up
1186 // here as destroyed bytes.
1187 deleted := time.Unix(1754000000, 0).Add(-31 * 24 * time.Hour)
1188 if err := release.TrashApp(ctx, s, sqid, "derp", "eve@evil.example", deleted); err != nil {
1189 t.Fatal(err)
1190 }
1191
1192 var logged bytes.Buffer
1193 log.SetOutput(&logged)
1194 t.Cleanup(func() { log.SetOutput(os.Stderr) })
1195
1196 c := cfg(t)
1197 c.ReleasePub = pub
1198 apex := "derp." + sqid + ".oncarlos.test"
1199 for pass := 1; pass <= 2; pass++ {
1200 if err := domains.Reconcile(ctx, s, reg, c); err != nil {
1201 t.Fatalf("pass %d: %v", pass, err)
1202 }
1203 // The custom domain survives, and the app is treated as live in every
1204 // other respect — its platform apex mints once and is not re-minted.
1205 mustGet(t, reg, "derp.custom")
1206 mustGet(t, reg, apex)
1207 if got := routesFor(t, reg, "derp"); len(got) != 2 {
1208 t.Fatalf("pass %d: routes = %+v, want the custom domain and exactly one apex", pass, got)
1209 }
1210 if got := appKeysLeft(t, s, sqid, "derp"); len(got) != 3 {
1211 t.Fatalf("pass %d: bytes destroyed under a production flag: %v", pass, got)
1212 }
1213 }
1214 // The refusal is the contract, not a side effect of some later check: an
1215 // operator who wrote that marker on purpose has to be told why nothing
1216 // happened, every pass, or they will assume deletion is broken.
1217 if !strings.Contains(logged.String(), "trash marker on production app") {
1218 t.Errorf("no refusal logged; log was:\n%s", logged.String())
1219 }
1220}
1221
1222// flagUnreadableStore breaks exactly one read — the app's production flag —
1223// the way an attacker with delete-but-not-sign, or a half-broken bucket,
1224// would. Everything else reads normally, so the marker is perfectly valid
1225// and only the flag is in doubt.
1226type flagUnreadableStore struct {
1227 release.Store
1228 key string
1229}
1230
1231func (s flagUnreadableStore) Get(ctx context.Context, key string) ([]byte, string, error) {
1232 if key == s.key {
1233 return nil, "", errors.New("bucket refused the read")
1234 }
1235 return s.Store.Get(ctx, key)
1236}
1237
1238// An UNREADABLE flag refuses in the same direction as an ON one. "Cannot
1239// prove it is not production" and "is production" get the same answer on
1240// every destructive path in this platform, because the state an attacker can
1241// manufacture is exactly the unreadable one.
1242func TestReconcileDoesNotDropRoutesWhenTheProductionFlagIsUnreadable(t *testing.T) {
1243 _, pub := releaseKeys(t)
1244 dir, reg := dirStore(t), regWith(t,
1245 registry.Route{Host: "derp.custom", Kind: "static", App: "derp",
1246 Status: "ready", Channel: "stable", Account: domains.Sqid(793481920157)},
1247 )
1248 sqid := writeAccountManifest(t, dir, 793481920157, "Kass")
1249 putKey(t, dir, "apps/"+sqid+"/derp/channels/stable.json")
1250 deleted := time.Unix(1754000000, 0).Add(-31 * 24 * time.Hour)
1251 if err := release.TrashApp(ctx, dir, sqid, "derp", "paul@keymail.dev", deleted); err != nil {
1252 t.Fatal(err)
1253 }
1254 s := flagUnreadableStore{Store: dir, key: release.ProductionKey(sqid, "derp")}
1255
1256 var logged bytes.Buffer
1257 log.SetOutput(&logged)
1258 t.Cleanup(func() { log.SetOutput(os.Stderr) })
1259
1260 c := cfg(t)
1261 c.ReleasePub = pub
1262 if err := domains.Reconcile(ctx, s, reg, c); err != nil {
1263 t.Fatal(err)
1264 }
1265 mustGet(t, reg, "derp.custom")
1266 if got := appKeysLeft(t, dir, sqid, "derp"); len(got) != 2 {
1267 t.Fatalf("bytes destroyed while the flag was unreadable: %v", got)
1268 }
1269 if !strings.Contains(logged.String(), "cannot read the production flag") {
1270 t.Errorf("no refusal logged; log was:\n%s", logged.String())
1271 }
1272}
1273
1274// #303: the health stamps are a per-box pass with no lease involved —
1275// callable on their own, stamping routes AND aliases from this box's view,
1276// and a no-op with nothing configured.
1277func TestStampHealthStandsAlone(t *testing.T) {
1278 reg := regWith(t, registry.Route{Host: "app.acct.oncarlos.test", Kind: "service", Addr: "10.0.0.1:1", Status: "ready"})
1279 if err := reg.UpsertAlias(registry.RouteAlias{Alias: "www.customer.test", Host: "app.acct.oncarlos.test", Account: "acct", App: "app"}); err != nil {
1280 t.Fatal(err)
1281 }
1282 cfg := domains.Config{
1283 AppsDomain: "oncarlos.test",
1284 EdgeIPs: func() []string { return []string{"203.0.113.7"} },
1285 Resolve: func(host string) ([]string, error) {
1286 if host == "www.customer.test" {
1287 return []string{"203.0.113.7"}, nil
1288 }
1289 return []string{"198.51.100.1"}, nil
1290 },
1291 }
1292 if err := domains.StampHealth(context.Background(), reg, cfg, nil); err != nil {
1293 t.Fatal(err)
1294 }
1295 if got := mustGet(t, reg, "app.acct.oncarlos.test"); got.DNSStatus != "elsewhere" {
1296 t.Errorf("route dns_status = %q; want elsewhere", got.DNSStatus)
1297 }
1298 aliases, err := reg.ListAliases()
1299 if err != nil || len(aliases) != 1 || aliases[0].DNSStatus != "pointing" {
1300 t.Errorf("alias dns_status: %+v, %v; want pointing", aliases, err)
1301 }
1302 // Nothing configured: nothing stamped, no error.
1303 reg2 := regWith(t, registry.Route{Host: "x.test", Kind: "service", Addr: "10.0.0.1:1", Status: "ready"})
1304 if err := domains.StampHealth(context.Background(), reg2, domains.Config{AppsDomain: "oncarlos.test"}, nil); err != nil {
1305 t.Fatal(err)
1306 }
1307 if got := mustGet(t, reg2, "x.test"); got.DNSStatus != "" || got.DNSCheckedAt != 0 {
1308 t.Errorf("unconfigured pass stamped %+v", got)
1309 }
1310}
1311
1312// OUR failure never overwrites a verdict: the last known verdict and its
1313// stamp time stay, so a bad DNS tick cannot unpublish a working custom
1314// domain (#287's gate) or launder staleness into "never checked".
1315//
1316// An authoritative NXDOMAIN is the opposite case and DOES stamp — the
1317// resolver answered, and the answer is that the record is gone.
1318func TestStampHealthHoldsLastVerdictOnResolverFailure(t *testing.T) {
1319 reg := regWith(t, registry.Route{Host: "app.acct.oncarlos.test", Kind: "service", Addr: "10.0.0.1:1", Status: "ready"})
1320 if err := reg.UpsertAlias(registry.RouteAlias{Alias: "www.customer.test", Host: "app.acct.oncarlos.test", Account: "acct", App: "app"}); err != nil {
1321 t.Fatal(err)
1322 }
1323 good := domains.Config{AppsDomain: "oncarlos.test", EdgeIPs: func() []string { return []string{"203.0.113.7"} },
1324 Resolve: func(string) ([]string, error) { return []string{"203.0.113.7"}, nil },
1325 Now: func() time.Time { return time.Unix(1000, 0) }}
1326 if err := domains.StampHealth(context.Background(), reg, good, nil); err != nil {
1327 t.Fatal(err)
1328 }
1329 broken := good
1330 broken.Resolve = func(string) ([]string, error) { return nil, errors.New("SERVFAIL") }
1331 broken.Now = func() time.Time { return time.Unix(2000, 0) }
1332 if err := domains.StampHealth(context.Background(), reg, broken, nil); err != nil {
1333 t.Fatal(err)
1334 }
1335 if got := mustGet(t, reg, "app.acct.oncarlos.test"); got.DNSStatus != "pointing" || got.DNSCheckedAt != 1000 {
1336 t.Errorf("route after resolver failure = %q @%d; want pointing @1000", got.DNSStatus, got.DNSCheckedAt)
1337 }
1338 aliases, _ := reg.ListAliases()
1339 if len(aliases) != 1 || aliases[0].DNSStatus != "pointing" || aliases[0].DNSCheckedAt != 1000 {
1340 t.Errorf("alias after resolver failure = %+v; want pointing @1000", aliases)
1341 }
1342 // A never-checked row records that we could not check — "unresolved",
1343 // not "" — so the operator can tell our outage from a missing record.
1344 fresh := regWith(t, registry.Route{Host: "new.acct.oncarlos.test", Kind: "service", Addr: "10.0.0.1:1", Status: "ready"})
1345 if err := domains.StampHealth(context.Background(), fresh, broken, nil); err != nil {
1346 t.Fatal(err)
1347 }
1348 if got := mustGet(t, fresh, "new.acct.oncarlos.test"); got.DNSStatus != "unresolved" || got.DNSCheckedAt != 2000 {
1349 t.Errorf("fresh row = %q @%d; want unresolved @2000", got.DNSStatus, got.DNSCheckedAt)
1350 }
1351
1352 // And the case the guard must NOT swallow: the customer deleted the
1353 // record. NXDOMAIN is an answer, so "pointing" gives way to it. Without
1354 // this, a domain that stopped resolving would read "pointing" forever.
1355 gone := good
1356 gone.Resolve = func(string) ([]string, error) {
1357 return nil, &net.DNSError{Err: "no such host", IsNotFound: true}
1358 }
1359 gone.Now = func() time.Time { return time.Unix(3000, 0) }
1360 if err := domains.StampHealth(context.Background(), reg, gone, nil); err != nil {
1361 t.Fatal(err)
1362 }
1363 if got := mustGet(t, reg, "app.acct.oncarlos.test"); got.DNSStatus != "" || got.DNSCheckedAt != 3000 {
1364 t.Errorf("route after NXDOMAIN = %q @%d; want \"\" @3000 — an authoritative answer is news", got.DNSStatus, got.DNSCheckedAt)
1365 }
1366}
1367
1368// Back-to-back passes are idempotent: a second StampHealth with the same
1369// view changes nothing but dns_checked_at.
1370func TestStampHealthIsIdempotent(t *testing.T) {
1371 reg := regWith(t, registry.Route{Host: "app.acct.oncarlos.test", Kind: "service", Addr: "10.0.0.1:1", Status: "ready"})
1372 c := cfg(t)
1373 for i := 0; i < 2; i++ {
1374 if err := domains.StampHealth(context.Background(), reg, c, nil); err != nil {
1375 t.Fatal(err)
1376 }
1377 }
1378 if got := mustGet(t, reg, "app.acct.oncarlos.test"); got.DNSStatus != "pointing" || got.DNSCheckedAt != c.Now().Unix() {
1379 t.Errorf("after two passes: %q @%d", got.DNSStatus, got.DNSCheckedAt)
1380 }
1381 // And inert with no apps domain, per Config's contract.
1382 if err := domains.StampHealth(context.Background(), reg, domains.Config{Resolve: c.Resolve, Now: func() time.Time { return time.Unix(9, 0) }}, nil); err != nil {
1383 t.Fatal(err)
1384 }
1385 if got := mustGet(t, reg, "app.acct.oncarlos.test"); got.DNSCheckedAt == 9 {
1386 t.Error("a pass with no AppsDomain wrote a stamp")
1387 }
1388}
1389
1390// The #344 shape, live on the flagship 2026-08-28: a route with an EMPTY
1391// account column, for an app whose name holds objects under ops (a stray
1392// pipeline.json from an abandoned attempt) AND under the tenant that
1393// actually ships it. The ops fallback used to win the tie; the host's own
1394// sqid label — birthday-alarm.<sqid>.oncarlos.com literally names its
1395// owner — must win instead. The mis-stamp deleted the tenant's delivered
1396// config (syncexec keyed apps/bab/...) and the app died on every wake.
1397func TestReconcileStampsTheHostsOwnAccountNotOpsOnATie(t *testing.T) {
1398 sqidNum := int64(793481920157)
1399 s, reg := dirStore(t), regWith(t)
1400 sqid := writeAccountManifest(t, s, sqidNum, "Paulca")
1401 if err := reg.Upsert(registry.Route{Host: "birthday-alarm." + sqid + ".oncarlos.test",
1402 Kind: "instance", App: "birthday-alarm", Status: "ready",
1403 Socket: "/run/carlos/birthday-alarm.sock"}); err != nil {
1404 t.Fatal(err)
1405 }
1406 putKey(t, s, "apps/bab/birthday-alarm/pipeline.json") // the stray
1407 putKey(t, s, "apps/"+sqid+"/birthday-alarm/config.json")
1408 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
1409 t.Fatal(err)
1410 }
1411 if rt := mustGet(t, reg, "birthday-alarm."+sqid+".oncarlos.test"); rt.Account != sqid {
1412 t.Errorf("account = %q, want %s — the ops fallback won a tie the host itself decides", rt.Account, sqid)
1413 }
1414}
1415
1416// The same tie on a host that names NO account (a custom domain): the pass
1417// must leave the row alone rather than defaulting it onto ops — an
1418// unstamped row already reads as ops at runtime, so stamping buys nothing
1419// and a wrong stamp deletes config (#344).
1420func TestReconcileLeavesAnAmbiguousCustomDomainUnstamped(t *testing.T) {
1421 s, reg := dirStore(t), regWith(t)
1422 sqid := writeAccountManifest(t, s, 793481920157, "Paulca")
1423 if err := reg.Upsert(registry.Route{Host: "birthdays.example.com",
1424 Kind: "instance", App: "birthday-alarm", Status: "ready",
1425 Socket: "/run/carlos/birthdays.sock"}); err != nil {
1426 t.Fatal(err)
1427 }
1428 putKey(t, s, "apps/bab/birthday-alarm/pipeline.json")
1429 putKey(t, s, "apps/"+sqid+"/birthday-alarm/config.json")
1430 if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
1431 t.Fatal(err)
1432 }
1433 if rt := mustGet(t, reg, "birthdays.example.com"); rt.Account != "" {
1434 t.Errorf("account = %q, want unstamped — ambiguity is not ops's to win", rt.Account)
1435 }
1436}
1437
1438// The plans limits record lives beside the manifest, at
1439// accounts/<sqid>/limits.json. It must never be mistaken for one.
1440func TestAccountManifestSqidIgnoresLimitsRecord(t *testing.T) {
1441 if _, ok := domains.AccountManifestSqid("accounts/abc/limits.json"); ok {
1442 t.Fatal("accounts/abc/limits.json parsed as an account manifest")
1443 }
1444}
1445