amadan
fix: a trash marker never outranks the production flag — and only schema 1 is believed
The trash marker is unsigned by design: anything holding PutObject on the bucket can write one. Until now reconcile's §1c executed any VALID marker unchecked, so a bucket writer with no signing key could park a marker on a production app and drop every one of its routes, on every box, at the next tick — a fleet-wide outage from a permission that was never meant to grant one. The purge path already refuses that contradiction; the route-drop now refuses it too, on the same terms: flag ON or flag UNREADABLE, and the marker is not acted on at all, loudly, every pass. A keyless box still acts on markers. It cannot verify a flag, and holding every trash forever wherever there is no pubkey would break hiding for legitimately keyless deployments whose owners have no way to make the guard pass. Every serving box carries the pubkey already (adoption needs it), so the guard holds where production traffic lives — and the destructive half refuses outright without a key, as before. ReadAppTombstone also pins the schema: an unknown one refuses rather than guesses. Refusal fails additive-safe — the app stays served and keeps its bytes until this box understands the shape.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
4 files changed,
+198
−7
internal/domains/reconcile.go+37 −7internal/domains/reconcile_test.go+109 −0internal/release/trash.go+11 −0internal/release/trash_test.go+41 −0
diff --git a/internal/domains/reconcile.go b/internal/domains/reconcile.go| index 4a35c4c2f..039191c35 100644 |
| --- a/internal/domains/reconcile.go |
| +++ b/internal/domains/reconcile.go |
| @@ -38,10 +38,11 @@ type Config struct { |
| AcmeDir string |
| // Now is the clock seam; nil = time.Now. |
| Now func() time.Time |
| - // ReleasePub verifies the production flag before the auto-purge pass |
| - // destroys anything. nil disables purging entirely — a box that cannot |
| - // verify must not destroy — and the app simply stays hidden, which is |
| - // the reversible half of deletion and costs nothing to leave in place. |
| + // ReleasePub verifies the production flag: before a trash marker is |
| + // allowed to drop an app's routes (§1c) and again before the auto-purge |
| + // pass destroys anything (§1d). nil disables purging entirely — a box |
| + // that cannot verify must not destroy — and leaves the route-drop |
| + // unguarded, which §1c explains and accepts. |
| ReleasePub ed25519.PublicKey |
| } |
| @@ -58,7 +59,8 @@ type Config struct { |
| // INSTRUCTED: every route of an app carrying a valid trash marker, custom |
| // domains included. The marker is not something this pass inferred; it is the |
| // owner saying stop serving this app, and a custom domain still serving a |
| -// "deleted" app would be the worse betrayal (§1c). |
| +// "deleted" app would be the worse betrayal (§1c) — unless the app is flagged |
| +// production, which outranks the marker on any box that can verify the flag. |
| func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg Config) error { |
| if cfg.AppsDomain == "" { |
| return nil // inert until configured |
| @@ -146,6 +148,15 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg |
| // believe must not destroy routes, so the app stays live and the log |
| // says why, every pass, until someone fixes or removes the object. |
| // |
| + // And a valid marker does NOT outrank the production flag. The marker is |
| + // unsigned by design — anything holding PutObject on the bucket can write |
| + // one — so without this check a writer with no signing key could park a |
| + // marker on a production app and take its routes down fleet-wide, on |
| + // every box, at the next tick. Dropping every route of a production app |
| + // is an outage, which is precisely what the flag exists to prevent, so |
| + // the same contradiction the purge path refuses (§1d) is refused here: |
| + // flag ON, or flag UNREADABLE, and the marker is not acted on at all. |
| + // |
| // Collected before the linkage below purely so the set is complete when |
| // the removal pass (right after 1b's stamping) reads it. The linkage |
| // itself does NOT skip trashed apps — see the stamping loop for why. |
| @@ -160,9 +171,28 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg |
| log.Printf("domains: ignoring trash marker for %s/%s: %v", acct, app, err) |
| continue |
| } |
| - if tomb != nil { |
| - trashed[appKey{acct, app}] = tomb |
| + if tomb == nil { |
| + continue |
| + } |
| + // A KEYLESS box acts on the marker, deliberately. It cannot verify a |
| + // flag, and the alternative — hold every trash forever wherever there |
| + // is no pubkey — would break hiding entirely on legitimately keyless |
| + // deployments, whose owners have no way to make the guard pass. The |
| + // guard is where it matters: every SERVING box carries the pubkey |
| + // already (adoption needs it), so production traffic is covered, and |
| + // the destructive half (§1d) refuses outright without a key. |
| + if cfg.ReleasePub != nil { |
| + flag, err := release.GetProduction(ctx, s, cfg.ReleasePub, acct, app) |
| + if err != nil { |
| + log.Printf("domains: cannot read the production flag for %s/%s — refusing to act on the trash marker: %v", acct, app, err) |
| + continue |
| + } |
| + if flag != nil && flag.Production { |
| + log.Printf("domains: trash marker on production app %s/%s — refusing to act on it; turn production off first", acct, app) |
| + continue |
| + } |
| } |
| + trashed[appKey{acct, app}] = tomb |
| } |
| // --- 1b. app→account linkage, from the KEY PATH -------------------------- |
diff --git a/internal/domains/reconcile_test.go b/internal/domains/reconcile_test.go| index 2afece8df..4c25b21b0 100644 |
| --- a/internal/domains/reconcile_test.go |
| +++ b/internal/domains/reconcile_test.go |
| @@ -12,11 +12,13 @@ import ( |
| "encoding/pem" |
| "errors" |
| "fmt" |
| + "log" |
| "math/big" |
| "os" |
| "path/filepath" |
| "reflect" |
| "slices" |
| + "strings" |
| "testing" |
| "time" |
| @@ -997,3 +999,110 @@ func TestReconcileNeverPurgesAProductionFlaggedApp(t *testing.T) { |
| t.Fatalf("marker gone: %v, %v", tomb, err) |
| } |
| } |
| + |
| +// The route-drop is as dangerous as the purge, and cheaper to reach: a trash |
| +// marker is UNSIGNED, so anyone holding PutObject can write one. If a marker |
| +// alone could take a production app's routes down, a bucket writer with no |
| +// signing key would have a fleet-wide outage button. So on a box that can |
| +// verify, the flag outranks the marker: nothing is dropped, nothing is |
| +// destroyed, and the contradiction is said out loud every pass. |
| +func TestReconcileNeverDropsRoutesOfAProductionFlaggedApp(t *testing.T) { |
| + priv, pub := releaseKeys(t) |
| + s, reg := dirStore(t), regWith(t, |
| + registry.Route{Host: "derp.custom", Kind: "static", App: "derp", |
| + Status: "ready", Channel: "stable", Account: domains.Sqid(793481920157)}, |
| + ) |
| + sqid := writeAccountManifest(t, s, 793481920157, "Kass") |
| + putKey(t, s, "apps/"+sqid+"/derp/channels/stable.json") |
| + // The flag goes on FIRST: SetProduction refuses a trashed app. |
| + if err := release.SetProduction(ctx, s, priv, sqid, "derp", true, "paul@keymail.dev"); err != nil { |
| + t.Fatal(err) |
| + } |
| + // Past the cooloff too, so a guard that only covered hiding would show up |
| + // here as destroyed bytes. |
| + deleted := time.Unix(1754000000, 0).Add(-31 * 24 * time.Hour) |
| + if err := release.TrashApp(ctx, s, sqid, "derp", "eve@evil.example", deleted); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + var logged bytes.Buffer |
| + log.SetOutput(&logged) |
| + t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| + |
| + c := cfg(t) |
| + c.ReleasePub = pub |
| + apex := "derp." + sqid + ".oncarlos.test" |
| + for pass := 1; pass <= 2; pass++ { |
| + if err := domains.Reconcile(ctx, s, reg, c); err != nil { |
| + t.Fatalf("pass %d: %v", pass, err) |
| + } |
| + // The custom domain survives, and the app is treated as live in every |
| + // other respect — its platform apex mints once and is not re-minted. |
| + mustGet(t, reg, "derp.custom") |
| + mustGet(t, reg, apex) |
| + if got := routesFor(t, reg, "derp"); len(got) != 2 { |
| + t.Fatalf("pass %d: routes = %+v, want the custom domain and exactly one apex", pass, got) |
| + } |
| + if got := appKeysLeft(t, s, sqid, "derp"); len(got) != 3 { |
| + t.Fatalf("pass %d: bytes destroyed under a production flag: %v", pass, got) |
| + } |
| + } |
| + // The refusal is the contract, not a side effect of some later check: an |
| + // operator who wrote that marker on purpose has to be told why nothing |
| + // happened, every pass, or they will assume deletion is broken. |
| + if !strings.Contains(logged.String(), "trash marker on production app") { |
| + t.Errorf("no refusal logged; log was:\n%s", logged.String()) |
| + } |
| +} |
| + |
| +// flagUnreadableStore breaks exactly one read — the app's production flag — |
| +// the way an attacker with delete-but-not-sign, or a half-broken bucket, |
| +// would. Everything else reads normally, so the marker is perfectly valid |
| +// and only the flag is in doubt. |
| +type flagUnreadableStore struct { |
| + release.Store |
| + key string |
| +} |
| + |
| +func (s flagUnreadableStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| + if key == s.key { |
| + return nil, "", errors.New("bucket refused the read") |
| + } |
| + return s.Store.Get(ctx, key) |
| +} |
| + |
| +// An UNREADABLE flag refuses in the same direction as an ON one. "Cannot |
| +// prove it is not production" and "is production" get the same answer on |
| +// every destructive path in this platform, because the state an attacker can |
| +// manufacture is exactly the unreadable one. |
| +func TestReconcileDoesNotDropRoutesWhenTheProductionFlagIsUnreadable(t *testing.T) { |
| + _, pub := releaseKeys(t) |
| + dir, reg := dirStore(t), regWith(t, |
| + registry.Route{Host: "derp.custom", Kind: "static", App: "derp", |
| + Status: "ready", Channel: "stable", Account: domains.Sqid(793481920157)}, |
| + ) |
| + sqid := writeAccountManifest(t, dir, 793481920157, "Kass") |
| + putKey(t, dir, "apps/"+sqid+"/derp/channels/stable.json") |
| + deleted := time.Unix(1754000000, 0).Add(-31 * 24 * time.Hour) |
| + if err := release.TrashApp(ctx, dir, sqid, "derp", "paul@keymail.dev", deleted); err != nil { |
| + t.Fatal(err) |
| + } |
| + s := flagUnreadableStore{Store: dir, key: release.ProductionKey(sqid, "derp")} |
| + |
| + var logged bytes.Buffer |
| + log.SetOutput(&logged) |
| + t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| + |
| + c := cfg(t) |
| + c.ReleasePub = pub |
| + if err := domains.Reconcile(ctx, s, reg, c); err != nil { |
| + t.Fatal(err) |
| + } |
| + mustGet(t, reg, "derp.custom") |
| + if got := appKeysLeft(t, dir, sqid, "derp"); len(got) != 2 { |
| + t.Fatalf("bytes destroyed while the flag was unreadable: %v", got) |
| + } |
| + if !strings.Contains(logged.String(), "cannot read the production flag") { |
| + t.Errorf("no refusal logged; log was:\n%s", logged.String()) |
| + } |
| +} |
diff --git a/internal/release/trash.go b/internal/release/trash.go| index f8a67b2ae..c5288a07c 100644 |
| --- a/internal/release/trash.go |
| +++ b/internal/release/trash.go |
| @@ -72,6 +72,17 @@ func ReadAppTombstone(ctx context.Context, s Store, acct, app string) (*AppTombs |
| if err := json.Unmarshal(body, &t); err != nil { |
| return nil, fmt.Errorf("release: malformed trash marker for %s/%s: %w", acct, app, err) |
| } |
| + // The schema is PINNED, and an unknown one refuses. A future schema 2 |
| + // could mean anything — a scoped marker, a partial deletion, an expiry |
| + // this box has never heard of — and the two things a marker drives are |
| + // hiding an app and destroying it. Refusing fails additive-safe, the way |
| + // the whole wire rule wants: an old box keeps serving and keeps the |
| + // bytes, loudly, until it is upgraded. Believing a shape we cannot read |
| + // would be the one direction that is not recoverable. |
| + if t.Schema != 1 { |
| + return nil, fmt.Errorf("release: trash marker for %s/%s has schema %d — this box only understands schema 1; refusing to act on it", |
| + acct, app, t.Schema) |
| + } |
| if t.Account != acct || t.App != app { |
| return nil, fmt.Errorf("release: trash marker at %s claims %s/%s — path and body disagree", |
| AppTombstoneKey(acct, app), t.Account, t.App) |
diff --git a/internal/release/trash_test.go b/internal/release/trash_test.go| index 116b2793a..8048630d1 100644 |
| --- a/internal/release/trash_test.go |
| +++ b/internal/release/trash_test.go |
| @@ -5,6 +5,7 @@ import ( |
| "crypto/ed25519" |
| "errors" |
| "slices" |
| + "strings" |
| "testing" |
| "time" |
| ) |
| @@ -77,6 +78,46 @@ func TestReadAppTombstoneParsesWireShape(t *testing.T) { |
| if tomb.Account != "bab" || tomb.App != "derp" || tomb.Actor != "paul@keymail.dev" || !tomb.DeletedAt.Equal(want) { |
| t.Fatalf("tombstone fields: %+v", tomb) |
| } |
| + // The schema is part of the shape, not decoration: the refusal below |
| + // only means anything if the happy path really carries a 1. |
| + if tomb.Schema != 1 { |
| + t.Fatalf("schema = %d, want 1", tomb.Schema) |
| + } |
| +} |
| + |
| +// A marker written by a future box, in a shape this one has never seen, is |
| +// refused — and refusal is the safe direction: the app keeps serving and |
| +// keeps its bytes until this box is upgraded. Believing an unknown schema |
| +// would hide or destroy on terms nobody here can read. |
| +func TestReadAppTombstoneRefusesUnknownSchema(t *testing.T) { |
| + ctx := context.Background() |
| + s := NewDirStore(t.TempDir()) |
| + body := []byte(`{"schema":2,"account":"bab","app":"derp","deleted_at":"2026-08-04T16:00:00Z","actor":"paul@keymail.dev"}`) |
| + if _, err := s.Put(ctx, AppTombstoneKey("bab", "derp"), body, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + tomb, err := ReadAppTombstone(ctx, s, "bab", "derp") |
| + if err == nil { |
| + t.Fatalf("schema 2 marker was believed: %+v", tomb) |
| + } |
| + if !strings.Contains(err.Error(), "schema 2") { |
| + t.Errorf("error %q does not name the schema it refused", err) |
| + } |
| + // And the refusal must reach the destructive path as a refusal, not as |
| + // "no marker" — PurgeApp errors rather than deciding there is nothing |
| + // to purge. |
| + if err := PurgeApp(ctx, s, nil, "bab", "derp", time.Now()); err == nil || errors.Is(err, ErrNotTrashed) { |
| + t.Errorf("PurgeApp on an unreadable-schema marker: %v, want a refusal", err) |
| + } |
| + // A schema-0 body (the field missing entirely) is refused for the same |
| + // reason: a marker that does not say what it is, is not a marker. |
| + if _, err := s.Put(ctx, AppTombstoneKey("bab", "noschema"), |
| + []byte(`{"account":"bab","app":"noschema","deleted_at":"2026-08-04T16:00:00Z"}`), ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := ReadAppTombstone(ctx, s, "bab", "noschema"); err == nil { |
| + t.Fatal("schema-less marker was not refused") |
| + } |
| } |
| func TestReadAppTombstoneRefusesDisagreement(t *testing.T) { |