amadan

carlos / platform Public

multi-account: merge-gate fixes — the migration must copy what is LIVE

Six findings, all at seams where an app-name key or a prefix assumption
outlived the thing it described.

C1 (would have rolled production back). `carlos accounts migrate` copied from
the LEGACY prefix. The moment this binary is adopted every ship/promote writes
to apps/bab/<app>/… and every read resolves qualified-first, so the legacy
pointer goes stale on the first deploy — migrating then would have moved a
stale pointer into the new account, lost every post-adopt release, and let the
next agent tick silently downgrade the app to a pre-adopt build before any
human looked. The source is now resolved with the runtime's own PER-KEY
precedence (qualified wins; legacy-only objects still come across so no
rollback target is lost), the verb says out loud when it finds both prefixes,
and after copying it re-reads every destination channel pointer and refuses if
it is not byte-identical to what the source serves — which also catches a
promote landing mid-migration. Moving INTO ops now requires an explicit -from,
because ops is also the default source and that is not a guess worth making.

C2. The console audit stream was app-name-keyed, so two accounts owning a
"blog" shared one activity log — including each other's member addresses.
audit gains account_id (additive, defaulting to ops), and Record/Events are
scoped.

I1. The activator's two LiveVersion copies still read "the parent directory",
which for a migrated tenant is the account SQID. ConvergeDecision would have
believed every migrated tenant converged forever — a tenant that silently
stops taking deploys. Both now delegate to release.LiveVersion: one definition
of which path segment is the version.

I2. The bake clock was account-blind, so one account's 72h stable observation
satisfied another account's identically-named app on the same box — the
supply-chain window skipped, which is the one thing that file exists to
prevent. Keyed on (account, app, channel, version) now, with the ops account
keeping the pre-accounts key shape so no existing box's observations are lost
(restarting the fleet's windows would freeze deploys for 72h on rollout).

I3. BackfillOpsMemberships made every app-grant holder an OWNER of ops — the
account that holds console, platform-site and status. A per-app grant was
never deployment-wide authority: grant holders backfill as members, the
operator flag backfills as owner, and if that would leave ops with no owner
the earliest grant holder is promoted and the promotion is logged loudly. A
later run never demotes.

I4. The silent blackhole: skip `console migrate-app` after `carlos accounts
migrate` and the console promotes into one account's keys while the hosts read
another's — "Promoted" flashes green and nothing deploys. The app page now
says the app is not deploying, and names the exact command that fixes it.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Paul Campbell ff5da8ee8ff21c74cac982444689389c8a5d6558 parent b6feaf3
17 files changed, +781 −115
  • accounts.go +160 −31
  • e2e_test.go +158 −0
  • internal/activator/activator_test.go +32 −0
  • internal/activator/backend_exec.go +12 −11
  • internal/activator/backend_unit.go +4 −11
  • internal/activator/backend_unit_test.go +8 −0
  • internal/console/account_test.go +81 −0
  • internal/console/server.go +8 −8
  • internal/console/server_account_test.go +91 −0
  • internal/console/store.go +81 −14
  • internal/console/store_test.go +4 −4
  • internal/console/templates/app.html +10 −0
  • internal/console/view.go +34 −5
  • internal/release/adopt.go +2 −2
  • internal/release/adopt_test.go +1 −1
  • internal/release/bake.go +27 −10
  • internal/release/bake_test.go +68 −18
diff --git a/accounts.go b/accounts.go
index 6fb52326b..d5e2b02a8 100644
--- a/accounts.go
+++ b/accounts.go
@@ -12,6 +12,7 @@ package main
import (
"context"
+ "crypto/ed25519"
"crypto/rand"
"encoding/binary"
"flag"
@@ -198,52 +199,44 @@ func runAccountsMigrate(args []string) {
log.Fatalf("accounts migrate: no account %s is materialized here — run `carlos accounts create` (or reconcile) first", *to)
}
- src := release.LegacyAppPrefix(*app)
- if *from != "" {
- if !release.ValidAccount(*from) {
- log.Fatalf("accounts migrate: %q is not an account sqid", *from)
- }
- src = release.AppPrefix(*from, *app)
- }
dst := release.AppPrefix(*to, *app)
- if src == dst {
- log.Fatalf("accounts migrate: %s is already under %s", *app, *to)
- }
-
ctx := context.Background()
store := openStore()
- keys, err := store.List(ctx, src)
- if err != nil {
- log.Fatalf("accounts migrate: list %s: %v", src, err)
- }
- if len(keys) == 0 {
- fmt.Printf("nothing under %s — nothing to copy\n", src)
- }
- for _, key := range keys {
- target := dst + strings.TrimPrefix(key, src)
+
+ // Where the app's objects are RIGHT NOW. This is the whole hazard of the
+ // verb, so it is resolved the way the runtime resolves them and not by
+ // assuming a prefix.
+ plan, srcAcct := migrationPlan(ctx, store, *app, *from, *to, dst)
+ if len(plan) == 0 {
+ fmt.Printf("nothing under %s — nothing to copy\n", release.AppPrefix(srcAcct, *app))
+ }
+ for _, c := range plan {
if *dryRun {
- fmt.Printf("would copy %s -> %s\n", key, target)
+ fmt.Printf("would copy %s -> %s\n", c.from, c.to)
continue
}
- body, _, err := store.Get(ctx, key)
+ body, _, err := store.Get(ctx, c.from)
if err != nil {
- log.Fatalf("accounts migrate: read %s: %v", key, err)
+ log.Fatalf("accounts migrate: read %s: %v", c.from, err)
}
// Create-only, then verify: re-running the migration over identical
// bytes is a no-op, and DIFFERENT bytes at a destination that already
// exists is refused loudly rather than overwritten.
- if _, err := store.Put(ctx, target, body, ""); err != nil {
- cur, _, gerr := store.Get(ctx, target)
+ if _, err := store.Put(ctx, c.to, body, ""); err != nil {
+ cur, _, gerr := store.Get(ctx, c.to)
if gerr != nil {
- log.Fatalf("accounts migrate: write %s: %v", target, err)
+ log.Fatalf("accounts migrate: write %s: %v", c.to, err)
}
if string(cur) != string(body) {
- log.Fatalf("accounts migrate: %s already exists with different content — refusing to overwrite", target)
+ log.Fatalf("accounts migrate: %s already exists with different content — refusing to overwrite", c.to)
}
- fmt.Printf("already copied %s -> %s\n", key, target)
+ fmt.Printf("already copied %s -> %s\n", c.from, c.to)
continue
}
- fmt.Printf("copied %s -> %s\n", key, target)
+ fmt.Printf("copied %s -> %s\n", c.from, c.to)
+ }
+ if !*dryRun {
+ verifyMigratedPointers(ctx, store, *app, srcAcct, *to)
}
// The routes' account column: what adopt and the edge build keys from.
@@ -276,12 +269,148 @@ func runAccountsMigrate(args []string) {
fmt.Printf("\nafterwards, on the console's box: %s\n", followUp)
return
}
- fmt.Printf("\n%s now lives under %s. The legacy objects at %s are UNTOUCHED —\n", *app, dst, src)
+ fmt.Printf("\n%s now lives under %s. Everything it was copied from is UNTOUCHED —\n", *app, dst)
fmt.Println("verify first (every channel pointer resolves qualified, one adopt pass is a")
- fmt.Println("no-op), and only then delete them by hand.")
+ fmt.Println("no-op), and only then delete the old objects by hand.")
fmt.Printf("\nThe console keeps its own app→account row: run this on the console's box to\ncomplete the move:\n\n %s\n", followUp)
}
+// copy is one object the migration moves.
+type copyOp struct{ from, to string }
+
+// migrationPlan works out which objects to copy, and from where. This is the
+// most dangerous decision the verb makes.
+//
+// Once this binary is adopted, every ship and promote writes to the
+// account-qualified keys and every read resolves qualified-first — so an app
+// that has not been migrated yet ends up with objects at BOTH prefixes, and
+// its LEGACY pointer goes stale the first time anyone deploys. Copying the
+// legacy prefix at that point would move a stale pointer into the new
+// account, lose every post-adopt release, and let the next agent tick
+// silently downgrade the app to a pre-adopt build.
+//
+// So the default source is not a prefix but the same PER-KEY PRECEDENCE the
+// runtime uses (release.GetWithFallback): for each relative key, the
+// ops-qualified object wins, and legacy-only objects still come across so no
+// rollback target is lost. `-from` overrides it with a single explicit
+// prefix, for moving between two non-ops accounts.
+//
+// Returns the copies to make and the account the app is being moved FROM (for
+// the post-copy verification).
+func migrationPlan(ctx context.Context, store release.Store, app, from, to, dst string) ([]copyOp, string) {
+ if from != "" {
+ if !release.ValidAccount(from) {
+ log.Fatalf("accounts migrate: %q is not an account sqid", from)
+ }
+ src := release.AppPrefix(from, app)
+ if src == dst {
+ log.Fatalf("accounts migrate: %s is already under %s", app, to)
+ }
+ keys, err := store.List(ctx, src)
+ if err != nil {
+ log.Fatalf("accounts migrate: list %s: %v", src, err)
+ }
+ var out []copyOp
+ for _, k := range keys {
+ out = append(out, copyOp{k, dst + strings.TrimPrefix(k, src)})
+ }
+ return out, from
+ }
+
+ if to == release.OpsAccount {
+ log.Fatalf("accounts migrate: moving into ops needs an explicit -from — ops is also the default SOURCE, and guessing which way you meant is not something this verb will do")
+ }
+ opsPrefix := release.AppPrefix(release.OpsAccount, app)
+ legacyPrefix := release.LegacyAppPrefix(app)
+ opsKeys, err := store.List(ctx, opsPrefix)
+ if err != nil {
+ log.Fatalf("accounts migrate: list %s: %v", opsPrefix, err)
+ }
+ legacyKeys, err := store.List(ctx, legacyPrefix)
+ if err != nil {
+ log.Fatalf("accounts migrate: list %s: %v", legacyPrefix, err)
+ }
+ if len(opsKeys) > 0 && len(legacyKeys) > 0 {
+ fmt.Printf("%s has objects under BOTH %s and %s — this app was shipped to since the\n", app, opsPrefix, legacyPrefix)
+ fmt.Printf("accounts build was adopted. Taking the qualified copy of every key that\n")
+ fmt.Printf("exists in both (that is what the runtime reads), and the legacy copy of the\n")
+ fmt.Printf("rest, so no rollback target is lost.\n")
+ } else if len(opsKeys) > 0 {
+ fmt.Printf("source: %s\n", opsPrefix)
+ } else if len(legacyKeys) > 0 {
+ fmt.Printf("source: %s\n", legacyPrefix)
+ }
+
+ // Qualified first, then legacy for anything it did not cover — the
+ // per-key precedence GetWithFallback applies on every read.
+ seen := map[string]bool{}
+ var out []copyOp
+ for _, k := range opsKeys {
+ rel := strings.TrimPrefix(k, opsPrefix)
+ seen[rel] = true
+ out = append(out, copyOp{k, dst + rel})
+ }
+ for _, k := range legacyKeys {
+ rel := strings.TrimPrefix(k, legacyPrefix)
+ if seen[rel] {
+ continue // the qualified copy is the live one
+ }
+ out = append(out, copyOp{k, dst + rel})
+ }
+ return out, release.OpsAccount
+}
+
+// verifyMigratedPointers re-reads every channel pointer after the copy and
+// refuses if the destination does not now serve exactly what the source
+// serves. Belt and braces over migrationPlan's precedence — and the one check
+// that catches a promote landing DURING the migration, which would otherwise
+// leave the new account one version behind with nothing to notice it.
+//
+// Byte comparison is the gate, so no signing key is needed. When
+// CARLOS_RELEASE_PUBKEY is set the versions are resolved and printed too, so
+// the operator sees what they just moved rather than a bare "ok".
+func verifyMigratedPointers(ctx context.Context, store release.Store, app, srcAcct, dstAcct string) {
+ prefix := release.ChannelsPrefix(dstAcct, app)
+ keys, err := store.List(ctx, prefix)
+ if err != nil {
+ log.Fatalf("accounts migrate: verify: list %s: %v", prefix, err)
+ }
+ var pub ed25519.PublicKey
+ if b := strings.TrimSpace(os.Getenv("CARLOS_RELEASE_PUBKEY")); b != "" {
+ if p, err := release.ParsePublicKey(b, "CARLOS_RELEASE_PUBKEY"); err == nil {
+ pub = p
+ }
+ }
+ for _, key := range keys {
+ channel := strings.TrimSuffix(strings.TrimPrefix(key, prefix), ".json")
+ got, _, err := store.Get(ctx, key)
+ if err != nil {
+ log.Fatalf("accounts migrate: verify %s: %v", key, err)
+ }
+ // What a reader on the OLD account resolves right now, by the same
+ // precedence the runtime uses.
+ want, _, _, err := release.GetWithFallback(ctx, store,
+ release.ChannelKey(srcAcct, app, channel), release.LegacyChannelKey(app, channel))
+ if err != nil {
+ log.Fatalf("accounts migrate: verify: re-read %s/%s at the source: %v", app, channel, err)
+ }
+ if string(got) != string(want) {
+ log.Fatalf("accounts migrate: VERIFICATION FAILED for channel %s: what %s now serves is not\n"+
+ "what %s serves. Something wrote to this app during the migration. Do NOT delete\n"+
+ "anything; re-run the migration once the writer has stopped.", channel, dstAcct, srcAcct)
+ }
+ if pub != nil {
+ p, err := release.Channel(ctx, store, pub, dstAcct, app, channel)
+ if err != nil {
+ log.Fatalf("accounts migrate: verify: %s/%s does not resolve under %s: %v", app, channel, dstAcct, err)
+ }
+ fmt.Printf("verified %s: %s serves %s under %s\n", channel, app, p.Version, dstAcct)
+ continue
+ }
+ fmt.Printf("verified %s: byte-identical to what %s serves (set CARLOS_RELEASE_PUBKEY to see versions)\n", channel, srcAcct)
+ }
+}
+
// resolveAccount turns a -account value into an account sqid: an exact sqid
// match wins, then a unique name match; an ambiguous name is an error that
// lists the candidates rather than picking one. Friendly without being
diff --git a/e2e_test.go b/e2e_test.go
index a7f5c497f..5179c4cd7 100644
--- a/e2e_test.go
+++ b/e2e_test.go
@@ -779,6 +779,164 @@ func TestAccountsCLI(t *testing.T) {
run("accounts", "migrate", "-registry", regPath, "-app", "demo", "-to", sqid)
}
+// TestAccountsMigrateTakesTheLiveObjects is the production-rollback trap.
+//
+// The moment this binary is adopted, every ship/promote writes to
+// apps/bab/<app>/… and every read resolves qualified-first — so the LEGACY
+// pointer goes stale the first time anyone deploys. A migrate that blindly
+// copied from the legacy prefix would move a stale pointer into the new
+// account, miss every post-adopt release, and the next agent tick would
+// silently downgrade the app to a pre-adopt build before any human looked at
+// it.
+//
+// So: with objects at BOTH prefixes, the verb must resolve them the way the
+// RUNTIME does (qualified wins per key), say that it saw both, and verify
+// afterwards that what the destination now serves is what the source served.
+func TestAccountsMigrateTakesTheLiveObjects(t *testing.T) {
+ goBin, err := exec.LookPath("go")
+ if err != nil {
+ t.Skip("go toolchain not found")
+ }
+ tmp := t.TempDir()
+ carlosBin := filepath.Join(tmp, "carlos")
+ build := exec.Command(goBin, "build", "-o", carlosBin, ".")
+ build.Env = os.Environ()
+ if out, err := build.CombinedOutput(); err != nil {
+ t.Fatalf("build carlos: %v\n%s", err, out)
+ }
+ storeDir := filepath.Join(tmp, "store")
+ regPath := filepath.Join(tmp, "registry.db")
+ store := release.NewDirStore(storeDir)
+ ctx := context.Background()
+
+ seed, pubB64, err := release.GenerateKey()
+ if err != nil {
+ t.Fatal(err)
+ }
+ priv, err := release.ParseSigningKey(seed, "e2e")
+ if err != nil {
+ t.Fatal(err)
+ }
+ pub, err := release.ParsePublicKey(pubB64, "e2e")
+ if err != nil {
+ t.Fatal(err)
+ }
+ env := append(os.Environ(),
+ "CARLOS_DEPLOYMENT_DIR="+storeDir,
+ "CARLOS_RELEASE_PUBKEY="+pubB64)
+ run := func(args ...string) (string, error) {
+ t.Helper()
+ cmd := exec.Command(carlosBin, args...)
+ cmd.Env = env
+ out, err := cmd.CombinedOutput()
+ return string(out), err
+ }
+
+ // The pre-adopt world: v1 shipped and promoted, its objects living at the
+ // legacy account-less keys. (Built by promoting under ops and copying the
+ // bytes down — a pointer's signature covers app/channel/version, not the
+ // key path, so those ARE valid legacy objects, signed by the same key a
+ // real deployment used.)
+ m1 := release.Manifest{Schema: 1, App: "demo", Version: "v1", Kind: "binary",
+ Artifacts: map[string]release.Artifact{"linux-amd64": {File: "demo"}}}
+ if err := release.Ship(ctx, store, release.OpsAccount, m1, map[string][]byte{"linux-amd64": []byte("v1 bytes")}); err != nil {
+ t.Fatal(err)
+ }
+ if err := release.Promote(ctx, store, priv, release.OpsAccount, "demo", "stable", "v1"); err != nil {
+ t.Fatal(err)
+ }
+ qualified, err := store.List(ctx, release.AppPrefix(release.OpsAccount, "demo"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, k := range qualified {
+ body, _, err := store.Get(ctx, k)
+ if err != nil {
+ t.Fatal(err)
+ }
+ legacy := release.LegacyAppPrefix("demo") + strings.TrimPrefix(k, release.AppPrefix(release.OpsAccount, "demo"))
+ if _, err := store.Put(ctx, legacy, body, ""); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ // Then the deployment adopts this binary and someone ships again: v2 lands
+ // ONLY at the qualified keys, and stable now points at it there. The
+ // legacy pointer still says v1 — stale, and invisible to every reader.
+ m2 := release.Manifest{Schema: 1, App: "demo", Version: "v2", Kind: "binary",
+ Artifacts: map[string]release.Artifact{"linux-amd64": {File: "demo"}}}
+ if err := release.Ship(ctx, store, release.OpsAccount, m2, map[string][]byte{"linux-amd64": []byte("v2 bytes")}); err != nil {
+ t.Fatal(err)
+ }
+ if err := release.Promote(ctx, store, priv, release.OpsAccount, "demo", "stable", "v2"); err != nil {
+ t.Fatal(err)
+ }
+ // Sanity: this is genuinely the trap — the two prefixes disagree.
+ if p, err := release.Channel(ctx, store, pub, release.OpsAccount, "demo", "stable"); err != nil || p.Version != "v2" {
+ t.Fatalf("fixture: runtime resolves %+v %v, want v2", p, err)
+ }
+ legacyRaw, _, err := store.Get(ctx, release.LegacyChannelKey("demo", "stable"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(legacyRaw) == "" {
+ t.Fatal("fixture: no legacy pointer")
+ }
+
+ reg, err := registry.Open(regPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := reg.Upsert(registry.Route{Host: "demo.example.com", Kind: "instance",
+ App: "demo", Status: "ready", Addr: "127.0.0.1:9000"}); err != nil {
+ t.Fatal(err)
+ }
+ reg.Close()
+
+ out, err := run("accounts", "create", "-registry", regPath, "-name", "Kass")
+ if err != nil {
+ t.Fatalf("accounts create: %v\n%s", err, out)
+ }
+ sqid := regexp.MustCompile(`(?m)^\s*sqid:\s*(\S+)`).FindStringSubmatch(out)[1]
+
+ // The verb must SAY it saw both prefixes — an operator staring at a
+ // half-migrated bucket needs to know that is what they are looking at.
+ out, err = run("accounts", "migrate", "-registry", regPath, "-app", "demo", "-to", sqid, "-dry-run")
+ if err != nil {
+ t.Fatalf("dry run: %v\n%s", err, out)
+ }
+ if !strings.Contains(out, release.AppPrefix(release.OpsAccount, "demo")) ||
+ !strings.Contains(out, release.LegacyAppPrefix("demo")) {
+ t.Errorf("dry run does not mention both source prefixes:\n%s", out)
+ }
+
+ out, err = run("accounts", "migrate", "-registry", regPath, "-app", "demo", "-to", sqid)
+ if err != nil {
+ t.Fatalf("migrate: %v\n%s", err, out)
+ }
+ // THE assertion: the destination serves what the runtime served, not the
+ // stale legacy pointer.
+ got, err := release.Channel(ctx, store, pub, sqid, "demo", "stable")
+ if err != nil {
+ t.Fatalf("resolve the migrated channel: %v", err)
+ }
+ if got.Version != "v2" {
+ t.Fatalf("migrated pointer = %s, want v2 — the verb copied the STALE legacy pointer, "+
+ "which would downgrade the app on the next agent tick", got.Version)
+ }
+ // Both versions' release objects came across, so a rollback target still
+ // resolves after the move.
+ for _, ver := range []string{"v1", "v2"} {
+ if _, err := release.GetManifest(ctx, store, sqid, "demo", ver); err != nil {
+ t.Errorf("manifest %s missing after the move: %v", ver, err)
+ }
+ }
+ // And nothing was deleted.
+ if _, _, err := store.Get(ctx, release.LegacyChannelKey("demo", "stable")); err != nil {
+ t.Errorf("migrate deleted a legacy object: %v", err)
+ }
+}
+
// TestAddUnitBackingRequiresDBForHibernate pins the I1 fix's operator-facing
// half: `carlos add -backing unit -hibernate` with no -db must refuse,
// because ops.FleetDBs (the box-wide litestream unit) has no way to
diff --git a/internal/activator/activator_test.go b/internal/activator/activator_test.go
index f462e0df0..5aee62fd7 100644
--- a/internal/activator/activator_test.go
+++ b/internal/activator/activator_test.go
@@ -1777,6 +1777,38 @@ func TestExecBackendLiveVersion(t *testing.T) {
}
}
+// A migrated tenant's live link points at <opt>/versions/<ver>/<acct>/<app>,
+// so reading "the parent directory" returns the ACCOUNT SQID as the version.
+// ConvergeDecision compares that against the route's version and would then
+// believe every migrated tenant is converged forever — a tenant that silently
+// stops taking deploys.
+func TestExecBackendLiveVersionOnAccountScopedPath(t *testing.T) {
+ b := testExecBackend(t)
+ stageLiveUnderAccount(t, b.OptDir, "kass.example.com", "kqmvexample", "abc1234", "kass")
+ if ver, err := b.LiveVersion("kass.example.com"); err != nil || ver != "abc1234" {
+ t.Fatalf("LiveVersion = %q, %v; want abc1234 (the version, not the account sqid)", ver, err)
+ }
+}
+
+// stageLiveUnderAccount builds the account-scoped version-store layout and
+// points a host's live symlink at it.
+func stageLiveUnderAccount(t *testing.T, optDir, host, acct, ver, app string) {
+ t.Helper()
+ dir := filepath.Join(optDir, "versions", ver, acct)
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(dir, app), []byte("bin"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.MkdirAll(filepath.Join(optDir, "live"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(filepath.Join(dir, app), filepath.Join(optDir, "live", host)); err != nil {
+ t.Fatal(err)
+ }
+}
+
// childEnv strips every CARLOS_* var from the inherited environment (the
// edge's own config must never leak into a child) and then layers the env
// file on top — including that file's own CARLOS_* entries.
diff --git a/internal/activator/backend_exec.go b/internal/activator/backend_exec.go
index 41f1d5912..ffc0b128b 100644
--- a/internal/activator/backend_exec.go
+++ b/internal/activator/backend_exec.go
@@ -21,6 +21,7 @@ import (
"github.com/carlosframework/platform/internal/lease"
"github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
)
// spawnHealthTimeout bounds how long a spawned instance may take to bind its
@@ -60,19 +61,19 @@ func (b *ExecBackend) defaultDBPath(host string) string {
}
// LiveVersion reports the version host's live symlink currently points at, or
-// "" (no error) if the host hasn't been put under version management yet.
-// (target = <OptDir>/versions/<ver>/<app> → the parent dir's name is <ver>.)
-// It doubles as the Service's convergence-sweep seam — New picks it up via an
+// "" (no error) if the host hasn't been put under version management yet. It
+// doubles as the Service's convergence-sweep seam — New picks it up via an
// optional interface.
+//
+// Delegated to release.LiveVersion rather than reading the parent directory,
+// because the version store gained an account level: a migrated tenant's link
+// is <opt>/versions/<ver>/<acct>/<app>, and "the parent dir" there is the
+// account SQID. ConvergeDecision compares this against the route's version,
+// so returning a sqid would convince it that every migrated tenant was
+// converged forever — a tenant that silently stops taking deploys. One
+// definition of "which segment is the version", in one place.
func (b *ExecBackend) LiveVersion(host string) (string, error) {
- target, err := os.Readlink(b.liveLink(host))
- if err != nil {
- if os.IsNotExist(err) {
- return "", nil
- }
- return "", err
- }
- return filepath.Base(filepath.Dir(target)), nil
+ return release.LiveVersion(b.OptDir, host)
}
// Acquire claims the per-host single-writer lease. A contested lease is
diff --git a/internal/activator/backend_unit.go b/internal/activator/backend_unit.go
index daede904d..a9b9a6c37 100644
--- a/internal/activator/backend_unit.go
+++ b/internal/activator/backend_unit.go
@@ -50,14 +50,13 @@ import (
"errors"
"fmt"
"net/http"
- "os"
"os/exec"
- "path/filepath"
"strings"
"sync"
"time"
"github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
)
// UnitName is the systemd unit backing a unit-backed tenant's route —
@@ -96,16 +95,10 @@ func (b *UnitBackend) systemctl(ctx context.Context, verb, unit string) error {
// LiveVersion mirrors ExecBackend.LiveVersion — the unit's ExecStart is the
// same live/<host> symlink the adopt pass repoints, so convergence reads the
-// same layout.
+// same layout, through the same one definition of which path segment is the
+// version (see ExecBackend.LiveVersion for why that matters now).
func (b *UnitBackend) LiveVersion(host string) (string, error) {
- target, err := os.Readlink(filepath.Join(b.OptDir, "live", host))
- if err != nil {
- if os.IsNotExist(err) {
- return "", nil
- }
- return "", err
- }
- return filepath.Base(filepath.Dir(target)), nil
+ return release.LiveVersion(b.OptDir, host)
}
// Acquire returns the inert lease (see the package comment: replication
diff --git a/internal/activator/backend_unit_test.go b/internal/activator/backend_unit_test.go
index 027417e8b..b985d7f4c 100644
--- a/internal/activator/backend_unit_test.go
+++ b/internal/activator/backend_unit_test.go
@@ -286,6 +286,14 @@ func TestUnitBackendLiveVersion(t *testing.T) {
if v, err := b.LiveVersion("kass.example.com"); err != nil || v != "abc1234" {
t.Fatalf("LiveVersion = %q, %v; want abc1234", v, err)
}
+
+ // A migrated tenant's link has the account sqid between the version and
+ // the binary; reading "the parent directory" would return the sqid and
+ // convince ConvergeDecision this tenant is converged forever.
+ stageLiveUnderAccount(t, opt, "moved.example.com", "kqmvexample", "def5678", "kass")
+ if v, err := b.LiveVersion("moved.example.com"); err != nil || v != "def5678" {
+ t.Fatalf("LiveVersion (account-scoped) = %q, %v; want def5678", v, err)
+ }
}
// The end-to-end shape the feature promises: a request for a SLEEPING unit
diff --git a/internal/console/account_test.go b/internal/console/account_test.go
index e15c1bef1..448a54022 100644
--- a/internal/console/account_test.go
+++ b/internal/console/account_test.go
@@ -220,6 +220,87 @@ func TestLastAccountRoundTrips(t *testing.T) {
}
}
+// A per-app grant was never deployment-wide authority. ops contains console,
+// platform-site and status, so backfilling every grant holder as an OWNER of
+// it would hand whoever once shipped one app the right to invite people into
+// the account that runs the platform itself. App-grant holders land as
+// members; the operator flag — which IS deployment-wide authority — is what
+// makes an owner.
+func TestBackfillGrantsAreMembersOperatorsAreOwners(t *testing.T) {
+ st := openTestStore(t)
+ if err := st.Grant("dev@keymail.dev", "kass"); err != nil {
+ t.Fatal(err)
+ }
+ if err := st.Grant("ops@keymail.dev", "console"); err != nil {
+ t.Fatal(err)
+ }
+ if err := st.SetOperator("ops@keymail.dev", true); err != nil {
+ t.Fatal(err)
+ }
+ if err := st.BackfillOpsMemberships(time.Now()); err != nil {
+ t.Fatal(err)
+ }
+ if role, _ := st.Role(OpsAccountID, "dev@keymail.dev"); role != RoleMember {
+ t.Errorf("app-grant holder role = %q, want member", role)
+ }
+ if role, _ := st.Role(OpsAccountID, "ops@keymail.dev"); role != RoleOwner {
+ t.Errorf("operator role = %q, want owner", role)
+ }
+ // Both still have their apps — the demotion is of authority, not access.
+ apps, _ := st.AppsIn(OpsAccountID)
+ if len(apps) != 2 {
+ t.Errorf("AppsIn(ops) = %v, want both apps claimed", apps)
+ }
+ // Idempotent, and it must not DEMOTE an owner on a later run.
+ if err := st.BackfillOpsMemberships(time.Now()); err != nil {
+ t.Fatal(err)
+ }
+ if role, _ := st.Role(OpsAccountID, "ops@keymail.dev"); role != RoleOwner {
+ t.Errorf("second backfill demoted the operator to %q", role)
+ }
+}
+
+// ops must never end up with nobody who can administer it. With grants but no
+// operator, the earliest grant holder is promoted — loudly, because it is a
+// fallback, not the rule.
+func TestBackfillPromotesAnOwnerWhenThereWouldBeNone(t *testing.T) {
+ st := openTestStore(t)
+ base := time.Unix(1700000000, 0)
+ if err := st.GrantAt("second@keymail.dev", "b", base.Add(time.Hour)); err != nil {
+ t.Fatal(err)
+ }
+ if err := st.GrantAt("first@keymail.dev", "a", base); err != nil {
+ t.Fatal(err)
+ }
+ if err := st.BackfillOpsMemberships(time.Now()); err != nil {
+ t.Fatal(err)
+ }
+ if role, _ := st.Role(OpsAccountID, "first@keymail.dev"); role != RoleOwner {
+ t.Errorf("earliest grant holder role = %q, want owner (nobody else could administer ops)", role)
+ }
+ if role, _ := st.Role(OpsAccountID, "second@keymail.dev"); role != RoleMember {
+ t.Errorf("later grant holder role = %q, want member", role)
+ }
+}
+
+// A deployment with no grants at all and no operator gets an ops account with
+// no members — which is correct: there is nobody to make an owner, and
+// inventing one would be worse than an empty account the first operator
+// claims.
+func TestBackfillLeavesOpsEmptyWhenThereIsNobody(t *testing.T) {
+ st := openTestStore(t)
+ if err := st.BackfillOpsMemberships(time.Now()); err != nil {
+ t.Fatal(err)
+ }
+ members, err := st.Members(OpsAccountID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(members) != 0 {
+ t.Errorf("Members(ops) = %+v, want none", members)
+ }
+}
+
// The console half of `carlos accounts migrate`: the app→account row moves,
// and re-running is a no-op rather than an error — an operator who is not
// sure whether the move landed must be able to just run it again.
diff --git a/internal/console/server.go b/internal/console/server.go
index 6968a117a..be121fcef 100644
--- a/internal/console/server.go
+++ b/internal/console/server.go
@@ -911,7 +911,7 @@ func (s *Server) handleApp(w http.ResponseWriter, r *http.Request) {
}
p.Versions = versions
p.Instances = s.instances.Report(r.Context(), views[0].Routes)
- if events, err := s.store.Events(app, eventsCap); err != nil {
+ if events, err := s.store.Events(acct.ID, app, eventsCap); err != nil {
log.Printf("console: events %s: %v", app, err)
} else {
p.Events = events
@@ -924,8 +924,8 @@ const eventsCap = 10
// audit best-effort-records an owner action: a failed audit line is logged,
// never surfaced, and never fails the action it records.
-func (s *Server) audit(app, addr, action, detail string) {
- if err := s.store.RecordEvent(app, addr, action, detail, time.Now()); err != nil {
+func (s *Server) audit(acct *Account, app, addr, action, detail string) {
+ if err := s.store.RecordEvent(acct.ID, app, addr, action, detail, time.Now()); err != nil {
log.Printf("console: audit %s %s: %v", action, app, err)
}
}
@@ -1035,7 +1035,7 @@ func (s *Server) handleNewApp(w http.ResponseWriter, r *http.Request) {
// account's own manifest is what the agent materializes, and minting
// already wrote it.
s.ensureAccountManifest(ctx, acct)
- s.audit(name, addr, "create", "created app "+name)
+ s.audit(acct, name, addr, "create", "created app "+name)
s.setFlash(w, "ok", "Created app "+name+" — it's yours.")
http.Redirect(w, r, home+"apps/"+url.PathEscape(name), http.StatusSeeOther)
}
@@ -1083,7 +1083,7 @@ func (s *Server) handlePromote(w http.ResponseWriter, r *http.Request) {
back("err", "Promote failed: "+err.Error())
return
}
- s.audit(app, addr, "promote", "promoted "+version+" to "+channel)
+ s.audit(acct, app, addr, "promote", "promoted "+version+" to "+channel)
msg := "Promoted " + version + " to " + channel + "."
if release.HoldFor(release.DefaultHolds, channel) > 0 {
msg += " Adopts after the bake window on each host — rollback never bakes."
@@ -1132,7 +1132,7 @@ func (s *Server) handleRollback(w http.ResponseWriter, r *http.Request) {
back("err", "Rollback failed: "+err.Error())
return
}
- s.audit(app, addr, "rollback", "rolled back "+channel+" to "+target)
+ s.audit(acct, app, addr, "rollback", "rolled back "+channel+" to "+target)
back("ok", "Rolled back "+channel+" to "+target+".")
}
@@ -1227,7 +1227,7 @@ func (s *Server) handleNewCanary(w http.ResponseWriter, r *http.Request) {
back("err", "Creating the canary failed: "+err.Error())
return
}
- s.audit(app, addr, "canary-create", "created canary "+name+" at "+version)
+ s.audit(acct, app, addr, "canary-create", "created canary "+name+" at "+version)
// Best-effort here: which URL to show in the success flash is cosmetic,
// so a registry hiccup just falls back to the generic "attach one" copy
// rather than failing the create that already succeeded.
@@ -1298,7 +1298,7 @@ func (s *Server) handleRetireCanary(w http.ResponseWriter, r *http.Request) {
back("err", "Retiring the canary failed: "+err.Error())
return
}
- s.audit(app, addr, "canary-retire", "retired canary "+name)
+ s.audit(acct, app, addr, "canary-retire", "retired canary "+name)
back("ok", "Retired canary "+name+" — its channel pointer is gone; shipped releases are untouched.")
}
diff --git a/internal/console/server_account_test.go b/internal/console/server_account_test.go
index 2ea0a33d1..b5815da21 100644
--- a/internal/console/server_account_test.go
+++ b/internal/console/server_account_test.go
@@ -6,6 +6,8 @@ import (
"strings"
"testing"
"time"
+
+ "github.com/carlosframework/platform/internal/registry"
)
// Account existence must not be probeable: a signed-in non-member asking for
@@ -195,6 +197,95 @@ func TestSwitcherListsOnlyMyAccounts(t *testing.T) {
}
}
+// The audit trail is per (account, app), not per app name. Two accounts can
+// both own a "blog"; sharing one activity stream would show each account the
+// other's actions — including the other account's member addresses, which is
+// a membership leak dressed up as a convenience.
+func TestAuditIsScopedToTheAccount(t *testing.T) {
+ fx := newConsoleFixture(t)
+ now := time.Now()
+ other := NewAccountID()
+ if err := fx.st.CreateAccount(other, accountSqid(other), "Other", "sam@keymail.dev", now); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := fx.st.CreateAppIn(other, "kass", "sam@keymail.dev", now); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := fx.st.RecordEvent(OpsAccountID, "kass", "paul@keymail.dev", "promote", "promoted v1 to stable", now); err != nil {
+ t.Fatal(err)
+ }
+ if err := fx.st.RecordEvent(other, "kass", "sam@keymail.dev", "promote", "promoted v9 to stable", now); err != nil {
+ t.Fatal(err)
+ }
+
+ ours, err := fx.st.Events(OpsAccountID, "kass", 10)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(ours) != 1 || ours[0].Address != "paul@keymail.dev" {
+ t.Fatalf("Events(ops, kass) = %+v, want only our own", ours)
+ }
+ theirs, _ := fx.st.Events(other, "kass", 10)
+ if len(theirs) != 1 || theirs[0].Address != "sam@keymail.dev" {
+ t.Fatalf("Events(other, kass) = %+v", theirs)
+ }
+
+ // And through the page, which is where the leak would actually be seen.
+ c := signInPaul(t, fx)
+ _, _, body := get(t, c, fx.u("/apps/kass"))
+ if !strings.Contains(body, "promoted v1 to stable") {
+ t.Error("the account's own activity is missing from its app page")
+ }
+ if strings.Contains(body, "sam@keymail.dev") || strings.Contains(body, "promoted v9 to stable") {
+ t.Errorf("another account's activity leaked onto this app page:\n%s", body)
+ }
+}
+
+// The silent blackhole: `carlos accounts migrate` re-stamps the ROUTES, and
+// if `console migrate-app` is then skipped, the console promotes into one
+// account's keys while every host reads another's. "Promoted v2 to stable"
+// flashes green and nothing deploys, forever, with no error anywhere.
+//
+// So the app page says so, and names the command that fixes it.
+func TestAppPageWarnsWhenAccountsDisagree(t *testing.T) {
+ fx := newConsoleFixture(t)
+ c := signInPaul(t, fx)
+
+ // Agreeing (the fixture's routes carry no account, which reads as ops):
+ // no warning, or the warning is worthless.
+ _, _, body := get(t, c, fx.u("/apps/kass"))
+ if strings.Contains(body, "not deploying") {
+ t.Fatalf("warned about an app whose accounts agree:\n%s", body)
+ }
+
+ // Now the routes move to another account and the console row is left
+ // behind — exactly the half-migrated state.
+ reg, err := registry.Open(fx.regPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ moved := NewAccountID()
+ sqid := accountSqid(moved)
+ if err := fx.st.CreateAccount(moved, sqid, "Kass", "paul@keymail.dev", time.Now()); err != nil {
+ t.Fatal(err)
+ }
+ for _, host := range []string{"kass.carlosapp.dev", "docs.kass.carlosapp.dev"} {
+ if err := reg.SetAccount(host, sqid); err != nil {
+ t.Fatal(err)
+ }
+ }
+ reg.Close()
+
+ _, _, body = get(t, c, fx.u("/apps/kass"))
+ if !strings.Contains(body, "not deploying") {
+ t.Errorf("no warning on a half-migrated app:\n%s", body)
+ }
+ if !strings.Contains(body, "console migrate-app -app kass -to "+sqid) {
+ t.Errorf("the warning does not name the command that fixes it:\n%s", body)
+ }
+}
+
// Creating an account from the console: minted id, bucket manifest, owner
// membership, and you land in it.
func TestCreateAccountFromConsole(t *testing.T) {
diff --git a/internal/console/store.go b/internal/console/store.go
index 2d2d19280..21d2e3cca 100644
--- a/internal/console/store.go
+++ b/internal/console/store.go
@@ -15,6 +15,7 @@ import (
"encoding/base64"
"encoding/hex"
"fmt"
+ "log"
"os"
"path/filepath"
"strings"
@@ -176,6 +177,10 @@ func (s *Store) migrate() error {
// until the migration verb rebuilds the table, because changing a PK
// is not an additive migration.
`ALTER TABLE apps ADD COLUMN account_id INTEGER NOT NULL DEFAULT 1;`,
+ // The audit trail is per (account, app) too. Existing rows default to
+ // ops, which is where every pre-accounts app lived — so no history is
+ // lost and none of it lands in somebody else's account.
+ `ALTER TABLE audit ADD COLUMN account_id INTEGER NOT NULL DEFAULT 1;`,
} {
if _, err := s.db.Exec(stmt); err != nil && !strings.Contains(err.Error(), "duplicate column") {
return err
@@ -235,8 +240,15 @@ func (s *Store) migrateAppsKey() error {
// Grant makes address an owner of app (idempotent).
func (s *Store) Grant(address, app string) error {
+ return s.GrantAt(address, app, time.Now())
+}
+
+// GrantAt is Grant with an explicit timestamp — the backfill promotes the
+// EARLIEST grant holder when there is no operator, so that ordering has to be
+// settable by a test rather than by sleeping.
+func (s *Store) GrantAt(address, app string, at time.Time) error {
_, err := s.db.Exec(`INSERT OR IGNORE INTO owners (address, app, created_at) VALUES (?, ?, ?)`,
- address, app, time.Now().Unix())
+ address, app, at.Unix())
return err
}
@@ -285,15 +297,31 @@ func (s *Store) Owners() (map[string][]string, error) {
// ---- accounts, memberships, invites -----------------------------------------
// BackfillOpsMemberships maps the pre-accounts world onto the new one, once:
-// the built-in ops account exists, every address that held ANY app grant is
-// an owner of it, and every granted app name is claimed in it. That is the
+// the built-in ops account exists, every address that held ANY app grant is a
+// member of it, and every granted app name is claimed in it. That is the
// spec's "every existing grant maps to a membership in the account that owns
// the app" — before accounts, every app was in ops by definition.
//
-// Idempotent (INSERT OR IGNORE throughout), so it runs on every console
-// start; a deployment that has already moved on gains nothing and loses
-// nothing. It deliberately does NOT delete the owners table: the grants stay
-// readable while the migration is being verified.
+// Grant holders become MEMBERS, not owners. A per-app grant was never
+// deployment-wide authority, and ops is not an ordinary account: it holds
+// console, platform-site and status. Backfilling every grant holder as an
+// owner would hand whoever once shipped one app the right to invite people
+// into the account that runs the platform itself. What IS deployment-wide
+// authority is the operator flag, so that — and only that — backfills as
+// owner.
+//
+// If that would leave ops with no owner at all, the earliest grant holder is
+// promoted and the promotion is logged loudly: an account nobody can
+// administer is a worse outcome than one operator who has to be told they
+// were given the role. A deployment with no grants and no operators gets an
+// empty ops account, which is correct — there is nobody to promote, and
+// inventing someone would be worse.
+//
+// Idempotent, so it runs on every console start: existing memberships are
+// left exactly as they are (a later run never demotes an owner), and a
+// deployment that has already moved on gains nothing and loses nothing. It
+// deliberately does NOT delete the owners table: the grants stay readable
+// while the migration is being verified.
func (s *Store) BackfillOpsMemberships(now time.Time) error {
tx, err := s.db.Begin()
if err != nil {
@@ -304,14 +332,47 @@ func (s *Store) BackfillOpsMemberships(now time.Time) error {
VALUES (?, ?, 'ops', ?)`, OpsAccountID, opsSqid, now.Unix()); err != nil {
return err
}
+ // OR IGNORE, never OR REPLACE: an address that is already a member keeps
+ // whatever role it has, so a second run cannot demote an owner.
if _, err := tx.Exec(`INSERT OR IGNORE INTO memberships (account_id, address, role, added_at)
-SELECT DISTINCT ?, address, ?, ? FROM owners`, OpsAccountID, RoleOwner, now.Unix()); err != nil {
+SELECT DISTINCT ?, address, ?, ? FROM owners`, OpsAccountID, RoleMember, now.Unix()); err != nil {
+ return err
+ }
+ if _, err := tx.Exec(`INSERT OR IGNORE INTO memberships (account_id, address, role, added_at)
+SELECT ?, address, ?, ? FROM users WHERE operator != 0`, OpsAccountID, RoleOwner, now.Unix()); err != nil {
+ return err
+ }
+ // The operators that were already members as grant holders: promote them.
+ if _, err := tx.Exec(`UPDATE memberships SET role = ? WHERE account_id = ?
+AND address IN (SELECT address FROM users WHERE operator != 0)`, RoleOwner, OpsAccountID); err != nil {
return err
}
if _, err := tx.Exec(`INSERT OR IGNORE INTO apps (app, account_id, created_by, created_at)
SELECT app, ?, MIN(address), ? FROM owners GROUP BY app`, OpsAccountID, now.Unix()); err != nil {
return err
}
+ var owners int
+ if err := tx.QueryRow(`SELECT COUNT(*) FROM memberships WHERE account_id = ? AND role = ?`,
+ OpsAccountID, RoleOwner).Scan(&owners); err != nil {
+ return err
+ }
+ if owners == 0 {
+ var first string
+ err := tx.QueryRow(`SELECT address FROM owners ORDER BY created_at, address LIMIT 1`).Scan(&first)
+ if err == sql.ErrNoRows {
+ return tx.Commit() // nobody to promote; an empty ops account is honest
+ }
+ if err != nil {
+ return err
+ }
+ if _, err := tx.Exec(`UPDATE memberships SET role = ? WHERE account_id = ? AND address = ?`,
+ RoleOwner, OpsAccountID, first); err != nil {
+ return err
+ }
+ log.Printf("console: no operator existed, so %s (the earliest app-grant holder) was made "+
+ "OWNER of the ops account — every account needs one. Review this: ops holds console, "+
+ "platform-site and status.", first)
+ }
return tx.Commit()
}
@@ -959,16 +1020,22 @@ type Event struct {
// RecordEvent appends one audit row. Callers treat it as best-effort:
// losing an audit line must never fail the action it records.
-func (s *Store) RecordEvent(app, address, action, detail string, now time.Time) error {
- _, err := s.db.Exec(`INSERT INTO audit (app, address, action, detail, created_at)
-VALUES (?, ?, ?, ?, ?)`, app, address, action, detail, now.Unix())
+//
+// Keyed on (account, app), never the app name alone: two accounts can both
+// own a "blog", and a shared activity stream would show each of them the
+// other's actions — including the other account's member addresses, which is
+// a membership leak dressed up as a convenience.
+func (s *Store) RecordEvent(accountID int64, app, address, action, detail string, now time.Time) error {
+ _, err := s.db.Exec(`INSERT INTO audit (account_id, app, address, action, detail, created_at)
+VALUES (?, ?, ?, ?, ?, ?)`, accountID, app, address, action, detail, now.Unix())
return err
}
-// Events returns app's audit rows, newest first, capped at limit.
-func (s *Store) Events(app string, limit int) ([]Event, error) {
+// Events returns one account's audit rows for an app, newest first, capped at
+// limit.
+func (s *Store) Events(accountID int64, app string, limit int) ([]Event, error) {
rows, err := s.db.Query(`SELECT app, address, action, detail, created_at
-FROM audit WHERE app = ? ORDER BY id DESC LIMIT ?`, app, limit)
+FROM audit WHERE account_id = ? AND app = ? ORDER BY id DESC LIMIT ?`, accountID, app, limit)
if err != nil {
return nil, err
}
diff --git a/internal/console/store_test.go b/internal/console/store_test.go
index 162fb2603..ceb8d56f4 100644
--- a/internal/console/store_test.go
+++ b/internal/console/store_test.go
@@ -235,11 +235,11 @@ func TestAuditEvents(t *testing.T) {
{App: "kass", Address: "paul@keymail.dev", Action: "canary-create", Detail: "created canary rc at v2"},
{App: "aria", Address: "bob@keymail.dev", Action: "promote", Detail: "promoted z1 to stable"},
} {
- if err := s.RecordEvent(ev.App, ev.Address, ev.Action, ev.Detail, base.Add(time.Duration(i)*time.Minute)); err != nil {
+ if err := s.RecordEvent(OpsAccountID, ev.App, ev.Address, ev.Action, ev.Detail, base.Add(time.Duration(i)*time.Minute)); err != nil {
t.Fatal(err)
}
}
- events, err := s.Events("kass", 10)
+ events, err := s.Events(OpsAccountID, "kass", 10)
if err != nil {
t.Fatal(err)
}
@@ -257,10 +257,10 @@ func TestAuditEvents(t *testing.T) {
t.Fatalf("CreatedAt = %v, want %v", events[0].CreatedAt, base.Add(time.Minute))
}
// The cap trims oldest-first.
- if events, _ := s.Events("kass", 1); len(events) != 1 || events[0].Action != "canary-create" {
+ if events, _ := s.Events(OpsAccountID, "kass", 1); len(events) != 1 || events[0].Action != "canary-create" {
t.Fatalf("Events(kass, 1) = %+v", events)
}
- if events, _ := s.Events("ghost", 10); len(events) != 0 {
+ if events, _ := s.Events(OpsAccountID, "ghost", 10); len(events) != 0 {
t.Fatalf("Events(ghost) = %+v, want empty", events)
}
}
diff --git a/internal/console/templates/app.html b/internal/console/templates/app.html
index ca0b07a6b..ab359aeae 100644
--- a/internal/console/templates/app.html
+++ b/internal/console/templates/app.html
@@ -2,6 +2,16 @@
<h1>{{.Detail.App}}</h1>
{{if .Flash}}<p class="banner">{{.Flash}}</p>{{end}}
{{if .FlashErr}}<p class="banner error">{{.FlashErr}}</p>{{end}}
+{{if .Detail.AccountMismatch}}
+<p class="banner error">
+ <strong>This app is not deploying.</strong> Its hosts read releases from
+ account <code>{{.Detail.RouteAccount}}</code>, but this page writes them to
+ <code>{{.Account.Sqid}}</code> — so a promote here succeeds and reaches
+ nothing. Its bucket objects were moved without the console's own record
+ following. On the console's box, run:
+ <br><code>console migrate-app -app {{.Detail.App}} -to {{.Detail.RouteAccount}}</code>
+</p>
+{{end}}
<h2>Domains</h2>
{{if .Detail.Domains}}
diff --git a/internal/console/view.go b/internal/console/view.go
index 07d17c06b..25856c73e 100644
--- a/internal/console/view.go
+++ b/internal/console/view.go
@@ -27,8 +27,21 @@ type AppView struct {
Domains []DomainView
Channels []ChannelView
Canaries []CanaryView
+ // RouteAccount is the account the app's ROUTES say it belongs to, when
+ // that disagrees with the account whose page this is; "" when they agree
+ // or there are no routes. It is the half-migrated state — see
+ // AccountMismatch.
+ RouteAccount string
}
+// AccountMismatch reports the half-migrated state that would otherwise be
+// silent: `carlos accounts migrate` re-stamped this app's routes onto another
+// account, and `console migrate-app` was never run, so the console promotes
+// into one account's bucket keys while every host reads another's. A promote
+// then flashes "Promoted" and deploys nothing, forever, with no error
+// anywhere. The page has to say so.
+func (v AppView) AccountMismatch() bool { return v.RouteAccount != "" }
+
// DomainView is one row of the app page's Domains table — the product
// surface for what routes always were. Unknowns stay empty strings here and
// render as the em-dash convention; Guidance is non-empty ONLY for a custom
@@ -156,11 +169,22 @@ func Assemble(regPath string, s release.Store, pub ed25519.PublicKey, acct strin
}
byApp := map[string][]RouteView{}
domainsByApp := map[string][]DomainView{}
+ // What the ROUTES say each app's account is. A disagreement with the
+ // account being viewed is the half-migrated state (see AccountMismatch);
+ // platform-minted routes are excluded because they are aliases that
+ // reconciliation re-stamps on its own schedule, so a lagging one is noise
+ // rather than a stuck deploy.
+ routeAccount := map[string]string{}
for _, rt := range routes {
ch := rt.Channel
if ch == "" {
ch = "stable" // registry contract: empty channel means stable
}
+ if rt.App != "" && !rt.Platform {
+ if a := release.RouteAccount(rt); routeAccount[rt.App] == "" {
+ routeAccount[rt.App] = a
+ }
+ }
byApp[rt.App] = append(byApp[rt.App], RouteView{
Host: rt.Host, Kind: rt.Kind, Status: rt.Status,
Version: rt.Version, Channel: ch, Hibernate: rt.Hibernate,
@@ -178,12 +202,17 @@ func Assemble(regPath string, s release.Store, pub ed25519.PublicKey, acct strin
}
seen[app] = true
channels := assembleChannels(ctx, s, pub, acct, app, now)
+ mismatch := ""
+ if ra := routeAccount[app]; ra != "" && ra != acct {
+ mismatch = ra
+ }
out = append(out, AppView{
- App: app,
- Routes: byApp[app], // List orders by host already
- Domains: domainsByApp[app],
- Channels: channels,
- Canaries: assembleCanaries(channels, byApp[app]),
+ App: app,
+ Routes: byApp[app], // List orders by host already
+ Domains: domainsByApp[app],
+ Channels: channels,
+ Canaries: assembleCanaries(channels, byApp[app]),
+ RouteAccount: mismatch,
})
}
return out, nil
diff --git a/internal/release/adopt.go b/internal/release/adopt.go
index 8e2f71d58..26c7ab487 100644
--- a/internal/release/adopt.go
+++ b/internal/release/adopt.go
@@ -83,7 +83,7 @@ func Pass(ctx context.Context, reg *registry.Registry, s Store, pub ed25519.Publ
bake := LoadFirstSeen(FirstSeenPath(optDir))
changed := false
for k, ptr := range pointers {
- if ptr != nil && bake.Observe(k.app, k.channel, ptr.Version, now) {
+ if ptr != nil && bake.Observe(k.acct, k.app, k.channel, ptr.Version, now) {
changed = true
}
}
@@ -179,7 +179,7 @@ func adoptHost(ctx context.Context, reg *registry.Registry, s Store, optDir stri
if cur == ver {
return // up to date — stay quiet on the periodic tick
}
- if hold := Hold(bake, rt.App, channel, ver, DefaultHolds, false, now); hold != "" {
+ if hold := Hold(bake, RouteAccount(rt), rt.App, channel, ver, DefaultHolds, false, now); hold != "" {
fmt.Printf(" %s: %s (%s) is %s\n", rt.Host, ver, channel, hold)
return
}
diff --git a/internal/release/adopt_test.go b/internal/release/adopt_test.go
index 3d81c8f38..61d8aafab 100644
--- a/internal/release/adopt_test.go
+++ b/internal/release/adopt_test.go
@@ -135,7 +135,7 @@ func TestPassStableHeldByBakeWindow(t *testing.T) {
// The pass persisted the first sighting so the window is measured from
// the box's own clock.
st := LoadFirstSeen(FirstSeenPath(opt))
- if _, ok := st.EligibleAt("myapp", "stable", "v1", DefaultHolds); !ok {
+ if _, ok := st.EligibleAt("bab", "myapp", "stable", "v1", DefaultHolds); !ok {
t.Fatalf("Pass did not tick the first-seen clock")
}
}
diff --git a/internal/release/bake.go b/internal/release/bake.go
index 7928d9e7d..c604df0cf 100644
--- a/internal/release/bake.go
+++ b/internal/release/bake.go
@@ -65,20 +65,37 @@ func FirstSeenPath(optDir string) string {
// eligible — evacuating a bad release must never itself sit in a bake window.
const firstSeenKeep = 12
-// FirstSeen is the bake clock: for each app/channel, when this box first
-// observed each version at its head.
+// FirstSeen is the bake clock: for each account/app/channel, when this box
+// first observed each version at its head.
type FirstSeen struct {
// Jitter desynchronises a fleet of boxes: each waits hold*(1+Jitter), so
// boxes don't all install in the same hour and the earliest become
// canaries for the rest. Minted once per box, in [0, 0.25).
Jitter float64 `json:"jitter"`
- // Channels is keyed "<app>/<channel>" -> version -> first-seen time.
+ // Channels is keyed "<account>/<app>/<channel>" -> version -> first-seen
+ // time. Pre-accounts clocks have bare "<app>/<channel>" keys; see
+ // channelClockKey.
Channels map[string]map[string]time.Time `json:"channels"`
path string // where Save persists; set by LoadFirstSeen
}
-func channelClockKey(app, channel string) string { return app + "/" + channel }
+// channelClockKey identifies one channel head's clock. The ACCOUNT is part of
+// it because app names are unique per account, not globally: without it, one
+// account's 72h stable observation would satisfy another account's
+// identically-named app on the same box, silently skipping the supply-chain
+// window that is the entire point of this file.
+//
+// The ops account keeps the pre-accounts key shape ("<app>/<channel>"), so
+// every existing box's observations carry across an upgrade. Restarting them
+// would freeze the whole fleet for 72h on the day this ships, which is a
+// worse failure than the one it would be protecting against.
+func channelClockKey(acct, app, channel string) string {
+ if acct == "" || acct == OpsAccount {
+ return app + "/" + channel
+ }
+ return acct + "/" + app + "/" + channel
+}
// LoadFirstSeen reads the bake clock from path. A missing or corrupt file is
// an empty clock — every head then bakes from "now", which fails closed (a
@@ -103,14 +120,14 @@ func LoadFirstSeen(path string) *FirstSeen {
// for signed feeds. An already-known version's timestamp is NEVER touched:
// that preserved first sighting is exactly what makes a pointer moved BACK
// to a previously-seen version (rollback, evacuation) immediately eligible.
-func (st *FirstSeen) Observe(app, channel, version string, now time.Time) bool {
+func (st *FirstSeen) Observe(acct, app, channel, version string, now time.Time) bool {
changed := false
if st.Jitter == 0 && len(st.Channels) == 0 {
// A fresh box mints its jitter once, alongside its first observation.
st.Jitter = rand.Float64() * 0.25
changed = true
}
- key := channelClockKey(app, channel)
+ key := channelClockKey(acct, app, channel)
seen := st.Channels[key]
if seen == nil {
seen = map[string]time.Time{}
@@ -148,8 +165,8 @@ func (st *FirstSeen) Save() error {
// record at this channel's head — brand new, so not eligible (callers that
// Observe first make that unreachable; a status read that doesn't tick the
// clock can legitimately see it).
-func (st *FirstSeen) EligibleAt(app, channel, version string, holds map[string]time.Duration) (time.Time, bool) {
- seen, ok := st.Channels[channelClockKey(app, channel)][version]
+func (st *FirstSeen) EligibleAt(acct, app, channel, version string, holds map[string]time.Duration) (time.Time, bool) {
+ seen, ok := st.Channels[channelClockKey(acct, app, channel)][version]
if !ok {
return time.Time{}, false
}
@@ -163,11 +180,11 @@ func (st *FirstSeen) EligibleAt(app, channel, version string, holds map[string]t
// Hold returns a non-empty human reason while a channel head must not yet be
// installed on this box, "" once it may be. ignore is the local, root-only
// override — deliberately never a publisher-side flag.
-func Hold(st *FirstSeen, app, channel, version string, holds map[string]time.Duration, ignore bool, now time.Time) string {
+func Hold(st *FirstSeen, acct, app, channel, version string, holds map[string]time.Duration, ignore bool, now time.Time) string {
if ignore {
return ""
}
- at, ok := st.EligibleAt(app, channel, version, holds)
+ at, ok := st.EligibleAt(acct, app, channel, version, holds)
if !ok {
// Fail closed: no record means no clock has run, so nothing installs.
return "in its supply-chain bake window (clock starts this pass)"
diff --git a/internal/release/bake_test.go b/internal/release/bake_test.go
index f94098d15..883521407 100644
--- a/internal/release/bake_test.go
+++ b/internal/release/bake_test.go
@@ -2,6 +2,7 @@ package release
import (
"context"
+ "encoding/json"
"os"
"path/filepath"
"testing"
@@ -20,29 +21,29 @@ func TestBakeHoldBlocksNewStableUntilHeld(t *testing.T) {
path := filepath.Join(t.TempDir(), "first-seen.json")
st := LoadFirstSeen(path)
t0 := time.Now()
- if !st.Observe("myapp", "stable", "v1", t0) {
+ if !st.Observe("bab", "myapp", "stable", "v1", t0) {
t.Fatalf("first Observe reported no change")
}
- if hold := Hold(st, "myapp", "stable", "v1", DefaultHolds, false, t0); hold == "" {
+ if hold := Hold(st, "bab", "myapp", "stable", "v1", DefaultHolds, false, t0); hold == "" {
t.Fatalf("brand-new stable head not held")
}
- if hold := Hold(st, "myapp", "stable", "v1", DefaultHolds, false, t0.Add(insideStableHold)); hold == "" {
+ if hold := Hold(st, "bab", "myapp", "stable", "v1", DefaultHolds, false, t0.Add(insideStableHold)); hold == "" {
t.Fatalf("stable head released before its 72h window")
}
- if hold := Hold(st, "myapp", "stable", "v1", DefaultHolds, false, t0.Add(pastStableHold)); hold != "" {
+ if hold := Hold(st, "bab", "myapp", "stable", "v1", DefaultHolds, false, t0.Add(pastStableHold)); hold != "" {
t.Fatalf("stable head still held past its window: %s", hold)
}
// A version with NO first-seen record fails closed.
- if hold := Hold(st, "myapp", "stable", "vGhost", DefaultHolds, false, t0.Add(pastStableHold)); hold == "" {
+ if hold := Hold(st, "bab", "myapp", "stable", "vGhost", DefaultHolds, false, t0.Add(pastStableHold)); hold == "" {
t.Fatalf("unseen version not held (must fail closed)")
}
// The local root-only override releases immediately.
- if hold := Hold(st, "myapp", "stable", "v1", DefaultHolds, true, t0); hold != "" {
+ if hold := Hold(st, "bab", "myapp", "stable", "v1", DefaultHolds, true, t0); hold != "" {
t.Fatalf("ignore=true still held: %s", hold)
}
// canary/* has no window: eligible the moment it is first seen.
- st.Observe("myapp", "canary/x", "v1", t0)
- if hold := Hold(st, "myapp", "canary/x", "v1", DefaultHolds, false, t0); hold != "" {
+ st.Observe("bab", "myapp", "canary/x", "v1", t0)
+ if hold := Hold(st, "bab", "myapp", "canary/x", "v1", DefaultHolds, false, t0); hold != "" {
t.Fatalf("canary held: %s", hold)
}
}
@@ -70,7 +71,7 @@ func TestRollbackNeverBakes(t *testing.T) {
if err != nil {
t.Fatalf("Channel: %v", err)
}
- st.Observe("myapp", "stable", ptr.Version, t0)
+ st.Observe("bab", "myapp", "stable", ptr.Version, t0)
// Time passes; v1's window elapses; v2 is promoted and first seen NOW.
t1 := t0.Add(pastStableHold)
@@ -78,8 +79,8 @@ func TestRollbackNeverBakes(t *testing.T) {
t.Fatalf("Promote v2: %v", err)
}
ptr, _ = Channel(ctx, s, pub, "bab", "myapp", "stable")
- st.Observe("myapp", "stable", ptr.Version, t1)
- if hold := Hold(st, "myapp", "stable", ptr.Version, DefaultHolds, false, t1); hold == "" {
+ st.Observe("bab", "myapp", "stable", ptr.Version, t1)
+ if hold := Hold(st, "bab", "myapp", "stable", ptr.Version, DefaultHolds, false, t1); hold == "" {
t.Fatalf("fresh v2 not held")
}
@@ -96,12 +97,12 @@ func TestRollbackNeverBakes(t *testing.T) {
if ptr.Version != "v1" {
t.Fatalf("rollback pointer = %s, want v1", ptr.Version)
}
- st.Observe("myapp", "stable", ptr.Version, t1) // re-seen at head: no reset
- if hold := Hold(st, "myapp", "stable", ptr.Version, DefaultHolds, false, t1); hold != "" {
+ st.Observe("bab", "myapp", "stable", ptr.Version, t1) // re-seen at head: no reset
+ if hold := Hold(st, "bab", "myapp", "stable", ptr.Version, DefaultHolds, false, t1); hold != "" {
t.Fatalf("rolled-back v1 held: %s (evacuation must never bake)", hold)
}
// And v2 stays held — the rollback didn't launder it.
- if hold := Hold(st, "myapp", "stable", "v2", DefaultHolds, false, t1); hold == "" {
+ if hold := Hold(st, "bab", "myapp", "stable", "v2", DefaultHolds, false, t1); hold == "" {
t.Fatalf("bad v2 no longer held after rollback")
}
}
@@ -114,12 +115,12 @@ func TestLoadFirstSeenCorruptFailsClosed(t *testing.T) {
st := LoadFirstSeen(path)
// A corrupt file is an empty clock: everything bakes from now — a wiped
// clock delays updates, it never lets one through early.
- if hold := Hold(st, "myapp", "stable", "v1", DefaultHolds, false, time.Now()); hold == "" {
+ if hold := Hold(st, "bab", "myapp", "stable", "v1", DefaultHolds, false, time.Now()); hold == "" {
t.Fatalf("corrupt clock did not fail closed")
}
// Missing file: same.
st2 := LoadFirstSeen(filepath.Join(t.TempDir(), "absent.json"))
- if hold := Hold(st2, "myapp", "stable", "v1", DefaultHolds, false, time.Now()); hold == "" {
+ if hold := Hold(st2, "bab", "myapp", "stable", "v1", DefaultHolds, false, time.Now()); hold == "" {
t.Fatalf("missing clock did not fail closed")
}
}
@@ -128,7 +129,7 @@ func TestFirstSeenSaveLoadRoundTrip(t *testing.T) {
path := filepath.Join(t.TempDir(), "first-seen.json")
st := LoadFirstSeen(path)
t0 := time.Now()
- st.Observe("myapp", "stable", "v1", t0)
+ st.Observe("bab", "myapp", "stable", "v1", t0)
if st.Jitter < 0 || st.Jitter >= 0.25 {
t.Fatalf("minted jitter %v outside [0, 0.25)", st.Jitter)
}
@@ -139,7 +140,7 @@ func TestFirstSeenSaveLoadRoundTrip(t *testing.T) {
if st2.Jitter != st.Jitter {
t.Fatalf("jitter not persisted: %v != %v", st2.Jitter, st.Jitter)
}
- at, ok := st2.EligibleAt("myapp", "stable", "v1", DefaultHolds)
+ at, ok := st2.EligibleAt("bab", "myapp", "stable", "v1", DefaultHolds)
if !ok {
t.Fatalf("first-seen record lost on reload")
}
@@ -169,3 +170,52 @@ func TestHoldForWildcardAndUnknown(t *testing.T) {
t.Fatalf("override stable = %v, want 1h", d)
}
}
+
+// The bake clock is per (account, app, channel, version). Keyed on the app
+// name alone, one account's 72h stable observation would satisfy ANOTHER
+// account's identically-named app on the same box — its supply-chain window
+// silently skipped, which is the one thing this clock exists to prevent.
+func TestBakeClockIsPerAccount(t *testing.T) {
+ st := LoadFirstSeen(filepath.Join(t.TempDir(), "clock.json"))
+ long := time.Now().Add(-100 * time.Hour)
+
+ // Account bab saw v1 at stable 100h ago; account kqmv has never seen it.
+ st.Observe("bab", "blog", "stable", "v1", long)
+ if hold := Hold(st, "bab", "blog", "stable", "v1", DefaultHolds, false, time.Now()); hold != "" {
+ t.Fatalf("the account that saw it 100h ago is held: %s", hold)
+ }
+ if hold := Hold(st, "kqmv", "blog", "stable", "v1", DefaultHolds, false, time.Now()); hold == "" {
+ t.Fatal("another account's identically-named app skipped its bake window " +
+ "on the strength of the first account's clock")
+ }
+}
+
+// A clock written before accounts existed has bare "<app>/<channel>" keys.
+// Reading those as the ops account keeps every existing box's observations —
+// otherwise the whole fleet's windows restart on upgrade, and nothing
+// installs for 72h.
+func TestBakeClockReadsPreAccountRecordsAsOps(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "clock.json")
+ long := time.Now().Add(-100 * time.Hour).UTC().Truncate(time.Second)
+ legacy := map[string]any{
+ "jitter": 0.1,
+ "channels": map[string]any{
+ "blog/stable": map[string]any{"v1": long},
+ },
+ }
+ raw, err := json.MarshalIndent(legacy, "", " ")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(path, raw, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ st := LoadFirstSeen(path)
+ if hold := Hold(st, OpsAccount, "blog", "stable", "v1", DefaultHolds, false, time.Now()); hold != "" {
+ t.Fatalf("a pre-accounts observation was lost on upgrade: %s", hold)
+ }
+ // A non-ops account gets no free ride from it.
+ if hold := Hold(st, "kqmv", "blog", "stable", "v1", DefaultHolds, false, time.Now()); hold == "" {
+ t.Fatal("a pre-accounts record satisfied a different account's window")
+ }
+}