amadan
accounts: a legacy-source migration never moves a qualified pointer backwards
Converge semantics exist because an ordinary migration's destination pointer can only be its own leftover. With -from legacy that inverts: a qualified pointer that exists alongside legacy keys is LIVE state, and the legacy copy is the stale one. This morning the flagship console's canary pointer went 3c7c7bc -> 771c2f0 exactly this way — a routine "-from legacy" cleanup converged live state onto a day-old copy, and the production rollback it armed was caught only because the next adopt pass printed the downgrade before anything restarted onto it. The legacy-source path now KEEPS a differing qualified pointer, says so loudly, and exempts it from source-comparison verification (which would otherwise fail against the very copy it refused). Artifacts still migrate — rollback targets are additive either way. The e2e test is the console incident verbatim: stale legacy v1, live qualified v2, migrate, assert v2 survives and v1's artifacts arrived.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2 files changed,
+141
−2
accounts.go+36 −2e2e_test.go+105 −0
diff --git a/accounts.go b/accounts.go| index 3a4679a2e..d7e46b3c3 100644 |
| --- a/accounts.go |
| +++ b/accounts.go |
| @@ -19,6 +19,7 @@ import ( |
| "log" |
| "net/http" |
| "os" |
| + "slices" |
| "strings" |
| "text/tabwriter" |
| "time" |
| @@ -272,6 +273,10 @@ func runAccountsMigrate(args []string) { |
| if len(plan) == 0 { |
| fmt.Printf("nothing under %s — nothing to copy\n", release.AppPrefix(srcAcct, *app)) |
| } |
| + // Pointers the legacy-source path refused to overwrite because a newer |
| + // qualified pointer already exists; verification must not compare these |
| + // against the legacy copy they deliberately did not take. |
| + var keptPointers []string |
| for _, c := range plan { |
| if *dryRun { |
| fmt.Printf("would copy %s -> %s\n", c.from, c.to) |
| @@ -293,6 +298,26 @@ func runAccountsMigrate(args []string) { |
| // again, with no way forward that did not involve hand-deleting |
| // objects mid-migration. |
| if c.pointer { |
| + // EXCEPT when the source is the legacy key space. There, the |
| + // converge rationale inverts: an ordinary migration's destination |
| + // pointer can only be this migration's own leftover, but a |
| + // QUALIFIED pointer that already exists alongside legacy keys is |
| + // live state — promotes have been landing on it — and the legacy |
| + // copy is the stale one. Converging would move production |
| + // BACKWARDS, which is precisely what happened to the console on |
| + // 2026-08-03 (qualified canary/console at 3c7c7bc overwritten by a |
| + // day-old legacy 771c2f0; caught by the next adopt pass's |
| + // "updated" line reading like a rollback). Keep the qualified |
| + // pointer, say so, and count it — the operator is told the legacy |
| + // pointer was NOT taken rather than left to find out from adopt. |
| + if srcAcct == release.LegacySource { |
| + if cur, _, err := store.Get(ctx, c.to); err == nil && string(cur) != string(body) { |
| + fmt.Printf("KEPT %s — a qualified pointer already exists and differs from the legacy\n"+ |
| + "copy; the qualified one is live state and wins. The legacy pointer was not taken.\n", c.to) |
| + keptPointers = append(keptPointers, c.to) |
| + continue |
| + } |
| + } |
| switch changed, err := putOverwrite(ctx, store, c.to, body); { |
| case err != nil: |
| log.Fatalf("accounts migrate: write %s: %v", c.to, err) |
| @@ -319,7 +344,7 @@ func runAccountsMigrate(args []string) { |
| fmt.Printf("copied %s -> %s\n", c.from, c.to) |
| } |
| if !*dryRun { |
| - verifyMigratedPointers(ctx, store, *app, srcAcct, *to) |
| + verifyMigratedPointers(ctx, store, *app, srcAcct, *to, keptPointers) |
| } |
| // The routes' account column: what adopt and the edge build keys from. |
| @@ -516,7 +541,7 @@ func migrationPlan(ctx context.Context, store release.Store, app, from, to, dst |
| // Byte comparison is the gate, so no signing key is needed. When |
| // CARLOS_RELEASE_PUBKEY is set the versions are resolved and printed too, so |
| // the operator sees what they just moved rather than a bare "ok". |
| -func verifyMigratedPointers(ctx context.Context, store release.Store, app, srcAcct, dstAcct string) { |
| +func verifyMigratedPointers(ctx context.Context, store release.Store, app, srcAcct, dstAcct string, kept []string) { |
| prefix := release.ChannelsPrefix(dstAcct, app) |
| keys, err := store.List(ctx, prefix) |
| if err != nil { |
| @@ -529,6 +554,15 @@ func verifyMigratedPointers(ctx context.Context, store release.Store, app, srcAc |
| } |
| } |
| for _, key := range keys { |
| + // A pointer the legacy-source path deliberately KEPT (a newer |
| + // qualified pointer outranked the stale legacy copy) must not be |
| + // compared against the source it refused — it is exempt by |
| + // construction, and the exemption was already printed. |
| + if slices.Contains(kept, key) { |
| + channel := strings.TrimSuffix(strings.TrimPrefix(key, prefix), ".json") |
| + fmt.Printf("verified %s: kept the existing qualified pointer (legacy copy was stale)\n", channel) |
| + continue |
| + } |
| channel := strings.TrimSuffix(strings.TrimPrefix(key, prefix), ".json") |
| got, _, err := store.Get(ctx, key) |
| if err != nil { |
diff --git a/e2e_test.go b/e2e_test.go| index 90b280edc..c49e0670f 100644 |
| --- a/e2e_test.go |
| +++ b/e2e_test.go |
| @@ -2824,3 +2824,108 @@ func TestRouteNeitherChannelNorEnvironmentIsUsageError(t *testing.T) { |
| t.Errorf("refusal does not explain \"\" at route creation vs. this verb:\n%s", out) |
| } |
| } |
| + |
| +// A legacy-source migration must never move a QUALIFIED pointer backwards. |
| +// The legacy copy of a channel pointer is stale the moment anything promotes |
| +// on the qualified key, and "-from legacy" taking legacy as truth converged |
| +// the flagship console's canary pointer from 3c7c7bc back to a day-old |
| +// 771c2f0 on 2026-08-03 — a production rollback armed by a routine cleanup, |
| +// caught only because the next adopt pass printed the downgrade. The verb |
| +// now keeps a differing qualified pointer and says so. |
| +func TestAccountsMigrateLegacyKeepsNewerQualifiedPointer(t *testing.T) { |
| + goBin, err := exec.LookPath("go") |
| + if err != nil { |
| + t.Skip("go toolchain not found") |
| + } |
| + tmp := t.TempDir() |
| + carlosBin := filepath.Join(tmp, "carlos") |
| + build := exec.Command(goBin, "build", "-o", carlosBin, ".") |
| + build.Env = os.Environ() |
| + if out, err := build.CombinedOutput(); err != nil { |
| + t.Fatalf("build carlos: %v\n%s", err, out) |
| + } |
| + storeDir := filepath.Join(tmp, "store") |
| + regPath := filepath.Join(tmp, "registry.db") |
| + store := release.NewDirStore(storeDir) |
| + ctx := context.Background() |
| + |
| + seed, pubB64, err := release.GenerateKey() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + priv, err := release.ParseSigningKey(seed, "e2e") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + pub, err := release.ParsePublicKey(pubB64, "e2e") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + env := append(os.Environ(), |
| + "CARLOS_DEPLOYMENT_DIR="+storeDir, |
| + "CARLOS_RELEASE_PUBKEY="+pubB64) |
| + run := func(args ...string) (string, error) { |
| + t.Helper() |
| + cmd := exec.Command(carlosBin, args...) |
| + cmd.Env = env |
| + out, err := cmd.CombinedOutput() |
| + return string(out), err |
| + } |
| + |
| + // The console shape: v1 exists at LEGACY keys (built via scratch store, |
| + // same signed-bytes trick as TestAccountsMigrateLegacyIntoOps), and v2 has |
| + // since been shipped and promoted on the QUALIFIED key — live state. |
| + scratch := release.NewDirStore(filepath.Join(tmp, "scratch")) |
| + m1 := release.Manifest{Schema: 1, App: "consoleish", Version: "v1", Kind: "binary", |
| + Artifacts: map[string]release.Artifact{"linux-amd64": {File: "consoleish"}}} |
| + if err := release.Ship(ctx, scratch, release.OpsAccount, m1, map[string][]byte{"linux-amd64": []byte("v1")}); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := release.Promote(ctx, scratch, priv, release.OpsAccount, "consoleish", "canary/main", "v1"); err != nil { |
| + t.Fatal(err) |
| + } |
| + qualified, err := scratch.List(ctx, release.AppPrefix(release.OpsAccount, "consoleish")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, k := range qualified { |
| + body, _, err := scratch.Get(ctx, k) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + legacy := release.LegacyAppPrefix("consoleish") + strings.TrimPrefix(k, release.AppPrefix(release.OpsAccount, "consoleish")) |
| + if _, err := store.Put(ctx, legacy, body, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + m2 := release.Manifest{Schema: 1, App: "consoleish", Version: "v2", Kind: "binary", |
| + Artifacts: map[string]release.Artifact{"linux-amd64": {File: "consoleish"}}} |
| + if err := release.Ship(ctx, store, release.OpsAccount, m2, map[string][]byte{"linux-amd64": []byte("v2")}); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := release.Promote(ctx, store, priv, release.OpsAccount, "consoleish", "canary/main", "v2"); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + out, err := run("accounts", "migrate", "-registry", regPath, "-app", "consoleish", |
| + "-from", release.LegacySource, "-to", release.OpsAccount) |
| + if err != nil { |
| + t.Fatalf("migrate: %v\n%s", err, out) |
| + } |
| + if !strings.Contains(out, "KEPT") { |
| + t.Errorf("the kept qualified pointer was not announced:\n%s", out) |
| + } |
| + // THE assertion: the qualified pointer still serves v2. Before the fix it |
| + // was converged back to v1 and this resolves the rollback. |
| + p, err := release.Channel(ctx, store, pub, release.OpsAccount, "consoleish", "canary/main") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if p.Version != "v2" { |
| + t.Fatalf("qualified pointer rolled back to %s — the legacy copy won", p.Version) |
| + } |
| + // And v1's artifacts still migrated (rollback targets preserved). |
| + if _, _, err := store.Get(ctx, release.AppPrefix(release.OpsAccount, "consoleish")+"releases/v1/manifest.json"); err != nil { |
| + t.Errorf("v1's manifest did not migrate: %v", err) |
| + } |
| +} |