amadan
release: change scopes and the change index document (#336)
Adds ChangeScope (which bucket keys bump which scope), the control/changes.json document, ReadChangeIndex, and bumpChangeIndex, a read-modify-CAS over raw get/put funcs so the store hooks can call it without recursion.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2 files changed,
+334
−0
internal/release/changeindex.go+151 −0internal/release/changeindex_test.go+183 −0
diff --git a/internal/release/changeindex.go b/internal/release/changeindex.go| new file mode 100644 |
| index 000000000..ccd4cfa2a |
| --- /dev/null |
| +++ b/internal/release/changeindex.go |
| @@ -0,0 +1,151 @@ |
| +// Package-file doc: control/changes.json is the change index (S3 change-index |
| +// design §3): a map from change scope to a generation number that every |
| +// successful bucket mutation inside that scope bumps. A box that has read the |
| +// index can compare one small object's etag, and the generations behind it, |
| +// instead of re-walking the deployment bucket. Like control/serial it is |
| +// UNSIGNED and carries no authority: it can only say "something may have |
| +// changed", so a reader that cannot trust it walks as before. |
| +package release |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "fmt" |
| + "math/rand/v2" |
| + "os" |
| + "strings" |
| + "time" |
| +) |
| + |
| +// ChangeIndexKey sits under control/ next to SerialKey, outside every prefix |
| +// that app or account listings walk. |
| +const ChangeIndexKey = "control/changes.json" |
| + |
| +// ErrChangeIndexSchema is an index written by a newer schema than this build |
| +// understands. Readers treat it as unknown (walk as today); writers refuse to |
| +// overwrite it. |
| +var ErrChangeIndexSchema = errors.New("release: change index schema unsupported") |
| + |
| +const changeIndexAttempts = 4 |
| + |
| +// ChangeIndex is the body of ChangeIndexKey. Generations only increase and |
| +// entries are never removed in v1. |
| +type ChangeIndex struct { |
| + Schema int `json:"schema"` |
| + Entries map[string]uint64 `json:"entries"` |
| +} |
| + |
| +// dataPrefixes are the per-app prefixes that hold high-volume data rather |
| +// than state a box loop consumes; writes there never bump a scope. |
| +var dataPrefixes = map[string]bool{ |
| + "logs": true, "errors": true, "analytics": true, "runs": true, "metrics": true, |
| +} |
| + |
| +// ChangeScope maps a bucket key to the scope whose generation a write to it |
| +// bumps. The scopes are apps/<a>/<b> (except the data prefixes), accounts/<a>, |
| +// control/certs and control/edges; every other key reports ok=false. |
| +func ChangeScope(key string) (scope string, ok bool) { |
| + parts := strings.Split(key, "/") |
| + switch parts[0] { |
| + case "apps": |
| + // apps/<a>/<b>/<rest...>: all three segments non-empty. |
| + if len(parts) < 4 || parts[1] == "" || parts[2] == "" || parts[3] == "" || dataPrefixes[parts[3]] { |
| + return "", false |
| + } |
| + return "apps/" + parts[1] + "/" + parts[2], true |
| + case "accounts": |
| + if len(parts) < 3 || parts[1] == "" || parts[2] == "" { |
| + return "", false |
| + } |
| + return "accounts/" + parts[1], true |
| + case "control": |
| + if len(parts) < 3 || parts[2] == "" { |
| + return "", false |
| + } |
| + if parts[1] == "certs" || parts[1] == "edges" { |
| + return "control/" + parts[1], true |
| + } |
| + } |
| + return "", false |
| +} |
| + |
| +// ReadChangeIndex returns the index and its etag for CAS. An absent object is |
| +// the ordinary pre-first-bump state: an empty schema-1 index and etag "". A |
| +// schema other than 1 is ErrChangeIndexSchema and an unparseable body is an |
| +// error too; neither is ever reported as empty, because "empty" would read as |
| +// "nothing changed". |
| +func ReadChangeIndex(ctx context.Context, s Store) (ChangeIndex, string, error) { |
| + return readChangeIndex(ctx, s.Get) |
| +} |
| + |
| +func readChangeIndex(ctx context.Context, get func(context.Context, string) ([]byte, string, error)) (ChangeIndex, string, error) { |
| + body, etag, err := get(ctx, ChangeIndexKey) |
| + if errors.Is(err, os.ErrNotExist) { |
| + return ChangeIndex{Schema: 1, Entries: map[string]uint64{}}, "", nil |
| + } |
| + if err != nil { |
| + return ChangeIndex{}, "", err |
| + } |
| + var idx ChangeIndex |
| + if err := json.Unmarshal(body, &idx); err != nil { |
| + return ChangeIndex{}, "", fmt.Errorf("release: change index malformed: %w", err) |
| + } |
| + if idx.Schema != 1 { |
| + return ChangeIndex{}, "", fmt.Errorf("%w: schema %d", ErrChangeIndexSchema, idx.Schema) |
| + } |
| + if idx.Entries == nil { |
| + idx.Entries = map[string]uint64{} |
| + } |
| + return idx, etag, nil |
| +} |
| + |
| +// bumpChangeIndex increments every distinct scope's generation in one read-modify-CAS, |
| +// retrying a lost race up to four times with 20-120ms of jitter between |
| +// attempts. It takes raw get/put funcs rather than a Store so the stores' |
| +// own mutation methods can call it with their unhooked internals. It makes no |
| +// context of its own: the caller supplies the (detached, time-boxed) one. |
| +// Callers treat failure as log-only; a lost bump costs latency, never |
| +// correctness, and must never fail the write that triggered it. |
| +func bumpChangeIndex(ctx context.Context, |
| + get func(context.Context, string) ([]byte, string, error), |
| + put func(context.Context, string, []byte, string) (string, error), |
| + scopes ...string) error { |
| + var lastErr error |
| + for attempt := 0; attempt < changeIndexAttempts; attempt++ { |
| + if attempt > 0 { |
| + jitter := 20*time.Millisecond + rand.N(100*time.Millisecond) |
| + t := time.NewTimer(jitter) |
| + select { |
| + case <-ctx.Done(): |
| + t.Stop() |
| + return ctx.Err() |
| + case <-t.C: |
| + } |
| + } |
| + idx, etag, err := readChangeIndex(ctx, get) |
| + if err != nil { |
| + return err |
| + } |
| + seen := make(map[string]bool, len(scopes)) |
| + for _, sc := range scopes { |
| + if !seen[sc] { // one write is one generation per scope, however many keys it touched |
| + seen[sc] = true |
| + idx.Entries[sc]++ |
| + } |
| + } |
| + body, err := json.Marshal(idx) |
| + if err != nil { |
| + return err |
| + } |
| + _, err = put(ctx, ChangeIndexKey, body, etag) |
| + if err == nil { |
| + return nil |
| + } |
| + if !errors.Is(err, ErrCASConflict) { |
| + return err |
| + } |
| + lastErr = err |
| + } |
| + return fmt.Errorf("bump %s: lost CAS race %d times: %w", ChangeIndexKey, changeIndexAttempts, lastErr) |
| +} |
diff --git a/internal/release/changeindex_test.go b/internal/release/changeindex_test.go| new file mode 100644 |
| index 000000000..f297816ce |
| --- /dev/null |
| +++ b/internal/release/changeindex_test.go |
| @@ -0,0 +1,183 @@ |
| +package release |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "testing" |
| +) |
| + |
| +func TestChangeScope(t *testing.T) { |
| + cases := map[string]string{ |
| + "apps/bab/shop/instances/a.example.com.json": "apps/bab/shop", |
| + "apps/bab/shop/config-gen.json": "apps/bab/shop", |
| + "apps/bab/shop/channels/canary/x": "apps/bab/shop", |
| + "apps/bab/shop/sidecars.json": "apps/bab/shop", |
| + "accounts/bab/account.json": "accounts/bab", |
| + "accounts/bab/telemetry.json": "accounts/bab", |
| + "control/certs/example.com": "control/certs", |
| + "control/edges/box-a.json": "control/edges", |
| + } |
| + for k, want := range cases { |
| + got, ok := ChangeScope(k) |
| + if !ok || got != want { |
| + t.Errorf("ChangeScope(%q) = %q,%v want %q", k, got, ok, want) |
| + } |
| + } |
| + for _, k := range []string{ |
| + "apps/bab/shop/logs/box/2026-10-21-10/x", "apps/bab/shop/errors/x", |
| + "apps/bab/shop/analytics/x", "apps/bab/shop/runs/r1/request.json", |
| + "apps/bab/shop/metrics/x", "apps/bab/shop", "apps/bab/", "apps/", |
| + ChangeIndexKey, "control/serial", "control/edge-mesh-key", |
| + "control/edge-mesh-key.next", "releases/x", "status/boxes/a.json", |
| + "fleets/f/apps/bab/shop/config.json", |
| + } { |
| + if s, ok := ChangeScope(k); ok { |
| + t.Errorf("ChangeScope(%q) = %q, want none", k, s) |
| + } |
| + } |
| +} |
| + |
| +// bumpFuncs adapts a Store to the raw get/put pair bumpChangeIndex takes. |
| +func bumpFuncs(s Store) (func(context.Context, string) ([]byte, string, error), func(context.Context, string, []byte, string) (string, error)) { |
| + return s.Get, s.Put |
| +} |
| + |
| +func TestBumpChangeIndexCreatesThenIncrements(t *testing.T) { |
| + t.Parallel() |
| + ctx := context.Background() |
| + s := NewDirStore(t.TempDir()) |
| + get, put := bumpFuncs(s) |
| + if err := bumpChangeIndex(ctx, get, put, "control/certs"); err != nil { |
| + t.Fatal(err) |
| + } |
| + for i := 0; i < 2; i++ { |
| + if err := bumpChangeIndex(ctx, get, put, "apps/bab/shop"); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + idx, etag, err := ReadChangeIndex(ctx, s) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if etag == "" || idx.Schema != 1 { |
| + t.Fatalf("etag %q schema %d", etag, idx.Schema) |
| + } |
| + if idx.Entries["apps/bab/shop"] != 2 || idx.Entries["control/certs"] != 1 || len(idx.Entries) != 2 { |
| + t.Fatalf("entries = %v", idx.Entries) |
| + } |
| +} |
| + |
| +func TestBumpChangeIndexAppliesAllScopesInOneWrite(t *testing.T) { |
| + t.Parallel() |
| + ctx := context.Background() |
| + s := NewDirStore(t.TempDir()) |
| + puts := 0 |
| + get := s.Get |
| + put := func(ctx context.Context, k string, b []byte, m string) (string, error) { |
| + puts++ |
| + return s.Put(ctx, k, b, m) |
| + } |
| + if err := bumpChangeIndex(ctx, get, put, "apps/bab/shop", "accounts/bab", "apps/bab/shop"); err != nil { |
| + t.Fatal(err) |
| + } |
| + idx, _, _ := ReadChangeIndex(ctx, s) |
| + if puts != 1 || idx.Entries["accounts/bab"] != 1 || idx.Entries["apps/bab/shop"] != 1 { |
| + t.Fatalf("puts %d entries %v", puts, idx.Entries) |
| + } |
| +} |
| + |
| +func TestBumpChangeIndexRetriesOnConflict(t *testing.T) { |
| + t.Parallel() |
| + ctx := context.Background() |
| + s := NewDirStore(t.TempDir()) |
| + puts := 0 |
| + put := func(ctx context.Context, k string, b []byte, m string) (string, error) { |
| + puts++ |
| + if puts <= 2 { |
| + return "", ErrCASConflict |
| + } |
| + return s.Put(ctx, k, b, m) |
| + } |
| + if err := bumpChangeIndex(ctx, s.Get, put, "apps/bab/shop"); err != nil { |
| + t.Fatal(err) |
| + } |
| + idx, _, _ := ReadChangeIndex(ctx, s) |
| + if puts != 3 || idx.Entries["apps/bab/shop"] != 1 { |
| + t.Fatalf("puts %d entries %v", puts, idx.Entries) |
| + } |
| +} |
| + |
| +func TestBumpChangeIndexGivesUpAfterFourAttempts(t *testing.T) { |
| + t.Parallel() |
| + s := NewDirStore(t.TempDir()) |
| + puts := 0 |
| + put := func(context.Context, string, []byte, string) (string, error) { |
| + puts++ |
| + return "", ErrCASConflict |
| + } |
| + err := bumpChangeIndex(context.Background(), s.Get, put, "apps/bab/shop") |
| + if err == nil || !errors.Is(err, ErrCASConflict) { |
| + t.Fatalf("err = %v", err) |
| + } |
| + if puts != 4 { |
| + t.Fatalf("puts = %d, want 4", puts) |
| + } |
| +} |
| + |
| +func TestBumpChangeIndexRefusesUnknownSchema(t *testing.T) { |
| + t.Parallel() |
| + ctx := context.Background() |
| + s := NewDirStore(t.TempDir()) |
| + if _, err := s.Put(ctx, ChangeIndexKey, []byte(`{"schema":2,"entries":{}}`), ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + puts := 0 |
| + put := func(ctx context.Context, k string, b []byte, m string) (string, error) { |
| + puts++ |
| + return s.Put(ctx, k, b, m) |
| + } |
| + err := bumpChangeIndex(ctx, s.Get, put, "apps/bab/shop") |
| + if !errors.Is(err, ErrChangeIndexSchema) || puts != 0 { |
| + t.Fatalf("err = %v, puts = %d", err, puts) |
| + } |
| +} |
| + |
| +func TestReadChangeIndexAbsentIsEmpty(t *testing.T) { |
| + t.Parallel() |
| + idx, etag, err := ReadChangeIndex(context.Background(), NewDirStore(t.TempDir())) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if idx.Schema != 1 || idx.Entries == nil || len(idx.Entries) != 0 || etag != "" { |
| + t.Fatalf("idx %+v etag %q", idx, etag) |
| + } |
| +} |
| + |
| +func TestReadChangeIndexMalformedAndSchema(t *testing.T) { |
| + t.Parallel() |
| + ctx := context.Background() |
| + s := NewDirStore(t.TempDir()) |
| + if _, err := s.Put(ctx, ChangeIndexKey, []byte(`not json`), ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, _, err := ReadChangeIndex(ctx, s); err == nil || errors.Is(err, ErrChangeIndexSchema) { |
| + t.Fatalf("malformed err = %v", err) |
| + } |
| + body, _ := json.Marshal(ChangeIndex{Schema: 3}) |
| + if _, err := s.Put(ctx, ChangeIndexKey, body, mustEtag(t, s)); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, _, err := ReadChangeIndex(ctx, s); !errors.Is(err, ErrChangeIndexSchema) { |
| + t.Fatalf("schema err = %v", err) |
| + } |
| +} |
| + |
| +func mustEtag(t *testing.T, s Store) string { |
| + t.Helper() |
| + _, e, err := s.Get(context.Background(), ChangeIndexKey) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + return e |
| +} |