amadan

carlos / platform Public

provision: find instance and sidecar records by walking app directories, not by listing all of apps/

Reconcile, ListAllRecords (the moves and steering passes) and the
sidecar reconcile each listed every object under apps/ and filtered for
records. On the flagship that is 59,288 objects, 50,695 of them log
chunks, to find 152 instance records: sixty list pages per reader, three
readers on every serial bump. It is most of why a box took 16 to 43s to
notice a promote (#231).

listAppSubtree walks apps/ by its directories (ListDirs, as the mail
sweep already does) and lists only <app>/instances/ or <app>/sidecars/,
eight at a time. It returns every key the full listing returned that the
callers' parsers accept, and the callers still parse every key.

A failed top-level listing is still an error. One account's or one app's
listing failing is partial: reconcile vetoes removal for the pass, as it
does for a record it cannot read, and ListAllRecords refuses the set, as
it already refused one with an unreadable account.

The domains reconcile still lists all of apps/; it reads several kinds of
key from that one listing and is left for its own change.
Human

Prompt

Keep investigating overnight until it's sorted.

From this commit's own message. The prompt ledger has no entries for it — which is what a squashed, rebased or amended commit looks like, and what every commit looks like in a repo without a ledger.

Summary

Instance and sidecar record readers walk app directories and list only instances/ or sidecars/, instead of listing every object under apps/ on every serial bump.
Agent notes
Measured: flagship apps/ = 59,288 keys, 80s to list from docs-t1. TestSteeringPassFailsClosedOnRecordsRead's fixture now fails ListDirs too, plus a per-app case. Console TestControlPlaneApp* fail in the sandbox on main as well.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude d20e27e9de05607de8abe0f662d8397d47bc3251 parent b2a76df
4 files changed, +277 −6
  • internal/host/steering_loop_test.go +16 −0
  • internal/provision/applisting_test.go +167 −0
  • internal/provision/reconcile.go +90 −5
  • internal/provision/sidecar_reconcile.go +4 −1
diff --git a/internal/host/steering_loop_test.go b/internal/host/steering_loop_test.go
index 3cf390200..6d7dda5fe 100644
--- a/internal/host/steering_loop_test.go
+++ b/internal/host/steering_loop_test.go
@@ -33,6 +33,16 @@ func (f failingListStore) List(ctx context.Context, prefix string) ([]string, er
return f.Store.List(ctx, prefix)
}
+// ListDirs fails the same prefix: the record readers walk apps/ by its
+// directories now (provision.listAppSubtree), so "the apps listing fails"
+// is a ListDirs of apps/ failing as much as a List of it.
+func (f failingListStore) ListDirs(ctx context.Context, prefix string) ([]string, error) {
+ if prefix == f.prefix {
+ return nil, f.err
+ }
+ return f.Store.ListDirs(ctx, prefix)
+}
+
// failingGetStore lists fine but refuses ONE named key — the half-read that
// would otherwise fold to a short desired set and delete the missing box's
// records.
@@ -284,6 +294,12 @@ func TestSteeringPassFailsClosedOnRecordsRead(t *testing.T) {
return failingListStore{Store: s, prefix: "apps/", err: boom}
},
},
+ {
+ name: "one app's records listing fails",
+ wrap: func(s release.Store) release.Store {
+ return failingListStore{Store: s, prefix: provision.RecordsPrefix(domains.Sqid(12), "app"), err: boom}
+ },
+ },
{
name: "a record object will not Get",
wrap: func(s release.Store) release.Store {
diff --git a/internal/provision/applisting_test.go b/internal/provision/applisting_test.go
new file mode 100644
index 000000000..d055094da
--- /dev/null
+++ b/internal/provision/applisting_test.go
@@ -0,0 +1,167 @@
+package provision
+
+import (
+ "context"
+ "errors"
+ "reflect"
+ "sort"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// listCountingStore records every List prefix asked of it.
+type listCountingStore struct {
+ release.Store
+ mu sync.Mutex
+ lists []string
+}
+
+func (s *listCountingStore) List(ctx context.Context, prefix string) ([]string, error) {
+ s.mu.Lock()
+ s.lists = append(s.lists, prefix)
+ s.mu.Unlock()
+ return s.Store.List(ctx, prefix)
+}
+
+// A bucket with everything a production one holds under apps/: records of
+// both kinds, the logs and release artifacts that dwarf them, an account that
+// has not materialized, a legacy account-less app, and an app named like a
+// release-plane word.
+func seedAppListingBucket(t *testing.T) release.Store {
+ t.Helper()
+ s := release.NewDirStore(t.TempDir())
+ keys := []string{
+ "apps/bab/shop/instances/a.example.com.json",
+ "apps/bab/shop/instances/b.example.com.json",
+ "apps/bab/shop/instances.json",
+ "apps/bab/shop/sidecars/bot.json",
+ "apps/bab/shop/logs/2026/10/05/chunk-1.jsonl",
+ "apps/bab/shop/logs/2026/10/05/chunk-2.jsonl",
+ "apps/bab/shop/releases/v1/manifest.json",
+ "apps/bab/shop/channels/edge.json",
+ "apps/bab/blog/instances/c.example.com.json",
+ "apps/bab/blog/instances/nested/too/deep.json",
+ "apps/bad/site/sidecars/w.json",
+ "apps/bad/site/instances/d.example.com.json",
+ "apps/zzz/ghost/instances/e.example.com.json", // account not materialized
+ "apps/legacyapp/channels/edge.json", // account-less
+ "apps/bab/channels/instances/f.example.com.json",
+ "apps/bab/releases/instances/g.example.com.json",
+ }
+ for _, k := range keys {
+ if _, err := s.Put(context.Background(), k, []byte(`{}`), ""); err != nil {
+ t.Fatalf("seed %s: %v", k, err)
+ }
+ }
+ return s
+}
+
+// The narrow walk finds exactly the keys the full listing did, for both
+// record kinds, and never lists the whole of apps/.
+func TestAppSubtreeListingMatchesTheFullListing(t *testing.T) {
+ inner := seedAppListingBucket(t)
+ known := map[string]bool{"bab": true, "bad": true}
+ ctx := context.Background()
+
+ all, err := inner.List(ctx, "apps/")
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, c := range []struct {
+ sub string
+ parse func(string) bool
+ }{
+ {"instances/", func(k string) bool { _, _, _, ok := ParseRecordKey(k, known); return ok }},
+ {"sidecars/", func(k string) bool { _, _, _, ok := ParseSidecarKey(k, known); return ok }},
+ } {
+ var want []string
+ for _, k := range all {
+ if c.parse(k) {
+ want = append(want, k)
+ }
+ }
+ sort.Strings(want)
+
+ s := &listCountingStore{Store: inner}
+ got, partial, err := listAppSubtree(ctx, s, known, c.sub)
+ if err != nil || partial {
+ t.Fatalf("%s: partial=%v err=%v", c.sub, partial, err)
+ }
+ var parsed []string
+ for _, k := range got {
+ if c.parse(k) {
+ parsed = append(parsed, k)
+ }
+ }
+ if !reflect.DeepEqual(parsed, want) {
+ t.Fatalf("%s: narrow walk found %v, the full listing %v", c.sub, parsed, want)
+ }
+ for _, p := range s.lists {
+ if !strings.HasSuffix(p, "/"+c.sub) {
+ t.Fatalf("%s: listed %q — the walk lists only <app>/%s prefixes", c.sub, p, c.sub)
+ }
+ }
+ }
+}
+
+// The passes that read records on every serial bump use the narrow walk, and
+// never the full listing: on the flagship, apps/ held 59,288 objects (50,695
+// of them logs) to find 152 records.
+func TestRecordReadersNeverListAllOfApps(t *testing.T) {
+ s := &listCountingStore{Store: seedAppListingBucket(t)}
+ // No account manifests: ops is always known, so the walk has bab to do.
+ if _, err := ListAllRecords(context.Background(), s); err != nil {
+ t.Fatalf("ListAllRecords: %v", err)
+ }
+ walked := false
+ for _, p := range s.lists {
+ if p == "apps/" {
+ t.Fatal("ListAllRecords listed the whole of apps/")
+ }
+ if strings.HasPrefix(p, "apps/bab/") {
+ walked = true
+ }
+ }
+ if !walked {
+ t.Fatalf("ListAllRecords listed %v and never an app of ops's", s.lists)
+ }
+}
+
+// failListStore fails List for one prefix only.
+type failListStore struct {
+ release.Store
+ fail string
+}
+
+func (s failListStore) List(ctx context.Context, prefix string) ([]string, error) {
+ if prefix == s.fail {
+ return nil, errors.New("list refused")
+ }
+ return s.Store.List(ctx, prefix)
+}
+
+// One app's listing failing is partial, not fatal: every other app's keys
+// still come back, and the caller is told some were not seen.
+func TestAppSubtreeListingOneAppFailingIsPartial(t *testing.T) {
+ s := failListStore{Store: seedAppListingBucket(t), fail: "apps/bab/blog/instances/"}
+ got, partial, err := listAppSubtree(context.Background(), s, map[string]bool{"bab": true}, "instances/")
+ if err != nil {
+ t.Fatalf("one app's failed listing failed the walk: %v", err)
+ }
+ if !partial {
+ t.Fatal("a failed app listing was not reported as partial")
+ }
+ var records []string
+ for _, k := range got {
+ if _, _, _, ok := ParseRecordKey(k, map[string]bool{"bab": true}); ok {
+ records = append(records, k)
+ }
+ }
+ want := []string{"apps/bab/shop/instances/a.example.com.json", "apps/bab/shop/instances/b.example.com.json"}
+ if !reflect.DeepEqual(records, want) {
+ t.Fatalf("records = %v, want %v", records, want)
+ }
+}
diff --git a/internal/provision/reconcile.go b/internal/provision/reconcile.go
index d6a55ef1b..a2e8c04a4 100644
--- a/internal/provision/reconcile.go
+++ b/internal/provision/reconcile.go
@@ -21,6 +21,7 @@ import (
"os"
"slices"
"strings"
+ "sync"
"time"
"github.com/carlosframework/platform/internal/domains"
@@ -716,10 +717,13 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
}
// --- collect in-scope records --------------------------------------------
- keys, err := s.List(ctx, "apps/")
+ keys, partial, err := listAppSubtree(ctx, s, known, "instances/")
if err != nil {
return fmt.Errorf("provision: list apps: %w", err)
}
+ if partial {
+ readFailed = true // unseen records must not read as retired
+ }
recs := map[string]Record{} // host -> record, in-scope only
// protect holds hosts whose record KEY parsed (ParseRecordKey succeeded)
// but whose body could not be trusted — unparseable JSON, or a body that
@@ -1323,13 +1327,15 @@ func ListAllRecords(ctx context.Context, s release.Store) ([]Record, error) {
if readFailed {
return nil, errors.New("provision: an account manifest could not be read cleanly — refusing to report a partial record set")
}
- keys, err := s.List(ctx, "apps/")
+ // Sorted (listAppSubtree sorts) so the result — and anything folded
+ // from it — is stable call to call, whatever order the store lists in.
+ keys, partial, err := listAppSubtree(ctx, s, known, "instances/")
if err != nil {
return nil, fmt.Errorf("provision: list apps: %w", err)
}
- // Sorted so the result — and anything folded from it — is stable
- // call to call, whatever order the store lists in.
- slices.Sort(keys)
+ if partial {
+ return nil, errors.New("provision: an app's records could not be listed — refusing to report a partial record set")
+ }
var recs []Record
for _, key := range keys {
acct, app, host, ok := ParseRecordKey(key, known)
@@ -1355,6 +1361,85 @@ func ListAllRecords(ctx context.Context, s release.Store) ([]Record, error) {
return recs, nil
}
+// listAppSubtree returns every key under apps/<acct>/<app>/<sub> for each
+// known account and each of its apps, sorted.
+//
+// It exists because the record readers used to List(ctx, "apps/") and filter:
+// that listing returns every object under every app — every release artifact
+// and every log chunk — to find the records among them. On the flagship on
+// 2026-10-05 apps/ held 59,288 objects, 50,695 of them logs, for 152 instance
+// records: sixty list pages, several times over, on every serial bump (#231).
+// This walks the same tree by its directories instead (ListDirs, the
+// delimiter-aware listing the mail sweep already uses): one page of accounts,
+// one page of apps per account, then one small listing per app.
+//
+// It finds the same records the full listing did. A record's account is
+// always one of known (ParseRecordKey and ParseSidecarKey refuse any other),
+// and its app is a directory directly under the account, so no record lives
+// anywhere this does not look. The callers still parse every key exactly as
+// before; this changes only which keys they are shown.
+//
+// Failure comes in two sizes. The top-level listing failing is an error, as
+// the single List failing was. One account's or one app's listing failing is
+// PARTIAL: the keys that were listed come back, and partial reports that some
+// were not. The callers treat that exactly as they treat a record they could
+// not read — removal is vetoed for the whole pass, everything that was seen
+// still converges — because a partial record set must never read as "the
+// bucket retired the rest", and one app's bad minute must not stop every
+// other app's routes from being minted.
+func listAppSubtree(ctx context.Context, s release.Store, known map[string]bool, sub string) (keys []string, partial bool, err error) {
+ accts, err := s.ListDirs(ctx, "apps/")
+ if err != nil {
+ return nil, false, err
+ }
+ var prefixes []string
+ for _, ap := range accts {
+ acct := strings.TrimSuffix(strings.TrimPrefix(ap, "apps/"), "/")
+ if !known[acct] {
+ continue
+ }
+ apps, err := s.ListDirs(ctx, ap)
+ if err != nil {
+ log.Printf("provision: listing %s: %v — its records are unseen this pass", ap, err)
+ partial = true
+ continue
+ }
+ for _, p := range apps {
+ prefixes = append(prefixes, p+sub)
+ }
+ }
+
+ // A handful at a time: each is one small round trip, and a box walks
+ // a few hundred apps.
+ const listers = 8
+ var (
+ mu sync.Mutex
+ out []string
+ wg sync.WaitGroup
+ sem = make(chan struct{}, listers)
+ )
+ for _, p := range prefixes {
+ wg.Add(1)
+ sem <- struct{}{}
+ go func(p string) {
+ defer wg.Done()
+ defer func() { <-sem }()
+ keys, err := s.List(ctx, p)
+ mu.Lock()
+ defer mu.Unlock()
+ if err != nil {
+ log.Printf("provision: listing %s: %v — its records are unseen this pass", p, err)
+ partial = true
+ return
+ }
+ out = append(out, keys...)
+ }(p)
+ }
+ wg.Wait()
+ slices.Sort(out)
+ return out, partial, nil
+}
+
// poolServed reports whether pool is one this box serves. An empty
// cfg.Pools means the shared "" pool ONLY, never "every pool" — the same
// convention domains.Config's AppsDomain=="" carries: unconfigured means the
diff --git a/internal/provision/sidecar_reconcile.go b/internal/provision/sidecar_reconcile.go
index 687dd9349..b8ee42868 100644
--- a/internal/provision/sidecar_reconcile.go
+++ b/internal/provision/sidecar_reconcile.go
@@ -239,10 +239,13 @@ func ReconcileSidecars(ctx context.Context, s release.Store, reg *registry.Regis
}
// --- collect in-scope records --------------------------------------------
- keys, err := s.List(ctx, "apps/")
+ keys, partial, err := listAppSubtree(ctx, s, known, "sidecars/")
if err != nil {
return fmt.Errorf("provision: list apps: %w", err)
}
+ if partial {
+ readFailed = true // unseen records must not read as retired
+ }
recs := map[string]SidecarRecord{} // host -> record, in-scope only
// protect holds hosts whose record KEY parsed but whose body could not be
// trusted — unparseable JSON, a body disagreeing with its own path, or a