amadan

carlos / platform Public

edge asset cache task 6: Reconcile reports which apps it read completely and applied

ReconcileConfig.Report (nil: today's pass) is filled with the apps whose
in-scope records were all read and positively applied. An app is left
out for a partial listing, an unreadable or unusable record, a host
collision, an unreadable account manifest (named, so the ops account
counts too), a skipped record (trash marker, move hold, refusals), any
failed convergence step, an adopting pass, and any retired row the pass
keeps: a failed claimRetire, a removal vetoed by a read failure, or one
withheld because the fresh bucket walk could not confirm it. A failed
pass and a dry run report nothing.

convergeOwned now returns its setter failures joined (still logging and
still converging the other columns); its call sites feed that into the
report and the pass-level error behaviour is unchanged.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude a5bc06f05252f845b70815f148fb8b189f84137e parent ae9fee7
2 files changed, +607 −53
  • internal/provision/reconcile.go +246 −53
  • internal/provision/reconcile_report_test.go +361 −0
diff --git a/internal/provision/reconcile.go b/internal/provision/reconcile.go
index 35d871fac..2466df045 100644
--- a/internal/provision/reconcile.go
+++ b/internal/provision/reconcile.go
@@ -136,6 +136,10 @@ type ReconcileConfig struct {
// Seeds is the seeded-instances wiring (seedreconcile.go). Nil: every
// seeded record is left alone.
Seeds *SeedHooks
+ // Report, when set, is filled before Reconcile returns with the apps
+ // this pass read completely and applied (edge asset cache design §4.2,
+ // ReconcileReport). Nil: nobody asks, and the pass is exactly as before.
+ Report *ReconcileReport
// beforeHoldWrite is the tests' pause under the host's hold lock, just
// before holdExisting's conditional write (TestReconcileHoldWrite...).
@@ -143,6 +147,21 @@ type ReconcileConfig struct {
beforeHoldWrite func(host string)
}
+// ReconcileReport is what one Reconcile pass says about each app's records
+// (edge asset cache design §4.2): Complete holds "<acct>/<app>" for every
+// app that has an in-scope record this pass and whose records the pass read
+// completely and applied to the registry without error. The box takes the
+// pass's start time as such an app's records confirmed time; an app left
+// out keeps its previous one. A wrong "complete" lets a stale asset
+// generation stay published, so every doubt leaves the app out: a partial
+// listing, a record that would not read or could not be used, a record the
+// pass skipped or did not converge in full, an unreadable account manifest,
+// a retirement that did not happen. A pass that returns an error, and a dry
+// run, leave Complete empty.
+type ReconcileReport struct {
+ Complete map[string]bool
+}
+
// InboundAny and InboundNone are the closed vocabulary ReconcileConfig.Inbound
// is read against — the BOX's half of design §8's two independent halves (the
// record's half is PlacementAny/PlacementNoInbound). "" reads as InboundAny:
@@ -705,12 +724,21 @@ func shapeMismatch(existing, want registry.Route) string {
// one of the two callers is exactly the single-unguarded-write failure mode
// the pass exists to prevent (design §4.3). Guarding here means both call
// sites are safe by construction, not by each one remembering to check.
-func convergeOwned(reg *registry.Registry, existing registry.Route, rec Record, policy *UpdatePolicy, policyKnown bool, dryRun bool) {
+//
+// A failed write is logged where it happens and the remaining columns still
+// converge; the failures come back joined (astra P7-1), so the caller can
+// tell a row it applied from one it did not (ReconcileReport). Nil means
+// every write it needed landed, including when none was needed; with
+// policyKnown false the update settings were not judged at all, which the
+// caller has to count for itself. A dry run writes nothing and returns nil.
+func convergeOwned(reg *registry.Registry, existing registry.Route, rec Record, policy *UpdatePolicy, policyKnown bool, dryRun bool) error {
+ var errs []error
if rec.Channel != "" && existing.Channel != rec.Channel {
if dryRun {
log.Printf("provision: dry-run: would converge %s channel %s -> %s", rec.Host, existing.Channel, rec.Channel)
} else if err := reg.SetChannel(rec.Host, rec.Channel); err != nil {
log.Printf("provision: converge channel on %s: %v", rec.Host, err)
+ errs = append(errs, fmt.Errorf("converge channel on %s: %w", rec.Host, err))
} else {
log.Printf("provision: converged %s channel %s -> %s", rec.Host, existing.Channel, rec.Channel)
}
@@ -720,6 +748,7 @@ func convergeOwned(reg *registry.Registry, existing registry.Route, rec Record,
log.Printf("provision: dry-run: would converge %s environment %q -> %q", rec.Host, existing.Environment, rec.Environment)
} else if err := reg.SetEnvironment(rec.Host, rec.Environment); err != nil {
log.Printf("provision: converge environment on %s: %v", rec.Host, err)
+ errs = append(errs, fmt.Errorf("converge environment on %s: %w", rec.Host, err))
} else {
log.Printf("provision: converged %s environment %q -> %q", rec.Host, existing.Environment, rec.Environment)
}
@@ -729,6 +758,7 @@ func convergeOwned(reg *registry.Registry, existing registry.Route, rec Record,
log.Printf("provision: dry-run: would converge %s status-named %v -> %v", rec.Host, existing.StatusNamed, *rec.StatusNamed)
} else if err := reg.SetStatusNamed(rec.Host, *rec.StatusNamed); err != nil {
log.Printf("provision: converge status-named on %s: %v", rec.Host, err)
+ errs = append(errs, fmt.Errorf("converge status-named on %s: %w", rec.Host, err))
} else {
log.Printf("provision: converged %s status-named %v -> %v", rec.Host, existing.StatusNamed, *rec.StatusNamed)
}
@@ -748,6 +778,7 @@ func convergeOwned(reg *registry.Registry, existing registry.Route, rec Record,
rec.Host, existing.Addr, addr, existing.Addrs, addrs, existing.Health, health)
} else if err := reg.SetUpstreams(rec.Host, addr, addrs, health); err != nil {
log.Printf("provision: converge upstreams on %s: %v", rec.Host, err)
+ errs = append(errs, fmt.Errorf("converge upstreams on %s: %w", rec.Host, err))
} else {
log.Printf("provision: converged %s upstreams addr %q -> %q, addrs %q -> %q (probe %q -> %q)",
rec.Host, existing.Addr, addr, existing.Addrs, addrs, existing.Health, health)
@@ -758,6 +789,7 @@ func convergeOwned(reg *registry.Registry, existing registry.Route, rec Record,
log.Printf("provision: dry-run: would converge %s steering %q -> %q", rec.Host, existing.Steering, rec.Steering)
} else if err := reg.SetSteering(rec.Host, rec.Steering); err != nil {
log.Printf("provision: converge steering on %s: %v", rec.Host, err)
+ errs = append(errs, fmt.Errorf("converge steering on %s: %w", rec.Host, err))
} else {
log.Printf("provision: converged %s steering %q -> %q", rec.Host, existing.Steering, rec.Steering)
}
@@ -771,6 +803,7 @@ func convergeOwned(reg *registry.Registry, existing registry.Route, rec Record,
rec.Host, existing.AssetsOff, want, existing.AssetGen, rec.AssetGen)
} else if err := reg.SetAssets(rec.Host, want, rec.AssetGen); err != nil {
log.Printf("provision: converge assets on %s: %v", rec.Host, err)
+ errs = append(errs, fmt.Errorf("converge assets on %s: %w", rec.Host, err))
} else {
log.Printf("provision: converged %s assets off %v -> %v, generation %q -> %q",
rec.Host, existing.AssetsOff, want, existing.AssetGen, rec.AssetGen)
@@ -788,6 +821,7 @@ func convergeOwned(reg *registry.Registry, existing registry.Route, rec Record,
log.Printf("provision: dry-run: would converge %s replicas %q -> %q", rec.Host, existing.Replicas, want)
} else if err := reg.SetReplicas(rec.Host, want); err != nil {
log.Printf("provision: converge replicas on %s: %v", rec.Host, err)
+ errs = append(errs, fmt.Errorf("converge replicas on %s: %w", rec.Host, err))
} else {
log.Printf("provision: converged %s replicas %q -> %q", rec.Host, existing.Replicas, want)
}
@@ -802,6 +836,7 @@ func convergeOwned(reg *registry.Registry, existing registry.Route, rec Record,
log.Printf("provision: dry-run: would converge %s replica mark %v -> %v", rec.Host, existing.Replica, want)
} else if err := reg.SetReplica(rec.Host, want); err != nil {
log.Printf("provision: converge replica mark on %s: %v", rec.Host, err)
+ errs = append(errs, fmt.Errorf("converge replica mark on %s: %w", rec.Host, err))
} else {
log.Printf("provision: converged %s replica mark %v -> %v", rec.Host, existing.Replica, want)
}
@@ -815,11 +850,13 @@ func convergeOwned(reg *registry.Registry, existing registry.Route, rec Record,
log.Printf("provision: dry-run: would converge %s update settings %q -> %q", rec.Host, existing.UpdateSettings, want)
} else if err := reg.SetUpdateSettings(rec.Host, want); err != nil {
log.Printf("provision: converge update settings on %s: %v", rec.Host, err)
+ errs = append(errs, fmt.Errorf("converge update settings on %s: %w", rec.Host, err))
} else {
log.Printf("provision: converged %s update settings %q -> %q", rec.Host, existing.UpdateSettings, want)
}
}
}
+ return errors.Join(errs...)
}
// Reconcile materializes bucket instance records into registry routes for
@@ -835,6 +872,10 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
if cfg.Now == nil {
cfg.Now = time.Now
}
+ // Emptied first, so a pass that returns an error reports nothing.
+ if cfg.Report != nil {
+ cfg.Report.Complete = map[string]bool{}
+ }
// scan is the record-collection phase over st: the in-scope records by
// host, the hosts whose record is there but cannot be used (protect),
@@ -843,16 +884,27 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
// view snapshot that has a removal to decide scans the bucket too
// (freshStore), so both judge every record by exactly these rules (S3
// change-index design §4.4).
+ //
+ // For the report (ReconcileReport) it also names what it could not
+ // vouch for: unread holds every "<acct>/<app>" with a record that would
+ // not read, could not be used, or collided with another's host;
+ // failedAccts the accounts whose manifest would not read; partial a
+ // listing that came back incomplete, which could hide any app's record.
type recordScan struct {
- recs map[string]Record
- protect map[string]bool
- readFailed bool
+ recs map[string]Record
+ protect map[string]bool
+ readFailed bool
+ unread map[string]bool
+ failedAccts map[string]bool
+ partial bool
}
scan := func(st release.Store) (recordScan, error) {
- known, readFailed, err := knownAccounts(ctx, st)
+ known, failedAccts, err := knownAccountsNaming(ctx, st)
if err != nil {
return recordScan{}, err
}
+ readFailed := len(failedAccts) > 0
+ unread := map[string]bool{}
// --- collect in-scope records --------------------------------------------
keys, partial, err := listAppSubtree(ctx, st, known, "instances/")
@@ -891,18 +943,21 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
// bucket retired it" — a transient Get failure disables removal
// for the whole pass, same as an unreadable account manifest.
readFailed = true
+ unread[acct+"/"+app] = true
continue
}
rec, err := DecodeRecord(raw)
if err != nil {
log.Printf("provision: skipping record %s: malformed: %v", key, err)
protect[host] = true
+ unread[acct+"/"+app] = true
continue
}
if rec.Host != host || rec.Account != acct || rec.App != app {
log.Printf("provision: skipping record %s: disagrees with its own path (host=%q account=%q app=%q)",
key, rec.Host, rec.Account, rec.App)
protect[host] = true
+ unread[acct+"/"+app] = true
continue
}
// The record IS the authorization, and this pass turns its host into
@@ -918,6 +973,7 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
if !ValidHostShape(rec.Host) {
log.Printf("provision: skipping record %s: host %q is not a hostname", key, rec.Host)
protect[host] = true
+ unread[acct+"/"+app] = true
continue
}
// ValidateShape is the console create door's policy check — but a
@@ -930,6 +986,7 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
if err := ValidateShape(rec); err != nil {
log.Printf("provision: skipping record %s: invalid shape: %v", key, err)
protect[host] = true
+ unread[acct+"/"+app] = true
continue
}
// ValidateShape deliberately does not inspect Channel (it validates
@@ -941,6 +998,7 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
if rec.Channel == "" {
log.Printf("provision: skipping record %s: empty channel", key)
protect[host] = true
+ unread[acct+"/"+app] = true
continue
}
// Scope. For a home-boxed record this is the region/pool test it
@@ -965,18 +1023,26 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
if prev, dup := recs[rec.Host]; dup {
log.Printf("provision: duplicate host %s: keeping %s, skipping %s", rec.Host,
RecordKey(prev.Account, prev.App, prev.Host), RecordKey(rec.Account, rec.App, rec.Host))
+ // The host's owner is in dispute: neither app vouches for it.
+ unread[prev.Account+"/"+prev.App] = true
+ unread[rec.Account+"/"+rec.App] = true
continue
}
recs[rec.Host] = rec
}
}
- return recordScan{recs: recs, protect: protect, readFailed: readFailed}, nil
+ return recordScan{recs: recs, protect: protect, readFailed: readFailed,
+ unread: unread, failedAccts: failedAccts, partial: partial}, nil
}
sc, err := scan(s)
if err != nil {
return err
}
recs, protect, readFailed := sc.recs, sc.protect, sc.readFailed
+ // What the report will say (ReconcileReport): every app starts out
+ // unvouched-for, and a host counts only once its record is positively
+ // applied below. A dry run applies nothing.
+ pr := &passReport{whole: sc.partial || cfg.DryRun, apps: sc.unread, accts: sc.failedAccts, applied: map[string]bool{}}
// --- manifest seed cache, per (acct, app), lazily -------------------------
// known=false (a genuine read failure, not "no manifest") means the seed
@@ -1069,6 +1135,7 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
// feature from a transient blip, so it treats it as the latter.
log.Printf("provision: manifest %s/%s: read failed: %v", acct, app, err)
readFailed = true
+ pr.fail(acct, app)
return "", false
}
seedCache[k], seedKnown[k] = rendered, true
@@ -1181,6 +1248,7 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
journalHandled, journaled, journalOK := journalPass(ctx, s, reg, cfg, recs, readFailed, protect)
if !journalOK {
readFailed = true
+ pr.whole = true // an unread journal could be holding any host's import gate
}
for _, rec := range recs {
@@ -1241,8 +1309,12 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
// place with a narrow status write, outside the claim hook as the
// other status writers are, and then converged as usual: the owned
// branch's narrow setters never touch status or hold_op.
+ // held: a running move owns this row this pass, so the record is not
+ // counted as applied even when every write below lands.
+ held := false
if cfg.Hold != nil {
- if status, op, held := cfg.Hold(rec); held {
+ if status, op, h := cfg.Hold(rec); h {
+ held = true
if !ok {
p, known := policyFor(rec.Account, rec.App)
mintHeld(ctx, reg, cfg, rec, seed, status, op, p, known)
@@ -1336,13 +1408,20 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
return nil
}
log.Printf("provision: minted %s route %s (app %q acct %q)", nr.Kind, rec.Host, rec.App, rec.Account)
+ // Minted with a known seed: the row is the record's own. A
+ // mint-only fallback seed is converged by a later pass.
+ pr.applied[rec.Host] = seedIsKnown && !held
case existing.Provisioned && existing.App == rec.App && existing.Account == rec.Account:
// The columns this record owns (design §4.1, issue #130). Narrow
// UPDATEs, never a re-Upsert: a read-modify-write racing a
// deploy's SetVersions or the activator's SetNextDue would
// clobber those, and Upsert cannot express "clear Environment"
// at all (it preserves a non-empty value by design).
- convergeOwned(reg, existing, rec, p, pKnown, cfg.DryRun)
+ //
+ // applied counts this record toward its app's report: every
+ // step below has to land, including those with nothing to
+ // write, and an unread policy or seed is a step not taken.
+ applied := convergeOwned(reg, existing, rec, p, pKnown, cfg.DryRun) == nil && pKnown && seedIsKnown && !held
// Absent means "this record does not speak to hibernation" —
// never a derivation stamped over an operator's own setting.
//
@@ -1366,6 +1445,7 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
if rec.Hibernate != nil && existing.Hibernate != *rec.Hibernate && !*rec.Hibernate &&
(NestedPaths(cfg.DataDir, existing) || LayoutJournalExists(cfg.DataDir, rec.Host)) {
log.Printf("provision: %s: record asks to clear hibernate but the row's paths are nested; revert the layout first", rec.Host)
+ applied = false
} else if rec.Hibernate != nil && existing.Hibernate != *rec.Hibernate {
dropped := ""
if !*rec.Hibernate && existing.Replication != "" {
@@ -1375,6 +1455,7 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
log.Printf("provision: dry-run: would converge %s hibernate %v -> %v%s", rec.Host, existing.Hibernate, *rec.Hibernate, dropped)
} else if err := reg.SetHibernate(rec.Host, *rec.Hibernate); err != nil {
log.Printf("provision: converge hibernate on %s: %v", rec.Host, err)
+ applied = false
} else {
log.Printf("provision: converged %s hibernate %v -> %v%s", rec.Host, existing.Hibernate, *rec.Hibernate, dropped)
}
@@ -1384,6 +1465,7 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
log.Printf("provision: dry-run: would converge %s gate %s -> %s", rec.Host, gateWord(existing.Gate), gateWord(rec.Gate))
} else if err := reg.SetGate(rec.Host, rec.Gate); err != nil {
log.Printf("provision: converge gate on %s: %v", rec.Host, err)
+ applied = false
} else {
log.Printf("provision: converged %s gate %s -> %s", rec.Host, gateWord(existing.Gate), gateWord(rec.Gate))
}
@@ -1402,6 +1484,7 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
} else if err := reg.SetSeedSource(rec.Host, col); err != nil {
log.Printf("provision: converge seed source on %s: %v", rec.Host, err)
sourceConverged = false
+ applied = false
}
}
// seedIsKnown guards against stamping a transient manifest-read
@@ -1411,6 +1494,7 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
log.Printf("provision: dry-run: would converge %s seed", rec.Host)
} else if err := reg.SetSeed(rec.Host, seed); err != nil {
log.Printf("provision: converge seed on %s: %v", rec.Host, err)
+ applied = false
}
}
// Unit is compared only when the RECORD states one: `carlos
@@ -1440,6 +1524,7 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
log.Printf("provision: %s: row is on the nested layout (db %q socket %q); the record's shared paths are not drift",
rec.Host, existing.DBPath, existing.Socket)
} else {
+ applied = false // the row is not the route this record describes
log.Printf("provision: %s: record describes a different route SHAPE than the row "+
"(kind %q/%q unit %q/%q socket %q/%q db %q/%q) — not converged in place; "+
"`carlos remove %s` purges this route's capability grants (registry.Delete's contract, "+
@@ -1448,6 +1533,7 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
want.Socket, existing.Socket, want.DBPath, existing.DBPath, rec.Host)
}
}
+ pr.applied[rec.Host] = applied
default:
// A route exists that this pass did not mint. If it is the same
// route the record describes, adopt it — that is what makes
@@ -1476,7 +1562,7 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
// already on the record's channel. Every write inside
// convergeOwned is dry-run guarded there, so this call
// writes nothing.
- convergeOwned(reg, existing, rec, p, pKnown, true)
+ _ = convergeOwned(reg, existing, rec, p, pKnown, true)
return nil
}
if err := reg.SetProvisioned(rec.Host, true); err != nil {
@@ -1491,11 +1577,17 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
// live process is allowed to sleep. It converges on the next
// pass through the owned branch above, by which point the
// adoption is already visible in the log.
- convergeOwned(reg, existing, rec, p, pKnown, cfg.DryRun)
+ //
+ // Nor are the gate and the seed, so the record is not counted
+ // as applied on the adopting pass whatever convergeOwned
+ // returns; the next pass, through the owned branch, counts it.
+ // convergeOwned has logged any failure itself.
+ _ = convergeOwned(reg, existing, rec, p, pKnown, cfg.DryRun)
}
return nil
}); err != nil {
log.Printf("provision: claim install %s: %v", rec.Host, err)
+ delete(pr.applied, rec.Host)
}
}
@@ -1549,50 +1641,65 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
// this column does not carry.
//
// retire runs that phase over one set of facts: the records, the
- // protected hosts and the trash lookup. With stopAtFirst it removes
- // nothing and reports whether any row would go. A record view
+ // protected hosts and the trash lookup. It returns the rows it would
+ // remove; with remove false it removes nothing. A record view
// snapshot's absence is not the bucket's (a write that missed its bump
// is missing from a Complete snapshot until the backstop rebuild), so a
// pass over a snapshot that would remove anything decides every removal
// again from the bucket's own walk, below (S3 change-index design §4.4).
- retire := func(recs map[string]Record, protect map[string]bool, trashFor func(acct, app string) trashState, stopAtFirst bool) (wouldRemove bool) {
- for _, rt := range routes {
- // Removal authority is scoped to this pass's OWN kind. A sidecar
- // row is Provisioned too — that flag is what grants a pass removal
- // authority — and it can never appear in the INSTANCE record set,
- // so without this guard the two passes would delete each other's
- // rows on every tick, forever, with no error anywhere and nothing
- // in the logs that reads as a failure. ReconcileSidecars carries
- // the mirror of this guard; both directions are asserted
- // (TestInstanceReconcileNeverRemovesSidecarRows).
- if rt.Kind == SidecarKind {
- continue
+ //
+ // For the report, a retired row this pass keeps, for whatever reason,
+ // leaves its app out (astra P3-3, P4-R3-3): a deleted hostname whose row
+ // survives must not ride a sibling's healthy convergence into a fresh
+ // records confirmed time. retire marks the rows it keeps itself; the
+ // caller marks those a withheld removal keeps.
+ retiredWhy := func(recs map[string]Record, trashFor func(acct, app string) trashState, rt registry.Route) (string, bool) {
+ // Removal authority is scoped to this pass's OWN kind. A sidecar
+ // row is Provisioned too — that flag is what grants a pass removal
+ // authority — and it can never appear in the INSTANCE record set,
+ // so without this guard the two passes would delete each other's
+ // rows on every tick, forever, with no error anywhere and nothing
+ // in the logs that reads as a failure. ReconcileSidecars carries
+ // the mirror of this guard; both directions are asserted
+ // (TestInstanceReconcileNeverRemovesSidecarRows).
+ if rt.Kind == SidecarKind {
+ return "", false
+ }
+ if !rt.Provisioned || !cfg.inScope(rt.Region, rt.Pool, everywhereRoute(rt)) {
+ return "", false
+ }
+ // Two triggers, asked in this order because the first is the
+ // cheap one: the bucket retired the instance (its record is
+ // gone), or the bucket retired the whole APP (a trash marker,
+ // while the record stays put). Both name the bucket's own
+ // decision; neither is a guess this pass made.
+ //
+ // The trash lookup needs both halves of the app's identity off
+ // the row. A Provisioned row always carries them — routeFor
+ // copies them from the record — so a row missing either is a
+ // shape this pass did not mint, and not one it will retire.
+ if _, ok := recs[rt.Host]; ok {
+ if rt.Account == "" || rt.App == "" || !trashFor(rt.Account, rt.App).removable {
+ return "", false
}
- if !rt.Provisioned || !cfg.inScope(rt.Region, rt.Pool, everywhereRoute(rt)) {
+ return "app in the trash", true
+ }
+ return "record gone", true
+ }
+ retire := func(recs map[string]Record, protect map[string]bool, trashFor func(acct, app string) trashState, remove bool) (doomed []registry.Route, removed map[string]bool) {
+ removed = map[string]bool{}
+ for _, rt := range routes {
+ why, ok := retiredWhy(recs, trashFor, rt)
+ if !ok {
continue
}
- // Two triggers, asked in this order because the first is the
- // cheap one: the bucket retired the instance (its record is
- // gone), or the bucket retired the whole APP (a trash marker,
- // while the record stays put). Both name the bucket's own
- // decision; neither is a guess this pass made.
- //
- // The trash lookup needs both halves of the app's identity off
- // the row. A Provisioned row always carries them — routeFor
- // copies them from the record — so a row missing either is a
- // shape this pass did not mint, and not one it will retire.
- why := "record gone"
- if _, ok := recs[rt.Host]; ok {
- if rt.Account == "" || rt.App == "" || !trashFor(rt.Account, rt.App).removable {
- continue
- }
- why = "app in the trash"
- }
if journaled[rt.Host] {
+ pr.fail(rt.Account, rt.App)
continue // the import journal owns the row (seedreconcile.go)
}
if protect[rt.Host] {
log.Printf("provision: not removing %s: its record exists but could not be read cleanly this pass", rt.Host)
+ pr.fail(rt.Account, rt.App)
continue
}
// A participant of a running operation keeps its row whatever
@@ -1601,10 +1708,12 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
// The operation deletes the row at its own finalization.
if cfg.KeepParticipant != nil && cfg.KeepParticipant(rt.Host) {
log.Printf("provision: not removing %s (%s): participant of a running operation", rt.Host, why)
+ pr.fail(rt.Account, rt.App)
continue
}
- if stopAtFirst {
- return true
+ doomed = append(doomed, rt)
+ if !remove {
+ continue
}
if cfg.DryRun {
log.Printf("provision: dry-run: would remove retired %s route %s (%s)", rt.Kind, rt.Host, why)
@@ -1612,14 +1721,41 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
}
if err := cfg.claimRetire(ctx, reg, rt); err != nil {
log.Printf("provision: remove retired %s route %s: %v", rt.Kind, rt.Host, err)
+ pr.fail(rt.Account, rt.App)
continue
}
+ removed[rt.Host] = true
log.Printf("provision: removed retired %s route %s (%s)", rt.Kind, rt.Host, why)
}
- return false
+ return doomed, removed
}
- if readFailed || !retire(recs, protect, trashFor, onSnapshot(s)) {
- return nil
+ // keep marks the apps of retired rows a withheld removal leaves standing.
+ keep := func(rows []registry.Route, removed map[string]bool) {
+ for _, rt := range rows {
+ if !removed[rt.Host] {
+ pr.fail(rt.Account, rt.App)
+ }
+ }
+ }
+ if readFailed {
+ // No removal this pass. Every row whose record is not in hand
+ // stays, so its app is not complete; a trashed app's records were
+ // all skipped above, so the trash lookup is not asked again here.
+ noTrash := func(string, string) trashState { return trashState{} }
+ for _, rt := range routes {
+ if _, ok := retiredWhy(recs, noTrash, rt); ok {
+ pr.fail(rt.Account, rt.App)
+ }
+ }
+ return pr.fill(cfg.Report, recs)
+ }
+ if !onSnapshot(s) {
+ retire(recs, protect, trashFor, true)
+ return pr.fill(cfg.Report, recs)
+ }
+ doomed, _ := retire(recs, protect, trashFor, false)
+ if len(doomed) == 0 {
+ return pr.fill(cfg.Report, recs)
}
// A snapshot pass with a removal to make: the record walk again, over
// the bucket, and the removal phase over what it read, by the walk's
@@ -1630,9 +1766,56 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
lsc, err := scan(live)
if err != nil || lsc.readFailed {
log.Printf("provision: not removing any route this pass: the record view lacks a record or holds a trash marker, and the bucket's own walk could not be read cleanly%s", walkErrSuffix(err))
+ keep(doomed, nil)
+ return pr.fill(cfg.Report, recs)
+ }
+ for id := range lsc.unread {
+ pr.apps[id] = true
+ }
+ // A row the snapshot would retire and the bucket keeps is a record the
+ // snapshot did not have: its app was not read completely either.
+ _, removed := retire(lsc.recs, lsc.protect, trashOver(live, nil), true)
+ keep(doomed, removed)
+ return pr.fill(cfg.Report, recs)
+}
+
+// passReport is Reconcile's bookkeeping for ReconcileReport. Nothing is
+// complete by default: a host counts only once its record is positively
+// applied, and an app only when every in-scope record of it was.
+type passReport struct {
+ whole bool // nothing is complete this pass
+ apps map[string]bool // "<acct>/<app>" that cannot be complete
+ accts map[string]bool // accounts whose manifest did not read cleanly
+ applied map[string]bool // hosts whose record this pass applied in full
+}
+
+func (p *passReport) fail(acct, app string) { p.apps[acct+"/"+app] = true }
+
+// fill writes the report for the in-scope records recs and returns nil, the
+// pass's own result, so each successful return reads "return pr.fill(...)".
+func (p *passReport) fill(rep *ReconcileReport, recs map[string]Record) error {
+ if rep == nil {
+ return nil
+ }
+ rep.Complete = map[string]bool{}
+ if p.whole {
return nil
}
- retire(lsc.recs, lsc.protect, trashOver(live, nil), false)
+ ok := map[string]bool{}
+ for _, rec := range recs {
+ id := rec.Account + "/" + rec.App
+ if _, seen := ok[id]; !seen {
+ ok[id] = true
+ }
+ if !p.applied[rec.Host] || p.apps[id] || p.accts[rec.Account] {
+ ok[id] = false
+ }
+ }
+ for id, c := range ok {
+ if c {
+ rep.Complete[id] = true
+ }
+ }
return nil
}
@@ -1829,10 +2012,20 @@ func gateWord(hash string) string {
// otherwise skipped — never fatal to the pass, but disabling enough that the
// removal phase in Reconcile refuses to run on it.
func knownAccounts(ctx context.Context, s release.Store) (known map[string]bool, readFailed bool, err error) {
+ known, failed, err := knownAccountsNaming(ctx, s)
+ return known, len(failed) > 0, err
+}
+
+// knownAccountsNaming is knownAccounts naming the accounts whose manifest
+// did not read cleanly, so Reconcile can leave every app of theirs out of
+// its report (ReconcileReport). The ops account is known whatever its
+// manifest says, so its failure has to be named to count at all.
+func knownAccountsNaming(ctx context.Context, s release.Store) (known, failed map[string]bool, err error) {
known = map[string]bool{domains.OpsSqid: true}
+ failed = map[string]bool{}
keys, err := s.List(ctx, "accounts/")
if err != nil {
- return nil, false, fmt.Errorf("provision: list accounts: %w", err)
+ return nil, nil, fmt.Errorf("provision: list accounts: %w", err)
}
var manifestKeys []string
for _, key := range keys {
@@ -1847,18 +2040,18 @@ func knownAccounts(ctx context.Context, s release.Store) (known map[string]bool,
body, err := bodies[key].body, bodies[key].err
if err != nil {
log.Printf("provision: skipping %s: read failed: %v", key, err)
- readFailed = true
+ failed[sqid] = true
continue
}
var m domains.AccountManifest
if err := json.Unmarshal(body, &m); err != nil || m.ID <= 0 || m.Sqid == "" {
log.Printf("provision: skipping %s: malformed account manifest: %v", key, err)
- readFailed = true
+ failed[sqid] = true
continue
}
if m.Sqid != sqid || domains.Sqid(m.ID) != sqid {
log.Printf("provision: skipping %s: sqid %q/%q is not Sqid(%d)=%q", key, sqid, m.Sqid, m.ID, domains.Sqid(m.ID))
- readFailed = true
+ failed[sqid] = true
continue
}
if m.Deleted() {
@@ -1867,7 +2060,7 @@ func knownAccounts(ctx context.Context, s release.Store) (known map[string]bool,
known[sqid] = true
}
}
- return known, readFailed, nil
+ return known, failed, nil
}
// trashGate builds the per-(acct, app) "is this app in the trash" predicate
diff --git a/internal/provision/reconcile_report_test.go b/internal/provision/reconcile_report_test.go
new file mode 100644
index 000000000..6686c3e3a
--- /dev/null
+++ b/internal/provision/reconcile_report_test.go
@@ -0,0 +1,361 @@
+package provision
+
+import (
+ "strings"
+ "testing"
+ "time"
+
+ "golang.org/x/crypto/bcrypt"
+
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// The completeness report (edge asset cache design §4.2): Reconcile says
+// which apps it read completely and applied, and the box's records
+// confirmed time advances only for those. A wrong "complete" lets a stale
+// asset generation stay published, so every doubt reads as incomplete.
+
+// reportScene is two apps of one account on a store that can be told to
+// fail one read: shop with instances a and b, notes with instance n, all
+// minted by a first pass.
+type reportScene struct {
+ s *viewCountingStore
+ reg *registry.Registry
+ sqid string
+ cfg ReconcileConfig
+}
+
+func (x reportScene) key(app, host string) string { return RecordKey(x.sqid, app, host) }
+
+func reportRecord(sqid, app, slug string) Record {
+ return Record{Schema: 1, Host: slug + "." + app + ".example.com", Slug: slug, App: app,
+ Account: sqid, Region: "ie", Channel: "stable", AssetGen: "0123456789abcdef", CreatedAt: time.Now()}
+}
+
+func newReportScene(t *testing.T, bucket bool) reportScene {
+ t.Helper()
+ s := &viewCountingStore{DirStore: dirStore(t), bucket: bucket}
+ sqid := writeAccountManifest(t, s, acctID, "Shop")
+ for _, app := range []string{"shop", "notes"} {
+ if err := PutManifest(ctx, s, sqid, app, Manifest{Schema: 1, App: app, Domain: app + ".example.com"}); err != nil {
+ t.Fatal(err)
+ }
+ }
+ for _, r := range []struct{ app, slug string }{{"shop", "a"}, {"shop", "b"}, {"notes", "n"}} {
+ if err := CreateRecord(ctx, s, reportRecord(sqid, r.app, r.slug)); err != nil {
+ t.Fatal(err)
+ }
+ }
+ x := reportScene{s: s, reg: regWith(t), sqid: sqid, cfg: ReconcileConfig{Region: "ie", DataDir: "/data"}}
+ if err := Reconcile(ctx, s, x.reg, x.cfg); err != nil {
+ t.Fatal(err)
+ }
+ for _, h := range []string{"a.shop.example.com", "b.shop.example.com", "n.notes.example.com"} {
+ mustGet(t, x.reg, h)
+ }
+ return x
+}
+
+// pass runs one reported pass and returns its report.
+func (x reportScene) pass(t *testing.T) (ReconcileReport, error) {
+ t.Helper()
+ rep := ReconcileReport{Complete: map[string]bool{"stale/entry": true}}
+ cfg := x.cfg
+ cfg.Report = &rep
+ err := Reconcile(ctx, x.s, x.reg, cfg)
+ return rep, err
+}
+
+func (x reportScene) mustPass(t *testing.T) ReconcileReport {
+ t.Helper()
+ rep, err := x.pass(t)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if rep.Complete["stale/entry"] {
+ t.Fatal("the report kept an entry from before the pass")
+ }
+ return rep
+}
+
+// rewrite replaces a record's body with f applied to the stored record.
+func (x reportScene) rewrite(t *testing.T, app, host string, f func(*Record)) {
+ t.Helper()
+ rec, err := GetRecord(ctx, x.s, x.sqid, app, host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ f(rec)
+ putReplace(t, x.s, x.key(app, host), mustJSON(t, rec))
+}
+
+func (x reportScene) shop() string { return x.sqid + "/shop" }
+func (x reportScene) notes() string { return x.sqid + "/notes" }
+
+func TestReportCompleteOnACleanPass(t *testing.T) {
+ x := newReportScene(t, false)
+ x.rewrite(t, "shop", "a.shop.example.com", func(r *Record) { r.AssetGen = "fedcba9876543210" })
+ rep := x.mustPass(t)
+ if !rep.Complete[x.shop()] || !rep.Complete[x.notes()] || len(rep.Complete) != 2 {
+ t.Fatalf("Complete = %v, want exactly %s and %s", rep.Complete, x.shop(), x.notes())
+ }
+ if got := mustGet(t, x.reg, "a.shop.example.com").AssetGen; got != "fedcba9876543210" {
+ t.Fatalf("asset generation = %q, want the record's new one", got)
+ }
+}
+
+func TestReportOmitsAnAppWithAnUnreadableRecord(t *testing.T) {
+ x := newReportScene(t, false)
+ x.s.setFailGet(x.key("shop", "b.shop.example.com"))
+ rep := x.mustPass(t)
+ if rep.Complete[x.shop()] {
+ t.Fatalf("shop is complete with one of its records unread: %v", rep.Complete)
+ }
+ if !rep.Complete[x.notes()] {
+ t.Fatalf("notes, read in full, is not complete: %v", rep.Complete)
+ }
+
+ // A record with no row yet, so no kept retirement speaks for it: the
+ // unread record alone keeps shop out.
+ x.s.setFailGet("")
+ if err := CreateRecord(ctx, x.s, reportRecord(x.sqid, "shop", "c")); err != nil {
+ t.Fatal(err)
+ }
+ x.s.setFailGet(x.key("shop", "c.shop.example.com"))
+ rep = x.mustPass(t)
+ if rep.Complete[x.shop()] {
+ t.Fatalf("shop is complete with a new record unread: %v", rep.Complete)
+ }
+ if !rep.Complete[x.notes()] {
+ t.Fatalf("notes is not complete: %v", rep.Complete)
+ }
+}
+
+func TestReportOmitsAProtectedRecordsApp(t *testing.T) {
+ x := newReportScene(t, false)
+ putReplace(t, x.s, x.key("shop", "b.shop.example.com"), []byte("{not json"))
+ rep := x.mustPass(t)
+ if rep.Complete[x.shop()] {
+ t.Fatalf("shop is complete with a malformed record: %v", rep.Complete)
+ }
+ if !rep.Complete[x.notes()] {
+ t.Fatalf("notes is not complete: %v", rep.Complete)
+ }
+ mustGet(t, x.reg, "b.shop.example.com") // protected, not retired
+
+ // A malformed record with no row: nothing but the protection speaks for it.
+ putReplace(t, x.s, x.key("shop", "b.shop.example.com"), mustJSON(t, reportRecord(x.sqid, "shop", "b")))
+ if _, err := x.s.Put(ctx, x.key("shop", "c.shop.example.com"), []byte("{not json"), ""); err != nil {
+ t.Fatal(err)
+ }
+ rep = x.mustPass(t)
+ if rep.Complete[x.shop()] {
+ t.Fatalf("shop is complete with a new malformed record: %v", rep.Complete)
+ }
+ if !rep.Complete[x.notes()] {
+ t.Fatalf("notes is not complete: %v", rep.Complete)
+ }
+}
+
+func TestReportOmitsEveryAppOnAPartialListing(t *testing.T) {
+ x := newReportScene(t, true)
+ x.s.setFailList("apps/" + x.sqid + "/notes/instances/")
+ rep := x.mustPass(t)
+ if len(rep.Complete) != 0 {
+ t.Fatalf("Complete = %v after a partial listing, want nothing", rep.Complete)
+ }
+}
+
+func TestReportEmptyWhenThePassFails(t *testing.T) {
+ x := newReportScene(t, false)
+ x.s.setFailList("accounts/")
+ rep, err := x.pass(t)
+ if err == nil {
+ t.Fatal("the pass returned nil with its account listing refused")
+ }
+ if len(rep.Complete) != 0 {
+ t.Fatalf("Complete = %v from a failed pass, want nothing", rep.Complete)
+ }
+}
+
+func TestReportOmitsAnAppWhoseRecordWasSkipped(t *testing.T) {
+ x := newReportScene(t, false)
+ x.rewrite(t, "shop", "a.shop.example.com", func(r *Record) { r.AssetGen = "fedcba9876543210" })
+ x.s.setFailGet(release.AppTombstoneKey(x.sqid, "shop"))
+ rep := x.mustPass(t)
+ if got := mustGet(t, x.reg, "a.shop.example.com").AssetGen; got != "0123456789abcdef" {
+ t.Fatalf("fixture: asset generation = %q, want the old one kept (the record was skipped)", got)
+ }
+ if rep.Complete[x.shop()] {
+ t.Fatalf("shop is complete with its records skipped: %v", rep.Complete)
+ }
+ if !rep.Complete[x.notes()] {
+ t.Fatalf("notes is not complete: %v", rep.Complete)
+ }
+}
+
+func TestReportOmitsAnAppWhoseGateConvergenceFailed(t *testing.T) {
+ x := newReportScene(t, false)
+ hash, err := bcrypt.GenerateFromPassword([]byte("letmein"), bcrypt.MinCost)
+ if err != nil {
+ t.Fatal(err)
+ }
+ x.rewrite(t, "shop", "a.shop.example.com", func(r *Record) { r.Gate = string(hash) })
+ addTrigger(t, x.reg.Path(), `CREATE TRIGGER nogate BEFORE UPDATE OF gate ON routes
+WHEN NEW.host = 'a.shop.example.com' BEGIN SELECT RAISE(FAIL, 'injected'); END`)
+ rep := x.mustPass(t)
+ row := mustGet(t, x.reg, "a.shop.example.com")
+ if row.Gate != "" || row.AssetGen != "0123456789abcdef" || row.AssetsOff {
+ t.Fatalf("fixture: row = gate %q gen %q off %v, want the gate unwritten and the asset columns already equal", row.Gate, row.AssetGen, row.AssetsOff)
+ }
+ if rep.Complete[x.shop()] {
+ t.Fatalf("shop is complete with its gate unconverged: %v", rep.Complete)
+ }
+ if !rep.Complete[x.notes()] {
+ t.Fatalf("notes is not complete: %v", rep.Complete)
+ }
+}
+
+func TestReportOmitsAnAppWhoseRetirementFailed(t *testing.T) {
+ x := newReportScene(t, false)
+ if err := DeleteRecord(ctx, x.s, x.sqid, "shop", "a.shop.example.com"); err != nil {
+ t.Fatal(err)
+ }
+ x.rewrite(t, "shop", "b.shop.example.com", func(r *Record) { r.AssetGen = "fedcba9876543210" })
+ addTrigger(t, x.reg.Path(), `CREATE TRIGGER nodelete BEFORE DELETE ON routes
+WHEN OLD.host = 'a.shop.example.com' BEGIN SELECT RAISE(FAIL, 'injected'); END`)
+ rep := x.mustPass(t)
+ mustGet(t, x.reg, "a.shop.example.com") // the retirement failed
+ if got := mustGet(t, x.reg, "b.shop.example.com").AssetGen; got != "fedcba9876543210" {
+ t.Fatalf("the sibling did not converge: asset generation %q", got)
+ }
+ if rep.Complete[x.shop()] {
+ t.Fatalf("shop is complete with a retired row left standing: %v", rep.Complete)
+ }
+ if !rep.Complete[x.notes()] {
+ t.Fatalf("notes is not complete: %v", rep.Complete)
+ }
+}
+
+func TestReportOmitsAnAppWhoseRetirementCouldNotBeConfirmed(t *testing.T) {
+ x := newReportScene(t, false)
+ if err := DeleteRecord(ctx, x.s, x.sqid, "shop", "a.shop.example.com"); err != nil {
+ t.Fatal(err)
+ }
+ x.rewrite(t, "shop", "b.shop.example.com", func(r *Record) { r.AssetGen = "fedcba9876543210" })
+ v := NewRecordView(x.s, release.NewChangeWatch(x.s))
+ sn := completeSnapshot(t, v)
+ // The snapshot holds every record; the bucket's own walk, which must
+ // confirm a's absence, cannot read notes' record.
+ x.s.setFailGet(x.key("notes", "n.notes.example.com"))
+ rep := ReconcileReport{}
+ cfg := x.cfg
+ cfg.Report = &rep
+ if err := Reconcile(ctx, sn.Store(x.s), x.reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ mustGet(t, x.reg, "a.shop.example.com") // removal withheld
+ if got := mustGet(t, x.reg, "b.shop.example.com").AssetGen; got != "fedcba9876543210" {
+ t.Fatalf("the sibling did not converge: asset generation %q", got)
+ }
+ if rep.Complete[x.shop()] {
+ t.Fatalf("shop is complete with its retirement unconfirmed: %v", rep.Complete)
+ }
+ if !rep.Complete[x.notes()] {
+ t.Fatalf("notes, read in full from the snapshot, is not complete: %v", rep.Complete)
+ }
+}
+
+// convergeOwned reports a setter's failure to its caller (astra P7-1): it
+// still logs it, still converges the columns on either side of it, and the
+// pass around it still returns nil while leaving the app incomplete.
+func TestConvergeOwnedReturnsASetterFailure(t *testing.T) {
+ x := newReportScene(t, false)
+ host := "a.shop.example.com"
+ addTrigger(t, x.reg.Path(), `CREATE TRIGGER noassets BEFORE UPDATE OF asset_gen ON routes
+WHEN NEW.host = '`+host+`' BEGIN SELECT RAISE(FAIL, 'injected'); END`)
+
+ rec, err := GetRecord(ctx, x.s, x.sqid, "shop", host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ rec.Steering, rec.AssetGen, rec.Updates = "latency", "fedcba9876543210", "manual"
+ existing := *mustGet(t, x.reg, host)
+ err = convergeOwned(x.reg, existing, *rec, nil, true, false)
+ if err == nil || !strings.Contains(err.Error(), "injected") {
+ t.Fatalf("convergeOwned = %v, want the injected SetAssets failure", err)
+ }
+ row := mustGet(t, x.reg, host)
+ if row.AssetGen != "0123456789abcdef" {
+ t.Fatalf("fixture: asset generation = %q, want the failed write's old value", row.AssetGen)
+ }
+ if row.Steering != "latency" || row.UpdateSettings != EffectiveUpdates(*rec, nil).Column() {
+ t.Fatalf("the columns around the failure did not converge: steering %q update settings %q", row.Steering, row.UpdateSettings)
+ }
+ if err := convergeOwned(x.reg, *row, *rec, nil, true, true); err != nil {
+ t.Fatalf("a dry run wrote nothing yet failed: %v", err)
+ }
+
+ // Through the pass: nil, and shop is not complete.
+ x.rewrite(t, "shop", host, func(r *Record) { r.AssetGen = "fedcba9876543210" })
+ rep := x.mustPass(t)
+ if rep.Complete[x.shop()] {
+ t.Fatalf("shop is complete with its asset columns unwritten: %v", rep.Complete)
+ }
+ if !rep.Complete[x.notes()] {
+ t.Fatalf("notes is not complete: %v", rep.Complete)
+ }
+ if got := mustGet(t, x.reg, host).AssetGen; got != "0123456789abcdef" {
+ t.Fatalf("fixture: the trigger did not hold: asset generation %q", got)
+ }
+}
+
+// An account manifest that will not read leaves every app of that account
+// out. The ops account is the one whose records still read (it is known
+// whatever its manifest says), so it is the case to pin; another account's
+// records go unread with its manifest.
+func TestReportOmitsAnAccountWhoseManifestFailed(t *testing.T) {
+ s := &viewCountingStore{DirStore: dirStore(t)}
+ ops := writeAccountManifest(t, s, 1, "Ops")
+ other := writeAccountManifest(t, s, acctID, "Shop")
+ for _, a := range []struct{ acct, app, slug string }{{ops, "shop", "a"}, {other, "notes", "n"}} {
+ if err := PutManifest(ctx, s, a.acct, a.app, Manifest{Schema: 1, App: a.app, Domain: a.app + ".example.com"}); err != nil {
+ t.Fatal(err)
+ }
+ if err := CreateRecord(ctx, s, reportRecord(a.acct, a.app, a.slug)); err != nil {
+ t.Fatal(err)
+ }
+ }
+ reg := regWith(t)
+ rep := ReconcileReport{}
+ cfg := ReconcileConfig{Region: "ie", DataDir: "/data", Report: &rep}
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ if !rep.Complete[ops+"/shop"] || !rep.Complete[other+"/notes"] {
+ t.Fatalf("fixture: Complete = %v, want both apps", rep.Complete)
+ }
+ s.setFailGet("accounts/" + ops + "/account.json")
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ if rep.Complete[ops+"/shop"] {
+ t.Fatalf("the ops account's app is complete with its account manifest unread: %v", rep.Complete)
+ }
+ if !rep.Complete[other+"/notes"] {
+ t.Fatalf("the other account's app is not complete: %v", rep.Complete)
+ }
+}
+
+// A nil Report is today's pass: the same rows, the same nil.
+func TestReportNilKeepsTodaysBehaviour(t *testing.T) {
+ x := newReportScene(t, false)
+ x.rewrite(t, "shop", "a.shop.example.com", func(r *Record) { r.AssetGen = "fedcba9876543210" })
+ if err := Reconcile(ctx, x.s, x.reg, x.cfg); err != nil {
+ t.Fatal(err)
+ }
+ if got := mustGet(t, x.reg, "a.shop.example.com").AssetGen; got != "fedcba9876543210" {
+ t.Fatalf("asset generation = %q, want the record's", got)
+ }
+}