amadan
agent: run the instance-key backfill after Start, off the edge's path (v0.26.1)
The v0.26.0 roll to a 127-instance box failed its 20-second update gate: activatorStart.Run backfilled every keyless row serially, one bucket read each, before it published the layout pass and started the activator, and runAgent brings the edge up only after Run returns. The backfill now runs in a goroutine after Start, and Run returns at once. What made "nothing moves a file under it" true before, nothing else running yet, is now the activator's per-host preparation slot (PrepareScheduledToken, as scheduled-token preparation uses): each row is read again and written inside it; a fenced host (layout pass, move, a restored hold) is skipped and counted with the skipped rows; a fence asked for during the write waits until it returns. A start's own mint beside the backfill is serialized by adminTokenMu, create-only, so one key results. The "agent: instance keys:" line is unchanged and is logged when the pass ends, after Start. Tests: a 300-row box with a slow bucket starts and returns within 2s (RED 9.07s before); slot skip/re-read and fence-waits-for-write against a real Service; a backfill/start-mint race on one host leaves exactly one key.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
5 files changed,
+365
−39
internal/activator/instancekey_test.go+65 −0internal/host/agent.go+7 −3internal/host/instancekeys.go+72 −11internal/host/instancekeys_test.go+185 −11internal/host/layoutleg.go+36 −14
diff --git a/internal/activator/instancekey_test.go b/internal/activator/instancekey_test.go| index 5bc223dfe..c632af017 100644 |
| --- a/internal/activator/instancekey_test.go |
| +++ b/internal/activator/instancekey_test.go |
| @@ -1002,3 +1002,68 @@ func TestBackfillInstanceKeyRefusesFencedAndConflictRows(t *testing.T) { |
| }) |
| } |
| } |
| + |
| +// v0.26.1 hotfix: the agent's backfill now runs beside the activator |
| +// rather than before it, so a backfill and a start's own mint can reach |
| +// the same host at once. Both read the files and ask a slow bucket outside |
| +// adminTokenMu, and both are held there until each has done so, so each |
| +// decides from a first read that found no key. They still decide and |
| +// append under the lock, create-only, so the host ends with exactly one |
| +// key, appended whole after the bytes that were there, and exactly one of |
| +// the two reports it minted. |
| +func TestBackfillAndAStartsMintRacingOnOneHostWriteOneKey(t *testing.T) { |
| + const rounds = 40 |
| + tok := "CARLOS_ADMIN_TOKEN=" + strings.Repeat("b", 64) + "\n" |
| + for i := range rounds { |
| + b := &ExecBackend{DataDir: t.TempDir()} |
| + b.ConfiguredKey = func(ctx context.Context, _ registry.Route) (string, error) { |
| + select { |
| + case <-time.After(5 * time.Millisecond): // a slow bucket that sets no key |
| + return "", nil |
| + case <-ctx.Done(): |
| + return "", ctx.Err() |
| + } |
| + } |
| + rt := registry.Route{Host: "shop.example.com", Kind: "instance", Status: "ready", Socket: "/run/carlos/shop.sock"} |
| + own := filepath.Join(b.DataDir, rt.Host+".env") |
| + if err := os.WriteFile(own, []byte(tok), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + // Both callers pass phase one before either takes the lock. |
| + var phase sync.WaitGroup |
| + phase.Add(2) |
| + instanceKeyHooks.betweenPhases = func(string) { phase.Done(); phase.Wait() } |
| + var wg sync.WaitGroup |
| + var backfilled, started InstanceKeyOutcome |
| + var berr, serr error |
| + wg.Add(2) |
| + go func() { |
| + defer wg.Done() |
| + backfilled, berr = b.BackfillInstanceKey(t.Context(), rt) |
| + }() |
| + go func() { |
| + defer wg.Done() |
| + started, serr = b.ensureInstanceKeyFor(t.Context(), rt, b.pathsFor(rt)) |
| + }() |
| + wg.Wait() |
| + instanceKeyHooks.betweenPhases = nil |
| + if berr != nil || serr != nil { |
| + t.Fatalf("round %d: backfill %v, %v; start %v, %v", i, backfilled, berr, started, serr) |
| + } |
| + minted := 0 |
| + for _, o := range []InstanceKeyOutcome{backfilled, started} { |
| + switch o { |
| + case InstanceKeyMinted: |
| + minted++ |
| + case InstanceKeyPresent: |
| + default: |
| + t.Fatalf("round %d: outcome %v, want minted or present", i, o) |
| + } |
| + } |
| + raw, _ := os.ReadFile(own) |
| + keys := keyLines(raw) |
| + if minted != 1 || len(keys) != 1 || !wellFormedInstanceKey(keys[0]) || string(raw) != tok+InstanceKeyVar+"="+keys[0]+"\n" { |
| + t.Fatalf("round %d: %d minted, %d keys, file %d bytes; want one whole key appended once", i, minted, len(keys), len(raw)) |
| + } |
| + } |
| +} |
diff --git a/internal/host/agent.go b/internal/host/agent.go| index d99c01c1b..fd026bbed 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -547,9 +547,11 @@ func runAgent(args []string, buildClaims ClaimFactory) { |
| // worse off than before this existed (a leftover unit that keeps |
| // running is exactly what an agent restart left before this pass). |
| // |
| - // Then, on a legacy box, layout recovery and the instance-key |
| - // backfill, before Start can wake anything; on a claim box, Start |
| - // alone. The order and the branch are activatorStart's. |
| + // Then, on a legacy box, layout recovery before Start can wake |
| + // anything, and the instance-key backfill in the background after |
| + // it, which this does not wait for (the edge below must listen |
| + // within the update gate, whatever the box's size); on a claim |
| + // box, Start alone. The order and the branch are activatorStart's. |
| activatorStart{ |
| Claims: claims.Writes.Enabled, |
| Sweep: func(ctx context.Context) error { |
| @@ -568,7 +570,9 @@ func runAgent(args []string, buildClaims ClaimFactory) { |
| return pass |
| }, |
| Routes: reg.List, |
| + Route: reg.Get, |
| Keys: eb, |
| + Slot: svc.PrepareScheduledToken, |
| Publish: armedLayoutPass.Store, |
| Start: svc.Start, |
| }.Run(context.Background()) |
diff --git a/internal/host/instancekeys.go b/internal/host/instancekeys.go| index d79ea90f4..74b19c6c0 100644 |
| --- a/internal/host/instancekeys.go |
| +++ b/internal/host/instancekeys.go |
| @@ -3,6 +3,7 @@ package host |
| import ( |
| "context" |
| "errors" |
| + "fmt" |
| "log" |
| "time" |
| @@ -13,9 +14,11 @@ import ( |
| // The instance-key backfill (L21, plan 2026-10-10-instance-key-provision.md |
| // ruling 7). Every launch path mints CARLOS_INSTANCE_KEY before it reads the |
| // env, so no process is ever started without one. The backfill gives every |
| -// instance already on a box its key now, at agent start, rather than at its |
| -// next start, so the rollout can be checked box by box before any app adopts |
| -// the key: a host that the backfill names "present" or "minted" is ready. |
| +// instance already on a box its key now, just after agent start, rather |
| +// than at its next start, so the rollout can be checked box by box before |
| +// any app adopts the key: a host that the backfill names "present" or |
| +// "minted" is ready. It runs in the background once the activator has |
| +// started (activatorStart.Run), never on the path the edge waits for. |
| // |
| // It runs only where the legacy activator writes <host>.env itself. With |
| // host claims on, every write to a host's installation runs under the |
| @@ -31,12 +34,64 @@ type instanceKeyBackfiller interface { |
| } |
| // instanceKeyBackfillBudget bounds one pass. Each keyless row costs one |
| -// read of the bucket's configuration before its key is minted, and the |
| -// pass runs before the activator starts, so with the bucket unreachable |
| -// the rows after the budget fail fast rather than each waiting out its |
| -// own read; every one of them mints at its next start. |
| +// read of the bucket's configuration before its key is minted. The pass |
| +// runs in the background after the activator starts (activatorStart.Run), |
| +// so it never holds the edge up, but with the bucket unreachable the rows |
| +// after the budget still fail fast rather than each waiting out its own |
| +// read; every one of them mints at its next start. |
| const instanceKeyBackfillBudget = time.Minute |
| +// hostPreparationSlot runs prepare in host's preparation slot, ordered |
| +// with the host's maintenance fence: (*activator.Service). |
| +// PrepareScheduledToken. It answers activator.ErrFenced without running |
| +// prepare when a fence is held for host, and a fence asked for while |
| +// prepare runs is installed only after it returns. Wakes do not wait on |
| +// the slot. |
| +type hostPreparationSlot func(host string, prepare func() (string, error)) (string, error) |
| + |
| +// errInstanceKeyRowSkipped is a row the backfill listed and, when its turn |
| +// came, did not write for: an operation held its host fenced, or its row |
| +// had changed (a move, a layout flip, a removal) so it is no longer one |
| +// the backfill selects. Counted with the skipped rows; its next start |
| +// mints the key if it still needs one. |
| +var errInstanceKeyRowSkipped = errors.New("skipped") |
| + |
| +// keyBackfillInSlot is the backfill's step for one row once the activator |
| +// is serving (the v0.26.1 hotfix moved the pass off the startup path). The |
| +// layout pass and every move fence a host before they move its files, and |
| +// install that fence only after any preparation already in the slot has |
| +// returned; the slot refuses a host already fenced. So inside the slot no |
| +// file of the host moves, and the row is read again there, so the write |
| +// goes to the paths the host has now, not those of the list taken before. |
| +// A start's own mint is not ordered by the slot: it and the backfill both |
| +// decide and append under the activator's adminTokenMu, create-only, so |
| +// whichever is second finds the key present. |
| +type keyBackfillInSlot struct { |
| + keys instanceKeyBackfiller |
| + slot hostPreparationSlot |
| + route func(string) (*registry.Route, error) |
| +} |
| + |
| +func (g keyBackfillInSlot) BackfillInstanceKey(ctx context.Context, rt registry.Route) (activator.InstanceKeyOutcome, error) { |
| + got, err := activator.InstanceKeyFailed, error(nil) |
| + _, serr := g.slot(rt.Host, func() (string, error) { |
| + fresh, rerr := g.route(rt.Host) |
| + switch { |
| + case rerr != nil: |
| + err = fmt.Errorf("read its row: %w", rerr) |
| + case fresh == nil || !instanceKeyRoute(*fresh): |
| + err = fmt.Errorf("%w: its row changed after the backfill listed it", errInstanceKeyRowSkipped) |
| + default: |
| + got, err = g.keys.BackfillInstanceKey(ctx, *fresh) |
| + } |
| + return "", nil |
| + }) |
| + if serr != nil { |
| + return activator.InstanceKeyFailed, fmt.Errorf("%w: an operation holds it (%v)", errInstanceKeyRowSkipped, serr) |
| + } |
| + return got, err |
| +} |
| + |
| // instanceKeyRoute selects the rows the backfill writes for: ready, |
| // activator-spawned instances, the rows execSyncable delivers config to, |
| // without sidecars and without replicas (a replica is never minted a key, |
| @@ -58,10 +113,12 @@ type instanceKeyBackfill struct { |
| } |
| // backfillInstanceKeys runs the backfill over routes, within |
| -// instanceKeyBackfillBudget. It never fails the agent: a host it cannot |
| -// write is logged and counted, and that host's next start mints its key (or |
| -// refuses, loudly, if the same fault is still there). Nothing logged names |
| -// a key. |
| +// instanceKeyBackfillBudget, and logs one count line when it ends, |
| +// "agent: instance keys: ...": on a legacy box that is after the activator |
| +// has started, within the budget plus one row's read. It never fails the |
| +// agent: a host it cannot write is logged and counted, and that host's |
| +// next start mints its key (or refuses, loudly, if the same fault is |
| +// still there). Nothing logged names a key. |
| // |
| // The error is read before the outcome: a refused or failed row answers |
| // InstanceKeyFailed (the zero outcome) beside its error, and counting an |
| @@ -77,6 +134,10 @@ func backfillInstanceKeys(ctx context.Context, routes []registry.Route, b instan |
| } |
| got, err := b.BackfillInstanceKey(ctx, rt) |
| switch { |
| + case errors.Is(err, errInstanceKeyRowSkipped): |
| + st.Skipped++ |
| + log.Printf("agent: instance key for %s: %v (its next start mints it)", rt.Host, err) |
| + continue |
| case errors.Is(err, activator.ErrInstanceKeyDeferred): |
| st.Deferred++ |
| log.Printf("agent: instance key for %s: %v", rt.Host, err) |
diff --git a/internal/host/instancekeys_test.go b/internal/host/instancekeys_test.go| index 1b6c63339..f09c8ac12 100644 |
| --- a/internal/host/instancekeys_test.go |
| +++ b/internal/host/instancekeys_test.go |
| @@ -5,6 +5,8 @@ import ( |
| "errors" |
| "fmt" |
| "reflect" |
| + "sync" |
| + "sync/atomic" |
| "testing" |
| "time" |
| @@ -121,14 +123,29 @@ func (f keyBackfillFunc) BackfillInstanceKey(ctx context.Context, rt registry.Ro |
| return f(ctx, rt) |
| } |
| -// Task 6 ruling (K1): the start sequence is activatorStart's, so the two |
| -// guards runAgent cannot be tested for are pinned here. A claim box starts |
| -// and nothing else: no sweep, no recovery, no backfill (ruling 7). A legacy |
| -// box sweeps, recovers, backfills, publishes the pass and then starts, in |
| -// that order: the backfill after every journal is settled and before the |
| -// pass is published or anything wakes. A route list that fails skips the |
| -// backfill only; the pass is still published and the service still starts. |
| -func TestActivatorStartBackfillsOnlyALegacyBoxBetweenRecoveryAndPublish(t *testing.T) { |
| +// openSlot is a preparation slot with no fence held: it runs prepare. |
| +func openSlot(_ string, prepare func() (string, error)) (string, error) { return prepare() } |
| + |
| +// routeOf answers the registry's Get from a fixed list. |
| +func routeOf(routes ...registry.Route) func(string) (*registry.Route, error) { |
| + return func(host string) (*registry.Route, error) { |
| + for _, rt := range routes { |
| + if rt.Host == host { |
| + return &rt, nil |
| + } |
| + } |
| + return nil, nil |
| + } |
| +} |
| + |
| +// Task 6 ruling (K1), as the v0.26.1 hotfix changed it: the start sequence |
| +// is activatorStart's, so the two guards runAgent cannot be tested for are |
| +// pinned here. A claim box starts and nothing else: no sweep, no recovery, |
| +// no backfill (ruling 7). A legacy box sweeps, recovers, publishes the |
| +// pass and starts, in that order, and the backfill runs after Start, once |
| +// every journal is settled. A route list that fails skips the backfill |
| +// only. |
| +func TestActivatorStartBackfillsOnlyALegacyBoxAfterRecoveryAndStart(t *testing.T) { |
| ready := registry.Route{Host: "shop.example.com", Kind: "instance", Status: "ready", Socket: "/run/carlos/shop.example.com.sock"} |
| for _, tc := range []struct { |
| name string |
| @@ -136,28 +153,185 @@ func TestActivatorStartBackfillsOnlyALegacyBoxBetweenRecoveryAndPublish(t *testi |
| listErr error |
| want []string |
| }{ |
| - {"legacy box", false, nil, []string{"sweep", "recover", "backfill", "publish", "start"}}, |
| + {"legacy box", false, nil, []string{"sweep", "recover", "publish", "start", "backfill"}}, |
| {"legacy box, routes unreadable", false, errors.New("db locked"), []string{"sweep", "recover", "publish", "start"}}, |
| {"claim box", true, nil, []string{"start"}}, |
| } { |
| t.Run(tc.name, func(t *testing.T) { |
| + var mu sync.Mutex |
| var events []string |
| - note := func(e string) { events = append(events, e) } |
| - activatorStart{ |
| + note := func(e string) { mu.Lock(); events = append(events, e); mu.Unlock() } |
| + done := activatorStart{ |
| Claims: tc.claims, |
| Sweep: func(context.Context) error { note("sweep"); return nil }, |
| Recover: func(context.Context, bool) *layout.Pass { note("recover"); return nil }, |
| Routes: func() ([]registry.Route, error) { return []registry.Route{ready}, tc.listErr }, |
| + Route: routeOf(ready), |
| Keys: keyBackfillFunc(func(context.Context, registry.Route) (activator.InstanceKeyOutcome, error) { |
| note("backfill") |
| return activator.InstanceKeyMinted, nil |
| }), |
| + Slot: openSlot, |
| Publish: func(*layout.Pass) { note("publish") }, |
| Start: func() { note("start") }, |
| }.Run(context.Background()) |
| + select { |
| + case <-done: |
| + case <-time.After(10 * time.Second): |
| + t.Fatal("the backfill never finished") |
| + } |
| + mu.Lock() |
| + defer mu.Unlock() |
| if !reflect.DeepEqual(events, tc.want) { |
| t.Fatalf("events = %v, want %v", events, tc.want) |
| } |
| }) |
| } |
| } |
| + |
| +// fenceCtx is a context with the deadline FenceAndStop requires. |
| +func fenceCtx(t *testing.T, d time.Duration) context.Context { |
| + ctx, cancel := context.WithTimeout(t.Context(), d) |
| + t.Cleanup(cancel) |
| + return ctx |
| +} |
| + |
| +// Off the startup path, the backfill shares the box with the layout pass |
| +// and moves, which fence a host before they move its files. A host that |
| +// is fenced when its turn comes is skipped and nothing is asked of it, and |
| +// a row whose status changed after the list (it began a move) is skipped |
| +// the same way; its next start mints the key. The rest are written from |
| +// the row as it stands, not as it was listed. |
| +func TestBackfillInSlotSkipsAFencedHostAndReadsTheRowAgain(t *testing.T) { |
| + ready := func(host string) registry.Route { |
| + return registry.Route{Host: host, Kind: "instance", Status: "ready", Socket: "/run/carlos/" + host + ".sock"} |
| + } |
| + fenced, moved, shop := ready("fenced.example.com"), ready("moved.example.com"), ready("shop.example.com") |
| + svc := &activator.Service{} |
| + if _, err := svc.FenceAndStop(fenceCtx(t, time.Second), fenced.Host, 1); err != nil && !errors.Is(err, activator.ErrStopUnproven) { |
| + t.Fatalf("fence: %v", err) |
| + } |
| + nowMoving := moved |
| + nowMoving.Status = "moving" |
| + nowShop := shop |
| + nowShop.Socket = "/run/carlos/nested/shop.example.com.sock" |
| + f := &fakeKeyBackfiller{got: map[string]activator.InstanceKeyOutcome{shop.Host: activator.InstanceKeyMinted}} |
| + var sockets []string |
| + g := keyBackfillInSlot{ |
| + keys: keyBackfillFunc(func(ctx context.Context, rt registry.Route) (activator.InstanceKeyOutcome, error) { |
| + sockets = append(sockets, rt.Socket) |
| + return f.BackfillInstanceKey(ctx, rt) |
| + }), |
| + slot: svc.PrepareScheduledToken, |
| + route: routeOf(fenced, nowMoving, nowShop), |
| + } |
| + st := backfillInstanceKeys(t.Context(), []registry.Route{fenced, moved, shop}, g) |
| + if !reflect.DeepEqual(f.asked, []string{shop.Host}) { |
| + t.Fatalf("asked %v, want only %s", f.asked, shop.Host) |
| + } |
| + if !reflect.DeepEqual(sockets, []string{nowShop.Socket}) { |
| + t.Fatalf("written from %v, want the row as it stands now", sockets) |
| + } |
| + if st != (instanceKeyBackfill{Minted: 1, Skipped: 2}) { |
| + t.Fatalf("stats = %+v, want one minted and two skipped", st) |
| + } |
| +} |
| + |
| +// The other half of "nothing moves a file under it": a fence asked for |
| +// while a row is being written (the layout pass or a move about to move |
| +// the host's files) is installed only once the write has returned. |
| +func TestBackfillInSlotHoldsOffAFenceUntilTheWriteReturns(t *testing.T) { |
| + rt := registry.Route{Host: "shop.example.com", Kind: "instance", Status: "ready", Socket: "/run/carlos/shop.example.com.sock"} |
| + svc := &activator.Service{} |
| + writing, release := make(chan struct{}), make(chan struct{}) |
| + var wrote atomic.Bool |
| + g := keyBackfillInSlot{ |
| + keys: keyBackfillFunc(func(context.Context, registry.Route) (activator.InstanceKeyOutcome, error) { |
| + close(writing) |
| + <-release |
| + wrote.Store(true) |
| + return activator.InstanceKeyMinted, nil |
| + }), |
| + slot: svc.PrepareScheduledToken, |
| + route: routeOf(rt), |
| + } |
| + done := make(chan instanceKeyBackfill) |
| + go func() { done <- backfillInstanceKeys(t.Context(), []registry.Route{rt}, g) }() |
| + <-writing |
| + fenced := make(chan bool) |
| + go func() { |
| + _, err := svc.FenceAndStopIfUnfenced(fenceCtx(t, 10*time.Second), rt.Host, 1) |
| + if err != nil && !errors.Is(err, activator.ErrStopUnproven) { |
| + t.Errorf("fence: %v", err) |
| + } |
| + fenced <- wrote.Load() |
| + }() |
| + select { |
| + case <-fenced: |
| + t.Fatal("the fence was installed while the backfill was writing the host") |
| + case <-time.After(200 * time.Millisecond): |
| + } |
| + close(release) |
| + if !<-fenced { |
| + t.Fatal("the fence was installed before the write returned") |
| + } |
| + if st := <-done; st != (instanceKeyBackfill{Minted: 1}) { |
| + t.Fatalf("stats = %+v", st) |
| + } |
| +} |
| + |
| +// v0.26.1 hotfix: the v0.26.0 roll to a 127-instance box failed its update |
| +// gate because the backfill ran serially before Start, one bucket read per |
| +// keyless row, and the edge never listened inside the gate's 20 seconds. |
| +// On a box with hundreds of keyless rows and a slow bucket, Start is |
| +// called, and Run returns (runAgent goes on to the agent socket and the |
| +// edge), without waiting for the backfill. |
| +func TestActivatorStartDoesNotWaitForTheBackfillOnALargeBox(t *testing.T) { |
| + const rows, perRow, bound = 300, 30 * time.Millisecond, 2 * time.Second |
| + var routes []registry.Route |
| + for i := range rows { |
| + host := fmt.Sprintf("shop%03d.example.com", i) |
| + routes = append(routes, registry.Route{Host: host, Kind: "instance", Status: "ready", Socket: "/run/carlos/" + host + ".sock"}) |
| + } |
| + ctx, cancel := context.WithCancel(context.Background()) |
| + defer cancel() |
| + slowBucket := keyBackfillFunc(func(ctx context.Context, rt registry.Route) (activator.InstanceKeyOutcome, error) { |
| + select { |
| + case <-time.After(perRow): |
| + return activator.InstanceKeyMinted, nil |
| + case <-ctx.Done(): |
| + return activator.InstanceKeyFailed, fmt.Errorf("%w: %w", activator.ErrInstanceKeyDeferred, ctx.Err()) |
| + } |
| + }) |
| + began := time.Now() |
| + var startedAfter time.Duration |
| + done := activatorStart{ |
| + Sweep: func(context.Context) error { return nil }, |
| + Recover: func(context.Context, bool) *layout.Pass { return nil }, |
| + Routes: func() ([]registry.Route, error) { return routes, nil }, |
| + Route: routeOf(routes...), |
| + Keys: slowBucket, |
| + Slot: (&activator.Service{}).PrepareScheduledToken, |
| + Publish: func(*layout.Pass) {}, |
| + Start: func() { startedAfter = time.Since(began) }, |
| + }.Run(ctx) |
| + returned := time.Since(began) |
| + if startedAfter == 0 || startedAfter > bound { |
| + t.Errorf("Start was called %v after Run began, want within %v (the backfill must not hold it)", startedAfter, bound) |
| + } |
| + if returned > bound { |
| + t.Errorf("Run returned after %v, want within %v: the edge waits for Run", returned, bound) |
| + } |
| + select { |
| + case <-done: |
| + t.Fatal("the backfill had finished when Run returned; it should still be running") |
| + default: |
| + } |
| + // Agent shutdown is not this test's: cancel the pass and let it end. |
| + cancel() |
| + select { |
| + case <-done: |
| + case <-time.After(30 * time.Second): |
| + t.Fatal("the backfill did not end once its context was cancelled") |
| + } |
| +} |
diff --git a/internal/host/layoutleg.go b/internal/host/layoutleg.go| index 2abec0341..fabd8861d 100644 |
| --- a/internal/host/layoutleg.go |
| +++ b/internal/host/layoutleg.go |
| @@ -354,36 +354,58 @@ func startActivator(ctx context.Context, sweep func(context.Context) error, reco |
| // |
| // On a legacy box it is the sweep, then layout recovery (design §8.2, |
| // "Recovery is the journal": every journal a previous agent left is |
| -// completed and the host converged to the fleet's intent), then the |
| -// instance-key backfill (L21), then the pass's publication, then Start. The |
| -// backfill runs after recovery has settled every journal and before the |
| -// pass is published or the activator wakes anything, so nothing moves a |
| -// file under it; the pass is published only after recovery, so the serial |
| -// leg never runs beside it. |
| +// completed and the host converged to the fleet's intent), then the pass's |
| +// publication, then Start. The pass is published only after recovery, so |
| +// the serial leg never runs beside it. |
| +// |
| +// The instance-key backfill (L21) runs after Start, in the background, and |
| +// Run does not wait for it (the v0.26.1 hotfix). Before, it ran between |
| +// recovery and the publication, one bucket read per keyless row, and on a |
| +// box with 127 instances the edge, which comes up after Run returns, did |
| +// not listen within the update gate's 20 seconds. The backfill still runs |
| +// after recovery has settled every journal. What kept anything from moving |
| +// a file under it while it ran first, nothing else running yet, is now |
| +// the activator's per-host preparation slot (keyBackfillInSlot): each row |
| +// is read again and written inside it; a host that an operation holds |
| +// fenced (a layout pass, a move) is skipped, and a fence asked for while |
| +// the row is written waits until the write is done. A start's own mint |
| +// beside the backfill is serialized by the activator's adminTokenMu, and |
| +// both are create-only, so the host ends with one key. |
| type activatorStart struct { |
| Claims bool // claims.Writes.Enabled: a claim box |
| Sweep func(context.Context) error // the instance-unit sweep |
| Recover func(context.Context, bool) *layout.Pass // builds the pass and recovers under it; the bool is whether the sweep succeeded |
| Routes func() ([]registry.Route, error) // the registry's List |
| + Route func(string) (*registry.Route, error) // the registry's Get: each row is read again in its host's slot |
| Keys instanceKeyBackfiller // the exec backend |
| + Slot hostPreparationSlot // the service's PrepareScheduledToken |
| Publish func(*layout.Pass) // armedLayoutPass.Store |
| Start func() // the service's Start |
| } |
| -func (s activatorStart) Run(ctx context.Context) { |
| +// Run starts the activator and returns. The channel closes when the |
| +// instance-key backfill has finished; on a claim box it is closed already. |
| +// runAgent does not wait on it. |
| +func (s activatorStart) Run(ctx context.Context) <-chan struct{} { |
| + done := make(chan struct{}) |
| if s.Claims { |
| s.Start() |
| - return |
| + close(done) |
| + return done |
| } |
| startActivator(ctx, s.Sweep, func(ctx context.Context, swept bool) { |
| - pass := s.Recover(ctx, swept) |
| - if routes, err := s.Routes(); err != nil { |
| + s.Publish(s.Recover(ctx, swept)) |
| + }, s.Start) |
| + go func() { |
| + defer close(done) |
| + routes, err := s.Routes() |
| + if err != nil { |
| log.Printf("agent: instance keys: list routes: %v (each host's next start mints its key)", err) |
| - } else { |
| - backfillInstanceKeys(ctx, routes, s.Keys) |
| + return |
| } |
| - s.Publish(pass) |
| - }, s.Start) |
| + backfillInstanceKeys(ctx, routes, keyBackfillInSlot{keys: s.Keys, slot: s.Slot, route: s.Route}) |
| + }() |
| + return done |
| } |
| // layoutIntentTimeout bounds the one intent read at startup, which the |