amadan
hotfix-instance-key-gate
Squashed from hotfix-instance-key-gate into main.
6 files changed,
+372
−43
internal/activator/instancekey_test.go+65 −0internal/console/layoutguard_test.go+7 −4internal/host/agent.go+7 −3internal/host/instancekeys.go+72 −11internal/host/instancekeys_test.go+185 −11internal/host/layoutleg.go+36 −14
CI passed — run details
CI log
=== gate ===
cd launchctx && go vet ./... && go test ./... -count=1
ok amadan.net/carlos/platform.git/launchctx 0.013s
? amadan.net/carlos/platform.git/launchctx/controlname [no test files]
? amadan.net/carlos/platform.git/launchctx/wireid [no test files]
go vet ./...
gofmt -l .: clean
go test ./... -count=1 -timeout 30m
ok github.com/carlosframework/platform 365.339s
? github.com/carlosframework/platform/cmd/changelog [no test files]
ok github.com/carlosframework/platform/cmd/console 4.378s
ok github.com/carlosframework/platform/cmd/moveprobe 93.784s
ok github.com/carlosframework/platform/internal/activator 229.290s
ok github.com/carlosframework/platform/internal/agentsock 0.027s
ok github.com/carlosframework/platform/internal/alerting 0.008s
ok github.com/carlosframework/platform/internal/analytics 0.058s
ok github.com/carlosframework/platform/internal/appconfig 2.921s
ok github.com/carlosframework/platform/internal/apperr 0.124s
ok github.com/carlosframework/platform/internal/applog 0.169s
ok github.com/carlosframework/platform/internal/awsinv 0.012s
ok github.com/carlosframework/platform/internal/boxchoice 0.008s
ok github.com/carlosframework/platform/internal/boxid 1.161s
? github.com/carlosframework/platform/internal/boxid/boxidtest [no test files]
? github.com/carlosframework/platform/internal/boxid/internal/signtest [no test files]
ok github.com/carlosframework/platform/internal/boxvolumes 0.009s
ok github.com/carlosframework/platform/internal/caps 0.070s
ok github.com/carlosframework/platform/internal/certwarm 0.006s
ok github.com/carlosframework/platform/internal/cgroup 0.219s
ok github.com/carlosframework/platform/internal/changelog 0.473s
ok github.com/carlosframework/platform/internal/clamd 0.064s
? github.com/carlosframework/platform/internal/cliflag [no test files]
ok github.com/carlosframework/platform/internal/console 372.307s
? github.com/carlosframework/platform/internal/controlname [no test files]
? github.com/carlosframework/platform/internal/controlsock [no test files]
ok github.com/carlosframework/platform/internal/dbdoor 6.430s
ok github.com/carlosframework/platform/internal/dbdoor/dbdoortest 0.015s
ok github.com/carlosframework/platform/internal/dbverify 0.016s
ok github.com/carlosframework/platform/internal/dnsrec 0.003s
ok github.com/carlosframework/platform/internal/docsite 0.012s
ok github.com/carlosframework/platform/internal/domains 2.840s
ok github.com/carlosframework/platform/internal/doorkey 0.007s
ok github.com/carlosframework/platform/internal/economics 0.012s
ok github.com/carlosframework/platform/internal/edge 27.491s
? github.com/carlosframework/platform/internal/edgeproof [no test files]
ok github.com/carlosframework/platform/internal/edgesfile 0.017s
ok github.com/carlosframework/platform/internal/fleetapi 0.185s
ok github.com/carlosframework/platform/internal/gate 2.152s
ok github.com/carlosframework/platform/internal/genselect 0.038s
ok github.com/carlosframework/platform/internal/geo 0.012s
ok github.com/carlosframework/platform/internal/geoip 0.053s
ok github.com/carlosframework/platform/internal/host 79.417s
ok github.com/carlosframework/platform/internal/hostactors 0.784s
ok github.com/carlosframework/platform/internal/hostbox 2.555s
ok github.com/carlosframework/platform/internal/hostclaim 1.267s
ok github.com/carlosframework/platform/internal/hostevidence 0.015s
ok github.com/carlosframework/platform/internal/hostexec 0.382s
ok github.com/carlosframework/platform/internal/hostfence 0.060s
ok github.com/carlosframework/platform/internal/hostfiles 10.656s
ok github.com/carlosframework/platform/internal/hostfloor 0.022s
ok github.com/carlosframework/platform/internal/hostguard 0.184s
ok github.com/carlosframework/platform/internal/hostops 2.601s
? github.com/carlosframework/platform/internal/hostops/fencerelease [no test files]
ok github.com/carlosframework/platform/internal/hostops/internal/fencerelease 0.004s
ok github.com/carlosframework/platform/internal/hostquarantine 0.002s
ok github.com/carlosframework/platform/internal/hostrelocate 0.980s
ok github.com/carlosframework/platform/internal/hostresolve 0.283s
ok github.com/carlosframework/platform/internal/hostwrite 0.006s
ok github.com/carlosframework/platform/internal/infraview 0.004s
ok github.com/carlosframework/platform/internal/instancectl 1.734s
ok github.com/carlosframework/platform/internal/instanceexec 2.918s
ok github.com/carlosframework/platform/internal/instancefs 0.009s
ok github.com/carlosframework/platform/internal/instancekey 0.012s
ok github.com/carlosframework/platform/internal/instanceupdates 0.009s
ok github.com/carlosframework/platform/internal/layout 2.371s
ok github.com/carlosframework/platform/internal/lease 9.565s
ok github.com/carlosframework/platform/internal/lease/leasetest 0.006s
ok github.com/carlosframework/platform/internal/ledger 0.024s
ok github.com/carlosframework/platform/internal/mail 0.377s
ok github.com/carlosframework/platform/internal/maintenance 0.120s
ok github.com/carlosframework/platform/internal/maintenanceexec 5.865s
ok github.com/carlosframework/platform/internal/maintenanceguard 0.187s
ok github.com/carlosframework/platform/internal/metrics 0.016s
ok github.com/carlosframework/platform/internal/moves 23.195s
ok github.com/carlosframework/platform/internal/moves/archive 1.254s
ok github.com/carlosframework/platform/internal/objstore 0.404s
ok github.com/carlosframework/platform/internal/ops 0.361s
ok github.com/carlosframework/platform/internal/park 0.516s
ok github.com/carlosframework/platform/internal/privatefile 0.007s
ok github.com/carlosframework/platform/internal/provision 4.572s
ok github.com/carlosframework/platform/internal/publicrelease 0.008s
? github.com/carlosframework/platform/internal/regionname [no test files]
ok github.com/carlosframework/platform/internal/registry 4.819s
ok github.com/carlosframework/platform/internal/relay 7.245s
ok github.com/carlosframework/platform/internal/release 21.225s
ok github.com/carlosframework/platform/internal/rollout 4.335s
ok github.com/carlosframework/platform/internal/rolloutwords 0.008s
ok github.com/carlosframework/platform/internal/rootaws 0.008s
ok github.com/carlosframework/platform/internal/s3compat 0.006s
ok github.com/carlosframework/platform/internal/safefile 0.004s
ok github.com/carlosframework/platform/internal/schedule 2.665s
ok github.com/carlosframework/platform/internal/sim/hostlife 18.580s
ok github.com/carlosframework/platform/internal/sim/litestreamfacts 4.555s
ok github.com/carlosframework/platform/internal/status 0.042s
ok github.com/carlosframework/platform/internal/steering 0.010s
ok github.com/carlosframework/platform/internal/stopevidence 0.223s
ok github.com/carlosframework/platform/internal/svcrole 0.006s
ok github.com/carlosframework/platform/internal/telemetry 0.011s
ok github.com/carlosframework/platform/internal/termfence 0.004s
ok github.com/carlosframework/platform/internal/tunnelclient 0.004s
ok github.com/carlosframework/platform/internal/unixdial 0.005s
ok github.com/carlosframework/platform/internal/updatewords 0.005s
ok github.com/carlosframework/platform/internal/usage 0.005s
ok github.com/carlosframework/platform/internal/version 0.003s
ok github.com/carlosframework/platform/internal/vet 1.992s
=== cross ===
--- shard 0 of 5 ---
cross-compiling darwin/arm64
--- shard 1 of 5 ---
cross-compiling darwin/amd64
--- shard 2 of 5 ---
cross-compiling linux/arm64
--- shard 3 of 5 ---
cross-compiling windows/amd64
--- shard 4 of 5 ---
cross-compiling windows/arm64
=== minio-contract ===
minio contract: arm64, minio RELEASE.2025-09-07T16-13-09Z, mc RELEASE.2025-08-13T08-35-41Z
minio: cached, digest verified
mc: cached, digest verified
minio healthy on 127.0.0.1:20999
Added `local` successfully.
Bucket created successfully `local/contract-bucket`.
Bucket created successfully `local/denied-bucket`.
Created policy `contract-only` successfully.
Added user `scopeduser` successfully.
Attached Policies: [contract-only]
To User: scopeduser
buckets and scoped user ready
=== RUN TestDeliveredContractMinio
=== RUN TestDeliveredContractMinio/put
=== RUN TestDeliveredContractMinio/get_round-trips_the_body
=== RUN TestDeliveredContractMinio/list_shows_the_key
=== RUN TestDeliveredContractMinio/head_and_list_agree_on_the_version
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_its_own_object
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/not_by_PutObjectTagging
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/and_not_by_x-amz-tagging_on_upload
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CreateMultipartUpload
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CopyObject_with_a_REPLACE_tagging_directive
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/but_multipart_itself_is_allowed
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_copy_itself_is_allowed
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/a_POST-policy_forged_tag_with_a_matching_version_is_still_not_a_verdict
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_an_ordinary_tagged-with-something-else_upload_is_allowed
=== RUN TestDeliveredContractMinio/delete_removes_it
=== RUN TestDeliveredContractMinio/cross-bucket_denial
--- PASS: TestDeliveredContractMinio (0.02s)
--- PASS: TestDeliveredContractMinio/put (0.00s)
--- PASS: TestDeliveredContractMinio/get_round-trips_the_body (0.00s)
--- PASS: TestDeliveredContractMinio/list_shows_the_key (0.00s)
--- PASS: TestDeliveredContractMinio/head_and_list_agree_on_the_version (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/not_by_PutObjectTagging (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/and_not_by_x-amz-tagging_on_upload (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route (0.01s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CreateMultipartUpload (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CopyObject_with_a_REPLACE_tagging_directive (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/but_multipart_itself_is_allowed (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_copy_itself_is_allowed (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/a_POST-policy_forged_tag_with_a_matching_version_is_still_not_a_verdict (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_an_ordinary_tagged-with-something-else_upload_is_allowed (0.00s)
--- PASS: TestDeliveredContractMinio/delete_removes_it (0.00s)
--- PASS: TestDeliveredContractMinio/cross-bucket_denial (0.00s)
PASS
ok github.com/carlosframework/platform/internal/objstore 0.023s
=== RUN TestT4StorageProbeMinio
2026/10/13 12:14:09 moves: T4: STORE: list: operation error S3: ListObjectsV2, https response error StatusCode: 403, RequestID: 18DE15C7B4A853ED, HostID: 7c420053e0fbcd7f41653fdd92ffa2802c903d05303b1aa15adc2e8bb65d183f, api error SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided. Check your key and signing method.
--- PASS: TestT4StorageProbeMinio (0.01s)
PASS
ok github.com/carlosframework/platform/internal/host 0.023s
minio contract: PASS
diff --git a/internal/activator/instancekey_test.go b/internal/activator/instancekey_test.go| index 5bc223dfe..c632af017 100644 |
| --- a/internal/activator/instancekey_test.go |
| +++ b/internal/activator/instancekey_test.go |
| @@ -1002,3 +1002,68 @@ func TestBackfillInstanceKeyRefusesFencedAndConflictRows(t *testing.T) { |
| }) |
| } |
| } |
| + |
| +// v0.26.1 hotfix: the agent's backfill now runs beside the activator |
| +// rather than before it, so a backfill and a start's own mint can reach |
| +// the same host at once. Both read the files and ask a slow bucket outside |
| +// adminTokenMu, and both are held there until each has done so, so each |
| +// decides from a first read that found no key. They still decide and |
| +// append under the lock, create-only, so the host ends with exactly one |
| +// key, appended whole after the bytes that were there, and exactly one of |
| +// the two reports it minted. |
| +func TestBackfillAndAStartsMintRacingOnOneHostWriteOneKey(t *testing.T) { |
| + const rounds = 40 |
| + tok := "CARLOS_ADMIN_TOKEN=" + strings.Repeat("b", 64) + "\n" |
| + for i := range rounds { |
| + b := &ExecBackend{DataDir: t.TempDir()} |
| + b.ConfiguredKey = func(ctx context.Context, _ registry.Route) (string, error) { |
| + select { |
| + case <-time.After(5 * time.Millisecond): // a slow bucket that sets no key |
| + return "", nil |
| + case <-ctx.Done(): |
| + return "", ctx.Err() |
| + } |
| + } |
| + rt := registry.Route{Host: "shop.example.com", Kind: "instance", Status: "ready", Socket: "/run/carlos/shop.sock"} |
| + own := filepath.Join(b.DataDir, rt.Host+".env") |
| + if err := os.WriteFile(own, []byte(tok), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + // Both callers pass phase one before either takes the lock. |
| + var phase sync.WaitGroup |
| + phase.Add(2) |
| + instanceKeyHooks.betweenPhases = func(string) { phase.Done(); phase.Wait() } |
| + var wg sync.WaitGroup |
| + var backfilled, started InstanceKeyOutcome |
| + var berr, serr error |
| + wg.Add(2) |
| + go func() { |
| + defer wg.Done() |
| + backfilled, berr = b.BackfillInstanceKey(t.Context(), rt) |
| + }() |
| + go func() { |
| + defer wg.Done() |
| + started, serr = b.ensureInstanceKeyFor(t.Context(), rt, b.pathsFor(rt)) |
| + }() |
| + wg.Wait() |
| + instanceKeyHooks.betweenPhases = nil |
| + if berr != nil || serr != nil { |
| + t.Fatalf("round %d: backfill %v, %v; start %v, %v", i, backfilled, berr, started, serr) |
| + } |
| + minted := 0 |
| + for _, o := range []InstanceKeyOutcome{backfilled, started} { |
| + switch o { |
| + case InstanceKeyMinted: |
| + minted++ |
| + case InstanceKeyPresent: |
| + default: |
| + t.Fatalf("round %d: outcome %v, want minted or present", i, o) |
| + } |
| + } |
| + raw, _ := os.ReadFile(own) |
| + keys := keyLines(raw) |
| + if minted != 1 || len(keys) != 1 || !wellFormedInstanceKey(keys[0]) || string(raw) != tok+InstanceKeyVar+"="+keys[0]+"\n" { |
| + t.Fatalf("round %d: %d minted, %d keys, file %d bytes; want one whole key appended once", i, minted, len(keys), len(raw)) |
| + } |
| + } |
| +} |
diff --git a/internal/console/layoutguard_test.go b/internal/console/layoutguard_test.go| index 2154ddcb3..b05056134 100644 |
| --- a/internal/console/layoutguard_test.go |
| +++ b/internal/console/layoutguard_test.go |
| @@ -106,10 +106,13 @@ func TestSystemDoorsRefuseAPreLayoutMoveWhileABoxIsNested(t *testing.T) { |
| t.Fatalf("update to v0.19.0 with a nested row = %d %v, want 409", st, out) |
| } |
| // A version at the floor or above is not the guard's business; the |
| - // request goes on to the job (and fails there, on this fixture's |
| - // absent feed, which is not what this test is about). |
| - if st, _ := postJSON(t, fx.ts.URL+"/api/cli/system/update", map[string]any{"channel": "canary", "version": "v0.26.0"}, owner); st == http.StatusConflict { |
| - t.Error("update to v0.26.0 was refused by the layout guard") |
| + // request goes on to the job (and fails there, which is not what this |
| + // test is about). The fixture has no SystemFeed, so the job asks the |
| + // vendor feed: the version must be one that can never be published, or |
| + // the job succeeds once it is (v0.26.0 was, on 2026-10-13), moves the |
| + // pointer, and the rollback below lands on the wrong version. |
| + if st, _ := postJSON(t, fx.ts.URL+"/api/cli/system/update", map[string]any{"channel": "canary", "version": "v99.0.0"}, owner); st == http.StatusConflict { |
| + t.Error("update to v99.0.0 was refused by the layout guard") |
| } |
| // Every row flat: the rollback goes through. |
diff --git a/internal/host/agent.go b/internal/host/agent.go| index 38df7d92e..0b5a9cca1 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -547,9 +547,11 @@ func runAgent(args []string, buildClaims ClaimFactory) { |
| // worse off than before this existed (a leftover unit that keeps |
| // running is exactly what an agent restart left before this pass). |
| // |
| - // Then, on a legacy box, layout recovery and the instance-key |
| - // backfill, before Start can wake anything; on a claim box, Start |
| - // alone. The order and the branch are activatorStart's. |
| + // Then, on a legacy box, layout recovery before Start can wake |
| + // anything, and the instance-key backfill in the background after |
| + // it, which this does not wait for (the edge below must listen |
| + // within the update gate, whatever the box's size); on a claim |
| + // box, Start alone. The order and the branch are activatorStart's. |
| activatorStart{ |
| Claims: claims.Writes.Enabled, |
| Sweep: func(ctx context.Context) error { |
| @@ -568,7 +570,9 @@ func runAgent(args []string, buildClaims ClaimFactory) { |
| return pass |
| }, |
| Routes: reg.List, |
| + Route: reg.Get, |
| Keys: eb, |
| + Slot: svc.PrepareScheduledToken, |
| Publish: armedLayoutPass.Store, |
| Start: svc.Start, |
| }.Run(context.Background()) |
diff --git a/internal/host/instancekeys.go b/internal/host/instancekeys.go| index d79ea90f4..74b19c6c0 100644 |
| --- a/internal/host/instancekeys.go |
| +++ b/internal/host/instancekeys.go |
| @@ -3,6 +3,7 @@ package host |
| import ( |
| "context" |
| "errors" |
| + "fmt" |
| "log" |
| "time" |
| @@ -13,9 +14,11 @@ import ( |
| // The instance-key backfill (L21, plan 2026-10-10-instance-key-provision.md |
| // ruling 7). Every launch path mints CARLOS_INSTANCE_KEY before it reads the |
| // env, so no process is ever started without one. The backfill gives every |
| -// instance already on a box its key now, at agent start, rather than at its |
| -// next start, so the rollout can be checked box by box before any app adopts |
| -// the key: a host that the backfill names "present" or "minted" is ready. |
| +// instance already on a box its key now, just after agent start, rather |
| +// than at its next start, so the rollout can be checked box by box before |
| +// any app adopts the key: a host that the backfill names "present" or |
| +// "minted" is ready. It runs in the background once the activator has |
| +// started (activatorStart.Run), never on the path the edge waits for. |
| // |
| // It runs only where the legacy activator writes <host>.env itself. With |
| // host claims on, every write to a host's installation runs under the |
| @@ -31,12 +34,64 @@ type instanceKeyBackfiller interface { |
| } |
| // instanceKeyBackfillBudget bounds one pass. Each keyless row costs one |
| -// read of the bucket's configuration before its key is minted, and the |
| -// pass runs before the activator starts, so with the bucket unreachable |
| -// the rows after the budget fail fast rather than each waiting out its |
| -// own read; every one of them mints at its next start. |
| +// read of the bucket's configuration before its key is minted. The pass |
| +// runs in the background after the activator starts (activatorStart.Run), |
| +// so it never holds the edge up, but with the bucket unreachable the rows |
| +// after the budget still fail fast rather than each waiting out its own |
| +// read; every one of them mints at its next start. |
| const instanceKeyBackfillBudget = time.Minute |
| +// hostPreparationSlot runs prepare in host's preparation slot, ordered |
| +// with the host's maintenance fence: (*activator.Service). |
| +// PrepareScheduledToken. It answers activator.ErrFenced without running |
| +// prepare when a fence is held for host, and a fence asked for while |
| +// prepare runs is installed only after it returns. Wakes do not wait on |
| +// the slot. |
| +type hostPreparationSlot func(host string, prepare func() (string, error)) (string, error) |
| + |
| +// errInstanceKeyRowSkipped is a row the backfill listed and, when its turn |
| +// came, did not write for: an operation held its host fenced, or its row |
| +// had changed (a move, a layout flip, a removal) so it is no longer one |
| +// the backfill selects. Counted with the skipped rows; its next start |
| +// mints the key if it still needs one. |
| +var errInstanceKeyRowSkipped = errors.New("skipped") |
| + |
| +// keyBackfillInSlot is the backfill's step for one row once the activator |
| +// is serving (the v0.26.1 hotfix moved the pass off the startup path). The |
| +// layout pass and every move fence a host before they move its files, and |
| +// install that fence only after any preparation already in the slot has |
| +// returned; the slot refuses a host already fenced. So inside the slot no |
| +// file of the host moves, and the row is read again there, so the write |
| +// goes to the paths the host has now, not those of the list taken before. |
| +// A start's own mint is not ordered by the slot: it and the backfill both |
| +// decide and append under the activator's adminTokenMu, create-only, so |
| +// whichever is second finds the key present. |
| +type keyBackfillInSlot struct { |
| + keys instanceKeyBackfiller |
| + slot hostPreparationSlot |
| + route func(string) (*registry.Route, error) |
| +} |
| + |
| +func (g keyBackfillInSlot) BackfillInstanceKey(ctx context.Context, rt registry.Route) (activator.InstanceKeyOutcome, error) { |
| + got, err := activator.InstanceKeyFailed, error(nil) |
| + _, serr := g.slot(rt.Host, func() (string, error) { |
| + fresh, rerr := g.route(rt.Host) |
| + switch { |
| + case rerr != nil: |
| + err = fmt.Errorf("read its row: %w", rerr) |
| + case fresh == nil || !instanceKeyRoute(*fresh): |
| + err = fmt.Errorf("%w: its row changed after the backfill listed it", errInstanceKeyRowSkipped) |
| + default: |
| + got, err = g.keys.BackfillInstanceKey(ctx, *fresh) |
| + } |
| + return "", nil |
| + }) |
| + if serr != nil { |
| + return activator.InstanceKeyFailed, fmt.Errorf("%w: an operation holds it (%v)", errInstanceKeyRowSkipped, serr) |
| + } |
| + return got, err |
| +} |
| + |
| // instanceKeyRoute selects the rows the backfill writes for: ready, |
| // activator-spawned instances, the rows execSyncable delivers config to, |
| // without sidecars and without replicas (a replica is never minted a key, |
| @@ -58,10 +113,12 @@ type instanceKeyBackfill struct { |
| } |
| // backfillInstanceKeys runs the backfill over routes, within |
| -// instanceKeyBackfillBudget. It never fails the agent: a host it cannot |
| -// write is logged and counted, and that host's next start mints its key (or |
| -// refuses, loudly, if the same fault is still there). Nothing logged names |
| -// a key. |
| +// instanceKeyBackfillBudget, and logs one count line when it ends, |
| +// "agent: instance keys: ...": on a legacy box that is after the activator |
| +// has started, within the budget plus one row's read. It never fails the |
| +// agent: a host it cannot write is logged and counted, and that host's |
| +// next start mints its key (or refuses, loudly, if the same fault is |
| +// still there). Nothing logged names a key. |
| // |
| // The error is read before the outcome: a refused or failed row answers |
| // InstanceKeyFailed (the zero outcome) beside its error, and counting an |
| @@ -77,6 +134,10 @@ func backfillInstanceKeys(ctx context.Context, routes []registry.Route, b instan |
| } |
| got, err := b.BackfillInstanceKey(ctx, rt) |
| switch { |
| + case errors.Is(err, errInstanceKeyRowSkipped): |
| + st.Skipped++ |
| + log.Printf("agent: instance key for %s: %v (its next start mints it)", rt.Host, err) |
| + continue |
| case errors.Is(err, activator.ErrInstanceKeyDeferred): |
| st.Deferred++ |
| log.Printf("agent: instance key for %s: %v", rt.Host, err) |
diff --git a/internal/host/instancekeys_test.go b/internal/host/instancekeys_test.go| index 1b6c63339..f09c8ac12 100644 |
| --- a/internal/host/instancekeys_test.go |
| +++ b/internal/host/instancekeys_test.go |
| @@ -5,6 +5,8 @@ import ( |
| "errors" |
| "fmt" |
| "reflect" |
| + "sync" |
| + "sync/atomic" |
| "testing" |
| "time" |
| @@ -121,14 +123,29 @@ func (f keyBackfillFunc) BackfillInstanceKey(ctx context.Context, rt registry.Ro |
| return f(ctx, rt) |
| } |
| -// Task 6 ruling (K1): the start sequence is activatorStart's, so the two |
| -// guards runAgent cannot be tested for are pinned here. A claim box starts |
| -// and nothing else: no sweep, no recovery, no backfill (ruling 7). A legacy |
| -// box sweeps, recovers, backfills, publishes the pass and then starts, in |
| -// that order: the backfill after every journal is settled and before the |
| -// pass is published or anything wakes. A route list that fails skips the |
| -// backfill only; the pass is still published and the service still starts. |
| -func TestActivatorStartBackfillsOnlyALegacyBoxBetweenRecoveryAndPublish(t *testing.T) { |
| +// openSlot is a preparation slot with no fence held: it runs prepare. |
| +func openSlot(_ string, prepare func() (string, error)) (string, error) { return prepare() } |
| + |
| +// routeOf answers the registry's Get from a fixed list. |
| +func routeOf(routes ...registry.Route) func(string) (*registry.Route, error) { |
| + return func(host string) (*registry.Route, error) { |
| + for _, rt := range routes { |
| + if rt.Host == host { |
| + return &rt, nil |
| + } |
| + } |
| + return nil, nil |
| + } |
| +} |
| + |
| +// Task 6 ruling (K1), as the v0.26.1 hotfix changed it: the start sequence |
| +// is activatorStart's, so the two guards runAgent cannot be tested for are |
| +// pinned here. A claim box starts and nothing else: no sweep, no recovery, |
| +// no backfill (ruling 7). A legacy box sweeps, recovers, publishes the |
| +// pass and starts, in that order, and the backfill runs after Start, once |
| +// every journal is settled. A route list that fails skips the backfill |
| +// only. |
| +func TestActivatorStartBackfillsOnlyALegacyBoxAfterRecoveryAndStart(t *testing.T) { |
| ready := registry.Route{Host: "shop.example.com", Kind: "instance", Status: "ready", Socket: "/run/carlos/shop.example.com.sock"} |
| for _, tc := range []struct { |
| name string |
| @@ -136,28 +153,185 @@ func TestActivatorStartBackfillsOnlyALegacyBoxBetweenRecoveryAndPublish(t *testi |
| listErr error |
| want []string |
| }{ |
| - {"legacy box", false, nil, []string{"sweep", "recover", "backfill", "publish", "start"}}, |
| + {"legacy box", false, nil, []string{"sweep", "recover", "publish", "start", "backfill"}}, |
| {"legacy box, routes unreadable", false, errors.New("db locked"), []string{"sweep", "recover", "publish", "start"}}, |
| {"claim box", true, nil, []string{"start"}}, |
| } { |
| t.Run(tc.name, func(t *testing.T) { |
| + var mu sync.Mutex |
| var events []string |
| - note := func(e string) { events = append(events, e) } |
| - activatorStart{ |
| + note := func(e string) { mu.Lock(); events = append(events, e); mu.Unlock() } |
| + done := activatorStart{ |
| Claims: tc.claims, |
| Sweep: func(context.Context) error { note("sweep"); return nil }, |
| Recover: func(context.Context, bool) *layout.Pass { note("recover"); return nil }, |
| Routes: func() ([]registry.Route, error) { return []registry.Route{ready}, tc.listErr }, |
| + Route: routeOf(ready), |
| Keys: keyBackfillFunc(func(context.Context, registry.Route) (activator.InstanceKeyOutcome, error) { |
| note("backfill") |
| return activator.InstanceKeyMinted, nil |
| }), |
| + Slot: openSlot, |
| Publish: func(*layout.Pass) { note("publish") }, |
| Start: func() { note("start") }, |
| }.Run(context.Background()) |
| + select { |
| + case <-done: |
| + case <-time.After(10 * time.Second): |
| + t.Fatal("the backfill never finished") |
| + } |
| + mu.Lock() |
| + defer mu.Unlock() |
| if !reflect.DeepEqual(events, tc.want) { |
| t.Fatalf("events = %v, want %v", events, tc.want) |
| } |
| }) |
| } |
| } |
| + |
| +// fenceCtx is a context with the deadline FenceAndStop requires. |
| +func fenceCtx(t *testing.T, d time.Duration) context.Context { |
| + ctx, cancel := context.WithTimeout(t.Context(), d) |
| + t.Cleanup(cancel) |
| + return ctx |
| +} |
| + |
| +// Off the startup path, the backfill shares the box with the layout pass |
| +// and moves, which fence a host before they move its files. A host that |
| +// is fenced when its turn comes is skipped and nothing is asked of it, and |
| +// a row whose status changed after the list (it began a move) is skipped |
| +// the same way; its next start mints the key. The rest are written from |
| +// the row as it stands, not as it was listed. |
| +func TestBackfillInSlotSkipsAFencedHostAndReadsTheRowAgain(t *testing.T) { |
| + ready := func(host string) registry.Route { |
| + return registry.Route{Host: host, Kind: "instance", Status: "ready", Socket: "/run/carlos/" + host + ".sock"} |
| + } |
| + fenced, moved, shop := ready("fenced.example.com"), ready("moved.example.com"), ready("shop.example.com") |
| + svc := &activator.Service{} |
| + if _, err := svc.FenceAndStop(fenceCtx(t, time.Second), fenced.Host, 1); err != nil && !errors.Is(err, activator.ErrStopUnproven) { |
| + t.Fatalf("fence: %v", err) |
| + } |
| + nowMoving := moved |
| + nowMoving.Status = "moving" |
| + nowShop := shop |
| + nowShop.Socket = "/run/carlos/nested/shop.example.com.sock" |
| + f := &fakeKeyBackfiller{got: map[string]activator.InstanceKeyOutcome{shop.Host: activator.InstanceKeyMinted}} |
| + var sockets []string |
| + g := keyBackfillInSlot{ |
| + keys: keyBackfillFunc(func(ctx context.Context, rt registry.Route) (activator.InstanceKeyOutcome, error) { |
| + sockets = append(sockets, rt.Socket) |
| + return f.BackfillInstanceKey(ctx, rt) |
| + }), |
| + slot: svc.PrepareScheduledToken, |
| + route: routeOf(fenced, nowMoving, nowShop), |
| + } |
| + st := backfillInstanceKeys(t.Context(), []registry.Route{fenced, moved, shop}, g) |
| + if !reflect.DeepEqual(f.asked, []string{shop.Host}) { |
| + t.Fatalf("asked %v, want only %s", f.asked, shop.Host) |
| + } |
| + if !reflect.DeepEqual(sockets, []string{nowShop.Socket}) { |
| + t.Fatalf("written from %v, want the row as it stands now", sockets) |
| + } |
| + if st != (instanceKeyBackfill{Minted: 1, Skipped: 2}) { |
| + t.Fatalf("stats = %+v, want one minted and two skipped", st) |
| + } |
| +} |
| + |
| +// The other half of "nothing moves a file under it": a fence asked for |
| +// while a row is being written (the layout pass or a move about to move |
| +// the host's files) is installed only once the write has returned. |
| +func TestBackfillInSlotHoldsOffAFenceUntilTheWriteReturns(t *testing.T) { |
| + rt := registry.Route{Host: "shop.example.com", Kind: "instance", Status: "ready", Socket: "/run/carlos/shop.example.com.sock"} |
| + svc := &activator.Service{} |
| + writing, release := make(chan struct{}), make(chan struct{}) |
| + var wrote atomic.Bool |
| + g := keyBackfillInSlot{ |
| + keys: keyBackfillFunc(func(context.Context, registry.Route) (activator.InstanceKeyOutcome, error) { |
| + close(writing) |
| + <-release |
| + wrote.Store(true) |
| + return activator.InstanceKeyMinted, nil |
| + }), |
| + slot: svc.PrepareScheduledToken, |
| + route: routeOf(rt), |
| + } |
| + done := make(chan instanceKeyBackfill) |
| + go func() { done <- backfillInstanceKeys(t.Context(), []registry.Route{rt}, g) }() |
| + <-writing |
| + fenced := make(chan bool) |
| + go func() { |
| + _, err := svc.FenceAndStopIfUnfenced(fenceCtx(t, 10*time.Second), rt.Host, 1) |
| + if err != nil && !errors.Is(err, activator.ErrStopUnproven) { |
| + t.Errorf("fence: %v", err) |
| + } |
| + fenced <- wrote.Load() |
| + }() |
| + select { |
| + case <-fenced: |
| + t.Fatal("the fence was installed while the backfill was writing the host") |
| + case <-time.After(200 * time.Millisecond): |
| + } |
| + close(release) |
| + if !<-fenced { |
| + t.Fatal("the fence was installed before the write returned") |
| + } |
| + if st := <-done; st != (instanceKeyBackfill{Minted: 1}) { |
| + t.Fatalf("stats = %+v", st) |
| + } |
| +} |
| + |
| +// v0.26.1 hotfix: the v0.26.0 roll to a 127-instance box failed its update |
| +// gate because the backfill ran serially before Start, one bucket read per |
| +// keyless row, and the edge never listened inside the gate's 20 seconds. |
| +// On a box with hundreds of keyless rows and a slow bucket, Start is |
| +// called, and Run returns (runAgent goes on to the agent socket and the |
| +// edge), without waiting for the backfill. |
| +func TestActivatorStartDoesNotWaitForTheBackfillOnALargeBox(t *testing.T) { |
| + const rows, perRow, bound = 300, 30 * time.Millisecond, 2 * time.Second |
| + var routes []registry.Route |
| + for i := range rows { |
| + host := fmt.Sprintf("shop%03d.example.com", i) |
| + routes = append(routes, registry.Route{Host: host, Kind: "instance", Status: "ready", Socket: "/run/carlos/" + host + ".sock"}) |
| + } |
| + ctx, cancel := context.WithCancel(context.Background()) |
| + defer cancel() |
| + slowBucket := keyBackfillFunc(func(ctx context.Context, rt registry.Route) (activator.InstanceKeyOutcome, error) { |
| + select { |
| + case <-time.After(perRow): |
| + return activator.InstanceKeyMinted, nil |
| + case <-ctx.Done(): |
| + return activator.InstanceKeyFailed, fmt.Errorf("%w: %w", activator.ErrInstanceKeyDeferred, ctx.Err()) |
| + } |
| + }) |
| + began := time.Now() |
| + var startedAfter time.Duration |
| + done := activatorStart{ |
| + Sweep: func(context.Context) error { return nil }, |
| + Recover: func(context.Context, bool) *layout.Pass { return nil }, |
| + Routes: func() ([]registry.Route, error) { return routes, nil }, |
| + Route: routeOf(routes...), |
| + Keys: slowBucket, |
| + Slot: (&activator.Service{}).PrepareScheduledToken, |
| + Publish: func(*layout.Pass) {}, |
| + Start: func() { startedAfter = time.Since(began) }, |
| + }.Run(ctx) |
| + returned := time.Since(began) |
| + if startedAfter == 0 || startedAfter > bound { |
| + t.Errorf("Start was called %v after Run began, want within %v (the backfill must not hold it)", startedAfter, bound) |
| + } |
| + if returned > bound { |
| + t.Errorf("Run returned after %v, want within %v: the edge waits for Run", returned, bound) |
| + } |
| + select { |
| + case <-done: |
| + t.Fatal("the backfill had finished when Run returned; it should still be running") |
| + default: |
| + } |
| + // Agent shutdown is not this test's: cancel the pass and let it end. |
| + cancel() |
| + select { |
| + case <-done: |
| + case <-time.After(30 * time.Second): |
| + t.Fatal("the backfill did not end once its context was cancelled") |
| + } |
| +} |
diff --git a/internal/host/layoutleg.go b/internal/host/layoutleg.go| index 2abec0341..fabd8861d 100644 |
| --- a/internal/host/layoutleg.go |
| +++ b/internal/host/layoutleg.go |
| @@ -354,36 +354,58 @@ func startActivator(ctx context.Context, sweep func(context.Context) error, reco |
| // |
| // On a legacy box it is the sweep, then layout recovery (design §8.2, |
| // "Recovery is the journal": every journal a previous agent left is |
| -// completed and the host converged to the fleet's intent), then the |
| -// instance-key backfill (L21), then the pass's publication, then Start. The |
| -// backfill runs after recovery has settled every journal and before the |
| -// pass is published or the activator wakes anything, so nothing moves a |
| -// file under it; the pass is published only after recovery, so the serial |
| -// leg never runs beside it. |
| +// completed and the host converged to the fleet's intent), then the pass's |
| +// publication, then Start. The pass is published only after recovery, so |
| +// the serial leg never runs beside it. |
| +// |
| +// The instance-key backfill (L21) runs after Start, in the background, and |
| +// Run does not wait for it (the v0.26.1 hotfix). Before, it ran between |
| +// recovery and the publication, one bucket read per keyless row, and on a |
| +// box with 127 instances the edge, which comes up after Run returns, did |
| +// not listen within the update gate's 20 seconds. The backfill still runs |
| +// after recovery has settled every journal. What kept anything from moving |
| +// a file under it while it ran first, nothing else running yet, is now |
| +// the activator's per-host preparation slot (keyBackfillInSlot): each row |
| +// is read again and written inside it; a host that an operation holds |
| +// fenced (a layout pass, a move) is skipped, and a fence asked for while |
| +// the row is written waits until the write is done. A start's own mint |
| +// beside the backfill is serialized by the activator's adminTokenMu, and |
| +// both are create-only, so the host ends with one key. |
| type activatorStart struct { |
| Claims bool // claims.Writes.Enabled: a claim box |
| Sweep func(context.Context) error // the instance-unit sweep |
| Recover func(context.Context, bool) *layout.Pass // builds the pass and recovers under it; the bool is whether the sweep succeeded |
| Routes func() ([]registry.Route, error) // the registry's List |
| + Route func(string) (*registry.Route, error) // the registry's Get: each row is read again in its host's slot |
| Keys instanceKeyBackfiller // the exec backend |
| + Slot hostPreparationSlot // the service's PrepareScheduledToken |
| Publish func(*layout.Pass) // armedLayoutPass.Store |
| Start func() // the service's Start |
| } |
| -func (s activatorStart) Run(ctx context.Context) { |
| +// Run starts the activator and returns. The channel closes when the |
| +// instance-key backfill has finished; on a claim box it is closed already. |
| +// runAgent does not wait on it. |
| +func (s activatorStart) Run(ctx context.Context) <-chan struct{} { |
| + done := make(chan struct{}) |
| if s.Claims { |
| s.Start() |
| - return |
| + close(done) |
| + return done |
| } |
| startActivator(ctx, s.Sweep, func(ctx context.Context, swept bool) { |
| - pass := s.Recover(ctx, swept) |
| - if routes, err := s.Routes(); err != nil { |
| + s.Publish(s.Recover(ctx, swept)) |
| + }, s.Start) |
| + go func() { |
| + defer close(done) |
| + routes, err := s.Routes() |
| + if err != nil { |
| log.Printf("agent: instance keys: list routes: %v (each host's next start mints its key)", err) |
| - } else { |
| - backfillInstanceKeys(ctx, routes, s.Keys) |
| + return |
| } |
| - s.Publish(pass) |
| - }, s.Start) |
| + backfillInstanceKeys(ctx, routes, keyBackfillInSlot{keys: s.Keys, slot: s.Slot, route: s.Route}) |
| + }() |
| + return done |
| } |
| // layoutIntentTimeout bounds the one intent read at startup, which the |