amadan
moves: commit into home through the production owners (home part 2)
Task 3's witnesses for a move ca -> home on the intoHome fixture, each through the production probe owners over the claim runtime: - TestMoveIntoHomeStagesTheHomeLane: the lane in data-home under the @home staging lease only; Touch is a union that only ever adds @home. - TestMoveIntoHomeTargetChecksPass: T1 to T13 ok on home-box; T8 and T12 name the home ie; T10 probes data-ca; T12 reads both quiet spellings. - TestMoveIntoHomeCommitsAndCloses: residue parks in @home, admission on home-box in ie, C8 establishment in data-home, commit, close in home, and a second move out of home. - TestMoveIntoHomeRestartedProbe: the restarted attempt is admitted after the first attempt's generation in ie and published fenced. - TestHomeTargetFreshStateShowsOnlyTheHomeToken: the runtime proof from clean claim state. - TestHomeTargetRetainedChoresKeepTheirSpelling: chores owed under ie or @home are neither duplicated nor rewritten, and acknowledged once. - TestMovesHomeRegionCallSiteInventory (repository root): the golden inventory of 24 call sites, each with its witness (14 pending Task 4). - TestMoveRecoveryCommitIntoHome (hostops): the console commit's witness. Test-only. The production probe mode gains a restore stand-in for a restarted attempt's <db>.gen and records what the runtime materialized.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
5 files changed,
+1722
−6
home_region_args_test.go+521 −0internal/host/movesfinal_test.go+8 −1internal/host/moveshome_target_test.go+1085 −0internal/host/movesparks_test.go+46 −5internal/hostops/move_recovery_commit_test.go+62 −0
CI passed — run details
CI log
=== gate ===
cd launchctx && go vet ./... && go test ./... -count=1
ok amadan.net/carlos/platform.git/launchctx 0.012s
? amadan.net/carlos/platform.git/launchctx/controlname [no test files]
? amadan.net/carlos/platform.git/launchctx/wireid [no test files]
go vet ./...
gofmt -l .: clean
go test ./... -count=1 -timeout 30m
ok github.com/carlosframework/platform 298.706s
? github.com/carlosframework/platform/cmd/changelog [no test files]
ok github.com/carlosframework/platform/cmd/console 2.818s
ok github.com/carlosframework/platform/cmd/moveprobe 93.282s
ok github.com/carlosframework/platform/cmd/zddprobe 0.718s
ok github.com/carlosframework/platform/internal/activator 249.494s
ok github.com/carlosframework/platform/internal/agentsock 0.015s
ok github.com/carlosframework/platform/internal/alerting 0.004s
ok github.com/carlosframework/platform/internal/analytics 0.019s
ok github.com/carlosframework/platform/internal/appconfig 2.495s
ok github.com/carlosframework/platform/internal/apperr 0.071s
ok github.com/carlosframework/platform/internal/applog 0.123s
ok github.com/carlosframework/platform/internal/awsinv 0.004s
ok github.com/carlosframework/platform/internal/boxchoice 0.004s
ok github.com/carlosframework/platform/internal/boxid 0.802s
? github.com/carlosframework/platform/internal/boxid/boxidtest [no test files]
? github.com/carlosframework/platform/internal/boxid/internal/signtest [no test files]
ok github.com/carlosframework/platform/internal/boxvolumes 0.008s
ok github.com/carlosframework/platform/internal/caps 0.012s
ok github.com/carlosframework/platform/internal/certwarm 0.002s
ok github.com/carlosframework/platform/internal/cgroup 0.694s
ok github.com/carlosframework/platform/internal/changelog 0.307s
ok github.com/carlosframework/platform/internal/clamd 0.055s
? github.com/carlosframework/platform/internal/cliflag [no test files]
ok github.com/carlosframework/platform/internal/console 393.426s
? github.com/carlosframework/platform/internal/controlname [no test files]
? github.com/carlosframework/platform/internal/controlsock [no test files]
ok github.com/carlosframework/platform/internal/dbdoor 5.883s
ok github.com/carlosframework/platform/internal/dbdoor/dbdoortest 0.009s
ok github.com/carlosframework/platform/internal/dbverify 0.005s
ok github.com/carlosframework/platform/internal/dnsrec 0.002s
ok github.com/carlosframework/platform/internal/docsite 0.004s
ok github.com/carlosframework/platform/internal/domains 2.427s
ok github.com/carlosframework/platform/internal/doorkey 0.006s
ok github.com/carlosframework/platform/internal/economics 0.003s
ok github.com/carlosframework/platform/internal/edge 26.760s
? github.com/carlosframework/platform/internal/edgeproof [no test files]
ok github.com/carlosframework/platform/internal/edgesfile 0.008s
ok github.com/carlosframework/platform/internal/fleetapi 0.132s
ok github.com/carlosframework/platform/internal/gate 1.655s
ok github.com/carlosframework/platform/internal/genselect 0.015s
ok github.com/carlosframework/platform/internal/geo 0.017s
ok github.com/carlosframework/platform/internal/geoip 0.031s
ok github.com/carlosframework/platform/internal/host 138.688s
ok github.com/carlosframework/platform/internal/hostactors 0.416s
ok github.com/carlosframework/platform/internal/hostbox 2.240s
ok github.com/carlosframework/platform/internal/hostclaim 0.513s
ok github.com/carlosframework/platform/internal/hostevidence 0.018s
ok github.com/carlosframework/platform/internal/hostexec 4.932s
ok github.com/carlosframework/platform/internal/hostfence 0.031s
ok github.com/carlosframework/platform/internal/hostfiles 10.513s
ok github.com/carlosframework/platform/internal/hostfloor 0.018s
ok github.com/carlosframework/platform/internal/hostguard 0.978s
ok github.com/carlosframework/platform/internal/hostops 1.635s
? github.com/carlosframework/platform/internal/hostops/fencerelease [no test files]
ok github.com/carlosframework/platform/internal/hostops/internal/fencerelease 0.017s
ok github.com/carlosframework/platform/internal/hostquarantine 0.019s
ok github.com/carlosframework/platform/internal/hostrelocate 0.444s
ok github.com/carlosframework/platform/internal/hostresolve 0.142s
ok github.com/carlosframework/platform/internal/hostwrite 0.019s
ok github.com/carlosframework/platform/internal/infraview 0.017s
ok github.com/carlosframework/platform/internal/instancectl 4.178s
ok github.com/carlosframework/platform/internal/instanceexec 2.760s
ok github.com/carlosframework/platform/internal/instancefs 0.018s
ok github.com/carlosframework/platform/internal/instancekey 0.021s
ok github.com/carlosframework/platform/internal/instanceupdates 0.025s
ok github.com/carlosframework/platform/internal/layout 1.564s
ok github.com/carlosframework/platform/internal/lease 9.837s
ok github.com/carlosframework/platform/internal/lease/leasetest 0.005s
ok github.com/carlosframework/platform/internal/ledger 0.011s
ok github.com/carlosframework/platform/internal/mail 0.165s
ok github.com/carlosframework/platform/internal/maintenance 0.090s
ok github.com/carlosframework/platform/internal/maintenanceexec 5.519s
ok github.com/carlosframework/platform/internal/maintenanceguard 0.082s
ok github.com/carlosframework/platform/internal/metrics 0.007s
ok github.com/carlosframework/platform/internal/moves 21.251s
ok github.com/carlosframework/platform/internal/moves/archive 0.812s
ok github.com/carlosframework/platform/internal/objstore 0.200s
ok github.com/carlosframework/platform/internal/ops 0.170s
ok github.com/carlosframework/platform/internal/park 0.210s
ok github.com/carlosframework/platform/internal/privatefile 0.006s
ok github.com/carlosframework/platform/internal/provision 16.037s
ok github.com/carlosframework/platform/internal/publicrelease 0.007s
? github.com/carlosframework/platform/internal/regionname [no test files]
ok github.com/carlosframework/platform/internal/registry 4.998s
ok github.com/carlosframework/platform/internal/relay 7.233s
ok github.com/carlosframework/platform/internal/release 20.106s
ok github.com/carlosframework/platform/internal/rollout 3.706s
ok github.com/carlosframework/platform/internal/rolloutwords 0.004s
ok github.com/carlosframework/platform/internal/rootaws 0.007s
ok github.com/carlosframework/platform/internal/rootparticipation 0.007s
ok github.com/carlosframework/platform/internal/rootstate 0.025s
ok github.com/carlosframework/platform/internal/runs 0.575s
ok github.com/carlosframework/platform/internal/s3compat 0.005s
ok github.com/carlosframework/platform/internal/safefile 0.005s
ok github.com/carlosframework/platform/internal/schedule 2.518s
ok github.com/carlosframework/platform/internal/seeds 0.015s
ok github.com/carlosframework/platform/internal/sim/hostlife 9.731s
ok github.com/carlosframework/platform/internal/sim/litestreamfacts 4.149s
ok github.com/carlosframework/platform/internal/status 0.040s
ok github.com/carlosframework/platform/internal/steering 0.006s
ok github.com/carlosframework/platform/internal/stopevidence 0.223s
ok github.com/carlosframework/platform/internal/svcrole 0.002s
ok github.com/carlosframework/platform/internal/telemetry 0.006s
ok github.com/carlosframework/platform/internal/termfence 0.004s
ok github.com/carlosframework/platform/internal/tunnelclient 0.004s
ok github.com/carlosframework/platform/internal/unixdial 0.003s
ok github.com/carlosframework/platform/internal/updatewords 0.005s
ok github.com/carlosframework/platform/internal/usage 0.004s
ok github.com/carlosframework/platform/internal/version 0.002s
ok github.com/carlosframework/platform/internal/vet 1.548s
=== cross ===
--- shard 0 of 5 ---
cross-compiling darwin/arm64
--- shard 1 of 5 ---
cross-compiling darwin/amd64
--- shard 2 of 5 ---
cross-compiling linux/arm64
linux/arm64 carlos binary: 46399650 bytes (ceiling 52000000)
--- shard 3 of 5 ---
cross-compiling windows/amd64
--- shard 4 of 5 ---
cross-compiling windows/arm64
=== minio-contract ===
minio contract: arm64, minio RELEASE.2025-09-07T16-13-09Z, mc RELEASE.2025-08-13T08-35-41Z
minio: cached, digest verified
mc: cached, digest verified
minio healthy on 127.0.0.1:20999
Added `local` successfully.
Bucket created successfully `local/contract-bucket`.
Bucket created successfully `local/denied-bucket`.
Created policy `contract-only` successfully.
Added user `scopeduser` successfully.
Attached Policies: [contract-only]
To User: scopeduser
buckets and scoped user ready
=== RUN TestDeliveredContractMinio
=== RUN TestDeliveredContractMinio/put
=== RUN TestDeliveredContractMinio/get_round-trips_the_body
=== RUN TestDeliveredContractMinio/list_shows_the_key
=== RUN TestDeliveredContractMinio/head_and_list_agree_on_the_version
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_its_own_object
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/not_by_PutObjectTagging
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/and_not_by_x-amz-tagging_on_upload
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CreateMultipartUpload
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CopyObject_with_a_REPLACE_tagging_directive
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/but_multipart_itself_is_allowed
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_copy_itself_is_allowed
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/a_POST-policy_forged_tag_with_a_matching_version_is_still_not_a_verdict
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_an_ordinary_tagged-with-something-else_upload_is_allowed
=== RUN TestDeliveredContractMinio/delete_removes_it
=== RUN TestDeliveredContractMinio/cross-bucket_denial
--- PASS: TestDeliveredContractMinio (0.02s)
--- PASS: TestDeliveredContractMinio/put (0.00s)
--- PASS: TestDeliveredContractMinio/get_round-trips_the_body (0.00s)
--- PASS: TestDeliveredContractMinio/list_shows_the_key (0.00s)
--- PASS: TestDeliveredContractMinio/head_and_list_agree_on_the_version (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/not_by_PutObjectTagging (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/and_not_by_x-amz-tagging_on_upload (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route (0.01s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CreateMultipartUpload (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CopyObject_with_a_REPLACE_tagging_directive (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/but_multipart_itself_is_allowed (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_copy_itself_is_allowed (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/a_POST-policy_forged_tag_with_a_matching_version_is_still_not_a_verdict (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_an_ordinary_tagged-with-something-else_upload_is_allowed (0.00s)
--- PASS: TestDeliveredContractMinio/delete_removes_it (0.00s)
--- PASS: TestDeliveredContractMinio/cross-bucket_denial (0.00s)
PASS
ok github.com/carlosframework/platform/internal/objstore 0.025s
=== RUN TestT4StorageProbeMinio
2026/10/15 18:46:56 moves: T4: STORE: list: operation error S3: ListObjectsV2, https response error StatusCode: 403, RequestID: 18DEC86018B7B8FB, HostID: 7c420053e0fbcd7f41653fdd92ffa2802c903d05303b1aa15adc2e8bb65d183f, api error SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided. Check your key and signing method.
--- PASS: TestT4StorageProbeMinio (0.01s)
PASS
ok github.com/carlosframework/platform/internal/host 0.022s
minio contract: PASS
diff --git a/home_region_args_test.go b/home_region_args_test.go| new file mode 100644 |
| index 000000000..bf2ab90d1 |
| --- /dev/null |
| +++ b/home_region_args_test.go |
| @@ -0,0 +1,521 @@ |
| +package main |
| + |
| +import ( |
| + "fmt" |
| + "go/ast" |
| + "go/parser" |
| + "go/token" |
| + "io/fs" |
| + "os" |
| + "path/filepath" |
| + "regexp" |
| + "sort" |
| + "strconv" |
| + "strings" |
| + "testing" |
| +) |
| + |
| +// Moves into the home region, part 2, Task 3: the static half of Review |
| +// Focus 1 (supplied regions are @home). This is a pin, not a proof. It |
| +// lists every production call site that hands the claim a region on a |
| +// move's path, with the argument expression text a reviewer saw and the |
| +// runtime witness that drives the site with a home target from clean claim |
| +// state, or "named-only" with the guard that keeps a home target away. The |
| +// proof is the runtime one (TestHomeTargetFreshStateShowsOnlyTheHomeToken |
| +// in internal/host, where a wrong spelling cannot hide). This test |
| +// classifies nothing as safe by its shape: a new site, an edited argument |
| +// or a vanished site fails until a reviewer updates the row and names a |
| +// witness. |
| + |
| +// homeRegionSite is one inventoried site. Callee is the call with the |
| +// region parameter in parentheses, the composite type with the field in |
| +// braces, or the assigned field; Arg is the argument expression as |
| +// go/printer prints it. |
| +type homeRegionSite struct { |
| + File, Func, Callee, Arg string |
| + // Witness is the runtime test that drives the site with a home target |
| + // from clean claim state, or "named-only: <the guard>". |
| + Witness string |
| + // Pending names the task that writes Witness while it is not written |
| + // yet; the test logs the row instead of requiring the test. |
| + Pending string |
| + // Note says why the row's argument is what it is, where that is not |
| + // the placement-derived @home a reviewer would expect. |
| + Note string |
| +} |
| + |
| +// The witnesses. |
| +const ( |
| + wFresh = "TestHomeTargetFreshStateShowsOnlyTheHomeToken" |
| + wBegin = "TestPlan13BeginClaimMoveHomeAndNamed" |
| + wConsoleCommt = "TestMoveRecoveryCommitIntoHome" |
| + wRevert = "TestMoveIntoHomePreFlipRevert" |
| + wAbort = "TestMoveIntoHomeAbortParksTheHomeLane" |
| + wClean = "TestMoveIntoHomeCleanRollback" |
| + wDirty = "TestMoveIntoHomeDirtyReturn" |
| + wFenced = "TestMoveIntoHomeFencedTargetRelease" |
| + wLost = "TestMoveIntoHomeDeclareLost" |
| + task4 = "Task 4" |
| + lossNote = "retained, validated intent: claimLossIntent copies TargetRegion from op.intent.to (@home, written by the start CAS) and the loss record signs it; SourceRegion is the record's From through park.PlacementRegion" |
| +) |
| + |
| +// homeRegionCallSites is the golden inventory, sorted by file, function and |
| +// callee. |
| +var homeRegionCallSites = []homeRegionSite{ |
| + {File: "internal/console/apiinstancemove.go", Func: "Server.handleAPIInstanceMoveRequest", Callee: "hostclaim.BeginMove(targetRegion)", Arg: "to", |
| + Witness: "named-only: the door refuses a home target before the plain path's BeginMove (MV-N21 today; MV-N510 under host claims, Task 5)"}, |
| + {File: "internal/host/movesabort.go", Func: "movesWorker.abortClaim", Callee: "hostclaim.Abort(targetRegion)", Arg: "park.PlacementRegion(rec.Move.To)", Witness: wAbort, Pending: task4}, |
| + {File: "internal/host/movescut.go", Func: "movesWorker.revertClaim", Callee: "hostclaim.Revert(targetRegion)", Arg: "park.PlacementRegion(rec.Move.To)", Witness: wRevert, Pending: task4}, |
| + {File: "internal/host/movesinstall.go", Func: "installRun.parking", Callee: "hostclaim.QueueResidueParks(region)", Arg: "cfg.parkRegion()", Witness: wFresh}, |
| + {File: "internal/host/movesland.go", Func: "landingRun.checking", Callee: "hostclaim.Touch(regions)", Arg: "nil", Witness: wFresh, |
| + Note: "the landing touches its own box only; the source round touched the region"}, |
| + {File: "internal/host/moveslandcommit.go", Func: "landingRun.commitClaim", Callee: "hostclaim.Commit(sourceRegion)", Arg: "park.PlacementRegion(l.rec.Move.From)", Witness: wFresh}, |
| + {File: "internal/host/moveslandcommit.go", Func: "landingRun.commitClaim", Callee: "hostclaim.Commit(targetRegion)", Arg: "cfg.parkRegion()", Witness: wFresh}, |
| + {File: "internal/host/movespass.go", Func: "stagingLeaseKey", Callee: "moves.StagingLeasePrefix(targetRegion)", Arg: "r", Witness: wFresh, |
| + Note: "r is park.PlacementRegion(targetRegion), the lane's record code"}, |
| + {File: "internal/host/movesphase1.go", Func: "movesWorker.sourceRound", Callee: "hostclaim.Touch(regions)", Arg: "[]string{park.PlacementRegion(rec.Move.To)}", Witness: wFresh}, |
| + {File: "internal/host/movesrecover.go", Func: "recoverRun.probed", Callee: "hostclaim.QueueReturnLanePark(sourceRegion)", Arg: "cfg.parkRegion()", Witness: wDirty, Pending: task4, |
| + Note: "runs on the named source; its region is the source's"}, |
| + {File: "internal/host/movesrollback.go", Func: "rollbackRun.claimReleased", Callee: "hostclaim.TargetReleased(targetRegion)", Arg: "cfg.parkRegion()", Witness: wClean, Pending: task4}, |
| + {File: "internal/host/movesrollback.go", Func: "rollbackRun.remote", Callee: "hostclaim.QueueStagingPark(region)", Arg: "cfg.parkRegion()", Witness: wClean, Pending: task4}, |
| + {File: "internal/hostops/move_enrollment.go", Func: "BeginClaimMove", Callee: "hostclaim.BeginAdmittedMove(targetRegion)", Arg: "park.PlacementRegion(to)", Witness: wBegin}, |
| + {File: "internal/hostops/move_fence_release.go", Func: "MoveRecoveryGrant.releaseFencedTarget", Callee: "hostclaim.FencedTargetRelease{SourceRegion}", Arg: "park.PlacementRegion(m.From)", Witness: wFenced, Pending: task4}, |
| + {File: "internal/hostops/move_fence_release.go", Func: "MoveRecoveryGrant.releaseFencedTarget", Callee: "hostclaim.FencedTargetRelease{TargetRegion}", Arg: "park.PlacementRegion(m.To)", Witness: wFenced, Pending: task4}, |
| + {File: "internal/hostops/move_fence_release.go", Func: "MoveRecoveryGrant.releaseFencedTarget", Callee: "hostclaim.ReleaseFencedTarget(r)", Arg: "r", Witness: wFenced, Pending: task4}, |
| + {File: "internal/hostops/move_loss_grant.go", Func: "lossScopedStore.claim", Callee: "hostclaim.ReleaseLostTarget(r)", Arg: "rel", Witness: wLost, Pending: task4, |
| + Note: "rel is moveLossRelease over the intent read back from the signed loss record (lossIntentOf), not a new derivation"}, |
| + {File: "internal/hostops/move_loss_owner.go", Func: "claimLossIntent", Callee: "moveLossIntent{TargetRegion}", Arg: `op.Intent["to"]`, Witness: wLost, Pending: task4, Note: lossNote}, |
| + {File: "internal/hostops/move_loss_owner.go", Func: "moveLossOwner.FinishLoss", Callee: "hostclaim.ReleaseLostTarget(r)", Arg: "moveLossRelease(in, durable, c.Holder, loss.Host)", Witness: wLost, Pending: task4}, |
| + {File: "internal/hostops/move_loss_owner.go", Func: "moveLossRelease", Callee: "hostclaim.LostTargetRelease{SourceRegion}", Arg: "in.SourceRegion", Witness: wLost, Pending: task4, Note: lossNote}, |
| + {File: "internal/hostops/move_loss_owner.go", Func: "moveLossRelease", Callee: "hostclaim.LostTargetRelease{TargetRegion}", Arg: "in.TargetRegion", Witness: wLost, Pending: task4, Note: lossNote}, |
| + {File: "internal/hostops/move_loss_owner.go", Func: "recordLossIntent", Callee: "in.SourceRegion =", Arg: "park.PlacementRegion(m.From)", Witness: wLost, Pending: task4, Note: lossNote}, |
| + {File: "internal/hostops/move_recovery_grant.go", Func: "MoveRecoveryGrant.Claim", Callee: "hostclaim.Commit(sourceRegion)", Arg: "park.PlacementRegion(m.From)", Witness: wConsoleCommt}, |
| + {File: "internal/hostops/move_recovery_grant.go", Func: "MoveRecoveryGrant.Claim", Callee: "hostclaim.Commit(targetRegion)", Arg: "park.PlacementRegion(m.To)", Witness: wConsoleCommt}, |
| +} |
| + |
| +// homeRegionCallees are the claim transitions and the lease key that take a |
| +// region, by package, with the index and name of each region parameter. |
| +var homeRegionCallees = map[string]map[string][]homeRegionParam{ |
| + "internal/hostclaim": { |
| + "Touch": {{1, "regions"}}, |
| + "Abort": {{1, "targetRegion"}}, |
| + "Revert": {{2, "targetRegion"}}, |
| + "Commit": {{1, "sourceRegion"}, {3, "targetRegion"}}, |
| + "QueueLocationPark": {{1, "region"}}, |
| + "QueueStagingPark": {{1, "region"}}, |
| + "QueueResidueParks": {{1, "region"}}, |
| + "QueueReturnLanePark": {{1, "sourceRegion"}}, |
| + "TargetReleased": {{2, "targetRegion"}}, |
| + "BeginMove": {{4, "targetRegion"}}, |
| + "BeginAdmittedMove": {{4, "targetRegion"}}, |
| + "ReleaseFencedTarget": {{0, "r"}}, |
| + "ReleaseLostTarget": {{0, "r"}}, |
| + }, |
| + "internal/moves": { |
| + "StagingLeasePrefix": {{0, "targetRegion"}}, |
| + }, |
| +} |
| + |
| +type homeRegionParam struct { |
| + index int |
| + name string |
| +} |
| + |
| +// homeRegionLiterals are the composite types whose region fields are |
| +// inventoried: the claim's two release literals, and the loss owner's own |
| +// intent (claimLossIntent builds it). The empty package is a type of the |
| +// scanned file's own package. |
| +var homeRegionLiterals = map[string]map[string]bool{ |
| + "internal/hostclaim": {"FencedTargetRelease": true, "LostTargetRelease": true}, |
| + "": {"moveLossIntent": true}, |
| +} |
| + |
| +// homeRegionFields are those literals' region fields. An assignment to one |
| +// (recordLossIntent's in.SourceRegion) is a site too. |
| +var homeRegionFields = map[string]bool{"TargetRegion": true, "SourceRegion": true} |
| + |
| +var homeRegionDirs = []string{"internal/host", "internal/hostops", "internal/console"} |
| + |
| +const homeRegionModule = "github.com/carlosframework/platform/" |
| + |
| +// scanHomeRegionSites lists the sites in every non-test Go file under the |
| +// scanned directories. overlay replaces a file's source by its |
| +// slash-separated path from root ("" removes the file) or adds a file. |
| +func scanHomeRegionSites(root string, overlay map[string]string) ([]homeRegionSite, error) { |
| + files := map[string]bool{} |
| + for _, dir := range homeRegionDirs { |
| + err := filepath.WalkDir(filepath.Join(root, dir), func(path string, d fs.DirEntry, err error) error { |
| + if err != nil { |
| + return err |
| + } |
| + if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { |
| + return nil |
| + } |
| + rel, err := filepath.Rel(root, path) |
| + if err != nil { |
| + return err |
| + } |
| + files[filepath.ToSlash(rel)] = true |
| + return nil |
| + }) |
| + if err != nil { |
| + return nil, err |
| + } |
| + } |
| + for path := range overlay { |
| + files[path] = true |
| + } |
| + var names []string |
| + for f := range files { |
| + names = append(names, f) |
| + } |
| + sort.Strings(names) |
| + var out []homeRegionSite |
| + for _, name := range names { |
| + var src any |
| + if body, ok := overlay[name]; ok { |
| + if body == "" { |
| + continue |
| + } |
| + src = body |
| + } |
| + f, err := parser.ParseFile(token.NewFileSet(), filepath.Join(root, filepath.FromSlash(name)), src, parser.SkipObjectResolution) |
| + if err != nil { |
| + return nil, err |
| + } |
| + out = append(out, homeRegionFileSites(name, f)...) |
| + } |
| + return out, nil |
| +} |
| + |
| +// homeRegionFileSites lists one file's sites, each under the function (or |
| +// method, by receiver type) that encloses it; a function literal's sites |
| +// are its enclosing declaration's. |
| +func homeRegionFileSites(name string, f *ast.File) []homeRegionSite { |
| + // The file's local names for the scanned packages. |
| + local := map[string]string{} |
| + for _, imp := range f.Imports { |
| + path, _ := strconv.Unquote(imp.Path.Value) |
| + rel, ok := strings.CutPrefix(path, homeRegionModule) |
| + if !ok { |
| + continue |
| + } |
| + if homeRegionCallees[rel] == nil && homeRegionLiterals[rel] == nil { |
| + continue |
| + } |
| + n := filepath.Base(rel) |
| + if imp.Name != nil { |
| + n = imp.Name.Name |
| + } |
| + local[n] = rel |
| + } |
| + resolve := func(e ast.Expr) (rel, sel string, ok bool) { |
| + switch x := e.(type) { |
| + case *ast.SelectorExpr: |
| + id, isID := x.X.(*ast.Ident) |
| + if !isID { |
| + return "", "", false |
| + } |
| + rel, ok = local[id.Name] |
| + return rel, x.Sel.Name, ok |
| + case *ast.Ident: |
| + return "", x.Name, true |
| + } |
| + return "", "", false |
| + } |
| + var out []homeRegionSite |
| + visit := func(fn string, root ast.Node) { |
| + ast.Inspect(root, func(n ast.Node) bool { |
| + switch x := n.(type) { |
| + case *ast.CallExpr: |
| + rel, callee, ok := resolve(x.Fun) |
| + if !ok || rel == "" { |
| + return true |
| + } |
| + for _, p := range homeRegionCallees[rel][callee] { |
| + arg := "<missing>" |
| + if p.index < len(x.Args) { |
| + arg = plan13Expr(x.Args[p.index]) |
| + } |
| + out = append(out, homeRegionSite{File: name, Func: fn, Callee: filepath.Base(rel) + "." + callee + "(" + p.name + ")", Arg: arg}) |
| + } |
| + case *ast.CompositeLit: |
| + rel, typ, ok := resolve(x.Type) |
| + if !ok || !homeRegionLiterals[rel][typ] { |
| + return true |
| + } |
| + if rel != "" { |
| + typ = filepath.Base(rel) + "." + typ |
| + } |
| + for _, el := range x.Elts { |
| + kv, isKV := el.(*ast.KeyValueExpr) |
| + if !isKV { |
| + continue |
| + } |
| + if key, isID := kv.Key.(*ast.Ident); isID && homeRegionFields[key.Name] { |
| + out = append(out, homeRegionSite{File: name, Func: fn, Callee: typ + "{" + key.Name + "}", Arg: plan13Expr(kv.Value)}) |
| + } |
| + } |
| + case *ast.AssignStmt: |
| + for i, lhs := range x.Lhs { |
| + sel, isSel := lhs.(*ast.SelectorExpr) |
| + if !isSel || !homeRegionFields[sel.Sel.Name] { |
| + continue |
| + } |
| + arg := "<multi-value>" |
| + if len(x.Rhs) == len(x.Lhs) { |
| + arg = plan13Expr(x.Rhs[i]) |
| + } |
| + out = append(out, homeRegionSite{File: name, Func: fn, Callee: plan13Expr(sel) + " =", Arg: arg}) |
| + } |
| + } |
| + return true |
| + }) |
| + } |
| + for _, d := range f.Decls { |
| + switch x := d.(type) { |
| + case *ast.FuncDecl: |
| + fn := x.Name.Name |
| + if x.Recv != nil && len(x.Recv.List) == 1 { |
| + t := x.Recv.List[0].Type |
| + if star, ok := t.(*ast.StarExpr); ok { |
| + t = star.X |
| + } |
| + if ix, ok := t.(*ast.IndexExpr); ok { |
| + t = ix.X |
| + } |
| + fn = plan13Expr(t) + "." + fn |
| + } |
| + if x.Body != nil { |
| + visit(fn, x.Body) |
| + } |
| + case *ast.GenDecl: |
| + visit("<package>", x) |
| + } |
| + } |
| + return out |
| +} |
| + |
| +// homeRegionDiff compares the sites found with the inventory. A site is |
| +// keyed by its file, function and callee; within one key the argument |
| +// texts are matched as a multiset. What is left over is a changed site |
| +// (one found and one row unmatched under the same key), an unknown site (a |
| +// found site with no row left), or a stale row (a row with no site left). |
| +func homeRegionDiff(found, inventory []homeRegionSite) []string { |
| + type key struct{ file, fn, callee string } |
| + want := map[key][]string{} |
| + for _, r := range inventory { |
| + k := key{r.File, r.Func, r.Callee} |
| + want[k] = append(want[k], r.Arg) |
| + } |
| + got := map[key][]string{} |
| + for _, s := range found { |
| + k := key{s.File, s.Func, s.Callee} |
| + got[k] = append(got[k], s.Arg) |
| + } |
| + keys := map[key]bool{} |
| + for k := range want { |
| + keys[k] = true |
| + } |
| + for k := range got { |
| + keys[k] = true |
| + } |
| + const fix = "; add or update its row in homeRegionCallSites (home_region_args_test.go) and name the runtime witness that drives it with a home target from clean claim state" |
| + var out []string |
| + for k := range keys { |
| + rows := append([]string(nil), want[k]...) |
| + var sites []string |
| + for _, a := range got[k] { |
| + if i := indexOf(rows, a); i >= 0 { |
| + rows = append(rows[:i], rows[i+1:]...) |
| + continue |
| + } |
| + sites = append(sites, a) |
| + } |
| + n := min(len(rows), len(sites)) |
| + for i := 0; i < n; i++ { |
| + out = append(out, fmt.Sprintf("changed site: %s %s %s passes %s, its row says %s%s", k.file, k.fn, k.callee, sites[i], rows[i], fix)) |
| + } |
| + for _, a := range sites[n:] { |
| + out = append(out, fmt.Sprintf("unknown site: %s %s %s passes %s%s", k.file, k.fn, k.callee, a, fix)) |
| + } |
| + for _, a := range rows[n:] { |
| + out = append(out, fmt.Sprintf("stale row: %s %s %s passing %s is no longer there%s", k.file, k.fn, k.callee, a, fix)) |
| + } |
| + } |
| + sort.Strings(out) |
| + return out |
| +} |
| + |
| +func indexOf(xs []string, x string) int { |
| + for i, s := range xs { |
| + if s == x { |
| + return i |
| + } |
| + } |
| + return -1 |
| +} |
| + |
| +// homeRegionTestNames is every top-level test function name in the |
| +// repository's test files. |
| +func homeRegionTestNames(t *testing.T) map[string]bool { |
| + t.Helper() |
| + re := regexp.MustCompile(`(?m)^func (Test\w+)\(t \*testing\.T\)`) |
| + out := map[string]bool{} |
| + err := filepath.WalkDir(".", func(path string, d fs.DirEntry, err error) error { |
| + if err != nil { |
| + return err |
| + } |
| + if d.IsDir() { |
| + if path != "." && (strings.HasPrefix(d.Name(), ".") || d.Name() == "node_modules" || d.Name() == "testdata") { |
| + return filepath.SkipDir |
| + } |
| + return nil |
| + } |
| + if !strings.HasSuffix(path, "_test.go") { |
| + return nil |
| + } |
| + body, err := os.ReadFile(path) |
| + if err != nil { |
| + return err |
| + } |
| + for _, m := range re.FindAllStringSubmatch(string(body), -1) { |
| + out[m[1]] = true |
| + } |
| + return nil |
| + }) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + return out |
| +} |
| + |
| +// homeRegionOverlay is path's source with each old, new pair applied: old |
| +// must be there exactly once. |
| +func homeRegionOverlay(t *testing.T, path string, pairs ...string) map[string]string { |
| + t.Helper() |
| + raw, err := os.ReadFile(filepath.FromSlash(path)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + body := string(raw) |
| + for i := 0; i+1 < len(pairs); i += 2 { |
| + if n := strings.Count(body, pairs[i]); n != 1 { |
| + t.Fatalf("fixture: %s holds %q %d times, want once", path, pairs[i], n) |
| + } |
| + body = strings.Replace(body, pairs[i], pairs[i+1], 1) |
| + } |
| + return map[string]string{path: body} |
| +} |
| + |
| +// homeRegionWant fails unless problems hold one starting with kind and |
| +// containing every part. |
| +func homeRegionWant(t *testing.T, problems []string, kind string, parts ...string) { |
| + t.Helper() |
| + for _, p := range problems { |
| + if !strings.HasPrefix(p, kind) { |
| + continue |
| + } |
| + all := true |
| + for _, part := range parts { |
| + all = all && strings.Contains(p, part) |
| + } |
| + if all { |
| + return |
| + } |
| + } |
| + t.Fatalf("no %q naming %q among %q", kind, parts, problems) |
| +} |
| + |
| +func TestMovesHomeRegionCallSiteInventory(t *testing.T) { |
| + found, err := scanHomeRegionSites(".", nil) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + t.Run("the tree matches the inventory", func(t *testing.T) { |
| + if problems := homeRegionDiff(found, homeRegionCallSites); len(problems) != 0 { |
| + for _, s := range found { |
| + t.Logf("found {File: %q, Func: %q, Callee: %q, Arg: %q}", s.File, s.Func, s.Callee, s.Arg) |
| + } |
| + for _, p := range problems { |
| + t.Error(p) |
| + } |
| + } |
| + }) |
| + |
| + t.Run("every row names a witness", func(t *testing.T) { |
| + tests := homeRegionTestNames(t) |
| + for _, r := range homeRegionCallSites { |
| + switch { |
| + case strings.HasPrefix(r.Witness, "named-only: ") && len(r.Witness) > len("named-only: "): |
| + case r.Witness == "": |
| + t.Errorf("%s %s %s names no witness", r.File, r.Func, r.Callee) |
| + case r.Pending != "": |
| + if tests[r.Witness] { |
| + t.Errorf("%s %s %s: %s is written; clear the row's Pending (%s)", r.File, r.Func, r.Callee, r.Witness, r.Pending) |
| + } |
| + t.Logf("pending (%s): %s %s %s, witness %s", r.Pending, r.File, r.Func, r.Callee, r.Witness) |
| + case !tests[r.Witness]: |
| + t.Errorf("%s %s %s: the witness %s is not a test in this repository", r.File, r.Func, r.Callee, r.Witness) |
| + } |
| + } |
| + }) |
| + |
| + // The fixtures: each edit the inventory exists to catch. |
| + t.Run("a new Commit site supplying a roster region is unknown", func(t *testing.T) { |
| + src := "package host\n\nimport \"github.com/carlosframework/platform/internal/hostclaim\"\n\n" + |
| + "func homeRegionFixture(rr regionRow) error {\n\treturn hostclaim.Commit(id, from, src, rr.row.Region, scope, staging, newID)(c)\n}\n" |
| + found, err := scanHomeRegionSites(".", map[string]string{"internal/host/zz_home_region_fixture.go": src}) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + homeRegionWant(t, homeRegionDiff(found, homeRegionCallSites), "unknown site", "internal/host/zz_home_region_fixture.go", "hostclaim.Commit(targetRegion)", "rr.row.Region") |
| + }) |
| + t.Run("home_region handed to QueueStagingPark at a new site is unknown", func(t *testing.T) { |
| + src := "package host\n\nimport (\n\t\"github.com/carlosframework/platform/internal/hostclaim\"\n\t\"github.com/carlosframework/platform/internal/park\"\n)\n\n" + |
| + "func homeRegionFixture(c *hostclaim.Claim) error {\n\tto := c.Op.Intent[\"home_region\"]\n\treturn hostclaim.QueueStagingPark(id, park.PlacementRegion(to), scope, newID)(c)\n}\n" |
| + found, err := scanHomeRegionSites(".", map[string]string{"internal/host/zz_home_region_fixture.go": src}) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + homeRegionWant(t, homeRegionDiff(found, homeRegionCallSites), "unknown site", "hostclaim.QueueStagingPark(region)", "park.PlacementRegion(to)") |
| + }) |
| + t.Run("home_region handed to QueueStagingPark at an inventoried site is changed", func(t *testing.T) { |
| + overlay := homeRegionOverlay(t, "internal/host/movesrollback.go", |
| + "scope := moves.StagingScope(host, id, obs.Claim.Holder.Incarnation, staged.Bound())", |
| + "scope := moves.StagingScope(host, id, obs.Claim.Holder.Incarnation, staged.Bound())\n\t\tto := obs.Claim.Op.Intent[\"home_region\"]", |
| + "hostclaim.QueueStagingPark(id, cfg.parkRegion(), scope, obligationID)", |
| + "hostclaim.QueueStagingPark(id, park.PlacementRegion(to), scope, obligationID)") |
| + found, err := scanHomeRegionSites(".", overlay) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + homeRegionWant(t, homeRegionDiff(found, homeRegionCallSites), "changed site", "rollbackRun.remote", "hostclaim.QueueStagingPark(region)", "park.PlacementRegion(to)") |
| + }) |
| + t.Run("a FencedTargetRelease naming home_region is changed", func(t *testing.T) { |
| + found, err := scanHomeRegionSites(".", homeRegionOverlay(t, "internal/hostops/move_fence_release.go", "TargetRegion: park.PlacementRegion(m.To)", |
| + `TargetRegion: o.Claim.Op.Intent["home_region"]`)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + homeRegionWant(t, homeRegionDiff(found, homeRegionCallSites), "changed site", "hostclaim.FencedTargetRelease{TargetRegion}", `o.Claim.Op.Intent["home_region"]`) |
| + }) |
| + t.Run("a site that no longer exists leaves a stale row", func(t *testing.T) { |
| + found, err := scanHomeRegionSites(".", homeRegionOverlay(t, "internal/host/movespass.go", "return moves.StagingLeasePrefix(r, host), nil", |
| + `return "control/staging-leases/" + r + "/" + host, nil`)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + homeRegionWant(t, homeRegionDiff(found, homeRegionCallSites), "stale row", "stagingLeaseKey", "moves.StagingLeasePrefix(targetRegion)") |
| + }) |
| + t.Run("an aliased import is still found", func(t *testing.T) { |
| + src := "package host\n\nimport hc \"github.com/carlosframework/platform/internal/hostclaim\"\n\n" + |
| + "func homeRegionFixture() error {\n\treturn hc.Touch(id, []string{\"ie\"}, nil)(c)\n}\n" |
| + found, err := scanHomeRegionSites(".", map[string]string{"internal/console/zz_home_region_fixture.go": src}) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + homeRegionWant(t, homeRegionDiff(found, homeRegionCallSites), "unknown site", "hostclaim.Touch(regions)", `[]string{"ie"}`) |
| + }) |
| +} |
diff --git a/internal/host/movesfinal_test.go b/internal/host/movesfinal_test.go| index 9f0f80d3f..a5e77767c 100644 |
| --- a/internal/host/movesfinal_test.go |
| +++ b/internal/host/movesfinal_test.go |
| @@ -53,9 +53,16 @@ func newFinalHarness(t *testing.T, opts ...p1Option) *finalHarness { |
| // newFinalHarnessFrom is newFinalHarness over a landing harness the caller |
| // prepared before the flip. |
| func newFinalHarnessFrom(t *testing.T, lh *landingHarness) *finalHarness { |
| + t.Helper() |
| + lh.flip(t) |
| + return newFinalHarnessFlipped(t, lh) |
| +} |
| + |
| +// newFinalHarnessFlipped is newFinalHarnessFrom over a landing harness the |
| +// caller already flipped (and may have changed after the flip). |
| +func newFinalHarnessFlipped(t *testing.T, lh *landingHarness) *finalHarness { |
| t.Helper() |
| h := &finalHarness{landingHarness: lh} |
| - h.flip(t) |
| h.land(t) |
| h.installed(t) |
| reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
diff --git a/internal/host/moveshome_target_test.go b/internal/host/moveshome_target_test.go| index 013f9b773..1500c0f7b 100644 |
| --- a/internal/host/moveshome_target_test.go |
| +++ b/internal/host/moveshome_target_test.go |
| @@ -2,13 +2,31 @@ package host |
| import ( |
| "context" |
| + "encoding/json" |
| + "errors" |
| + "fmt" |
| + "os" |
| + "path/filepath" |
| + "sort" |
| + "strings" |
| + "sync" |
| "testing" |
| + "time" |
| "github.com/carlosframework/platform/internal/activator" |
| + "github.com/carlosframework/platform/internal/genselect" |
| + "github.com/carlosframework/platform/internal/hostactors" |
| + "github.com/carlosframework/platform/internal/hostbox" |
| "github.com/carlosframework/platform/internal/hostclaim" |
| + "github.com/carlosframework/platform/internal/hostfiles" |
| + "github.com/carlosframework/platform/internal/hostops" |
| + "github.com/carlosframework/platform/internal/instancekey" |
| + "github.com/carlosframework/platform/internal/lease" |
| "github.com/carlosframework/platform/internal/moves" |
| "github.com/carlosframework/platform/internal/park" |
| "github.com/carlosframework/platform/internal/provision" |
| + "github.com/carlosframework/platform/internal/release" |
| + "github.com/superfly/ltx" |
| ) |
| // startIntoHomeT3 writes the record and claim for a move from the harness's |
| @@ -108,3 +126,1070 @@ func TestHomeTargetT3RefusesABoxThatSetsARegion(t *testing.T) { |
| t.Fatalf("T3 = %+v, want %q", t3, want) |
| } |
| } |
| + |
| +// Home part 2, Task 3: the commit into home, through the production owners. |
| +// Every witness below runs the move ca -> home on the intoHome fixture: the |
| +// source ca-box in ca, the target home-box whose roster name is ie, whose |
| +// placement code is "" and whose engine region for the move is @home. |
| + |
| +// claimWatch is the harness's control store with a recorder on the host |
| +// claim: every claim a transition writes is decoded and kept, so a chore |
| +// the run created is seen even after an acknowledgement removed it, and so |
| +// are every touched region and box and every quiet_after key the claim ever |
| +// held. It embeds the DirStore so the store's optional interfaces stay. |
| +type claimWatch struct { |
| + *release.DirStore |
| + mu sync.Mutex |
| + writes []hostclaim.Claim |
| +} |
| + |
| +func (w *claimWatch) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { |
| + etag, err := w.DirStore.Put(ctx, key, body, ifMatch) |
| + if err == nil && key == "control/hosts/"+p1Host+".json" { |
| + var c hostclaim.Claim |
| + if jerr := json.Unmarshal(body, &c); jerr == nil { |
| + w.mu.Lock() |
| + w.writes = append(w.writes, c) |
| + w.mu.Unlock() |
| + } |
| + } |
| + return etag, err |
| +} |
| + |
| +// since is every claim written after the n-th write. |
| +func (w *claimWatch) since(n int) []hostclaim.Claim { |
| + w.mu.Lock() |
| + defer w.mu.Unlock() |
| + return append([]hostclaim.Claim(nil), w.writes[n:]...) |
| +} |
| + |
| +// watchClaims puts a claimWatch in front of the harness's store before |
| +// anything captures it. |
| +func watchClaims(out **claimWatch) p1Option { |
| + return func(h *phase1Harness) { |
| + w := &claimWatch{DirStore: h.s.(*release.DirStore)} |
| + h.s = w |
| + *out = w |
| + } |
| +} |
| + |
| +// stagingLeaseKeys records the key of every staging lease the harness's |
| +// boxes take, as the production hook keys it (stagingLeaseKey over the |
| +// record code the round passes). |
| +type stagingLeaseKeys struct { |
| + mu sync.Mutex |
| + keys []string |
| +} |
| + |
| +func (k *stagingLeaseKeys) wrap(c *moveCfg) { |
| + inner := c.StagingLease |
| + c.StagingLease = func(ctx context.Context, code, host string) (moves.SafeUntilGuard, func(context.Context) error, error) { |
| + key, err := stagingLeaseKey(code, host) |
| + if err != nil { |
| + return nil, nil, err |
| + } |
| + k.mu.Lock() |
| + k.keys = append(k.keys, key) |
| + k.mu.Unlock() |
| + return inner(ctx, code, host) |
| + } |
| +} |
| + |
| +func (k *stagingLeaseKeys) all() []string { |
| + k.mu.Lock() |
| + defer k.mu.Unlock() |
| + return append([]string(nil), k.keys...) |
| +} |
| + |
| +// homeLeaseKey is the home lane's staging lease for the harness's host. |
| +var homeLeaseKey = "control/staging-leases/" + park.HomeRegion + "/" + p1Host |
| + |
| +// onlyTheHomeToken is TestHomeTargetFreshStateShowsOnlyTheHomeToken's |
| +// assertion over a witness that ran from clean claim state (the start CAS |
| +// wrote intent.to @home and home_region ie, and nothing else names home): |
| +// every chore created for the target side names @home and is owned by |
| +// region:@home, touched only ever gained @home and the target box beside |
| +// the source's, no quiet_after key is ie or "", and every staging lease |
| +// taken for the home lane is keyed @home. A supplied ie would show here as |
| +// a chore, a touched region, a quiet key or a lease key of its own. |
| +func onlyTheHomeToken(t *testing.T, writes []hostclaim.Claim, leases []string, src, srcBox string) { |
| + t.Helper() |
| + if len(writes) == 0 { |
| + t.Fatal("fixture: the witness wrote no claim") |
| + } |
| + seen := map[string]bool{} |
| + for _, c := range writes { |
| + for _, o := range c.Obligations { |
| + if seen[o.ID] { |
| + continue |
| + } |
| + seen[o.ID] = true |
| + switch { |
| + case o.Kind == "park-local": |
| + if o.Owner != "box:"+o.Box || o.Region != "" { |
| + t.Fatalf("a park-local chore %+v", o) |
| + } |
| + case o.Region == src: |
| + if o.Owner != "region:"+src { |
| + t.Fatalf("a source-side chore %+v", o) |
| + } |
| + case o.Region != park.HomeRegion || o.Owner != "region:"+park.HomeRegion: |
| + t.Fatalf("a target-side chore not named %s: %+v", park.HomeRegion, o) |
| + } |
| + } |
| + for _, r := range c.Touched.Regions { |
| + if r != src && r != park.HomeRegion { |
| + t.Fatalf("touched gained %q: %+v", r, c.Touched) |
| + } |
| + } |
| + for _, b := range c.Touched.Boxes { |
| + if b != srcBox && b != "home-box" { |
| + t.Fatalf("touched gained box %q: %+v", b, c.Touched) |
| + } |
| + } |
| + for k := range c.QuietAfter { |
| + if k != src && k != park.HomeRegion { |
| + t.Fatalf("quiet_after gained %q: %v", k, c.QuietAfter) |
| + } |
| + } |
| + if c.Op.Intent != nil && c.Op.Intent["to"] != "" && c.Op.Intent["to"] != park.HomeRegion { |
| + t.Fatalf("the op's to is %q", c.Op.Intent["to"]) |
| + } |
| + } |
| + for _, k := range leases { |
| + if k != homeLeaseKey && k != "control/staging-leases/"+src+"/"+p1Host { |
| + t.Fatalf("a staging lease keyed %q", k) |
| + } |
| + } |
| +} |
| + |
| +// laneKeysIn lists the keys under the move's staging lane in bucket. |
| +func laneKeysIn(h *phase1Harness, bucket, id string) []string { |
| + prefix := moves.StagingPrefix(p1Host, id) |
| + var keys []string |
| + for k := range h.f.buckets[bucket] { |
| + if rest, ok := strings.CutPrefix(k, prefix); ok { |
| + keys = append(keys, rest) |
| + } |
| + } |
| + sort.Strings(keys) |
| + return keys |
| +} |
| + |
| +// TestMoveIntoHomeStagesTheHomeLane: the source round on ca-box stages the |
| +// lane into the home box's bucket, data-home, under moves/<host>/<id>/, |
| +// under the staging lease of the home lane and no other. Touch is a union: |
| +// from a claim that names no home it adds @home and home-box; with ie and |
| +// home-box already there it writes nothing; with ie and no home-box it adds |
| +// @home and home-box and keeps ie. The only region it ever adds is @home. |
| +func TestMoveIntoHomeStagesTheHomeLane(t *testing.T) { |
| + type tc struct { |
| + name string |
| + opts []p1Option |
| + seedBox bool |
| + writes int |
| + } |
| + for _, c := range []tc{ |
| + {name: "touched names no home", writes: 1}, |
| + {name: "touched holds ie and home-box", opts: []p1Option{retainTouched("ie")}, seedBox: true, writes: 0}, |
| + {name: "touched holds ie without home-box", opts: []p1Option{retainTouched("ie")}, writes: 1}, |
| + } { |
| + t.Run(c.name, func(t *testing.T) { |
| + var watch *claimWatch |
| + h := newPhase1Harness(t, append([]p1Option{intoHome, watchClaims(&watch)}, c.opts...)...) |
| + leases := &stagingLeaseKeys{} |
| + leases.wrap(&h.cfg) |
| + h.start(t, "manual") |
| + if c.seedBox { |
| + rewriteP1Claim(t, h.s, func(cl *hostclaim.Claim) { cl.Touched.Boxes = append(cl.Touched.Boxes, "home-box") }) |
| + } |
| + before := h.claim(t) |
| + beforeTouched, _ := json.Marshal(before.Touched) |
| + var atFirst *hostclaim.Claim |
| + h.f.failAt = func(op, key string) error { |
| + if atFirst == nil { |
| + atFirst = h.claim(t) |
| + } |
| + return nil |
| + } |
| + h.pass(t) |
| + if atFirst == nil { |
| + t.Fatal("no staging write happened") |
| + } |
| + if d := int(atFirst.Gen - before.Gen); d != c.writes { |
| + t.Fatalf("%d claim writes before the first staging write, want %d (touched %+v -> %+v)", d, c.writes, before.Touched, atFirst.Touched) |
| + } |
| + after := h.claim(t) |
| + for _, r := range before.Touched.Regions { |
| + if !containsString(after.Touched.Regions, r) { |
| + t.Fatalf("touched lost %q: %+v -> %+v", r, before.Touched, after.Touched) |
| + } |
| + } |
| + for _, b := range before.Touched.Boxes { |
| + if !containsString(after.Touched.Boxes, b) { |
| + t.Fatalf("touched lost box %q: %+v -> %+v", b, before.Touched, after.Touched) |
| + } |
| + } |
| + for _, r := range after.Touched.Regions { |
| + if !containsString(before.Touched.Regions, r) && r != park.HomeRegion { |
| + t.Fatalf("touched gained %q: %+v -> %+v", r, before.Touched, after.Touched) |
| + } |
| + } |
| + if !containsString(after.Touched.Boxes, "home-box") { |
| + t.Fatalf("touched has no home-box: %+v", after.Touched) |
| + } |
| + if c.writes == 0 { |
| + if afterTouched, _ := json.Marshal(after.Touched); string(afterTouched) != string(beforeTouched) { |
| + t.Fatalf("touched was rewritten: %s -> %s", beforeTouched, afterTouched) |
| + } |
| + } else if !containsString(after.Touched.Regions, park.HomeRegion) { |
| + t.Fatalf("the Touch did not add %s: %+v", park.HomeRegion, after.Touched) |
| + } |
| + // The lane is in data-home under moves/<host>/<id>/, and every |
| + // staging request went there. |
| + keys := laneKeysIn(h, "data-home", p1ID) |
| + for _, k := range []string{"move.json", "replica/0000/0000000000000001-0000000000000002.ltx", "replica/0001/0000000000000001-0000000000000002.ltx"} { |
| + if !containsString(keys, k) { |
| + t.Fatalf("the home lane lacks %s: %v", k, keys) |
| + } |
| + } |
| + for _, call := range h.f.stagingCalls() { |
| + if call.Bucket != "data-home" || !strings.HasPrefix(call.Key, moves.StagingPrefix(p1Host, p1ID)) { |
| + t.Fatalf("a staging request outside the home lane: %+v", call) |
| + } |
| + } |
| + // The lease of the home lane, and no other. |
| + got := leases.all() |
| + if len(got) == 0 { |
| + t.Fatal("the round took no staging lease") |
| + } |
| + for _, k := range got { |
| + if k != homeLeaseKey { |
| + t.Fatalf("a staging lease keyed %q, want %q", k, homeLeaseKey) |
| + } |
| + } |
| + if c.name == "touched names no home" { |
| + onlyTheHomeToken(t, watch.since(0), got, "ca", "ca-box") |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// TestMoveIntoHomeTargetChecksPass: on home-box, every readiness check is |
| +// ok over the production composition (the home box sets no region). T8 |
| +// names the home by its name, ie, when it fails; T10 probes the source's |
| +// bucket, data-ca; T12 refuses inside a quiet window recorded under ie, and |
| +// again under @home, naming ie. |
| +func TestMoveIntoHomeTargetChecksPass(t *testing.T) { |
| + newHarness := func(t *testing.T) (*phase1Harness, moveCfg, *[]string) { |
| + h := newPhase1Harness(t, intoHome, realLaunchEnv) |
| + h.start(t, "manual") |
| + h.pass(t) |
| + tc := h.targetCfg() |
| + probed := &[]string{} |
| + tc.LaneProbe = func(_ context.Context, bucket, region, id string) error { |
| + *probed = append(*probed, bucket+"/"+region+"/"+id) |
| + return nil |
| + } |
| + return h, tc, probed |
| + } |
| + h, tc, probed := newHarness(t) |
| + h.targetPass(t, tc) |
| + r := h.targetReport(t) |
| + if r.Box != "home-box" || r.Region != "" { |
| + t.Fatalf("the report is not the home box's: box %q region %q", r.Box, r.Region) |
| + } |
| + for _, id := range []string{moves.CheckProtocol, moves.CheckSecrets, moves.CheckEnv, moves.CheckStorage, moves.CheckArchive, moves.CheckDisk, moves.CheckRelease, |
| + moves.CheckOneOwner, moves.CheckClaim, moves.CheckLane, moves.CheckBudget, moves.CheckQuiet, moves.CheckArmed} { |
| + if c, ok := r.Checks[id]; !ok || !c.OK { |
| + t.Fatalf("%s on the home box: %+v (present %v)", id, c, ok) |
| + } |
| + } |
| + if ok, why := r.Passing(); !ok { |
| + t.Fatalf("the home box's report does not pass: %s", why) |
| + } |
| + if len(*probed) == 0 { |
| + t.Fatal("T10 probed nothing") |
| + } |
| + for _, p := range *probed { |
| + if p != "data-ca/ca-central-1/"+p1ID { |
| + t.Fatalf("T10 probed %s, want the source's bucket data-ca", p) |
| + } |
| + } |
| + // T8 names the home by its name: a second placementless box makes the |
| + // home ambiguous, and the reason says ie, never @home or "". |
| + if _, err := h.s.Put(t.Context(), "control/edges/home-box-2.json", |
| + []byte(`{"v":1,"box":"home-box-2","eip":"1.2.3.10","data_bucket":"data-home-2","data_bucket_region":"eu-west-1","hosts":{}}`), ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + h.clock.Advance(time.Second) |
| + h.targetPass(t, tc) |
| + if t8 := h.targetReport(t).Checks[moves.CheckOneOwner]; t8.OK || t8.Reason != t8Count(2, "ie") { |
| + t.Fatalf("T8 with two home boxes: %+v, want %q", t8, t8Count(2, "ie")) |
| + } |
| + |
| + // T12 inside a quiet window, recorded under either spelling. |
| + for _, spelling := range []string{"ie", park.HomeRegion} { |
| + h, tc, _ := newHarness(t) |
| + until := h.clock.now.Add(time.Hour).UTC() |
| + rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) { c.QuietAfter = map[string]time.Time{spelling: until} }) |
| + h.targetPass(t, tc) |
| + if t12 := h.targetReport(t).Checks[moves.CheckQuiet]; t12.OK || t12.Reason != t12NotQuiet("ie", until) { |
| + t.Fatalf("T12 with quiet_after under %s: %+v, want %q", spelling, t12, t12NotQuiet("ie", until)) |
| + } |
| + } |
| +} |
| + |
| +// intoHomeClose is the close harness for the move ca -> home with the |
| +// production probe mode (Task 2): the real cut, flip and landing through |
| +// restored, the probe through movesProbeOwners over the claim runtime. |
| +// With residue, an earlier home stay of this incarnation is seeded in |
| +// data-home first (a root generation and one named generation, the |
| +// landing harness's residue), so the landing's parking step has work. It |
| +// returns the residue's keys. |
| +func intoHomeClose(t *testing.T, residue bool, opts ...p1Option) *intoHomeRun { |
| + t.Helper() |
| + return intoHomeCloseWith(t, residue, nil, opts...) |
| +} |
| + |
| +// intoHomeCloseWith is intoHomeClose with afterFlip run on the landing |
| +// harness between the flip and the first landing pass. |
| +func intoHomeCloseWith(t *testing.T, residue bool, afterFlip func(*landingHarness), opts ...p1Option) *intoHomeRun { |
| + t.Helper() |
| + r := &intoHomeRun{leases: &stagingLeaseKeys{}} |
| + lh := newLandingHarness(t, append([]p1Option{intoHome, realLaunchEnv, watchClaims(&r.watch)}, opts...)...) |
| + // Both boxes' configurations derive from the source's (targetCfg), so |
| + // one wrap records every staging lease the run takes. |
| + r.leases.wrap(&lh.cfg) |
| + if residue { |
| + r.residue = earlierHomeStay(t, lh) |
| + } |
| + lh.flip(t) |
| + if afterFlip != nil { |
| + afterFlip(lh) |
| + } |
| + h := newFinalHarnessFlipped(t, lh) |
| + if _, err := provision.SetRecordMove(context.Background(), h.s, "bab", "app", p1Host, func(rec *provision.Record) error { |
| + rec.Move.CutAttempts = 2 |
| + return nil |
| + }); err != nil { |
| + t.Fatal(err) |
| + } |
| + h.ownAuthorize = true |
| + r.h, r.p = h, h.productionProbe(t) |
| + // Before the probe the home box's env file gains bytes the prepared |
| + // manifest does not carry (a comment line, which changes no variable |
| + // and so not the env pin), and an adoption record sits beside it, as |
| + // TestLandingWakesEstablishThroughC8 stages them: the establishment then |
| + // has a revision to commit and a local effect to show. |
| + envPath := filepath.Join(h.tdir, p1Host+".env") |
| + before, err := os.ReadFile(envPath) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + r.env = append(append([]byte(nil), before...), []byte("# established on home-box\n")...) |
| + if err := os.WriteFile(envPath, r.env, 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + r.adoption = filepath.Join(h.tdir, p1Host+instancekey.AdoptionSuffix) |
| + if err := os.WriteFile(r.adoption, []byte(`{"v":2}`), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + return r |
| +} |
| + |
| +// intoHomeRun is one into-home run: the close harness, its production |
| +// probe, the residue planted, the env file and adoption record staged for |
| +// C8, and the recorders of the claims written and the staging leases taken. |
| +type intoHomeRun struct { |
| + h *finalHarness |
| + p *prodProbe |
| + residue []string |
| + env []byte |
| + adoption string |
| + watch *claimWatch |
| + leases *stagingLeaseKeys |
| +} |
| + |
| +// earlierHomeStay plants an earlier stay of this incarnation in data-home |
| +// at the holder's location, as the landing harness's residue does: the |
| +// root generation's level file, two level files of the named generation |
| +// old, and the earlier stay's generation.json, here a marker that names |
| +// old (the probe reads the location's current marker, and a home box that |
| +// served the instance before left a well-formed one). |
| +func earlierHomeStay(t *testing.T, h *landingHarness) []string { |
| + t.Helper() |
| + ctx := context.Background() |
| + marker, err := json.Marshal(genselect.Marker{Op: "old", Incarnation: h.inc, Host: p1Host, Box: "home-box", InstalledAt: h.clock.now.Add(-48 * time.Hour).UTC()}) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + keys := []string{ |
| + cutRoot + "/g-old/0000/0000000000000001-0000000000000001.ltx", |
| + cutRoot + "/g-old/0009/0000000000000001-0000000000000001.ltx", |
| + cutRoot + "/0000/0000000000000001-0000000000000001.ltx", |
| + cutRoot + "/generation.json", |
| + } |
| + for _, k := range keys { |
| + body := []byte("residue " + k) |
| + if strings.HasSuffix(k, "/generation.json") { |
| + body = marker |
| + } |
| + if _, err := h.vs.PutIfAbsent(ctx, "data-home", k, body); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + return keys |
| +} |
| + |
| +// TestMoveIntoHomeCommitsAndCloses: the whole move ca -> home through the |
| +// real cut, flip and landing, then the production probe owners and the |
| +// claim runtime, as TestMoveFromHomeClosesThroughTheSourcePark does for |
| +// part 1 with the fake probe. |
| +func TestMoveIntoHomeCommitsAndCloses(t *testing.T) { |
| + r := intoHomeClose(t, true) |
| + // While the attempt is live, the owners' admission evidence (the one the |
| + // writer was bound to) accepts an admission on home-box in ie against |
| + // the op's @home, and refuses the same admission in another region. |
| + checked := false |
| + r.p.afterMaterialize = func(int) error { |
| + ctx := context.Background() |
| + o, err := hostclaim.Read(ctx, r.h.s, p1Host) |
| + if err != nil { |
| + return err |
| + } |
| + in, err := r.p.owners.Select(ctx, r.h.landingHarness.route, o, poProbe) |
| + if err != nil { |
| + return err |
| + } |
| + if in.Admission.Region != "ie" || o.Claim.Op.Intent["to"] != park.HomeRegion { |
| + t.Fatalf("fixture: an admission in %q against the op's %q", in.Admission.Region, o.Claim.Op.Intent["to"]) |
| + } |
| + if err := r.p.owners.AdmissionEvidence(ctx, o, in.Admission, in.Baseline); err != nil { |
| + t.Fatalf("the evidence refused ie against %s: %v", park.HomeRegion, err) |
| + } |
| + bad := in.Admission |
| + bad.Region = "ca" |
| + if err := r.p.owners.AdmissionEvidence(ctx, o, bad, in.Baseline); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("the evidence accepted an admission in ca: %v", err) |
| + } |
| + checked = true |
| + return nil |
| + } |
| + intoHomeCommitsAndCloses(t, r, 1) |
| + if !checked { |
| + t.Fatal("the evidence was never checked") |
| + } |
| + intoHomeMovesOutAgain(t, r) |
| +} |
| + |
| +// homeRegional is the home box's regional runner as the claim assembly |
| +// builds it (claimassembly.go): hostbox.Regional over a park engine in the |
| +// box's roster region ie with the fleet's home designation ie (so it owns |
| +// @home too), on data-home, with the real moves park view seen from ie. The |
| +// claim's authorizer and the regional owner check stand in as allowing. |
| +func homeRegional(h *finalHarness) hostbox.Regional { |
| + return hostbox.Regional{Engine: park.Engine{HomeRegion: "ie", Store: h.vs, Claims: h.s, Bucket: "data-home", Region: "ie", Leases: claimParkLeases(h.leases), |
| + Now: h.clock.Now, NewID: obligationID, View: movesParkView(MovesRecordsFrom(h.s), h.vs, "data-home", "ie"), Authorize: allowParkAuthorize}, |
| + Owner: func(context.Context, hostclaim.Observed) error { return nil }} |
| +} |
| + |
| +// residueChores is the op's park-generation chores, by generation. |
| +func residueChores(c *hostclaim.Claim) map[string]hostclaim.Obligation { |
| + out := map[string]hostclaim.Obligation{} |
| + for _, o := range c.Obligations { |
| + if o.Op == p1ID && o.Kind == "park-generation" { |
| + out[o.Generation] = o |
| + } |
| + } |
| + return out |
| +} |
| + |
| +// intoHomeCommitsAndCloses drives an intoHomeClose run from restored (or |
| +// probed) through the probe, the commit and the close, asserting each |
| +// step. attempts is how many probe attempts the runtime admitted by the |
| +// commit. |
| +func intoHomeCommitsAndCloses(t *testing.T, r *intoHomeRun, attempts int) { |
| + t.Helper() |
| + ctx := context.Background() |
| + h, p, residue, env, adoption := r.h, r.p, r.residue, r.env, r.adoption |
| + // The landing's parking step queued one park-generation per residue |
| + // generation, in @home, owned by region:@home, and none for the |
| + // install's own. |
| + if residue != nil { |
| + gens := residueChores(h.claim(t)) |
| + if len(gens) != 2 { |
| + t.Fatalf("residue chores: %+v", gens) |
| + } |
| + for _, g := range []string{hostclaim.RootGeneration, "old"} { |
| + o, ok := gens[g] |
| + if !ok || o.Region != park.HomeRegion || o.Owner != "region:"+park.HomeRegion || o.Park == nil || o.Park.Location != cutRoot { |
| + t.Fatalf("the residue chore for %s: %+v", g, o) |
| + } |
| + } |
| + if _, ok := gens[p1ID]; ok { |
| + t.Fatal("the landing queued a park of its own generation") |
| + } |
| + } |
| + |
| + // The probe, the commit and the open, through the production owners. |
| + h.land(t) |
| + h.committedAndOpen(t) |
| + if len(p.admissions) != attempts { |
| + t.Fatalf("admissions through the claim runtime: %+v", p.admissions) |
| + } |
| + a := p.admissions[len(p.admissions)-1] |
| + if a.Box != "home-box" || a.Region != "ie" || a.Mode != "cold" || (attempts == 1 && a.Generation != p1ID) { |
| + t.Fatalf("the probe's admission: %+v", a) |
| + } |
| + c := h.claim(t) |
| + if c.Op.Intent["to"] != park.HomeRegion || c.Op.Intent["home_region"] != "ie" || c.Holder.WriterBox != "home-box" { |
| + t.Fatalf("the claim after the probe: to %q home %q writer %q", c.Op.Intent["to"], c.Op.Intent["home_region"], c.Holder.WriterBox) |
| + } |
| + // The writer asked the owners' admission evidence for each admission, |
| + // and it passed: the admission's ie and the op's @home are one region |
| + // under home_region ie (SameRegion). The same evidence refuses the |
| + // admission in a region that is not the target's. |
| + if p.evidence != attempts { |
| + t.Fatalf("the writer's admission evidence ran %d times, want %d", p.evidence, attempts) |
| + } |
| + if !park.SameRegion(c.Op.Intent["home_region"], a.Region, c.Op.Intent["to"]) || a.Region == c.Op.Intent["to"] { |
| + t.Fatalf("fixture: the admission's %q and the op's %q are not two spellings of one region", a.Region, c.Op.Intent["to"]) |
| + } |
| + |
| + // C8 established the prepared generation's files manifest in data-home |
| + // under <location>/files/g-<id>/. |
| + m, found, err := hostfiles.Load(ctx, p1FilesStore{vs: h.vs, bucket: "data-home"}, moves.GenerationFilesPrefix(cutRoot, p1ID)) |
| + if err != nil || !found || m.Seq != 1 { |
| + t.Fatalf("the generation's files manifest in data-home: %+v %v %v", m, found, err) |
| + } |
| + if e, ok := m.Entries["upload.txt"]; !ok || !strings.HasPrefix(e.Object, moves.GenerationFilesPrefix(cutRoot, p1ID)) { |
| + t.Fatalf("the files manifest does not carry the instance's files from data-home: %+v", m.Entries) |
| + } |
| + if m.Platform[hostfiles.EnvItem].SHA256 != sha(env) { |
| + t.Fatalf("the established env item is not the env file on disk: %+v", m.Platform) |
| + } |
| + if _, has := m.Platform[instancekey.AdoptionItem]; has { |
| + t.Fatal("the adoption item was carried") |
| + } |
| + if _, err := os.Stat(adoption); !errors.Is(err, os.ErrNotExist) { |
| + t.Fatalf("the adoption record remains: %v", err) |
| + } |
| + for k := range h.f.buckets["data-ca"] { |
| + if strings.Contains(k, "/files/g-"+p1ID+"/") { |
| + t.Fatalf("the generation's files were written to the source's bucket: %s", k) |
| + } |
| + } |
| + |
| + // The commit: the source's location park in ca, the lane park in |
| + // @home, the source's predecessor covered, quiet_after gains ca. |
| + src := h.chore(t, "ca", false) |
| + if src == nil || src.Owner != "region:ca" || src.Park.Location != cutRoot { |
| + t.Fatalf("the source's location park: %+v", src) |
| + } |
| + lane := h.chore(t, park.HomeRegion, true) |
| + if lane == nil || lane.Owner != "region:"+park.HomeRegion || lane.Park.Location != moves.ScopeLocation(p1Host, p1ID) { |
| + t.Fatalf("the lane park: %+v (%+v)", lane, c.Obligations) |
| + } |
| + if h.chore(t, "ie", true) != nil || h.chore(t, "ie", false) != nil { |
| + t.Fatalf("a chore named ie: %+v", c.Obligations) |
| + } |
| + covered := false |
| + for _, ad := range c.Holder.Admissions { |
| + if ad.Generation == p1ID && ad.Predecessor == "cur" { |
| + if ad.Region != "ie" || ad.PredecessorRegion != "ca" || !ad.Covered { |
| + t.Fatalf("the probe's admission after the commit: %+v", ad) |
| + } |
| + covered = true |
| + } |
| + } |
| + if !covered { |
| + t.Fatalf("no admission retires the source's generation: %+v", c.Holder.Admissions) |
| + } |
| + |
| + // After the horizon the home box's engine acknowledges the lane park |
| + // and the residue parks; the source parks its location; the record |
| + // closes in home. |
| + h.clock.Advance(24 * time.Hour) |
| + h.land(t) |
| + if h.chore(t, park.HomeRegion, true) != nil { |
| + t.Fatalf("the lane park was not acknowledged: %+v", h.claim(t).Obligations) |
| + } |
| + if residue != nil { |
| + // The residue parks are the home region's regional runner's: the |
| + // real hostbox.Regional on home-box, its engine in the roster's ie |
| + // with the @home alias, over the real park view seen from ie. |
| + r := homeRegional(h) |
| + for _, o := range residueChores(h.claim(t)) { |
| + if err := r.Run(ctx, park.Work{Host: p1Host, ID: o.ID, Op: p1ID}); err != nil { |
| + t.Fatalf("the home box's regional runner on the residue park of %s: %v", o.Generation, err) |
| + } |
| + } |
| + if gens := residueChores(h.claim(t)); len(gens) != 0 { |
| + t.Fatalf("the residue parks were not acknowledged: %+v", gens) |
| + } |
| + for _, k := range residue { |
| + if _, ok := h.f.buckets["data-home"][k]; ok && !strings.HasSuffix(k, "/generation.json") { |
| + t.Fatalf("the earlier stay's %s is still at the location", k) |
| + } |
| + } |
| + } |
| + h.notClosed(t, "the source has not parked") |
| + h.sourcePass(t) |
| + h.done(t) |
| + rec := h.read(t, p1Host) |
| + if rec.Region != "" || rec.LastMove == nil || rec.LastMove.Outcome != "done" { |
| + t.Fatalf("the record after the close: region %q last %+v", rec.Region, rec.LastMove) |
| + } |
| + for k := range h.f.buckets["data-ca"] { |
| + if strings.HasPrefix(k, cutRoot+"/") { |
| + t.Fatalf("still at the source's location: %s", k) |
| + } |
| + } |
| + // The acknowledged parks set quiet_after: ca for the source's location, |
| + // and never the unset region. |
| + q := h.claim(t).QuietAfter |
| + if _, ok := q["ca"]; !ok { |
| + t.Fatalf("quiet_after %v names no ca", q) |
| + } |
| + if _, ok := q[""]; ok { |
| + t.Fatalf("quiet_after %v names the unset region", q) |
| + } |
| +} |
| + |
| +// intoHomeMovesOutAgain: the closed move leaves the instance in home, and |
| +// it can move out again by part 1's path. The roster places a move from |
| +// home-box (roster ie, the fleet's home) to ca-box as "" -> ca; the start |
| +// CAS begins it on the closed claim; and the home box's source round |
| +// accepts it: it touches ca and ca-box, takes ca's staging lease and |
| +// stages the new move's lane into data-ca. |
| +func intoHomeMovesOutAgain(t *testing.T, r *intoHomeRun) { |
| + t.Helper() |
| + h := r.h |
| + ctx := context.Background() |
| + roster := hostactors.Roster{V: 1, DeploymentBucket: "deploy", Revision: 1, HomeRegion: "ie", Boxes: map[string]hostactors.Box{ |
| + "home-box": {Region: "ie", State: "active", Enrollment: p1DstEnrollment}, |
| + "ca-box": {Region: "ca", State: "active", Enrollment: p1SrcEnrollment}, |
| + }} |
| + if !hostops.ClaimMoveRosterPlaces(roster, "home-box", "ca-box", "", "ca", "ie") { |
| + t.Fatal("the roster does not place home-box -> ca-box as a move out of home") |
| + } |
| + if hostops.ClaimMoveRosterPlaces(roster, "home-box", "ca-box", "ie", "ca", "ie") { |
| + t.Fatal("the roster placed the home box's record as the named region ie") |
| + } |
| + rec := h.read(t, p1Host) |
| + if _, err := provision.SetRecordMove(ctx, h.s, "bab", "app", p1Host, provision.MoveStartTransition(provision.MoveStart{ |
| + ID: p1ID2, Incarnation: h.inc, From: rec.Region, To: "ca", Cut: "manual", By: "op", Now: h.clock.Now()})); err != nil { |
| + t.Fatal(err) |
| + } |
| + obs, err := hostclaim.Read(ctx, h.s, p1Host) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := hostclaim.Update(ctx, h.s, obs, hostclaim.BeginAdmittedMove(p1ID2, "op", "home-box", "ca-box", park.PlacementRegion("ca"), h.clock.Now(), |
| + hostclaim.MoveAdmission{HomeRegion: "ie", AppID: "app_" + p1Host, AdmissionID: p1ID2, SourceEnrollment: p1DstEnrollment, TargetEnrollment: p1SrcEnrollment})); err != nil { |
| + t.Fatalf("the start CAS of the move out of home: %v", err) |
| + } |
| + // The home box as the source: its own bucket, data directory, route |
| + // and launch environment. |
| + c := h.landingCfg(t) |
| + c.LaunchEnv = func(string) (map[string]string, map[string]bool, error) { |
| + env, instance, _, err := (&activator.ExecBackend{DataDir: h.tdir, Region: ""}).LaunchLayers(h.landingHarness.route) |
| + return env, instance, err |
| + } |
| + c.ReadyProbe, c.StrictNext = h.cfg.ReadyProbe, h.cfg.StrictNext |
| + before := len(r.leases.all()) |
| + h.lease.mu.Lock() |
| + h.lease.until = h.clock.now.Add(moves.StagingLeaseTTL) |
| + h.lease.mu.Unlock() |
| + w := &movesWorker{cfg: c, mirrors: map[string]lease.MirrorStore{}} |
| + if err := movesPass(ctx, h.s, w); err != nil { |
| + t.Fatal(err) |
| + } |
| + m := h.read(t, p1Host).Move |
| + if m == nil || m.ID != p1ID2 || m.State != provision.MovePreparing { |
| + t.Fatalf("the move out of home after the source round: %+v", m) |
| + } |
| + keys := laneKeysIn(h.phase1Harness, "data-ca", p1ID2) |
| + replica := false |
| + for _, k := range keys { |
| + replica = replica || strings.HasPrefix(k, "replica/") |
| + } |
| + if !containsString(keys, "move.json") || !replica { |
| + t.Fatalf("the lane into ca: %v", keys) |
| + } |
| + if got := r.leases.all()[before:]; len(got) == 0 || got[0] != "control/staging-leases/ca/"+p1Host { |
| + t.Fatalf("the move out of home's staging leases: %v", got) |
| + } |
| + cl := h.claim(t) |
| + if !containsString(cl.Touched.Regions, "ca") || !containsString(cl.Touched.Boxes, "ca-box") || cl.Op.Intent["to"] != "ca" || cl.Op.Exec != "home-box" { |
| + t.Fatalf("the claim of the move out of home: %+v %+v", cl.Op, cl.Touched) |
| + } |
| +} |
| + |
| +// TestMoveIntoHomeRestartedProbe: the first probe attempt is admitted on |
| +// home-box and the agent dies before readiness. The restarted attempt is |
| +// admitted into a fresh generation whose predecessor is the first |
| +// attempt's, in the region that admission recorded, ie; it is published |
| +// fenced (TestMovesProbeOwnersPublishARestartedAttemptFenced's shape), on |
| +// home-box into data-home; and the commit and the close proceed as for one |
| +// attempt. |
| +func TestMoveIntoHomeRestartedProbe(t *testing.T) { |
| + r := intoHomeClose(t, true) |
| + h, p := r.h, r.p |
| + published := false |
| + p.afterMaterialize = func(n int) error { |
| + switch n { |
| + case 1: |
| + return errors.New("test: the agent died before readiness") |
| + case 2: |
| + restartedAttemptPublishesFenced(t, h, p) |
| + published = true |
| + } |
| + return nil |
| + } |
| + h.holdProbed = true |
| + h.land(t) |
| + if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandProbing || len(p.admissions) != 1 || p.admissions[0].Generation != p1ID { |
| + t.Fatalf("the first attempt: %+v admissions %+v", m, p.admissions) |
| + } |
| + first := p.admissions[0] |
| + if first.Box != "home-box" || first.Region != "ie" || first.PredecessorRegion != "ca" || first.Predecessor != "cur" { |
| + t.Fatalf("the first attempt's admission: %+v", first) |
| + } |
| + |
| + // The agent restarts: a fresh worker, and the claim runtime's second |
| + // attempt, published fenced while it is live. |
| + h.freshTarget() |
| + h.land(t) |
| + if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandProbed || !published { |
| + t.Fatalf("the restarted attempt: %+v published %v", m, published) |
| + } |
| + if len(p.admissions) != 2 { |
| + t.Fatalf("admissions: %+v", p.admissions) |
| + } |
| + |
| + // The commit and the close, as for one attempt. |
| + h.holdProbed = false |
| + intoHomeCommitsAndCloses(t, r, 2) |
| +} |
| + |
| +// restartedAttemptPublishesFenced runs inside the restarted attempt, after |
| +// Materialize: the admission is a fresh generation whose predecessor is the |
| +// first attempt's generation in the first admission's spelling, ie, and |
| +// the fresh generation is published fenced, never unfenced and never |
| +// before its own first root, then over the location's marker in data-home. |
| +func restartedAttemptPublishesFenced(t *testing.T, h *finalHarness, p *prodProbe) { |
| + t.Helper() |
| + ctx := context.Background() |
| + first, again := p.admissions[0], p.admissions[1] |
| + if again.Box != "home-box" || again.Region != "ie" || again.Mode != "cold" || again.Generation == p1ID || again.Predecessor != p1ID || again.PredecessorRegion != first.Region { |
| + t.Fatalf("the restarted attempt's admission: %+v (first %+v)", again, first) |
| + } |
| + admitted, in := p.materialized, p.materializedIn |
| + if g := admitted.Claim.Holder.Generation; g != again.Generation || in.Admission.NewGeneration != g { |
| + t.Fatalf("fixture: the materialized holder is at %s, its inputs admit %s", g, in.Admission.NewGeneration) |
| + } |
| + if err := p.owners.Publish(ctx, admitted, in); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("an unfenced marker for a fresh generation: %v", err) |
| + } |
| + synced := 0 |
| + pub := ClaimPublication{Sync: func(context.Context) error { synced++; return nil }, Check: func(context.Context) error { return nil }} |
| + if _, err := p.owners.PublishFenced(ctx, admitted, in, pub); !errors.Is(err, hostbox.ErrChanged) || synced != 1 { |
| + t.Fatalf("a marker before the root: %v synced=%d", err, synced) |
| + } |
| + root := cutRoot + "/g-" + again.Generation + "/0009/" + ltx.FormatFilename(1, ltx.TXID(h.n)) |
| + hdr, err := (<x.Header{Version: ltx.Version, Flags: ltx.HeaderFlagNoChecksum, PageSize: 4096, Commit: 4, MinTXID: 1, MaxTXID: ltx.TXID(h.n)}).MarshalBinary() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := h.vs.PutIfAbsent(ctx, "data-home", root, hdr); err != nil { |
| + t.Fatal(err) |
| + } |
| + receipt, err := p.owners.PublishFenced(ctx, admitted, in, pub) |
| + if err != nil || receipt.Root.Key != root || receipt.MarkerVersion == "" { |
| + t.Fatalf("publish: %+v %v", receipt, err) |
| + } |
| + if err := p.owners.PublicationEvidence(ctx, admitted); err != nil { |
| + t.Fatal(err) |
| + } |
| + raw, err := h.vs.GetVersion(ctx, "data-home", cutRoot+"/generation.json", receipt.MarkerVersion) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var marker genselect.Marker |
| + if err := json.Unmarshal(raw, &marker); err != nil || marker.Op != again.Generation || marker.Box != "home-box" || marker.Incarnation != h.inc { |
| + t.Fatalf("the published marker: %+v %v", marker, err) |
| + } |
| + for k := range h.f.buckets["data-ca"] { |
| + if strings.HasPrefix(k, cutRoot+"/g-"+again.Generation+"/") { |
| + t.Fatalf("the restarted attempt wrote into the source's bucket: %s", k) |
| + } |
| + } |
| +} |
| + |
| +// cleanHomeStart asserts the claim a witness starts from: the start CAS |
| +// wrote intent.to @home and home_region ie, and nothing else in the claim |
| +// names home in either spelling (no touched entry, no quiet key, no chore). |
| +func cleanHomeStart(t *testing.T, c *hostclaim.Claim) { |
| + t.Helper() |
| + if c.Op.Intent["to"] != park.HomeRegion || c.Op.Intent["home_region"] != "ie" { |
| + t.Fatalf("the start CAS: %+v", c.Op.Intent) |
| + } |
| + for _, r := range c.Touched.Regions { |
| + if r == "ie" || r == park.HomeRegion { |
| + t.Fatalf("touched names home at the start: %+v", c.Touched) |
| + } |
| + } |
| + if containsString(c.Touched.Boxes, "home-box") || len(c.QuietAfter) != 0 || len(c.Obligations) != 0 { |
| + t.Fatalf("the start is not clean: touched %+v quiet %v obligations %+v", c.Touched, c.QuietAfter, c.Obligations) |
| + } |
| +} |
| + |
| +// homeTokenSeen counts what a witness created for the target side, so the |
| +// fresh-state proof is never vacuous: chores in @home, and whether touched |
| +// ever held @home and home-box. |
| +func homeTokenSeen(writes []hostclaim.Claim) (chores int, touched bool) { |
| + seen := map[string]bool{} |
| + for _, c := range writes { |
| + for _, o := range c.Obligations { |
| + if o.Region == park.HomeRegion && !seen[o.ID] { |
| + seen[o.ID] = true |
| + chores++ |
| + } |
| + } |
| + if containsString(c.Touched.Regions, park.HomeRegion) && containsString(c.Touched.Boxes, "home-box") { |
| + touched = true |
| + } |
| + } |
| + return chores, touched |
| +} |
| + |
| +// TestHomeTargetFreshStateShowsOnlyTheHomeToken (Review Focus 1, supplied |
| +// regions are @home; the runtime proof): each into-home witness of Task 3 |
| +// runs once from clean claim state, where a wrong spelling cannot hide. A |
| +// chore queued with ie would be a new obligation named ie; a Touch with ie |
| +// would add ie to touched; a staging lease keyed with ie would show in the |
| +// recorded keys; an exact comparison with intent.to (Commit's target and |
| +// the release paths) would refuse and stop the witness. After each, every |
| +// target-side chore the run ever created is @home and owned by |
| +// region:@home, touched gained only @home and home-box, quiet_after never |
| +// named ie or "", and every home lane lease is keyed @home. Task 4 adds its |
| +// witnesses to this table. |
| +func TestHomeTargetFreshStateShowsOnlyTheHomeToken(t *testing.T) { |
| + type run struct { |
| + writes []hostclaim.Claim |
| + leases []string |
| + chores int // the least number of @home chores the witness creates |
| + } |
| + for name, witness := range map[string]func(t *testing.T) run{ |
| + "the source round stages the home lane": func(t *testing.T) run { |
| + var watch *claimWatch |
| + h := newPhase1Harness(t, intoHome, watchClaims(&watch)) |
| + leases := &stagingLeaseKeys{} |
| + leases.wrap(&h.cfg) |
| + h.start(t, "manual") |
| + cleanHomeStart(t, h.claim(t)) |
| + h.pass(t) |
| + h.targetPass(t, h.targetCfg()) |
| + h.pass(t) |
| + if m := h.read(t, p1Host).Move; m.State != provision.MoveReady { |
| + t.Fatalf("the move did not reach ready: %+v", m) |
| + } |
| + return run{writes: watch.since(0), leases: leases.all()} |
| + }, |
| + "the landing, the commit and the close": func(t *testing.T) run { |
| + r := intoHomeClose(t, true) |
| + intoHomeCommitsAndCloses(t, r, 1) |
| + // Two residue parks and the lane park. |
| + return run{writes: r.watch.since(0), leases: r.leases.all(), chores: 3} |
| + }, |
| + "the restarted probe": func(t *testing.T) run { |
| + r := intoHomeClose(t, false) |
| + r.p.afterMaterialize = func(n int) error { |
| + if n == 1 { |
| + return errors.New("test: the agent died before readiness") |
| + } |
| + return nil |
| + } |
| + r.h.land(t) |
| + r.h.freshTarget() |
| + intoHomeCommitsAndCloses(t, r, 2) |
| + return run{writes: r.watch.since(0), leases: r.leases.all(), chores: 1} |
| + }, |
| + } { |
| + t.Run(name, func(t *testing.T) { |
| + got := witness(t) |
| + // The claim the start CAS left is the first written with the |
| + // move's op: clean, as cleanHomeStart requires. |
| + for i, c := range got.writes { |
| + if c.Op.Kind == hostclaim.KindMoving { |
| + cleanHomeStart(t, &got.writes[i]) |
| + break |
| + } |
| + } |
| + onlyTheHomeToken(t, got.writes, got.leases, "ca", "ca-box") |
| + chores, touched := homeTokenSeen(got.writes) |
| + if chores < got.chores || !touched { |
| + t.Fatalf("the witness created %d chores in %s (want at least %d), touched held it %v", chores, park.HomeRegion, got.chores, touched) |
| + } |
| + if len(got.leases) == 0 { |
| + t.Fatal("the witness took no staging lease") |
| + } |
| + for _, k := range got.leases { |
| + if k != homeLeaseKey { |
| + t.Fatalf("a staging lease keyed %q", k) |
| + } |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// obligationsJSON is the claim's obligation list as stored, for byte |
| +// comparison. |
| +func obligationsJSON(t *testing.T, obs []hostclaim.Obligation) string { |
| + t.Helper() |
| + b, err := json.Marshal(obs) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + return string(b) |
| +} |
| + |
| +// otherThan is the obligations not in region. |
| +func otherThan(obs []hostclaim.Obligation, region string) []hostclaim.Obligation { |
| + var out []hostclaim.Obligation |
| + for _, o := range obs { |
| + if o.Region != region { |
| + out = append(out, o) |
| + } |
| + } |
| + return out |
| +} |
| + |
| +// retainedID is a fixed obligation id for a seeded chore. |
| +func retainedID(n int) string { return fmt.Sprintf("%032x", 0xa000+n) } |
| + |
| +// TestHomeTargetRetainedChoresKeepTheirSpelling (Review Focus 1, retained |
| +// state keeps its spelling): the residue chores and the lane chore are |
| +// already owed when the landing and the commit queue theirs, under ie and |
| +// then under @home. The landing's QueueResidueParks(@home) leaves the |
| +// obligation list byte-identical; the commit's QueueStagingPark(@home) |
| +// leaves every chore but the source's new location park byte-identical. No |
| +// second chore, no rewrite of ie to @home. The home box's engines then |
| +// acknowledge each seeded chore once, and the move closes in home. |
| +func TestHomeTargetRetainedChoresKeepTheirSpelling(t *testing.T) { |
| + for _, spelling := range []string{"ie", park.HomeRegion} { |
| + t.Run(spelling, func(t *testing.T) { |
| + ctx := context.Background() |
| + ids := map[string]hostclaim.Obligation{} |
| + var seeded, atQueue string |
| + r := intoHomeCloseWith(t, true, func(lh *landingHarness) { |
| + // Owed before the landing's parking step: one park-generation |
| + // per residue generation, with the landing's own scope. |
| + rec := lh.read(t, p1Host) |
| + rewriteP1Claim(t, lh.s, func(c *hostclaim.Claim) { |
| + scope := moves.ResidueScope(*c.Holder, *rec.Move.FlippedAt) |
| + for i, g := range []string{hostclaim.RootGeneration, "old"} { |
| + s := scope |
| + o := hostclaim.Obligation{ID: retainedID(2 * i), Op: p1ID, Kind: "park-generation", Region: spelling, Generation: g, Retirement: retainedID(2*i + 1), Owner: "region:" + spelling, Park: &s} |
| + ids[g] = o |
| + c.Obligations = append(c.Obligations, o) |
| + } |
| + }) |
| + seeded = obligationsJSON(t, lh.claim(t).Obligations) |
| + lh.landFault = func(point string) error { |
| + if point == "after-residue-queued" && atQueue == "" { |
| + atQueue = obligationsJSON(t, lh.claim(t).Obligations) |
| + } |
| + return nil |
| + } |
| + }) |
| + h, p := r.h, r.p |
| + if atQueue == "" { |
| + t.Fatal("the landing never queued its residue parks") |
| + } |
| + if atQueue != seeded { |
| + t.Fatalf("QueueResidueParks(%s) over chores owed under %s changed the obligations:\n%s\nwas\n%s", park.HomeRegion, spelling, atQueue, seeded) |
| + } |
| + |
| + // Owed before the commit: the lane's park with the commit's own |
| + // scope. |
| + var beforeCommit, afterCommit []hostclaim.Obligation |
| + h.landFault = func(point string) error { |
| + switch point { |
| + case "before-commit": |
| + if beforeCommit != nil { |
| + return nil |
| + } |
| + rec := h.read(t, p1Host) |
| + rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) { |
| + _, staging, err := moves.CommitScopes(c, rec.Move.FlippedAt) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + o := hostclaim.Obligation{ID: retainedID(10), Op: p1ID, Kind: "park-replica", Region: spelling, Retirement: retainedID(11), Owner: "region:" + spelling, Park: &staging} |
| + ids["lane"] = o |
| + c.Obligations = append(c.Obligations, o) |
| + }) |
| + beforeCommit = h.claim(t).Obligations |
| + case "after-claim-committed": |
| + afterCommit = h.claim(t).Obligations |
| + } |
| + return nil |
| + } |
| + h.land(t) |
| + h.committedAndOpen(t) |
| + if len(p.admissions) != 1 || afterCommit == nil { |
| + t.Fatalf("the commit: admissions %+v, claim captured %v", p.admissions, afterCommit != nil) |
| + } |
| + if got, want := obligationsJSON(t, otherThan(afterCommit, "ca")), obligationsJSON(t, beforeCommit); got != want { |
| + t.Fatalf("the commit over a lane park owed under %s changed the home chores:\n%s\nwas\n%s", spelling, got, want) |
| + } |
| + if src := afterCommit[len(afterCommit)-1]; len(afterCommit) != len(beforeCommit)+1 || src.Region != "ca" || src.Kind != "park-replica" || src.Park.Location != cutRoot { |
| + t.Fatalf("the commit queued %+v beside the retained chores", afterCommit[len(beforeCommit):]) |
| + } |
| + |
| + // Each retained chore is acknowledged once, in its spelling: the |
| + // lane by the home box's moves engine, the residue by its |
| + // regional runner. |
| + h.clock.Advance(24 * time.Hour) |
| + h.land(t) |
| + reg := homeRegional(h) |
| + for _, g := range []string{hostclaim.RootGeneration, "old"} { |
| + if err := reg.Run(ctx, park.Work{Host: p1Host, ID: ids[g].ID, Op: p1ID}); err != nil { |
| + t.Fatalf("the regional runner on the residue park of %s owed under %s: %v", g, spelling, err) |
| + } |
| + } |
| + c := h.claim(t) |
| + for name, o := range ids { |
| + for _, owed := range c.Obligations { |
| + if owed.ID == o.ID { |
| + t.Fatalf("the %s chore owed under %s was not acknowledged: %+v", name, spelling, owed) |
| + } |
| + } |
| + n := 0 |
| + for k := range h.f.buckets["data-home"] { |
| + if strings.HasPrefix(k, "parked/"+h.inc+"/"+p1Host+"/"+o.Retirement+"/") && strings.HasSuffix(k, "/manifest.json") { |
| + n++ |
| + } |
| + } |
| + if n != 1 { |
| + t.Fatalf("the %s chore's retirement %s holds %d manifests, want 1", name, o.Retirement, n) |
| + } |
| + } |
| + for _, o := range c.Obligations { |
| + if o.Region == park.HomeRegion || o.Region == "ie" { |
| + t.Fatalf("a home chore is still owed: %+v", o) |
| + } |
| + } |
| + // A second run of an acknowledged chore writes nothing. |
| + gen := c.Gen |
| + if err := reg.Run(ctx, park.Work{Host: p1Host, ID: ids["old"].ID, Op: p1ID}); err != nil || h.claim(t).Gen != gen { |
| + t.Fatalf("a second acknowledgement: %v (claim gen %d to %d)", err, gen, h.claim(t).Gen) |
| + } |
| + h.sourcePass(t) |
| + h.done(t) |
| + }) |
| + } |
| +} |
diff --git a/internal/host/movesparks_test.go b/internal/host/movesparks_test.go| index eb68782c5..3dcab785f 100644 |
| --- a/internal/host/movesparks_test.go |
| +++ b/internal/host/movesparks_test.go |
| @@ -550,11 +550,24 @@ type prodProbe struct { |
| // the hash of the launch it is asked to start. |
| selected ClaimLaunchInputs |
| attempt hostclaim.WakeRetryAttempt |
| - // admissions are the admissions the runtime made, one per attempt. |
| + // admissions are the admissions the runtime made, one per attempt, and |
| + // admitted the claim each left, the last attempt's last. |
| admissions []hostclaim.AdmissionRecord |
| - last activator.ProbeResult |
| - live bool |
| - seq int |
| + admitted hostclaim.Observed |
| + // materialized is the claim and the launch inputs the runtime last |
| + // materialized: the admitted inputs, which for a fresh generation are |
| + // not the selected ones (each Select mints its own generation). |
| + materialized hostclaim.Observed |
| + materializedIn ClaimLaunchInputs |
| + // afterMaterialize, when set, runs in the n-th wake (from 1) after |
| + // Materialize and before readiness, while the attempt is live; an error |
| + // ends the wake there, as an agent that died there leaves it (home part |
| + // 2, Task 3). |
| + afterMaterialize func(n int) error |
| + wakes int |
| + last activator.ProbeResult |
| + live bool |
| + seq int |
| } |
| // productionProbe switches the harness's probe to the production mode: the |
| @@ -655,7 +668,7 @@ func (h *landingHarness) productionProbe(t *testing.T) *prodProbe { |
| } |
| p.writerEvidence = f.admissionEvidence |
| c := f.controller("boot-1", "controller-1", 4101, "start-1") |
| - c.wake.Inputs = dispatch |
| + c.wake.Inputs = p1RecordingInputs{ClaimOwnerDispatch: dispatch, p: p} |
| c.wake.Continue = hostclaim.SessionContinues |
| c.wake.Operations = bound.Operations |
| c.wake.ProbeReady = func(context.Context, *exec.Cmd, registry.Route, ClaimLaunchInputs, string) error { return nil } |
| @@ -664,6 +677,18 @@ func (h *landingHarness) productionProbe(t *testing.T) *prodProbe { |
| return p |
| } |
| +// p1RecordingInputs is the owner dispatch as the runtime's inputs, every |
| +// method its own, with Materialize recording what the runtime admitted. |
| +type p1RecordingInputs struct { |
| + ClaimOwnerDispatch |
| + p *prodProbe |
| +} |
| + |
| +func (r p1RecordingInputs) Materialize(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs, work hostwrite.Work) error { |
| + r.p.materialized, r.p.materializedIn = o, in |
| + return r.ClaimOwnerDispatch.Materialize(ctx, o, in, work) |
| +} |
| + |
| // wake is one probe attempt through the claim runtime: the activator's |
| // ProbeWake with its start policy, as far as the claim runs on main. |
| func (p *prodProbe) wake(ctx context.Context, cfg activator.ProbeConfig) (activator.ProbeResult, error) { |
| @@ -692,6 +717,8 @@ func (p *prodProbe) wake(ctx context.Context, cfg activator.ProbeConfig) (activa |
| } |
| ah := admitted.Claim.Holder |
| p.admissions = append(p.admissions, ah.Admissions[len(ah.Admissions)-1]) |
| + p.admitted = admitted |
| + p.wakes++ |
| s, err := p.ctl.runtime.Prepare(ctx, rt, admitted, policy) |
| if err != nil { |
| return activator.ProbeResult{}, err |
| @@ -705,9 +732,23 @@ func (p *prodProbe) wake(ctx context.Context, cfg activator.ProbeConfig) (activa |
| return activator.ProbeResult{}, err |
| } |
| p.attempt = s.CurrentAttempt() |
| + // The activator's cold restore from the selected baseline stands in |
| + // here: it rewrites <dbPath>.gen to the admitted generation. The |
| + // landing's install already wrote the move id, so only a restarted |
| + // attempt's fresh generation changes it (home part 2, Task 3). |
| + if gen, err := hostbox.ReadGenerationMarker(rt.DBPath + ".gen"); err != nil || gen != ah.Generation { |
| + if err := hostbox.WriteGenerationMarker(rt.DBPath+".gen", ah.Generation); err != nil { |
| + return activator.ProbeResult{}, err |
| + } |
| + } |
| if err := s.Materialize(ctx); err != nil { |
| return activator.ProbeResult{}, err |
| } |
| + if p.afterMaterialize != nil { |
| + if err := p.afterMaterialize(p.wakes); err != nil { |
| + return activator.ProbeResult{}, err |
| + } |
| + } |
| p.last = activator.ProbeResult{Session: ah.Session, Stamp: ah.Stamp, Generation: ah.Generation} |
| p.live = true |
| return p.last, nil |
diff --git a/internal/hostops/move_recovery_commit_test.go b/internal/hostops/move_recovery_commit_test.go| index 36cc707b1..a505ca398 100644 |
| --- a/internal/hostops/move_recovery_commit_test.go |
| +++ b/internal/hostops/move_recovery_commit_test.go |
| @@ -112,3 +112,65 @@ func TestMoveRecoveryCommitFromHome(t *testing.T) { |
| } |
| } |
| } |
| + |
| +// Home part 2, Task 3 (the inventory's witness for this Commit site): the |
| +// console's commit of a move into home, from clean claim state. The record |
| +// names the target by the home's code, ""; the start CAS wrote the op's to |
| +// as @home with home_region uk, the roster name of uk-box, whose admissions |
| +// keep their engine region uk. The commit queues the lane park in @home, |
| +// owned by region:@home, and the source's location park in ie; nothing is |
| +// queued under uk or "". |
| +func TestMoveRecoveryCommitIntoHome(t *testing.T) { |
| + w := newLossWorld(t, "landing", func(c *hostclaim.Claim, r *provision.Record, _ *AppAdmission) { |
| + c.Op.Phase = hostclaim.PhaseLandingProbed |
| + c.Op.Intent["to"], c.Op.Intent["home_region"] = park.HomeRegion, "uk" |
| + r.Move.State = provision.MoveCommitted |
| + r.Move.CommittedAt = r.Move.FlippedAt |
| + r.Move.To, r.Region = "", "" |
| + }) |
| + // The console names the same home (CONSOLE_HOME_REGION), as the |
| + // operation's binding requires. |
| + w.control.homeRegion = "uk" |
| + before := w.claim(t) |
| + if len(before.Claim.Obligations) != 0 { |
| + t.Fatalf("fixture: the claim already owes %+v", before.Claim.Obligations) |
| + } |
| + rec, _ := w.record(t) |
| + err := w.control.RunRecovery(context.Background(), func(ctx context.Context) error { |
| + return runMoveRecovery(ctx, before, func(ctx context.Context, g *MoveRecoveryGrant) error { |
| + return g.Claim(ctx, RecoveryMoveCommit) |
| + }) |
| + }) |
| + after := w.claim(t) |
| + if err != nil || after.Claim.Op.Phase != hostclaim.PhaseCommitted { |
| + t.Fatalf("commit: %v, claim=%+v", err, after.Claim.Op) |
| + } |
| + source, staging, err := moves.CommitScopes(before.Claim, rec.Move.FlippedAt) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var lane, loc bool |
| + for _, o := range after.Claim.Obligations { |
| + if o.Region == "" || o.Region == "uk" { |
| + t.Fatalf("a chore named %q: %+v", o.Region, o) |
| + } |
| + switch { |
| + case o.Kind == "park-replica" && o.Park != nil && *o.Park == staging: |
| + if o.Region != park.HomeRegion || o.Owner != "region:"+park.HomeRegion { |
| + t.Fatalf("the lane park %+v, want it in %s", o, park.HomeRegion) |
| + } |
| + lane = true |
| + case o.Kind == "park-replica" && o.Park != nil && *o.Park == source: |
| + if o.Region != "ie" || o.Owner != "region:ie" { |
| + t.Fatalf("the source's location park %+v, want it in ie", o) |
| + } |
| + loc = true |
| + } |
| + } |
| + if !lane || !loc { |
| + t.Fatalf("the commit's parks: lane %v location %v in %+v", lane, loc, after.Claim.Obligations) |
| + } |
| + if after.Claim.Op.Intent["to"] != park.HomeRegion { |
| + t.Fatalf("the op's to after the commit: %+v", after.Claim.Op.Intent) |
| + } |
| +} |