| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/base64" |
| 6 | "errors" |
| 7 | "fmt" |
| 8 | "reflect" |
| 9 | "strings" |
| 10 | "sync" |
| 11 | "testing" |
| 12 | "time" |
| 13 | |
| 14 | "github.com/carlosframework/platform/internal/activator" |
| 15 | "github.com/carlosframework/platform/internal/hostbox" |
| 16 | "github.com/carlosframework/platform/internal/hostclaim" |
| 17 | "github.com/carlosframework/platform/internal/hostguard" |
| 18 | "github.com/carlosframework/platform/internal/hostops" |
| 19 | "github.com/carlosframework/platform/internal/hostwrite" |
| 20 | "github.com/carlosframework/platform/internal/lease" |
| 21 | "github.com/carlosframework/platform/internal/lease/leasetest" |
| 22 | "github.com/carlosframework/platform/internal/moves" |
| 23 | "github.com/carlosframework/platform/internal/park" |
| 24 | "github.com/carlosframework/platform/internal/registry" |
| 25 | "github.com/carlosframework/platform/internal/release" |
| 26 | ) |
| 27 | |
| 28 | // Plan 5 Task 9b (moves-after-Plan-12 Task 8; rulings Q1, Q12, Q15): the |
| 29 | // moves wake source is registered right after the parks, with both |
| 30 | // switches on, and its registration is what opens the one gate |
| 31 | // (movesWakeSourceRegistered), so the parks arm in the same agent start as |
| 32 | // the wake owners and never before. |
| 33 | |
| 34 | // mwSecretsKey is a CARLOS_SECRETS_KEY for the registered source's env |
| 35 | // report key and held render: a valid P-256 scalar. |
| 36 | var mwSecretsKey = base64.StdEncoding.EncodeToString([]byte("0123456789abcdef0123456789abcdef")) |
| 37 | |
| 38 | // mwEnv is an environment with the two switches as given. |
| 39 | func mwEnv(claim, phase1 bool) func(string) string { |
| 40 | vals := map[string]string{"CARLOS_SECRETS_KEY": mwSecretsKey} |
| 41 | if claim { |
| 42 | vals["CARLOS_HOST_CLAIM"] = "1" |
| 43 | } |
| 44 | if phase1 { |
| 45 | vals[moves.SwitchEnv] = "on" |
| 46 | } |
| 47 | return func(k string) string { return vals[k] } |
| 48 | } |
| 49 | |
| 50 | // mwReadAccess is the claim's move read access as a fixture: the route and |
| 51 | // launch it answers, and the held works it selects, counted. |
| 52 | type mwReadAccess struct { |
| 53 | route registry.Route |
| 54 | launch ClaimServingLaunch |
| 55 | report moves.EnvReport |
| 56 | works []string |
| 57 | } |
| 58 | |
| 59 | func (a *mwReadAccess) HeldWork(_ context.Context, _, action string, _ hostguard.Captured) (hostwrite.Work, error) { |
| 60 | a.works = append(a.works, action) |
| 61 | return plan9D5Work{}, nil |
| 62 | } |
| 63 | func (a *mwReadAccess) Route(_ context.Context, host string) (*registry.Route, error) { |
| 64 | if host != a.route.Host { |
| 65 | return nil, hostbox.ErrChanged |
| 66 | } |
| 67 | rt := a.route |
| 68 | return &rt, nil |
| 69 | } |
| 70 | func (a *mwReadAccess) Launch(_ context.Context, host string) (ClaimServingLaunch, moves.EnvReport, error) { |
| 71 | if host != a.route.Host { |
| 72 | return ClaimServingLaunch{}, moves.EnvReport{}, hostbox.ErrChanged |
| 73 | } |
| 74 | return a.launch, a.report, nil |
| 75 | } |
| 76 | |
| 77 | // mwArm arms a moveCfg for a box whose placement is placement over the |
| 78 | // claim's engine in engineRegion, with the fleet's home binding home. |
| 79 | func mwArm(t *testing.T, view func(context.Context, hostclaim.Observed) (park.View, error), engineRegion, home, placement string) (moveCfg, error) { |
| 80 | t.Helper() |
| 81 | auth := func(context.Context, hostclaim.Observed, hostclaim.Obligation) error { return nil } |
| 82 | providers := ClaimProviders{moveParks: &claimMoveParks{ |
| 83 | local: claimMoveLocalAuthorizer(func(context.Context) error { return nil }, auth), |
| 84 | engine: park.Engine{Bucket: "data", Region: engineRegion, HomeRegion: home, Leases: (&renewTable{}).leases(), View: view, Authorize: auth}, |
| 85 | leases: hostbox.ReplicaParkLeaseSet, check: func(context.Context) error { return nil }}} |
| 86 | c := moveCfg{OwnBucket: "data", Region: placement} |
| 87 | err := armMovesParks(t.Context(), &c, providers) |
| 88 | return c, err |
| 89 | } |
| 90 | |
| 91 | // TestBothFlagsOnRegistersAndArms (Q1, Q15): with CARLOS_HOST_CLAIM and |
| 92 | // CARLOS_MOVES_PHASE1 both on, the agent's two registrations, the parks and |
| 93 | // then the wake source, open the gate: the claim's registered move source |
| 94 | // is this box's moves source, and the moves arms take the claim's engine. A |
| 95 | // named box arms over its own engine. A home box arms only when the claim |
| 96 | // grants it the @home alias (claim Plan 13's signed fleet home |
| 97 | // designation); without it the home box stays unarmed. The parks alone do |
| 98 | // not open the gate, and a second registration is refused. |
| 99 | func TestBothFlagsOnRegistersAndArms(t *testing.T) { |
| 100 | withMovesWakeSource(t, false) |
| 101 | ctx := t.Context() |
| 102 | env := mwEnv(true, true) |
| 103 | hib := activator.Config{Bucket: "data", Region: "eu-west-1"} |
| 104 | store := release.NewDirStore(t.TempDir()) |
| 105 | roster := func(context.Context) (string, error) { return "ie", nil } |
| 106 | parks, err := registerProductionMovesParks(ctx, env, store, hib, roster, newMovesRefusals("ie-box", nil, t.Logf, time.Now)) |
| 107 | if err != nil { |
| 108 | t.Fatal(err) |
| 109 | } |
| 110 | if parks.View == nil || parks.Evidence == nil || claimMoveSources.Parks == nil { |
| 111 | t.Fatalf("the parks are not registered: %+v", parks) |
| 112 | } |
| 113 | var dep *ClaimDependencyError |
| 114 | if err := movesWakeSourceRegistered(); !errors.As(err, &dep) { |
| 115 | t.Fatalf("the parks alone opened the gate: %v", err) |
| 116 | } |
| 117 | if c, err := mwArm(t, parks.View, "ie", "", "ie"); !errors.As(err, &dep) || parkHooksArmed(c) { |
| 118 | t.Fatalf("armed on the parks alone: %v", err) |
| 119 | } |
| 120 | |
| 121 | if err := registerProductionMovesWakeSource(env, parks, t.TempDir()); err != nil { |
| 122 | t.Fatal(err) |
| 123 | } |
| 124 | if err := movesWakeSourceRegistered(); err != nil { |
| 125 | t.Fatalf("the gate is shut after the wake source registered: %v", err) |
| 126 | } |
| 127 | reg, ok := claimMoveSources.Source.(*movesWakeSourceRegistration) |
| 128 | if !ok { |
| 129 | t.Fatalf("the claim's move source is %T, not this box's moves source", claimMoveSources.Source) |
| 130 | } |
| 131 | if _, ok := reg.ClaimProductionMoveSources.(*movesWakeSources); !ok { |
| 132 | t.Fatalf("the registered source is %T", reg.ClaimProductionMoveSources) |
| 133 | } |
| 134 | |
| 135 | // A named box arms over its own engine. |
| 136 | if c, err := mwArm(t, parks.View, "ie", "", "ie"); err != nil || !parkHooksArmed(c) || c.ParkRunHeld == nil || c.ParkLocalAuthorize == nil { |
| 137 | t.Fatalf("a named box did not arm: %v", err) |
| 138 | } |
| 139 | // A home box with the @home grant arms, and keeps the home binding. |
| 140 | c, err := mwArm(t, parks.View, "ie", "ie", "") |
| 141 | if err != nil || !parkHooksArmed(c) || c.ParkHomeRegion != "ie" { |
| 142 | t.Fatalf("a home box the claim grants @home did not arm: %v (%q)", err, c.ParkHomeRegion) |
| 143 | } |
| 144 | // Without the grant a home box stays unarmed, over its named engine or |
| 145 | // over one that names itself @home. |
| 146 | for _, engine := range []string{"ie", park.HomeRegion} { |
| 147 | if c, err := mwArm(t, parks.View, engine, "", ""); !errors.Is(err, hostbox.ErrChanged) || parkHooksArmed(c) { |
| 148 | t.Fatalf("a home box without the grant (engine %q) armed: %v", engine, err) |
| 149 | } |
| 150 | } |
| 151 | |
| 152 | // One registration per process: a second is refused and the gate stays. |
| 153 | if err := registerProductionMovesWakeSource(env, parks, t.TempDir()); !errors.Is(err, hostwrite.ErrUnavailable) { |
| 154 | t.Fatalf("a second registration: %v", err) |
| 155 | } |
| 156 | if err := movesWakeSourceRegistered(); err != nil { |
| 157 | t.Fatalf("a refused second registration shut the gate: %v", err) |
| 158 | } |
| 159 | } |
| 160 | |
| 161 | // TestEitherFlagOffRegistersNothing (ruling H1, Q1): with either switch |
| 162 | // off, or both, neither registration does anything, the gate stays shut, |
| 163 | // the moves arms stay unarmed, and the claim keeps its named refusals. |
| 164 | func TestEitherFlagOffRegistersNothing(t *testing.T) { |
| 165 | hib := activator.Config{Bucket: "data", Region: "eu-west-1"} |
| 166 | roster := func(context.Context) (string, error) { return "ie", nil } |
| 167 | f := newVwFixture(t) |
| 168 | live := movesParkSources{View: f.view("ie"), Evidence: movesRetirementEvidence{Records: f.records(), Now: time.Now}} |
| 169 | for name, env := range map[string]func(string) string{ |
| 170 | "claim off": mwEnv(false, true), |
| 171 | "phase 1 off": mwEnv(true, false), |
| 172 | "both off": mwEnv(false, false), |
| 173 | } { |
| 174 | t.Run(name, func(t *testing.T) { |
| 175 | withMovesWakeSource(t, false) |
| 176 | store := release.NewDirStore(t.TempDir()) |
| 177 | parks, err := registerProductionMovesParks(t.Context(), env, store, hib, roster, newMovesRefusals("ie-box", nil, t.Logf, time.Now)) |
| 178 | if err != nil || parks.View != nil || parks.Evidence != nil { |
| 179 | t.Fatalf("the parks: %+v %v", parks, err) |
| 180 | } |
| 181 | // Off, even a complete park source registers no wake source. |
| 182 | for _, p := range []movesParkSources{parks, live} { |
| 183 | if err := registerProductionMovesWakeSource(env, p, t.TempDir()); err != nil { |
| 184 | t.Fatal(err) |
| 185 | } |
| 186 | } |
| 187 | if claimMoveSources.Source != nil || claimMoveSources.Parks != nil { |
| 188 | t.Fatalf("registered: source %v parks %v", claimMoveSources.Source, claimMoveSources.Parks) |
| 189 | } |
| 190 | var dep *ClaimDependencyError |
| 191 | if err := movesWakeSourceRegistered(); !errors.As(err, &dep) || !strings.Contains(err.Error(), movesWakeSourceContract) { |
| 192 | t.Fatalf("the gate: %v", err) |
| 193 | } |
| 194 | if c, err := mwArm(t, live.View, "ie", "", "ie"); !errors.As(err, &dep) || parkHooksArmed(c) { |
| 195 | t.Fatalf("armed with a switch off: %v", err) |
| 196 | } |
| 197 | }) |
| 198 | } |
| 199 | } |
| 200 | |
| 201 | // TestTheClaimsParkViewIsTheMovesView (Q15): the wake source's View and |
| 202 | // VerifyRetirement are never read by the claim, because the parks 9a |
| 203 | // registered first stay in place; both delegate to the same park sources, |
| 204 | // so whichever the claim reads, the answer is the moves view's (and the |
| 205 | // moves evidence's). The claim's regional view, which every host's park |
| 206 | // chores and the at-rest sweep read, is that view too. |
| 207 | func TestTheClaimsParkViewIsTheMovesView(t *testing.T) { |
| 208 | withMovesWakeSource(t, false) |
| 209 | f := newVwFixture(t) |
| 210 | evidence := &countingEvidence{RegionalEvidence: movesRetirementEvidence{Records: f.records(), Now: time.Now}} |
| 211 | parks, err := newMovesParkSources(f.records(), f.bucket, "data", fixedRegion("ie"), evidence) |
| 212 | if err != nil { |
| 213 | t.Fatal(err) |
| 214 | } |
| 215 | if err := registerMovesParks(parks); err != nil { |
| 216 | t.Fatal(err) |
| 217 | } |
| 218 | registered := claimMoveSources.Parks |
| 219 | if err := registerProductionMovesWakeSource(mwEnv(true, true), parks, t.TempDir()); err != nil { |
| 220 | t.Fatal(err) |
| 221 | } |
| 222 | if claimMoveSources.Parks != registered { |
| 223 | t.Fatal("the wake source replaced the parks 9a registered") |
| 224 | } |
| 225 | f.marker(t, "cur", f.inc) |
| 226 | obs := vwObs(&hostclaim.Claim{Holder: f.holder("cur", "cur@ie"), Op: hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box"}}) |
| 227 | direct, err := movesParkView(f.records(), f.bucket, "data", "ie")(t.Context(), obs) |
| 228 | if err != nil || direct.AuthoritativeRegion != "ie" || !direct.MarkerKnown || direct.MarkerGeneration != "cur" { |
| 229 | t.Fatalf("the moves view: %+v %v", direct, err) |
| 230 | } |
| 231 | src := claimMoveSources.Source |
| 232 | for name, view := range map[string]func(context.Context, hostclaim.Observed) (park.View, error){ |
| 233 | "the claim's park view": claimMoveSources.Parks.View, |
| 234 | "the wake source's View": src.View, |
| 235 | "the registered parks": parks.View, |
| 236 | "the claim's regional view": productionClaimRegional{Identity: handoverRecoveryIdentity(t, hostbox.Actor{Box: "box-1", Enrollment: "enrollment-1", Revision: 7, Binding: "7", Region: "ie"}), Parks: claimMoveSources.Parks}.View, |
| 237 | } { |
| 238 | got, err := view(t.Context(), obs) |
| 239 | if err != nil { |
| 240 | t.Fatalf("%s: %v", name, err) |
| 241 | } |
| 242 | got.GenerationOf, direct.GenerationOf = nil, nil |
| 243 | got.ArchivedIncarnations, got.RefusedLocations = nil, nil |
| 244 | if !reflect.DeepEqual(got, direct) { |
| 245 | t.Fatalf("%s is not the moves view:\n got %+v\nwant %+v", name, got, direct) |
| 246 | } |
| 247 | } |
| 248 | retired := vwObs(&hostclaim.Claim{}) |
| 249 | if _, _, err := src.VerifyRetirement(t.Context(), retired, nil); evidence.calls != 1 { |
| 250 | t.Fatalf("the wake source's VerifyRetirement did not delegate to the moves evidence: %d calls (%v)", evidence.calls, err) |
| 251 | } |
| 252 | if _, _, err := claimMoveSources.Parks.Evidence.VerifyRetirement(t.Context(), retired, nil); evidence.calls != 2 { |
| 253 | t.Fatalf("the claim's evidence is not the moves evidence: %d calls (%v)", evidence.calls, err) |
| 254 | } |
| 255 | } |
| 256 | |
| 257 | // countingEvidence counts VerifyRetirement calls. |
| 258 | type countingEvidence struct { |
| 259 | hostbox.RegionalEvidence |
| 260 | calls int |
| 261 | } |
| 262 | |
| 263 | func (e *countingEvidence) VerifyRetirement(ctx context.Context, o hostclaim.Observed, v []lease.ObjectVersion) (string, time.Time, error) { |
| 264 | e.calls++ |
| 265 | return e.RegionalEvidence.VerifyRetirement(ctx, o, v) |
| 266 | } |
| 267 | |
| 268 | // TestRegisteredSourceOpensTheMovesOwnersWithTheClaimsFiles (Task 8's |
| 269 | // controller overrides): the registered source's Open returns the moves |
| 270 | // owner set the claim binds (the landing, committed and recovery owners, |
| 271 | // with Operations, LossRecovery and LossSources), over the claim's read |
| 272 | // access, the actor's bucket and region, and Task 7b's held release adapter; |
| 273 | // it opens no files provider of its own. The C8 provider the recovery |
| 274 | // owners establish through is the one Task 6's registration binds for the |
| 275 | // claim's serving owners: the same value. A second provider is a wiring |
| 276 | // error. |
| 277 | func TestRegisteredSourceOpensTheMovesOwnersWithTheClaimsFiles(t *testing.T) { |
| 278 | withMovesWakeSource(t, false) |
| 279 | ctx := t.Context() |
| 280 | f := newVwFixture(t) |
| 281 | parks, err := newMovesParkSources(f.records(), f.bucket, "data", fixedRegion("ie"), movesRetirementEvidence{Records: f.records(), Now: time.Now}) |
| 282 | if err != nil { |
| 283 | t.Fatal(err) |
| 284 | } |
| 285 | if err := registerMovesParks(parks); err != nil { |
| 286 | t.Fatal(err) |
| 287 | } |
| 288 | env := mwEnv(true, true) |
| 289 | dataDir := t.TempDir() |
| 290 | if err := registerProductionMovesWakeSource(env, parks, dataDir); err != nil { |
| 291 | t.Fatal(err) |
| 292 | } |
| 293 | src := claimMoveSources.Source.(*movesWakeSourceRegistration).ClaimProductionMoveSources.(*movesWakeSources) |
| 294 | src.Getenv = env |
| 295 | src.markers = func(context.Context, string, string) (ClaimMarkerStore, error) { return plan9D5Markers{f.bucket}, nil } |
| 296 | actor := hostbox.Actor{Box: "ie-box", Region: "ie", Bucket: "data", Enrollment: "enr", Cloud: hostbox.CloudIdentity{Region: "eu-west-1"}} |
| 297 | store := release.NewDirStore(t.TempDir()) |
| 298 | data := leasetest.NewVersionedBucket() |
| 299 | access := &mwReadAccess{route: registry.Route{Host: vwHost}} |
| 300 | out := ClaimOwnerBindings{Data: data} |
| 301 | // OpenClaimOwners hands Open its move read access; here the fixture's. |
| 302 | owners, files, err := claimMoveSources.Source.Open(ctx, store, data, actor, access) |
| 303 | if err != nil || files != nil { |
| 304 | t.Fatalf("open: files %v, %v", files, err) |
| 305 | } |
| 306 | if ok, err := claimMovesC1(env, owners); !ok || err != nil { |
| 307 | t.Fatalf("the owner set is not complete: %v %v", ok, err) |
| 308 | } |
| 309 | if owners.LossRecovery == nil || owners.LossSources == nil || owners.Operations == nil || owners.ProbeReady != nil { |
| 310 | t.Fatalf("owners %+v", owners) |
| 311 | } |
| 312 | if reflect.TypeOf(owners.LossRecovery) != reflect.TypeOf(hostops.MoveLossRecoveryOwner()) { |
| 313 | t.Fatalf("the loss owner is %T", owners.LossRecovery) |
| 314 | } |
| 315 | m, ok := owners.Inputs.(*movesProbeOwners) |
| 316 | if !ok { |
| 317 | t.Fatalf("the inputs are %T", owners.Inputs) |
| 318 | } |
| 319 | if !m.valid() || m.Box != "ie-box" || m.Region != "ie" || m.Bucket != "data" || m.Data != data || m.Claims != store || m.DataDir != movesDataDir(dataDir) || m.Files != nil || m.Release == nil || m.Page == nil { |
| 320 | t.Fatalf("the probe owners: %+v", m) |
| 321 | } |
| 322 | ops, ok := owners.Operations.(*movesProbeAttempts) |
| 323 | if !ok || ops.Claims != store || ops.DataDir != m.DataDir || ops.Admitted == nil || ops.RecoveryAttempt == nil { |
| 324 | t.Fatalf("the attempts: %#v", owners.Operations) |
| 325 | } |
| 326 | // The read access is the claim's: route, launch and held works. |
| 327 | if rt, err := m.Route(vwHost); err != nil || rt.Host != vwHost { |
| 328 | t.Fatalf("route %v %v", rt, err) |
| 329 | } |
| 330 | if _, err := m.Works(ctx, vwHost, movesActionHeldConfig, hostguard.Captured{}); err != nil || !reflect.DeepEqual(access.works, []string{movesActionHeldConfig}) { |
| 331 | t.Fatalf("held work %v %v", access.works, err) |
| 332 | } |
| 333 | |
| 334 | // The claim's C8 provider binds the serving owners and the recovery |
| 335 | // owners alike: one value. |
| 336 | out.Moves = owners |
| 337 | provider := &movesFiles{} |
| 338 | closeFiles, err := openClaimFilesManifest(ctx, env, &fakeFilesSource{files: provider}, store, data, actor, &out, func(context.Context) error { return nil }) |
| 339 | if err != nil || closeFiles == nil { |
| 340 | t.Fatalf("files: %v", err) |
| 341 | } |
| 342 | if out.Serving.Files != ClaimFilesManifest(provider) || m.Files != ClaimFilesManifest(provider) { |
| 343 | t.Fatalf("the recovery owners' files %v, the serving owners' %v", m.Files, out.Serving.Files) |
| 344 | } |
| 345 | // Another provider for the same owners is refused, never swapped in. |
| 346 | again := ClaimOwnerBindings{Moves: owners} |
| 347 | if _, err := openClaimFilesManifest(ctx, env, &fakeFilesSource{files: &movesFiles{}}, store, data, actor, &again, func(context.Context) error { return nil }); !errors.Is(err, hostwrite.ErrUnavailable) || m.Files != ClaimFilesManifest(provider) { |
| 348 | t.Fatalf("a second provider: %v, files %v", err, m.Files) |
| 349 | } |
| 350 | // Without the read access, or the secrets key, Open refuses: the wake |
| 351 | // is then refused by name, never half-wired. |
| 352 | if _, _, err := src.Open(ctx, store, data, actor, nil); !errors.Is(err, hostwrite.ErrUnavailable) { |
| 353 | t.Fatalf("no read access: %v", err) |
| 354 | } |
| 355 | src.Getenv = mwEnv(true, true) |
| 356 | noKey := *src |
| 357 | noKey.Getenv = func(k string) string { |
| 358 | if k == "CARLOS_SECRETS_KEY" { |
| 359 | return "" |
| 360 | } |
| 361 | return env(k) |
| 362 | } |
| 363 | if _, _, err := noKey.Open(ctx, store, data, actor, access); err == nil { |
| 364 | t.Fatal("opened without the env report key") |
| 365 | } |
| 366 | if _, _, err := src.Open(ctx, store, data, hostbox.Actor{Box: "ie-box"}, access); !errors.Is(err, hostwrite.ErrUnavailable) { |
| 367 | t.Fatalf("a bare actor: %v", err) |
| 368 | } |
| 369 | } |
| 370 | |
| 371 | // mwDataSource is a moves source that records the bucket the claim hands |
| 372 | // its Open. |
| 373 | type mwDataSource struct { |
| 374 | parkRegistrationWake |
| 375 | data lease.VersionedStore |
| 376 | } |
| 377 | |
| 378 | func (s *mwDataSource) Open(_ context.Context, _ release.Store, data lease.VersionedStore, _ hostbox.Actor, _ ClaimMoveReadAccess) (ClaimMovesOwners, ClaimFilesManifest, error) { |
| 379 | s.data = data |
| 380 | return ClaimMovesOwners{}, nil, nil |
| 381 | } |
| 382 | |
| 383 | // mwBucket is the claim's data bucket, recording each listing's context. |
| 384 | // When started is set, a listing closes it once it has reached the bucket |
| 385 | // and then holds until its context ends. The test sets started before the |
| 386 | // listing's goroutine starts, and reads the counters only after that |
| 387 | // goroutine's result has been received, so neither side races. |
| 388 | type mwBucket struct { |
| 389 | *leasetest.VersionedBucket |
| 390 | lists int |
| 391 | deadlines []time.Time |
| 392 | started chan struct{} |
| 393 | } |
| 394 | |
| 395 | func (b *mwBucket) ListVersions(ctx context.Context, bucket, prefix string) ([]lease.ObjectVersion, error) { |
| 396 | b.lists++ |
| 397 | d, _ := ctx.Deadline() |
| 398 | b.deadlines = append(b.deadlines, d) |
| 399 | if b.started != nil { |
| 400 | close(b.started) |
| 401 | <-ctx.Done() |
| 402 | return nil, ctx.Err() |
| 403 | } |
| 404 | return b.VersionedBucket.ListVersions(ctx, bucket, prefix) |
| 405 | } |
| 406 | |
| 407 | // TestMovesOwnersReadTheBucketThroughTheRequestStore (Task 8 fix round 1, |
| 408 | // Astra and platform-1e): the moves owners get the claim's data bucket as |
| 409 | // the serving owners do, through hostbox.RequestStore: every request is |
| 410 | // bounded by the claim's request limit, the box's identity is checked |
| 411 | // before each one (a refused identity issues no later bucket call), and |
| 412 | // the operation's own context still cancels a request in flight. |
| 413 | func TestMovesOwnersReadTheBucketThroughTheRequestStore(t *testing.T) { |
| 414 | inner := &mwBucket{VersionedBucket: leasetest.NewVersionedBucket()} |
| 415 | inner.AddBucket("data") |
| 416 | var refuse error |
| 417 | checks := 0 |
| 418 | check := func(context.Context) error { checks++; return refuse } |
| 419 | src := &mwDataSource{} |
| 420 | out := ClaimOwnerBindings{Data: inner} |
| 421 | openProductionClaimMoveWake(t.Context(), release.NewDirStore(t.TempDir()), hostbox.Actor{Box: "ie-box"}, src, &out, check) |
| 422 | if out.wakeUnavailable != nil || src.data == nil { |
| 423 | t.Fatalf("open: %v", out.wakeUnavailable) |
| 424 | } |
| 425 | if _, ok := src.data.(hostbox.RequestStore); !ok { |
| 426 | t.Fatalf("the moves owners' bucket is %T, not the claim's request store", src.data) |
| 427 | } |
| 428 | before := time.Now() |
| 429 | if _, err := src.data.ListVersions(t.Context(), "data", "instances/"); err != nil { |
| 430 | t.Fatal(err) |
| 431 | } |
| 432 | if inner.lists != 1 || checks != 1 || inner.deadlines[0].IsZero() || inner.deadlines[0].After(time.Now().Add(hostbox.RequestLimit)) || inner.deadlines[0].Before(before.Add(hostbox.RequestLimit-time.Minute)) { |
| 433 | t.Fatalf("the request's deadline %v (lists %d, checks %d), want the request limit", inner.deadlines, inner.lists, checks) |
| 434 | } |
| 435 | refuse = hostbox.ErrChanged |
| 436 | if _, err := src.data.ListVersions(t.Context(), "data", "instances/"); !errors.Is(err, hostbox.ErrChanged) || inner.lists != 1 { |
| 437 | t.Fatalf("a refused identity: %v, %d bucket calls", err, inner.lists) |
| 438 | } |
| 439 | if _, err := src.data.GetVersion(t.Context(), "data", "k", "v"); !errors.Is(err, hostbox.ErrChanged) { |
| 440 | t.Fatalf("a refused identity's read: %v", err) |
| 441 | } |
| 442 | refuse = nil |
| 443 | // A request in flight: wait (bounded) until it has reached the |
| 444 | // bucket, or has ended early, then cancel the operation. |
| 445 | started := make(chan struct{}) |
| 446 | inner.started = started |
| 447 | ctx, cancel := context.WithCancel(t.Context()) |
| 448 | defer cancel() |
| 449 | done := make(chan error, 1) |
| 450 | go func() { _, err := src.data.ListVersions(ctx, "data", "instances/"); done <- err }() |
| 451 | select { |
| 452 | case <-started: |
| 453 | case err := <-done: |
| 454 | t.Fatalf("the request ended before it reached the bucket: %v", err) |
| 455 | case <-time.After(10 * time.Second): |
| 456 | t.Fatal("the request never reached the bucket") |
| 457 | } |
| 458 | cancel() |
| 459 | select { |
| 460 | case err := <-done: |
| 461 | if !errors.Is(err, context.Canceled) { |
| 462 | t.Fatalf("a cancelled operation's request: %v", err) |
| 463 | } |
| 464 | case <-time.After(10 * time.Second): |
| 465 | t.Fatal("the operation's cancellation did not reach the request") |
| 466 | } |
| 467 | if inner.lists != 2 { |
| 468 | t.Fatalf("%d bucket calls, want the first and the cancelled one", inner.lists) |
| 469 | } |
| 470 | } |
| 471 | |
| 472 | // TestMovesSourceOpenRefusalIsLoggedOnce (Task 8 fix round 1, |
| 473 | // platform-1e): a moves source that does not open keeps every wake on the |
| 474 | // box refused (fail closed), and the cause is logged once per agent start, |
| 475 | // so an operator can see why the box serves nothing. |
| 476 | func TestMovesSourceOpenRefusalIsLoggedOnce(t *testing.T) { |
| 477 | var lines []string |
| 478 | oldLogf, oldOnce := claimMoveWakeLogf, claimMoveWakeLogOnce |
| 479 | claimMoveWakeLogf = func(format string, args ...any) { lines = append(lines, fmt.Sprintf(format, args...)) } |
| 480 | claimMoveWakeLogOnce = new(sync.Once) |
| 481 | t.Cleanup(func() { claimMoveWakeLogf, claimMoveWakeLogOnce = oldLogf, oldOnce }) |
| 482 | cause := errors.New("moves: secrets key: not a 32-byte key") |
| 483 | for i := 0; i < 3; i++ { |
| 484 | out := ClaimOwnerBindings{} |
| 485 | openProductionClaimMoveWake(t.Context(), nil, hostbox.Actor{}, &parkRegistrationWake{failure: cause}, &out, func(context.Context) error { return nil }) |
| 486 | if !errors.Is(out.wakeUnavailable, cause) { |
| 487 | t.Fatalf("open %d: the refusal is not kept: %v", i, out.wakeUnavailable) |
| 488 | } |
| 489 | } |
| 490 | if len(lines) != 1 || !strings.Contains(lines[0], cause.Error()) || !strings.Contains(lines[0], "every wake") { |
| 491 | t.Fatalf("logged %q, want the cause once", lines) |
| 492 | } |
| 493 | } |
| 494 | |
| 495 | // TestBindClaimMovesFilesRefusesASecondProvider (Task 8 fix round 1, |
| 496 | // platform-1e): the claim's C8 provider binds the moves owners once; the |
| 497 | // same provider again changes nothing, and another is refused and never |
| 498 | // swapped in. A binding without moves owners, or without a provider, does |
| 499 | // nothing. |
| 500 | func TestBindClaimMovesFilesRefusesASecondProvider(t *testing.T) { |
| 501 | m := &movesProbeOwners{} |
| 502 | first, second := &movesFiles{Box: "first"}, &movesFiles{Box: "second"} |
| 503 | out := ClaimOwnerBindings{Moves: m.MovesOwners(&movesProbeAttempts{})} |
| 504 | out.Serving.Files = first |
| 505 | if err := bindClaimMovesFiles(&out); err != nil || m.Files != ClaimFilesManifest(first) { |
| 506 | t.Fatalf("the first provider: %v, files %v", err, m.Files) |
| 507 | } |
| 508 | if err := bindClaimMovesFiles(&out); err != nil || m.Files != ClaimFilesManifest(first) { |
| 509 | t.Fatalf("the same provider again: %v", err) |
| 510 | } |
| 511 | out.Serving.Files = second |
| 512 | if err := bindClaimMovesFiles(&out); !errors.Is(err, hostwrite.ErrUnavailable) || m.Files != ClaimFilesManifest(first) { |
| 513 | t.Fatalf("a second provider: %v, files %v", err, m.Files) |
| 514 | } |
| 515 | if err := bindClaimMovesFiles(&ClaimOwnerBindings{Serving: ClaimServingSources{Files: second}}); err != nil { |
| 516 | t.Fatalf("no moves owners: %v", err) |
| 517 | } |
| 518 | if err := bindClaimMovesFiles(&ClaimOwnerBindings{Moves: out.Moves}); err != nil || m.Files != ClaimFilesManifest(first) { |
| 519 | t.Fatalf("no provider: %v", err) |
| 520 | } |
| 521 | } |
| 522 | |
amadan