| 1 | package console |
| 2 | |
| 3 | // The operator's resolve route answers three actions. With the claim on, |
| 4 | // declare-lost and readmit are refused before any effect (the refused fixture |
| 5 | // in claimFixtures, and the readmit refusal below). reset-wake-retry is Plan |
| 6 | // 6b's operator reset: it writes one host claim (a lifecycle CAS root) under |
| 7 | // its own reset-wake-retry admission registration, never an app permit. |
| 8 | // |
| 9 | // It is a lifecycle variant of the route: driven foreign and open like a |
| 10 | // door, but its only effect is the claim root, so its open run is judged by |
| 11 | // claimLifecycleOpenProblems (admission written, exactly its own claim moved, |
| 12 | // no app effect) instead of claimOpenProblems. |
| 13 | |
| 14 | import ( |
| 15 | "context" |
| 16 | "errors" |
| 17 | "fmt" |
| 18 | "go/ast" |
| 19 | "go/parser" |
| 20 | "go/token" |
| 21 | "net/http" |
| 22 | "slices" |
| 23 | "sort" |
| 24 | "strconv" |
| 25 | "strings" |
| 26 | "sync" |
| 27 | "testing" |
| 28 | "time" |
| 29 | |
| 30 | "github.com/carlosframework/platform/internal/hostclaim" |
| 31 | "github.com/carlosframework/platform/internal/hostops" |
| 32 | "github.com/carlosframework/platform/internal/release" |
| 33 | ) |
| 34 | |
| 35 | const claimResolveKey = "route POST /api/cli/ops/hosts/{host}/resolve" |
| 36 | |
| 37 | // claimLifecycleVariant is one more request on a registered entry whose |
| 38 | // fixture covers another of its actions. Its only effect is the host claim on |
| 39 | // host, written under apps[0]'s own registration grant. |
| 40 | type claimLifecycleVariant struct { |
| 41 | name string |
| 42 | apps []claimAppRef // apps[0] holds the claim on host and registers |
| 43 | host string |
| 44 | config func(*Config) |
| 45 | setup func(t *testing.T, cx *claimCase) |
| 46 | do func(t *testing.T, cx *claimCase) (int, string) |
| 47 | // check proves from the world that the foreign runs did nothing |
| 48 | // (open=false) and the open run did exactly the action (open=true). |
| 49 | check func(t *testing.T, cx *claimCase, open bool) string |
| 50 | } |
| 51 | |
| 52 | var claimLifecycleVariants = map[string][]claimLifecycleVariant{ |
| 53 | claimResolveKey: {claimWakeResetVariant()}, |
| 54 | } |
| 55 | |
| 56 | // claimResolveActions is every action handleAPIHostResolve dispatches on, |
| 57 | // from source (TestClaimHostResolveActions), with what covers it. |
| 58 | var claimResolveActions = map[string]string{ |
| 59 | "declare-lost": "claimFixtures' refused fixture: claimHostResolveUnsupportedMsg, foreign and open", |
| 60 | "readmit": "TestClaimHostResolveActions: the same refusal, foreign and open", |
| 61 | "reset-wake-retry": "claimLifecycleVariants: the claim write under its own reset-wake-retry registration", |
| 62 | } |
| 63 | |
| 64 | // claimVariantCensus: every variant names a registered entry that also has a |
| 65 | // fixture, and carries its app, host, request and world check. |
| 66 | func claimVariantCensus(entries []claimEntry, fixtures map[string]claimFixture, variants map[string][]claimLifecycleVariant) []string { |
| 67 | got := map[string]bool{} |
| 68 | for _, e := range entries { |
| 69 | got[e.Kind+" "+e.Pattern] = true |
| 70 | } |
| 71 | var problems []string |
| 72 | for key, vs := range variants { |
| 73 | if !got[key] { |
| 74 | problems = append(problems, fmt.Sprintf("lifecycle variants name %s, which is not registered", key)) |
| 75 | } |
| 76 | if _, ok := fixtures[key]; !ok { |
| 77 | problems = append(problems, fmt.Sprintf("lifecycle variants name %s, which has no claim fixture", key)) |
| 78 | } |
| 79 | for _, v := range vs { |
| 80 | if v.name == "" || len(v.apps) == 0 || v.host == "" || v.do == nil || v.check == nil { |
| 81 | problems = append(problems, fmt.Sprintf("lifecycle variant %s %q has no name, app, host, request or check", key, v.name)) |
| 82 | } |
| 83 | } |
| 84 | } |
| 85 | sort.Strings(problems) |
| 86 | return problems |
| 87 | } |
| 88 | |
| 89 | func runClaimLifecycleVariants(t *testing.T) { |
| 90 | keys := make([]string, 0, len(claimLifecycleVariants)) |
| 91 | for k := range claimLifecycleVariants { |
| 92 | keys = append(keys, k) |
| 93 | } |
| 94 | sort.Strings(keys) |
| 95 | for _, key := range keys { |
| 96 | for _, v := range claimLifecycleVariants[key] { |
| 97 | t.Run(key+" "+v.name, func(t *testing.T) { |
| 98 | for _, p := range runClaimLifecycleVariant(t, v) { |
| 99 | t.Error(p) |
| 100 | } |
| 101 | }) |
| 102 | } |
| 103 | } |
| 104 | } |
| 105 | |
| 106 | // runClaimLifecycleVariant drives one variant like a door: setup, one |
| 107 | // foreign-exclusive run per affected incarnation (each must answer 409 and |
| 108 | // leave nothing), then the open run. |
| 109 | func runClaimLifecycleVariant(t *testing.T, v claimLifecycleVariant) []string { |
| 110 | t.Helper() |
| 111 | var opts []func(*Config) |
| 112 | if v.config != nil { |
| 113 | opts = append(opts, v.config) |
| 114 | } |
| 115 | cx := newClaimCaseWith(t, opts...) |
| 116 | if v.setup != nil { |
| 117 | v.setup(t, cx) |
| 118 | } |
| 119 | cx.quiesce() |
| 120 | run := func() (claimWindow, string) { |
| 121 | var body string |
| 122 | w := cx.measureRun(t, v.apps, func() int { |
| 123 | status, out := v.do(t, cx) |
| 124 | body = out |
| 125 | return status |
| 126 | }) |
| 127 | return w, body |
| 128 | } |
| 129 | var problems []string |
| 130 | for _, ref := range v.apps { |
| 131 | restore := cx.foreign(t, ref) |
| 132 | w, body := run() |
| 133 | label := "foreign exclusive on " + ref.String() |
| 134 | if w.status != http.StatusConflict { |
| 135 | problems = append(problems, fmt.Sprintf("%s: answered %d %q, want the 409 refusal", label, w.status, body)) |
| 136 | } |
| 137 | problems = append(problems, claimForeignProblems(ref, v.apps, w)...) |
| 138 | if p := v.check(t, cx, false); p != "" { |
| 139 | problems = append(problems, label+": "+p) |
| 140 | } |
| 141 | restore() |
| 142 | } |
| 143 | w, body := run() |
| 144 | problems = append(problems, claimLifecycleOpenProblems(v, w, body)...) |
| 145 | if p := v.check(t, cx, true); p != "" { |
| 146 | problems = append(problems, "open run: "+p) |
| 147 | } |
| 148 | return problems |
| 149 | } |
| 150 | |
| 151 | // claimLifecycleOpenProblems: with every incarnation open, the request must |
| 152 | // register with apps[0]'s admission, move exactly its own claim root (which |
| 153 | // the production classifier admitted), and reach no app effect at all. |
| 154 | func claimLifecycleOpenProblems(v claimLifecycleVariant, w claimWindow, body string) []string { |
| 155 | var problems []string |
| 156 | say := func(format string, args ...any) { |
| 157 | problems = append(problems, "open run: "+fmt.Sprintf(format, args...)) |
| 158 | } |
| 159 | if w.status != http.StatusOK { |
| 160 | say("the authenticated valid request answered %d %q", w.status, body) |
| 161 | } |
| 162 | key, err := hostclaim.Key(v.host) |
| 163 | if err != nil { |
| 164 | say("host %s: %v", v.host, err) |
| 165 | } |
| 166 | admitted, moved := false, false |
| 167 | for _, e := range w.events { |
| 168 | switch { |
| 169 | case e.admissionWrite(v.apps[0]): |
| 170 | admitted = true |
| 171 | case e.Control: |
| 172 | case e.Err != nil: |
| 173 | say("raw %s reached a key the production classifier refuses (%v)", e, e.Err) |
| 174 | case e.NonApp == claimLifecycleRoot && e.Key == key: |
| 175 | moved = true |
| 176 | case e.NonApp == claimLifecycleRoot: |
| 177 | say("raw %s moved another lifecycle root", e) |
| 178 | case e.Unscoped || len(e.Scopes) != 0: |
| 179 | say("raw %s is an app effect; this action writes only its claim", e) |
| 180 | } |
| 181 | } |
| 182 | for ref, changed := range w.sqlChanged { |
| 183 | say("SQL rows of %s changed in %v", ref, changed) |
| 184 | } |
| 185 | if !admitted { |
| 186 | say("no write to %s's admission, so the request never registered with the gate", v.apps[0]) |
| 187 | } |
| 188 | if !moved { |
| 189 | say("the claim %s did not move", key) |
| 190 | } |
| 191 | return problems |
| 192 | } |
| 193 | |
| 194 | // ---- reset-wake-retry ------------------------------------------------------- |
| 195 | |
| 196 | const ( |
| 197 | claimResetEpisode = "claim-reset-episode" |
| 198 | claimResetNext = "claim-reset-next" |
| 199 | claimResetRequest = "claim-reset-request" |
| 200 | ) |
| 201 | |
| 202 | var claimResetAttempt = hostclaim.WakeRetryAttempt{ID: "claim-reset-attempt", Box: "box-ie-default", Enrollment: "claim-reset-enrollment", |
| 203 | BootID: "claim-reset-boot", ControllerID: "claim-reset-controller", InvocationID: "claim-reset-invocation", LaunchInputs: "claim-reset-inputs"} |
| 204 | |
| 205 | // claimResetStops records the attempts the door asked the box to stop. |
| 206 | type claimResetStops struct { |
| 207 | mu sync.Mutex |
| 208 | seen []hostclaim.WakeRetryAttempt |
| 209 | } |
| 210 | |
| 211 | func (s *claimResetStops) stop(_ context.Context, _ hostclaim.Observed, a hostclaim.WakeRetryAttempt) error { |
| 212 | s.mu.Lock() |
| 213 | s.seen = append(s.seen, a) |
| 214 | s.mu.Unlock() |
| 215 | return nil |
| 216 | } |
| 217 | |
| 218 | func (s *claimResetStops) list() []hostclaim.WakeRetryAttempt { |
| 219 | s.mu.Lock() |
| 220 | defer s.mu.Unlock() |
| 221 | return slices.Clone(s.seen) |
| 222 | } |
| 223 | |
| 224 | func (s *claimResetStops) clear() { |
| 225 | s.mu.Lock() |
| 226 | s.seen = nil |
| 227 | s.mu.Unlock() |
| 228 | } |
| 229 | |
| 230 | // claimUnusedClaims is the handle the reset door is configured with. Every |
| 231 | // call fails: the mounted door must read and write claims only through the |
| 232 | // console's guarded data boundary. |
| 233 | type claimUnusedClaims struct{} |
| 234 | |
| 235 | var errClaimUnusedClaims = errors.New("claim fixture: the reset door used its configured raw claims handle") |
| 236 | |
| 237 | var _ release.Store = claimUnusedClaims{} |
| 238 | |
| 239 | func (claimUnusedClaims) Get(context.Context, string) ([]byte, string, error) { |
| 240 | return nil, "", errClaimUnusedClaims |
| 241 | } |
| 242 | func (claimUnusedClaims) Put(context.Context, string, []byte, string) (string, error) { |
| 243 | return "", errClaimUnusedClaims |
| 244 | } |
| 245 | func (claimUnusedClaims) List(context.Context, string) ([]string, error) { |
| 246 | return nil, errClaimUnusedClaims |
| 247 | } |
| 248 | func (claimUnusedClaims) ListDirs(context.Context, string) ([]string, error) { |
| 249 | return nil, errClaimUnusedClaims |
| 250 | } |
| 251 | func (claimUnusedClaims) Delete(context.Context, string) error { return errClaimUnusedClaims } |
| 252 | |
| 253 | func claimWakeResetDoor(stops *claimResetStops) func(*Config) { |
| 254 | return func(cfg *Config) { |
| 255 | cfg.WakeRetryReset = &WakeRetryResetDoor{Claims: claimUnusedClaims{}, Authority: func(context.Context) error { return nil }, |
| 256 | StopAttempt: stops.stop, NewID: func() string { return claimResetNext }} |
| 257 | } |
| 258 | } |
| 259 | |
| 260 | // stageWakeReset stages hostLossWorld's serving claim for bab/jam with one |
| 261 | // reserved wake retry attempt, and the reset request for it in cx.vars. |
| 262 | func (cx *claimCase) stageWakeReset(t *testing.T) hostclaim.Observed { |
| 263 | t.Helper() |
| 264 | ctx := context.Background() |
| 265 | cx.enroll(t, "bab", "jam") |
| 266 | cx.hostLossWorld(t) |
| 267 | o, err := hostclaim.Read(ctx, cx.rel, claimLossHost) |
| 268 | if err != nil { |
| 269 | t.Fatal(err) |
| 270 | } |
| 271 | inc := o.Claim.Holder.Incarnation |
| 272 | if o, err = hostclaim.Update(ctx, cx.rel, o, hostclaim.ReserveWakeRetryAttempt(hostclaim.WakeRetryReservation{ |
| 273 | HolderIncarnation: inc, EpisodeID: claimResetEpisode, Attempt: claimResetAttempt, Now: time.Now()})); err != nil { |
| 274 | t.Fatalf("reserve the wake retry attempt on %s: %v", claimLossHost, err) |
| 275 | } |
| 276 | cx.vars["reset"] = `{"action":"reset-wake-retry","incarnation":` + strconv.Quote(inc) + `,"episode":"` + claimResetEpisode + `","request_id":"` + claimResetRequest + `"}` |
| 277 | cx.vars["reset-etag"] = o.ETag |
| 278 | cx.vars["reset-incarnation"] = inc |
| 279 | return o |
| 280 | } |
| 281 | |
| 282 | func claimWakeResetVariant() claimLifecycleVariant { |
| 283 | stops := &claimResetStops{} |
| 284 | return claimLifecycleVariant{ |
| 285 | name: "reset-wake-retry", |
| 286 | apps: claimRefs("bab/jam"), |
| 287 | host: claimLossHost, |
| 288 | config: claimWakeResetDoor(stops), |
| 289 | setup: func(t *testing.T, cx *claimCase) { |
| 290 | stops.clear() |
| 291 | cx.stageWakeReset(t) |
| 292 | }, |
| 293 | do: claimRefusedAPI(http.MethodPost, "/api/cli/ops/hosts/"+claimLossHost+"/resolve", "<reset>"), |
| 294 | check: func(t *testing.T, cx *claimCase, open bool) string { |
| 295 | ctx := context.Background() |
| 296 | o, err := hostclaim.Read(ctx, cx.rel, claimLossHost) |
| 297 | if err != nil || o.Claim == nil || o.Claim.Holder == nil { |
| 298 | return fmt.Sprintf("read the claim: %v", err) |
| 299 | } |
| 300 | stopped := stops.list() |
| 301 | if !open { |
| 302 | if len(stopped) != 0 { |
| 303 | return fmt.Sprintf("stopped %v before the gate refused", stopped) |
| 304 | } |
| 305 | if o.ETag != cx.vars["reset-etag"] { |
| 306 | return "the claim moved" |
| 307 | } |
| 308 | return "" |
| 309 | } |
| 310 | if len(stopped) != 1 || stopped[0] != claimResetAttempt { |
| 311 | return fmt.Sprintf("stopped %v, want exactly the reserved attempt", stopped) |
| 312 | } |
| 313 | h := o.Claim.Holder |
| 314 | r := h.WakeRetry |
| 315 | if h.Incarnation != cx.vars["reset-incarnation"] || r == nil || r.EpisodeID != claimResetNext || r.State != "open" || r.Attempt != nil || |
| 316 | !strings.Contains(r.ResetRequest, `"RequestID":"`+claimResetRequest+`"`) { |
| 317 | return fmt.Sprintf("the claim was not reset: holder %s retry %+v", h.Incarnation, r) |
| 318 | } |
| 319 | // The grant came from the reset's own registration: the holder's |
| 320 | // exact incarnation and the observation the door read. |
| 321 | adm, err := hostops.ReadCanonicalAdmission(ctx, cx.rel, "bab", "jam") |
| 322 | if err != nil { |
| 323 | return fmt.Sprintf("read the admission: %v", err) |
| 324 | } |
| 325 | for _, reg := range adm.Value.Registrations { |
| 326 | if reg.Kind == wakeRetryResetKind && reg.Host == claimLossHost && reg.Incarnation == h.Incarnation && reg.Claim.ETag == cx.vars["reset-etag"] { |
| 327 | return "" |
| 328 | } |
| 329 | } |
| 330 | return fmt.Sprintf("no %s registration for %s at the observed claim: %+v", wakeRetryResetKind, h.Incarnation, adm.Value.Registrations) |
| 331 | }, |
| 332 | } |
| 333 | } |
| 334 | |
| 335 | // TestClaimHostResolveActions pins, from source, every action the resolve |
| 336 | // route dispatches on, and drives the readmit refusal the census names. |
| 337 | func TestClaimHostResolveActions(t *testing.T) { |
| 338 | fs := token.NewFileSet() |
| 339 | f, err := parser.ParseFile(fs, "apihostclaim.go", nil, 0) |
| 340 | if err != nil { |
| 341 | t.Fatal(err) |
| 342 | } |
| 343 | got := map[string]bool{} |
| 344 | for _, d := range f.Decls { |
| 345 | fn, ok := d.(*ast.FuncDecl) |
| 346 | if !ok || fn.Name.Name != "handleAPIHostResolve" { |
| 347 | continue |
| 348 | } |
| 349 | ast.Inspect(fn.Body, func(n ast.Node) bool { |
| 350 | b, ok := n.(*ast.BinaryExpr) |
| 351 | if !ok || (b.Op != token.EQL && b.Op != token.NEQ) { |
| 352 | return true |
| 353 | } |
| 354 | sel, ok := b.X.(*ast.SelectorExpr) |
| 355 | lit, lok := b.Y.(*ast.BasicLit) |
| 356 | if !ok || !lok || sel.Sel.Name != "Action" || lit.Kind != token.STRING { |
| 357 | return true |
| 358 | } |
| 359 | action, err := strconv.Unquote(lit.Value) |
| 360 | if err != nil { |
| 361 | t.Fatal(err) |
| 362 | } |
| 363 | got[action] = true |
| 364 | return true |
| 365 | }) |
| 366 | } |
| 367 | for action := range got { |
| 368 | if strings.TrimSpace(claimResolveActions[action]) == "" { |
| 369 | t.Errorf("the resolve route dispatches on %q, which claimResolveActions does not cover", action) |
| 370 | } |
| 371 | } |
| 372 | for action := range claimResolveActions { |
| 373 | if !got[action] { |
| 374 | t.Errorf("claimResolveActions names %q, which the resolve route no longer dispatches on", action) |
| 375 | } |
| 376 | } |
| 377 | if f := claimFixtures[claimResolveKey].refused; f == nil || f.msg != claimHostResolveUnsupportedMsg { |
| 378 | t.Errorf("%s is not the declare-lost refusal", claimResolveKey) |
| 379 | } |
| 380 | if !slices.ContainsFunc(claimLifecycleVariants[claimResolveKey], func(v claimLifecycleVariant) bool { return v.name == "reset-wake-retry" }) { |
| 381 | t.Errorf("%s has no reset-wake-retry lifecycle variant", claimResolveKey) |
| 382 | } |
| 383 | readmit := &claimRefusalFixture{ |
| 384 | apps: claimRefs("bab/jam"), |
| 385 | setup: func(t *testing.T, cx *claimCase) { |
| 386 | cx.enroll(t, "bab", "jam") |
| 387 | cx.hostLossWorld(t) |
| 388 | }, |
| 389 | do: claimRefusedAPI(http.MethodPost, "/api/cli/ops/hosts/"+claimLossHost+"/resolve", |
| 390 | `{"action":"readmit","box":"`+claimLossBox+`","enrollment":"claim-readmit","revision":1,"loss_ids":["claim-readmit-loss"]}`), |
| 391 | status: http.StatusConflict, msg: claimHostResolveUnsupportedMsg, |
| 392 | } |
| 393 | for _, p := range runClaimRefusal(t, readmit) { |
| 394 | t.Error(p) |
| 395 | } |
| 396 | } |
| 397 | |
| 398 | // TestClaimWakeResetWritesOnlyUnderItsGrant: the configured raw handle never |
| 399 | // survives ConfigureClaimRuntime, and the same reset written through the |
| 400 | // console's data boundary without the reset's registration grant is refused. |
| 401 | func TestClaimWakeResetWritesOnlyUnderItsGrant(t *testing.T) { |
| 402 | stops := &claimResetStops{} |
| 403 | cx := newClaimCaseWith(t, claimWakeResetDoor(stops)) |
| 404 | if d := cx.srv.cfg.WakeRetryReset; d == nil || d.Claims == nil { |
| 405 | t.Fatal("ConfigureClaimRuntime dropped the reset door") |
| 406 | } else if _, raw := d.Claims.(claimUnusedClaims); raw { |
| 407 | t.Fatalf("the reset door kept its configured claims handle after ConfigureClaimRuntime") |
| 408 | } |
| 409 | o := cx.stageWakeReset(t) |
| 410 | ctx := context.Background() |
| 411 | _, err := hostclaim.Update(ctx, cx.srv.cfg.Releases, o, hostclaim.ResetWakeRetry(hostclaim.WakeRetryReset{HolderIncarnation: o.Claim.Holder.Incarnation, |
| 412 | ObservedEpisode: claimResetEpisode, NewEpisodeID: claimResetNext, OperatorRequest: "claim-ungranted", AttemptSettled: true})) |
| 413 | if err == nil { |
| 414 | t.Fatal("a reset with no lifecycle grant wrote the claim through Config.Releases") |
| 415 | } |
| 416 | fresh, err := hostclaim.Read(ctx, cx.rel, claimLossHost) |
| 417 | if err != nil || fresh.ETag != o.ETag { |
| 418 | t.Fatalf("the refused reset moved the claim: %v", err) |
| 419 | } |
| 420 | if len(stops.list()) != 0 { |
| 421 | t.Fatal("a direct claim write stopped an attempt") |
| 422 | } |
| 423 | } |
| 424 | |
| 425 | func TestClaimWakeResetRegistrationSurvivesSweepAndDrain(t *testing.T) { |
| 426 | stops := &claimResetStops{} |
| 427 | cx := newClaimCaseWith(t, claimWakeResetDoor(stops)) |
| 428 | cx.stageWakeReset(t) |
| 429 | var deletion *hostops.OperationHandle |
| 430 | var resetID, appID string |
| 431 | control := cx.srv.cfg.ClaimAuthority.Control |
| 432 | cx.srv.cfg.WakeRetryReset.StopAttempt = func(ctx context.Context, observed hostclaim.Observed, attempt hostclaim.WakeRetryAttempt) error { |
| 433 | adm, err := hostops.ReadCanonicalAdmission(ctx, cx.rel, "bab", "jam") |
| 434 | if err != nil { |
| 435 | t.Fatal(err) |
| 436 | } |
| 437 | appID = adm.Value.AppID |
| 438 | for id, reg := range adm.Value.Registrations { |
| 439 | if reg.Kind == hostops.RegistrationWakeRetryReset { |
| 440 | resetID = id |
| 441 | } |
| 442 | } |
| 443 | if resetID == "" { |
| 444 | t.Fatal("stop started without registration") |
| 445 | } |
| 446 | if err := control.SettleRegistration(ctx, "bab", "jam", appID, resetID, time.Now()); !errors.Is(err, hostops.ErrPending) { |
| 447 | t.Fatalf("sweep removed reset during stop: %v", err) |
| 448 | } |
| 449 | owner := hostops.RegistrationOwner{Console: claimFxConsole, Enrollment: claimFxConsoleEnrollment, Controller: "concurrent-delete"} |
| 450 | deletion, err = control.BeginExclusive(ctx, "bab", "jam", appID, hostops.AppOperation{ID: "concurrent-delete", Kind: hostops.OpDeleting, AppID: appID}, owner) |
| 451 | if err != nil { |
| 452 | t.Fatal(err) |
| 453 | } |
| 454 | if err := deletion.Drain(ctx, time.Now(), nil); !errors.Is(err, hostops.ErrPending) { |
| 455 | t.Fatalf("delete drained reset during stop: %v", err) |
| 456 | } |
| 457 | return stops.stop(ctx, observed, attempt) |
| 458 | } |
| 459 | status, body := claimRefusedAPI(http.MethodPost, "/api/cli/ops/hosts/"+claimLossHost+"/resolve", "<reset>")(t, cx) |
| 460 | if status != http.StatusOK { |
| 461 | t.Fatalf("reset after concurrent sweep/drain: %d %s", status, body) |
| 462 | } |
| 463 | o, err := hostclaim.Read(t.Context(), cx.rel, claimLossHost) |
| 464 | if err != nil || o.Claim.Holder.WakeRetry.EpisodeID != claimResetNext || len(stops.list()) != 1 { |
| 465 | t.Fatalf("reset did not complete: %+v %v", o, err) |
| 466 | } |
| 467 | if err := deletion.Drain(t.Context(), time.Now(), nil); err != nil { |
| 468 | t.Fatalf("finished reset blocked deletion: %v", err) |
| 469 | } |
| 470 | adm, err := hostops.ReadAdmission(t.Context(), cx.rel, "bab", "jam", appID) |
| 471 | if err != nil { |
| 472 | t.Fatal(err) |
| 473 | } |
| 474 | if _, ok := adm.Value.Registrations[resetID]; ok { |
| 475 | t.Fatal("finished reset registration survived drain") |
| 476 | } |
| 477 | } |
| 478 | |
| 479 | func TestClaimWakeResetCancelsWhenRenewalStalls(t *testing.T) { |
| 480 | started := make(chan struct{}) |
| 481 | ctx, closeRegistration := keepWakeResetRegistration(t.Context(), time.Now(), 100*time.Millisecond, time.Millisecond, func(ctx context.Context, _ time.Time) error { |
| 482 | close(started) |
| 483 | <-ctx.Done() |
| 484 | return ctx.Err() |
| 485 | }) |
| 486 | defer closeRegistration() |
| 487 | select { |
| 488 | case <-started: |
| 489 | case <-time.After(time.Second): |
| 490 | t.Fatal("renewal never started") |
| 491 | } |
| 492 | select { |
| 493 | case <-ctx.Done(): |
| 494 | case <-time.After(time.Second): |
| 495 | t.Fatal("stalled renewal outlived confirmed registration") |
| 496 | } |
| 497 | } |
| 498 | |
amadan