| 1 | package hostclaim |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "encoding/json" |
| 7 | "errors" |
| 8 | "testing" |
| 9 | "time" |
| 10 | |
| 11 | "github.com/carlosframework/platform/internal/release" |
| 12 | ) |
| 13 | |
| 14 | func retryAttempt(id string, session uint64) WakeRetryAttempt { |
| 15 | return WakeRetryAttempt{ID: id, Box: "adi-knitting", Enrollment: "enroll-1", BootID: "boot-1", |
| 16 | ControllerID: "controller-1", Session: session, InvocationID: "invoke-" + id, LaunchInputs: "sha256-inputs-" + id} |
| 17 | } |
| 18 | |
| 19 | func TestWakeRetryDurableBudgetAndBackoff(t *testing.T) { |
| 20 | ctx := context.Background() |
| 21 | s := release.NewDirStore(t.TempDir()) |
| 22 | o := servingClaim(t, ctx, s) |
| 23 | inc := o.Claim.Holder.Incarnation |
| 24 | now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) |
| 25 | |
| 26 | for n := 1; n <= WakeRetryLimit; n++ { |
| 27 | a := retryAttempt(string(rune('0'+n)), o.Claim.Holder.Session) |
| 28 | var err error |
| 29 | o, err = Update(ctx, s, o, ReserveWakeRetryAttempt(WakeRetryReservation{ |
| 30 | HolderIncarnation: inc, EpisodeID: "episode-1", Attempt: a, Now: now, |
| 31 | })) |
| 32 | if err != nil { |
| 33 | t.Fatalf("reserve %d: %v", n, err) |
| 34 | } |
| 35 | if o.Claim.Holder.WakeRetry.AttemptsReserved != n || o.Claim.Holder.WakeRetry.State != "attempting" { |
| 36 | t.Fatalf("reserved %d = %+v", n, o.Claim.Holder.WakeRetry) |
| 37 | } |
| 38 | // A reread proves the budget is stored in the claim rather than process memory. |
| 39 | if o, err = Read(ctx, s, testHost); err != nil { |
| 40 | t.Fatal(err) |
| 41 | } |
| 42 | o, err = Update(ctx, s, o, FailWakeRetryAttempt(WakeRetryResult{ |
| 43 | HolderIncarnation: inc, EpisodeID: "episode-1", Attempt: a, Now: now, |
| 44 | })) |
| 45 | if err != nil { |
| 46 | t.Fatalf("fail %d: %v", n, err) |
| 47 | } |
| 48 | wantState := "open" |
| 49 | if n == WakeRetryLimit { |
| 50 | wantState = "failed" |
| 51 | } |
| 52 | if r := o.Claim.Holder.WakeRetry; r.State != wantState || !r.NotBefore.Equal(now.Add(wakeRetryDelays[n-1])) { |
| 53 | t.Fatalf("settled %d = %+v", n, r) |
| 54 | } |
| 55 | if _, err := Update(ctx, s, o, ReserveWakeRetryAttempt(WakeRetryReservation{HolderIncarnation: inc, Attempt: retryAttempt("early", 0), Now: now})); !errors.Is(err, ErrWrongState) { |
| 56 | t.Fatalf("early reserve %d = %v", n, err) |
| 57 | } |
| 58 | now = now.Add(wakeRetryDelays[n-1]) |
| 59 | } |
| 60 | if _, err := Update(ctx, s, o, ReserveWakeRetryAttempt(WakeRetryReservation{HolderIncarnation: inc, Attempt: retryAttempt("six", 0), Now: now})); !errors.Is(err, ErrWrongState) { |
| 61 | t.Fatalf("sixth reserve = %v", err) |
| 62 | } |
| 63 | if _, err := Update(ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: "adi-knitting", Mode: "cold", NewGeneration: "new-after-failure", PredecessorCertified: true})); !errors.Is(err, ErrWrongState) { |
| 64 | t.Fatalf("serving admission after exhausted budget = %v", err) |
| 65 | } |
| 66 | } |
| 67 | |
| 68 | func TestWakeRetryColdAdmissionBindingAndExactHealth(t *testing.T) { |
| 69 | ctx := context.Background() |
| 70 | s := release.NewDirStore(t.TempDir()) |
| 71 | o := servingClaim(t, ctx, s) |
| 72 | inc := o.Claim.Holder.Incarnation |
| 73 | a := retryAttempt("first", 0) |
| 74 | now := time.Date(2026, 10, 6, 13, 0, 0, 0, time.UTC) |
| 75 | var err error |
| 76 | o, err = Update(ctx, s, o, ReserveWakeRetryAttempt(WakeRetryReservation{HolderIncarnation: inc, EpisodeID: "episode", Attempt: a, Now: now})) |
| 77 | if err != nil { |
| 78 | t.Fatal(err) |
| 79 | } |
| 80 | o, err = Update(ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: "adi-knitting", Mode: "cold", NewGeneration: "generation-1", PredecessorCertified: true})) |
| 81 | if err != nil { |
| 82 | t.Fatal(err) |
| 83 | } |
| 84 | session := o.Claim.Holder.Session |
| 85 | if o.Claim.Holder.WakeRetry.AttemptsReserved != 1 { // cold admission did not reset the episode |
| 86 | t.Fatalf("episode after admission = %+v", o.Claim.Holder.WakeRetry) |
| 87 | } |
| 88 | if _, err := Update(ctx, s, o, ClearWakeRetryHealthy(WakeRetryHealth{WakeRetryResult: WakeRetryResult{HolderIncarnation: inc, EpisodeID: "episode", Attempt: a}, ReadinessPassed: true})); !errors.Is(err, ErrWrongState) { |
| 89 | t.Fatalf("health before session binding = %v", err) |
| 90 | } |
| 91 | o, err = Update(ctx, s, o, BindWakeRetrySession(inc, "episode", "first", session)) |
| 92 | if err != nil { |
| 93 | t.Fatal(err) |
| 94 | } |
| 95 | a.Session = session |
| 96 | wrong := a |
| 97 | wrong.InvocationID = "other-invocation" |
| 98 | if _, err := Update(ctx, s, o, ClearWakeRetryHealthy(WakeRetryHealth{WakeRetryResult: WakeRetryResult{HolderIncarnation: inc, EpisodeID: "episode", Attempt: wrong}, ReadinessPassed: true})); !errors.Is(err, ErrWrongState) { |
| 99 | t.Fatalf("wrong invocation health = %v", err) |
| 100 | } |
| 101 | if _, err := Update(ctx, s, o, ClearWakeRetryHealthy(WakeRetryHealth{WakeRetryResult: WakeRetryResult{HolderIncarnation: inc, EpisodeID: "episode", Attempt: a}})); !errors.Is(err, ErrWrongState) { |
| 102 | t.Fatalf("failed readiness = %v", err) |
| 103 | } |
| 104 | o, err = Update(ctx, s, o, ClearWakeRetryHealthy(WakeRetryHealth{WakeRetryResult: WakeRetryResult{HolderIncarnation: inc, EpisodeID: "episode", Attempt: a}, ReadinessPassed: true})) |
| 105 | if err != nil || o.Claim.Holder.WakeRetry != nil { |
| 106 | t.Fatalf("healthy clear = %+v, %v", o.Claim.Holder.WakeRetry, err) |
| 107 | } |
| 108 | } |
| 109 | |
| 110 | func TestWakeRetryOperatorResetRequiresSettledAttempt(t *testing.T) { |
| 111 | ctx := context.Background() |
| 112 | s := release.NewDirStore(t.TempDir()) |
| 113 | o := servingClaim(t, ctx, s) |
| 114 | inc := o.Claim.Holder.Incarnation |
| 115 | a := retryAttempt("active", o.Claim.Holder.Session) |
| 116 | var err error |
| 117 | o, err = Update(ctx, s, o, ReserveWakeRetryAttempt(WakeRetryReservation{HolderIncarnation: inc, EpisodeID: "old", Attempt: a, Now: time.Now()})) |
| 118 | if err != nil { |
| 119 | t.Fatal(err) |
| 120 | } |
| 121 | reset := WakeRetryReset{HolderIncarnation: inc, ObservedEpisode: "old", NewEpisodeID: "new", OperatorRequest: "request-42"} |
| 122 | if _, err := Update(ctx, s, o, ResetWakeRetry(reset)); !errors.Is(err, ErrWrongState) { |
| 123 | t.Fatalf("unsettled reset = %v", err) |
| 124 | } |
| 125 | reset.AttemptSettled = true |
| 126 | o, err = Update(ctx, s, o, ResetWakeRetry(reset)) |
| 127 | if err != nil { |
| 128 | t.Fatal(err) |
| 129 | } |
| 130 | if got := o.Claim.Holder.WakeRetry; got.EpisodeID != "new" || got.State != "open" || got.AttemptsReserved != 0 || got.ResetRequest != "request-42" { |
| 131 | t.Fatalf("reset episode = %+v", got) |
| 132 | } |
| 133 | } |
| 134 | |
| 135 | func TestWakeRetryValidationAndWireShape(t *testing.T) { |
| 136 | ctx := context.Background() |
| 137 | s := release.NewDirStore(t.TempDir()) |
| 138 | o := servingClaim(t, ctx, s) |
| 139 | bad := []WakeRetry{ |
| 140 | {EpisodeID: "e", State: "open", AttemptsReserved: 5}, |
| 141 | {EpisodeID: "e", State: "attempting", AttemptsReserved: 1}, |
| 142 | {EpisodeID: "e", State: "failed", AttemptsReserved: 4, NotBefore: time.Now()}, |
| 143 | {EpisodeID: "e", State: "unknown"}, |
| 144 | } |
| 145 | for i := range bad { |
| 146 | if _, err := Update(ctx, s, o, func(c *Claim) error { r := bad[i]; c.Holder.WakeRetry = &r; return nil }); !errors.Is(err, ErrMalformed) { |
| 147 | t.Errorf("bad %d = %v", i, err) |
| 148 | } |
| 149 | } |
| 150 | raw, err := json.Marshal(o.Claim.Holder) |
| 151 | if err != nil { |
| 152 | t.Fatal(err) |
| 153 | } |
| 154 | if bytes.Contains(raw, []byte(`"wake_retry"`)) { |
| 155 | t.Fatalf("legacy holder gained wake_retry: %s", raw) |
| 156 | } |
| 157 | } |
| 158 | |
amadan