| 1 | package hostclaim |
| 2 | |
| 3 | import ( |
| 4 | "fmt" |
| 5 | "time" |
| 6 | ) |
| 7 | |
| 8 | const WakeRetryLimit = 5 |
| 9 | |
| 10 | var wakeRetryDelays = [...]time.Duration{time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second, 16 * time.Second} |
| 11 | |
| 12 | type WakeRetry struct { |
| 13 | EpisodeID string `json:"episode_id"` |
| 14 | AttemptsReserved int `json:"attempts_reserved"` |
| 15 | State string `json:"state"` |
| 16 | NotBefore time.Time `json:"not_before,omitempty"` |
| 17 | Attempt *WakeRetryAttempt `json:"attempt,omitempty"` |
| 18 | ResetRequest string `json:"reset_request,omitempty"` |
| 19 | } |
| 20 | |
| 21 | type WakeRetryAttempt struct { |
| 22 | ID string `json:"id"` |
| 23 | Box string `json:"box"` |
| 24 | Enrollment string `json:"enrollment"` |
| 25 | BootID string `json:"boot_id"` |
| 26 | ControllerID string `json:"controller_id"` |
| 27 | Session uint64 `json:"session,omitempty"` |
| 28 | InvocationID string `json:"invocation_id"` |
| 29 | LaunchInputs string `json:"launch_inputs"` |
| 30 | } |
| 31 | |
| 32 | func (r *WakeRetry) validate() error { |
| 33 | if r == nil { |
| 34 | return nil |
| 35 | } |
| 36 | if r.EpisodeID == "" || r.AttemptsReserved < 0 || r.AttemptsReserved > WakeRetryLimit { |
| 37 | return fmt.Errorf("%w: wake retry episode", ErrMalformed) |
| 38 | } |
| 39 | switch r.State { |
| 40 | case "open": |
| 41 | if r.Attempt != nil || r.AttemptsReserved >= WakeRetryLimit { |
| 42 | return fmt.Errorf("%w: open wake retry", ErrMalformed) |
| 43 | } |
| 44 | case "attempting": |
| 45 | if r.AttemptsReserved == 0 || r.Attempt == nil || !r.NotBefore.IsZero() || !r.Attempt.valid() { |
| 46 | return fmt.Errorf("%w: attempting wake retry", ErrMalformed) |
| 47 | } |
| 48 | case "failed": |
| 49 | if r.AttemptsReserved != WakeRetryLimit || r.Attempt != nil || r.NotBefore.IsZero() { |
| 50 | return fmt.Errorf("%w: failed wake retry", ErrMalformed) |
| 51 | } |
| 52 | default: |
| 53 | return fmt.Errorf("%w: wake retry state %q", ErrMalformed, r.State) |
| 54 | } |
| 55 | return nil |
| 56 | } |
| 57 | |
| 58 | func (a *WakeRetryAttempt) valid() bool { |
| 59 | return a.ID != "" && a.Box != "" && a.Enrollment != "" && a.BootID != "" && |
| 60 | a.ControllerID != "" && a.InvocationID != "" && a.LaunchInputs != "" |
| 61 | } |
| 62 | |
| 63 | type WakeRetryReservation struct { |
| 64 | HolderIncarnation string |
| 65 | EpisodeID string |
| 66 | Attempt WakeRetryAttempt |
| 67 | Now time.Time |
| 68 | } |
| 69 | |
| 70 | // ReserveWakeRetryAttempt spends one serving retry after the caller has |
| 71 | // established all read-only prerequisites. Session zero denotes the first |
| 72 | // admission and must be bound with BindWakeRetrySession before app code runs. |
| 73 | func ReserveWakeRetryAttempt(in WakeRetryReservation) func(*Claim) error { |
| 74 | return func(c *Claim) error { |
| 75 | if c.Holder == nil || c.Op.Kind != KindServing || c.Holder.Incarnation != in.HolderIncarnation || |
| 76 | in.Now.IsZero() || !in.Attempt.valid() || (in.Attempt.Session != 0 && in.Attempt.Session != c.Holder.Session) { |
| 77 | return ErrWrongState |
| 78 | } |
| 79 | r := c.Holder.WakeRetry |
| 80 | if r == nil { |
| 81 | if in.EpisodeID == "" { |
| 82 | return ErrWrongState |
| 83 | } |
| 84 | r = &WakeRetry{EpisodeID: in.EpisodeID, State: "open"} |
| 85 | c.Holder.WakeRetry = r |
| 86 | } else if in.EpisodeID != "" && in.EpisodeID != r.EpisodeID { |
| 87 | return ErrWrongState |
| 88 | } |
| 89 | if r.State != "open" || r.AttemptsReserved >= WakeRetryLimit || in.Now.Before(r.NotBefore) { |
| 90 | return ErrWrongState |
| 91 | } |
| 92 | r.AttemptsReserved++ |
| 93 | r.State, r.NotBefore = "attempting", time.Time{} |
| 94 | a := in.Attempt |
| 95 | r.Attempt = &a |
| 96 | return nil |
| 97 | } |
| 98 | } |
| 99 | |
| 100 | // BindWakeRetrySession attaches the cold admission resulting from a first |
| 101 | // reservation. It is required before that invocation may execute app code. |
| 102 | func BindWakeRetrySession(incarnation, episode, attempt string, session uint64) func(*Claim) error { |
| 103 | return func(c *Claim) error { |
| 104 | if c.Holder == nil || c.Holder.Incarnation != incarnation || session == 0 || c.Holder.Session != session { |
| 105 | return ErrWrongState |
| 106 | } |
| 107 | r := c.Holder.WakeRetry |
| 108 | if r == nil || r.EpisodeID != episode || r.State != "attempting" || r.Attempt == nil || |
| 109 | r.Attempt.ID != attempt || r.Attempt.Session != 0 { |
| 110 | return ErrWrongState |
| 111 | } |
| 112 | r.Attempt.Session = session |
| 113 | return nil |
| 114 | } |
| 115 | } |
| 116 | |
| 117 | type WakeRetryResult struct { |
| 118 | HolderIncarnation string |
| 119 | EpisodeID string |
| 120 | Attempt WakeRetryAttempt |
| 121 | Now time.Time |
| 122 | } |
| 123 | |
| 124 | func sameWakeAttempt(a *WakeRetryAttempt, b WakeRetryAttempt) bool { return a != nil && *a == b } |
| 125 | |
| 126 | // FailWakeRetryAttempt settles the exact reserved attempt. A crash or |
| 127 | // ambiguous result uses this transition too; the reservation is never refunded. |
| 128 | func FailWakeRetryAttempt(in WakeRetryResult) func(*Claim) error { |
| 129 | return func(c *Claim) error { |
| 130 | if c.Holder == nil || c.Holder.Incarnation != in.HolderIncarnation || in.Now.IsZero() { |
| 131 | return ErrWrongState |
| 132 | } |
| 133 | r := c.Holder.WakeRetry |
| 134 | if r == nil || r.EpisodeID != in.EpisodeID || r.State != "attempting" || !sameWakeAttempt(r.Attempt, in.Attempt) { |
| 135 | return ErrWrongState |
| 136 | } |
| 137 | r.Attempt = nil |
| 138 | r.NotBefore = in.Now.Add(wakeRetryDelays[r.AttemptsReserved-1]) |
| 139 | if r.AttemptsReserved == WakeRetryLimit { |
| 140 | r.State = "failed" |
| 141 | } else { |
| 142 | r.State = "open" |
| 143 | } |
| 144 | return nil |
| 145 | } |
| 146 | } |
| 147 | |
| 148 | type WakeRetryHealth struct { |
| 149 | WakeRetryResult |
| 150 | ReadinessPassed bool |
| 151 | } |
| 152 | |
| 153 | // ClearWakeRetryHealthy accepts only a readiness result for the exact live |
| 154 | // reservation. The caller is responsible for performing the backend checks. |
| 155 | func ClearWakeRetryHealthy(in WakeRetryHealth) func(*Claim) error { |
| 156 | return func(c *Claim) error { |
| 157 | if !in.ReadinessPassed || c.Holder == nil || c.Holder.Incarnation != in.HolderIncarnation || |
| 158 | in.Attempt.Session == 0 || c.Holder.Session != in.Attempt.Session { |
| 159 | return ErrWrongState |
| 160 | } |
| 161 | r := c.Holder.WakeRetry |
| 162 | if r == nil || r.EpisodeID != in.EpisodeID || r.State != "attempting" || !sameWakeAttempt(r.Attempt, in.Attempt) { |
| 163 | return ErrWrongState |
| 164 | } |
| 165 | c.Holder.WakeRetry = nil |
| 166 | return nil |
| 167 | } |
| 168 | } |
| 169 | |
| 170 | type WakeRetryReset struct { |
| 171 | HolderIncarnation string |
| 172 | ObservedEpisode string |
| 173 | NewEpisodeID string |
| 174 | OperatorRequest string |
| 175 | AttemptSettled bool |
| 176 | } |
| 177 | |
| 178 | // ResetWakeRetry is the operator-authorized primitive. Authorization and any |
| 179 | // required fencing happen outside this package and are asserted by the input. |
| 180 | func ResetWakeRetry(in WakeRetryReset) func(*Claim) error { |
| 181 | return func(c *Claim) error { |
| 182 | if c.Holder == nil || c.Holder.Incarnation != in.HolderIncarnation || in.ObservedEpisode == "" || |
| 183 | in.NewEpisodeID == "" || in.NewEpisodeID == in.ObservedEpisode || in.OperatorRequest == "" { |
| 184 | return ErrWrongState |
| 185 | } |
| 186 | r := c.Holder.WakeRetry |
| 187 | if r == nil || r.EpisodeID != in.ObservedEpisode || (r.State == "attempting" && !in.AttemptSettled) { |
| 188 | return ErrWrongState |
| 189 | } |
| 190 | c.Holder.WakeRetry = &WakeRetry{EpisodeID: in.NewEpisodeID, State: "open", ResetRequest: in.OperatorRequest} |
| 191 | return nil |
| 192 | } |
| 193 | } |
| 194 | |
amadan