| 1 | package hostexec |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/rand" |
| 6 | "encoding/hex" |
| 7 | "errors" |
| 8 | "sync" |
| 9 | "time" |
| 10 | ) |
| 11 | |
| 12 | type RootClientConfig struct { |
| 13 | Project func(State) error |
| 14 | Binding Binding |
| 15 | Clock Clock |
| 16 | Observe func(context.Context) (ExecutionRequest, error) |
| 17 | Call func(context.Context, ExecutionRequest) (ExecutionResponse, error) |
| 18 | CloseAdmission func() |
| 19 | Cleanup func(context.Context) error |
| 20 | } |
| 21 | |
| 22 | type RootClient struct { |
| 23 | cfg RootClientConfig |
| 24 | mu sync.Mutex |
| 25 | gate chan struct{} |
| 26 | request ExecutionRequest |
| 27 | uncertain bool |
| 28 | state State |
| 29 | err error |
| 30 | next int64 |
| 31 | cancel context.CancelFunc |
| 32 | done chan struct{} |
| 33 | watchDone chan struct{} |
| 34 | autoDone chan struct{} |
| 35 | closeMu sync.Mutex |
| 36 | cleaned, settled bool |
| 37 | settleID string |
| 38 | } |
| 39 | |
| 40 | func requestIdentity() (string, error) { |
| 41 | var raw [24]byte |
| 42 | if _, err := rand.Read(raw[:]); err != nil { |
| 43 | return "", err |
| 44 | } |
| 45 | return hex.EncodeToString(raw[:]), nil |
| 46 | } |
| 47 | |
| 48 | func OpenRootClient(ctx context.Context, cfg RootClientConfig) (*RootClient, error) { |
| 49 | if cfg.Clock == nil || cfg.Call == nil || cfg.Observe == nil || cfg.CloseAdmission == nil || cfg.Cleanup == nil { |
| 50 | return nil, ErrState |
| 51 | } |
| 52 | worker, cancel := context.WithCancel(ctx) |
| 53 | c := &RootClient{cfg: cfg, gate: make(chan struct{}, 1), cancel: cancel, done: make(chan struct{}), watchDone: make(chan struct{}), autoDone: make(chan struct{})} |
| 54 | if err := c.refresh(worker, true); err != nil { |
| 55 | cancel() |
| 56 | close(c.done) |
| 57 | close(c.watchDone) |
| 58 | close(c.autoDone) |
| 59 | return c, err |
| 60 | } |
| 61 | go func() { |
| 62 | defer close(c.watchDone) |
| 63 | for { |
| 64 | select { |
| 65 | case <-worker.Done(): |
| 66 | return |
| 67 | case <-cfg.Clock.After(100 * time.Millisecond): |
| 68 | if _, err := c.Ready(worker); err != nil { |
| 69 | c.cancel() |
| 70 | return |
| 71 | } |
| 72 | } |
| 73 | } |
| 74 | }() |
| 75 | go func() { |
| 76 | defer close(c.done) |
| 77 | for { |
| 78 | select { |
| 79 | case <-worker.Done(): |
| 80 | c.lose(worker.Err()) |
| 81 | return |
| 82 | case <-cfg.Clock.After(Renewal): |
| 83 | if err := c.Refresh(worker); err != nil { |
| 84 | c.lose(err) |
| 85 | return |
| 86 | } |
| 87 | } |
| 88 | } |
| 89 | }() |
| 90 | go func() { |
| 91 | defer close(c.autoDone) |
| 92 | <-worker.Done() |
| 93 | cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) |
| 94 | defer cancel() |
| 95 | _ = c.close(cleanup) |
| 96 | }() |
| 97 | return c, nil |
| 98 | } |
| 99 | |
| 100 | func (c *RootClient) lose(err error) { |
| 101 | c.mu.Lock() |
| 102 | if c.err == nil { |
| 103 | c.err = errors.Join(ErrLost, err) |
| 104 | } |
| 105 | c.mu.Unlock() |
| 106 | c.cancel() |
| 107 | c.cfg.CloseAdmission() |
| 108 | } |
| 109 | |
| 110 | func (c *RootClient) Refresh(ctx context.Context) error { return c.refresh(ctx, false) } |
| 111 | func (c *RootClient) refresh(ctx context.Context, begin bool) error { |
| 112 | select { |
| 113 | case c.gate <- struct{}{}: |
| 114 | case <-ctx.Done(): |
| 115 | return ctx.Err() |
| 116 | } |
| 117 | defer func() { <-c.gate }() |
| 118 | c.mu.Lock() |
| 119 | prior, next, previousErr := c.state, c.next, c.err |
| 120 | c.mu.Unlock() |
| 121 | if previousErr != nil { |
| 122 | return previousErr |
| 123 | } |
| 124 | _, start, err := c.cfg.Clock.Now() |
| 125 | if err != nil { |
| 126 | return err |
| 127 | } |
| 128 | if !begin && start >= prior.SafeUntil { |
| 129 | c.lose(ErrLost) |
| 130 | return ErrLost |
| 131 | } |
| 132 | q, err := c.cfg.Observe(ctx) |
| 133 | if err != nil { |
| 134 | c.lose(err) |
| 135 | return err |
| 136 | } |
| 137 | if q.Host != c.cfg.Binding.Host || q.Incarnation != c.cfg.Binding.Incarnation || q.Op != c.cfg.Binding.Op { |
| 138 | c.lose(ErrLost) |
| 139 | return ErrLost |
| 140 | } |
| 141 | q.Schema = 1 |
| 142 | q.ID, err = requestIdentity() |
| 143 | if err != nil { |
| 144 | return err |
| 145 | } |
| 146 | q.Operation = "executionView" |
| 147 | q.Token = prior.ControllerID |
| 148 | if begin { |
| 149 | q.Operation = "executionBegin" |
| 150 | } else if start >= next { |
| 151 | q.Operation = "executionRenew" |
| 152 | } |
| 153 | c.mu.Lock() |
| 154 | old := c.request |
| 155 | c.mu.Unlock() |
| 156 | if !begin && (old.Stamp != q.Stamp || old.Session != q.Session) { |
| 157 | q.Operation = "executionRenew" |
| 158 | } |
| 159 | c.mu.Lock() |
| 160 | c.request = q |
| 161 | c.uncertain = true |
| 162 | c.mu.Unlock() |
| 163 | out, err := c.cfg.Call(ctx, q) |
| 164 | if err != nil { |
| 165 | c.lose(err) |
| 166 | return err |
| 167 | } |
| 168 | _, now, clockErr := c.cfg.Clock.Now() |
| 169 | if clockErr != nil || (!begin && now >= prior.SafeUntil) || out.Schema != 1 || out.Host != q.Host || out.Incarnation != q.Incarnation || out.Stamp != q.Stamp || out.Session != q.Session || out.Op != q.Op || out.BootID != c.cfg.Binding.BootID || out.Token == "" || out.Status != "active" || out.SafeUntil <= now || out.LeaseETag == "" || out.Expires.IsZero() { |
| 170 | c.lose(ErrLost) |
| 171 | return ErrLost |
| 172 | } |
| 173 | binding := c.cfg.Binding |
| 174 | binding.ControllerID = out.Token |
| 175 | c.mu.Lock() |
| 176 | if c.err != nil { |
| 177 | err := c.err |
| 178 | c.mu.Unlock() |
| 179 | return err |
| 180 | } |
| 181 | c.state = State{V: 1, Binding: binding, State: out.Status, SafeUntil: out.SafeUntil, Expires: out.Expires, LeaseETag: out.LeaseETag} |
| 182 | c.request = q |
| 183 | c.uncertain = false |
| 184 | if q.Operation != "executionView" { |
| 185 | c.next = start + int64(Renewal) |
| 186 | } |
| 187 | state := c.state |
| 188 | c.mu.Unlock() |
| 189 | if c.cfg.Project != nil { |
| 190 | if err := c.cfg.Project(state); err != nil { |
| 191 | c.lose(err) |
| 192 | return err |
| 193 | } |
| 194 | } |
| 195 | return nil |
| 196 | } |
| 197 | |
| 198 | func (c *RootClient) Ready(ctx context.Context) (State, error) { |
| 199 | if ctx.Err() != nil { |
| 200 | return State{}, ctx.Err() |
| 201 | } |
| 202 | c.mu.Lock() |
| 203 | state, err := c.state, c.err |
| 204 | c.mu.Unlock() |
| 205 | if err != nil { |
| 206 | return state, err |
| 207 | } |
| 208 | _, now, err := c.cfg.Clock.Now() |
| 209 | if err != nil || state.State != "active" || now >= state.SafeUntil { |
| 210 | c.lose(ErrLost) |
| 211 | return state, ErrLost |
| 212 | } |
| 213 | return state, nil |
| 214 | } |
| 215 | |
| 216 | func (c *RootClient) Check(ctx context.Context, q ExecutionRequest) error { |
| 217 | if _, err := c.Ready(ctx); err != nil { |
| 218 | return err |
| 219 | } |
| 220 | c.mu.Lock() |
| 221 | defer c.mu.Unlock() |
| 222 | if c.err != nil { |
| 223 | return c.err |
| 224 | } |
| 225 | old := c.request |
| 226 | if c.uncertain || old.Host != q.Host || old.Incarnation != q.Incarnation || old.Stamp != q.Stamp || old.Session != q.Session || old.Op != q.Op { |
| 227 | return ErrLost |
| 228 | } |
| 229 | return nil |
| 230 | } |
| 231 | |
| 232 | func (c *RootClient) Close(ctx context.Context) error { |
| 233 | err := c.close(ctx) |
| 234 | select { |
| 235 | case <-c.autoDone: |
| 236 | return err |
| 237 | case <-ctx.Done(): |
| 238 | return errors.Join(err, ctx.Err()) |
| 239 | } |
| 240 | } |
| 241 | |
| 242 | func (c *RootClient) close(ctx context.Context) error { |
| 243 | c.closeMu.Lock() |
| 244 | defer c.closeMu.Unlock() |
| 245 | if c.settled { |
| 246 | return nil |
| 247 | } |
| 248 | c.lose(ErrLost) |
| 249 | c.cancel() |
| 250 | select { |
| 251 | case <-c.done: |
| 252 | case <-ctx.Done(): |
| 253 | return ctx.Err() |
| 254 | } |
| 255 | select { |
| 256 | case <-c.watchDone: |
| 257 | case <-ctx.Done(): |
| 258 | return ctx.Err() |
| 259 | } |
| 260 | select { |
| 261 | case c.gate <- struct{}{}: |
| 262 | case <-ctx.Done(): |
| 263 | return ctx.Err() |
| 264 | } |
| 265 | defer func() { <-c.gate }() |
| 266 | if !c.cleaned { |
| 267 | if err := c.cfg.Cleanup(ctx); err != nil { |
| 268 | return err |
| 269 | } |
| 270 | c.cleaned = true |
| 271 | } |
| 272 | c.mu.Lock() |
| 273 | q, token, uncertain := c.request, c.state.ControllerID, c.uncertain |
| 274 | c.mu.Unlock() |
| 275 | if uncertain { |
| 276 | if !q.Valid() { |
| 277 | return ErrLost |
| 278 | } |
| 279 | out, err := c.cfg.Call(ctx, q) |
| 280 | if err != nil { |
| 281 | return err |
| 282 | } |
| 283 | if !executionReplyMatches(q, c.cfg.Binding.BootID, out) || out.Status != "active" { |
| 284 | return ErrLost |
| 285 | } |
| 286 | token = out.Token |
| 287 | c.mu.Lock() |
| 288 | c.state.Binding = c.cfg.Binding |
| 289 | c.state.ControllerID = token |
| 290 | c.uncertain = false |
| 291 | c.mu.Unlock() |
| 292 | } |
| 293 | if token == "" { |
| 294 | return ErrLost |
| 295 | } |
| 296 | q.Token, q.Operation = token, "executionSettle" |
| 297 | var err error |
| 298 | if c.settleID == "" { |
| 299 | c.settleID, err = requestIdentity() |
| 300 | if err != nil { |
| 301 | return err |
| 302 | } |
| 303 | } |
| 304 | q.ID = c.settleID |
| 305 | settle, err := c.cfg.Call(ctx, q) |
| 306 | if err != nil { |
| 307 | return err |
| 308 | } |
| 309 | if !executionReplyMatches(q, c.cfg.Binding.BootID, settle) || settle.Status != "settled" || settle.Token != token { |
| 310 | return ErrLost |
| 311 | } |
| 312 | if c.cfg.Project != nil { |
| 313 | c.mu.Lock() |
| 314 | state := c.state |
| 315 | c.mu.Unlock() |
| 316 | state.State = "settled" |
| 317 | if err := c.cfg.Project(state); err != nil { |
| 318 | return err |
| 319 | } |
| 320 | } |
| 321 | c.settled = true |
| 322 | return nil |
| 323 | } |
| 324 | |
| 325 | func executionReplyMatches(q ExecutionRequest, boot string, out ExecutionResponse) bool { |
| 326 | return out.Schema == 1 && out.Host == q.Host && out.Incarnation == q.Incarnation && out.Stamp == q.Stamp && out.Session == q.Session && out.Op == q.Op && out.BootID == boot && out.Token != "" |
| 327 | } |
| 328 | |
amadan