carlos / platform Public

Clone
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package hostops
2
3import (
4 "context"
5 "encoding/json"
6 "errors"
7 "fmt"
8 "os"
9 "slices"
10 "strconv"
11 "strings"
12 "time"
13
14 "github.com/carlosframework/platform/internal/controlname"
15 "github.com/carlosframework/platform/internal/hostactors"
16 "github.com/carlosframework/platform/internal/hostbox"
17 "github.com/carlosframework/platform/internal/hostclaim"
18 "github.com/carlosframework/platform/internal/hostevidence"
19 "github.com/carlosframework/platform/internal/release"
20)
21
22var ErrWakeResetUncertain = errors.New("wake reset: stop outcome uncertain")
23
24const WakeResetLifetime = 2 * time.Minute
25const WakeResetPrefix = "control/wake-resets/"
26const WakeResetArchivePrefix = "control/wake-reset-archive/"
27
28// WakeResetRequest is an immutable signed instruction. Cancellation is a second
29// signed object; its presence only prevents new dispatch, never proves no stop.
30type WakeResetAdmission struct {
31 Account string `json:"account"`
32 App string `json:"app"`
33 AppID string `json:"app_id"`
34 Registration string `json:"registration"`
35 Epoch uint64 `json:"epoch"`
36 Owner RegistrationOwner `json:"owner"`
37}
38
39type WakeResetRequest struct {
40 Admission WakeResetAdmission `json:"admission"`
41 V int `json:"v"`
42 ID string `json:"id"`
43 Bucket string `json:"bucket"`
44 Console string `json:"console"`
45 ConsoleEnrollment string `json:"console_enrollment"`
46 Namespace string `json:"namespace"`
47 Host string `json:"host"`
48 ClaimETag string `json:"claim_etag"`
49 ClaimGen uint64 `json:"claim_gen"`
50 Incarnation string `json:"incarnation"`
51 Stamp uint64 `json:"stamp"`
52 Session uint64 `json:"session"`
53 Operation string `json:"operation"`
54 Phase string `json:"phase"`
55 Episode string `json:"episode"`
56 Attempt hostclaim.WakeRetryAttempt `json:"attempt"`
57 Round uint64 `json:"round,omitempty"`
58 IssuedAt time.Time `json:"issued_at"`
59 Expires time.Time `json:"expires"`
60}
61
62type WakeResetCancellation struct {
63 V int `json:"v"`
64 RequestSHA256 string `json:"request_sha256"`
65}
66
67type WakeResetTransport struct {
68 Store release.Store
69 Directory hostactors.Directory
70 Now func() time.Time
71}
72
73func wakeResetID(r WakeResetRequest) string {
74 r.ID = ""
75 r.IssuedAt = time.Time{}
76 r.Expires = time.Time{}
77 raw, _ := json.Marshal(r)
78 return evidenceDigest(raw)
79}
80
81func (r WakeResetRequest) Key() string { return WakeResetPrefix + r.Attempt.Box + "/" + r.ID + ".json" }
82func (r WakeResetRequest) validate() error {
83 if !controlname.ValidAccount(r.Admission.Account) || !controlname.ValidApp(r.Admission.App) || r.Admission.AppID == "" || r.Admission.Registration == "" || r.Admission.Epoch == 0 || r.Admission.Owner.Console != r.Console || r.Admission.Owner.Enrollment != r.ConsoleEnrollment || r.Admission.Owner.Controller == "" || r.V != 1 || !digestOK(r.ID) || !controlname.ValidHostShape(r.Host) || r.ClaimETag == "" || r.ClaimGen == 0 || !safeSegment(r.Bucket) || !safeSegment(r.Console) || r.ConsoleEnrollment == "" || r.Namespace == "" || !safeSegment(r.Incarnation) || r.Episode == "" || r.Attempt.ID == "" || !safeSegment(r.Attempt.Box) || r.Attempt.Enrollment == "" || r.Attempt.BootID == "" || r.Attempt.ControllerID == "" || r.IssuedAt.IsZero() || !r.Expires.After(r.IssuedAt) || r.Expires.Sub(r.IssuedAt) > WakeResetLifetime {
84 return ErrIdentity
85 }
86 return nil
87}
88
89// Authority rechecks both enrollments, namespace delegation and termination
90// fences from the current signed roster. An invitation alone grants nothing.
91func (t WakeResetTransport) Authority(ctx context.Context, console, enrollment, namespace string) error {
92 if t.Store == nil || t.Now == nil || t.Directory.Store == nil {
93 return ErrContract
94 }
95 o, err := t.Directory.Read(ctx)
96 if err != nil {
97 return err
98 }
99 c, ok := o.Roster.Consoles[console]
100 if !ok || c.Enrollment != enrollment || c.DeploymentBucket != t.Directory.Bucket || !slices.Contains(c.Namespaces, namespace) || c.Compute == nil || o.Roster.RequestLimitNS != int64(hostbox.RequestLimit) {
101 return ErrIdentity
102 }
103 b, ok := o.Roster.Boxes[c.Compute.Box]
104 if !ok || b.Enrollment != c.Compute.Enrollment || b.State != "active" {
105 return ErrIdentity
106 }
107 fenced, err := t.Directory.TerminationFence(ctx, o, c.Compute.Box, t.Now())
108 if err != nil {
109 return err
110 }
111 if fenced {
112 return ErrIdentity
113 }
114 return nil
115}
116
117func (t WakeResetTransport) Authorize(ctx context.Context, r WakeResetRequest) error {
118 if err := r.validate(); err != nil {
119 return err
120 }
121 if t.Store == nil || t.Now == nil || r.Bucket != t.Directory.Bucket || r.IssuedAt.After(t.Now().Add(5*time.Second)) || !t.Now().Before(r.Expires) {
122 return ErrIdentity
123 }
124 if _, _, err := t.Store.Get(ctx, wakeResetArchiveKey(r)); err == nil {
125 return ErrIdentity
126 } else if !errors.Is(err, os.ErrNotExist) {
127 return err
128 }
129 if err := t.Authority(ctx, r.Console, r.ConsoleEnrollment, r.Namespace); err != nil {
130 return err
131 }
132 roster, err := t.Directory.Read(ctx)
133 if err != nil {
134 return err
135 }
136 b, ok := roster.Roster.Boxes[r.Attempt.Box]
137 if !ok || b.State != "active" || b.Enrollment != r.Attempt.Enrollment || b.Namespace != r.Namespace {
138 return ErrIdentity
139 }
140 fenced, err := t.Directory.TerminationFence(ctx, roster, r.Attempt.Box, t.Now())
141 if err != nil {
142 return err
143 }
144 if fenced {
145 return ErrIdentity
146 }
147 o, err := hostclaim.Read(ctx, t.Store, r.Host)
148 if err != nil {
149 return err
150 }
151 if !r.matches(o) {
152 return ErrIdentity
153 }
154 h := o.Claim.Holder
155 if h.Account != r.Admission.Account || h.App != r.Admission.App {
156 return ErrIdentity
157 }
158 admission, err := ReadAdmission(ctx, t.Store, r.Admission.Account, r.Admission.App, r.Admission.AppID)
159 if err != nil {
160 return err
161 }
162 reg, ok := admission.Value.Registrations[r.Admission.Registration]
163 if !ok || admission.Value.Epoch != r.Admission.Epoch || reg.Epoch != r.Admission.Epoch || reg.Owner != r.Admission.Owner || reg.Kind != RegistrationWakeRetryReset || reg.Incarnation != r.Incarnation || reg.Host != r.Host || reg.Claim != SavedObservation(o) || !RegistrationLive(reg, t.Now()) {
164 return ErrIdentity
165 }
166 return ctx.Err()
167}
168func (r WakeResetRequest) matches(o hostclaim.Observed) bool {
169 if o.Claim == nil || o.Claim.Holder == nil || o.ETag != r.ClaimETag || o.Claim.Gen != r.ClaimGen || o.Claim.Op.Kind != hostclaim.KindServing || o.Claim.Op.ID != r.Operation || o.Claim.Op.Phase != r.Phase {
170 return false
171 }
172 h := o.Claim.Holder
173 wr := h.WakeRetry
174 return h.Incarnation == r.Incarnation && h.Stamp == r.Stamp && h.Session == r.Session && wr != nil && wr.State == "attempting" && wr.EpisodeID == r.Episode && wr.Attempt != nil && *wr.Attempt == r.Attempt
175}
176
177// Read checks the signed envelope and key binding before the worker uses a
178// request. The deployment signing key authenticates only console requests;
179// the box's separate enrolled evidence key authenticates the stop result.
180func (t WakeResetTransport) Read(ctx context.Context, key string) (WakeResetRequest, error) {
181 var r WakeResetRequest
182 raw, _, err := t.Store.Get(ctx, key)
183 if err != nil {
184 return r, err
185 }
186 if len(raw) > 32<<10 || release.VerifyRootEnvelope(t.Directory.PublicKey, raw, &r) != nil || r.validate() != nil || r.ID != wakeResetID(r) || r.Key() != key {
187 return r, ErrIdentity
188 }
189 return r, nil
190}
191func (t WakeResetTransport) cancelled(ctx context.Context, r WakeResetRequest) (bool, error) {
192 raw, _, err := t.Store.Get(ctx, r.Key()+".cancel")
193 if errors.Is(err, os.ErrNotExist) {
194 return false, nil
195 }
196 if err != nil {
197 return false, err
198 }
199 var cancel WakeResetCancellation
200 body, _ := json.Marshal(r)
201 if release.VerifyRootEnvelope(t.Directory.PublicKey, raw, &cancel) != nil || cancel.V != 1 || cancel.RequestSHA256 != evidenceDigest(body) {
202 return false, ErrIdentity
203 }
204 return true, nil
205}
206
207// Pass invokes only the selected box's local stop owner. Positive completion
208// comes exclusively from VerifyReceipt. Replay reuses root's durable stop
209// fence; a failed call or publication never manufactures a completion record.
210func (t WakeResetTransport) Pass(ctx context.Context, box string, stop func(context.Context, hostclaim.Observed, hostclaim.WakeRetryAttempt) error) error {
211 _, err := t.Scan(ctx, box, stop)
212 return err
213}
214
215// Scan is Pass that also reports how many request objects it listed, so an
216// idle worker can poll the bucket less often.
217func (t WakeResetTransport) Scan(ctx context.Context, box string, stop func(context.Context, hostclaim.Observed, hostclaim.WakeRetryAttempt) error) (int, error) {
218 if t.Store == nil || t.Now == nil || stop == nil || !safeSegment(box) {
219 return 0, ErrContract
220 }
221 keys, err := t.Store.List(ctx, WakeResetPrefix+box+"/")
222 if err != nil {
223 return 0, err
224 }
225 var errs []error
226 listed := 0
227 for _, key := range keys {
228 if !strings.HasSuffix(key, ".json") {
229 continue
230 }
231 listed++
232 r, err := t.Read(ctx, key)
233 if err != nil {
234 errs = append(errs, err)
235 continue
236 }
237 if r.Attempt.Box != box {
238 errs = append(errs, ErrIdentity)
239 continue
240 }
241 if !t.Now().Before(r.Expires) {
242 continue
243 }
244 if err = t.VerifyReceipt(ctx, r); err == nil {
245 continue
246 }
247 cancelled, err := t.cancelled(ctx, r)
248 if err != nil {
249 errs = append(errs, err)
250 continue
251 }
252 if cancelled {
253 continue
254 }
255 if err = t.Authorize(ctx, r); err != nil {
256 errs = append(errs, err)
257 continue
258 }
259 o, err := hostclaim.Read(ctx, t.Store, r.Host)
260 if err != nil || !r.matches(o) {
261 errs = append(errs, errors.Join(err, ErrIdentity))
262 continue
263 }
264 request, done := context.WithDeadline(ctx, minResetDeadline(t.Now(), r.Expires))
265 err = stop(request, o, r.Attempt)
266 done()
267 if err != nil {
268 errs = append(errs, errors.Join(ErrWakeResetUncertain, err))
269 continue
270 }
271 if err = t.VerifyReceipt(ctx, r); err != nil {
272 errs = append(errs, err)
273 }
274 }
275 return listed, errors.Join(errs...)
276}
277
278// VerifyReceipt accepts only a signed root receipt for the full captured writer
279// tuple and claim ETag. An expired request can recover a positive receipt: expiry
280// closes dispatch, not evidence. Missing signatures and missing objects remain
281// uncertain. The unsigned list is discovery, never authority.
282func (t WakeResetTransport) VerifyReceipt(ctx context.Context, r WakeResetRequest) error {
283 if r.validate() != nil || t.Store == nil || t.Now == nil {
284 return ErrIdentity
285 }
286 roster, err := t.Directory.Read(ctx)
287 if err != nil {
288 return errors.Join(ErrWakeResetUncertain, err)
289 }
290 actor, ok := roster.Roster.Boxes[r.Attempt.Box]
291 if !ok || actor.State != "active" || actor.Enrollment != r.Attempt.Enrollment {
292 return ErrIdentity
293 }
294 fenced, err := t.Directory.TerminationFence(ctx, roster, r.Attempt.Box, t.Now())
295 if err != nil || fenced {
296 return errors.Join(ErrIdentity, err)
297 }
298 prefix := fmt.Sprintf("control/hosts/%s/serving-evidence/%s/", r.Host, r.Attempt.Box)
299 keys, err := t.Store.List(ctx, prefix)
300 if err != nil {
301 return errors.Join(ErrWakeResetUncertain, err)
302 }
303 for _, key := range keys {
304 if !strings.HasSuffix(key, ".json") {
305 continue
306 }
307 raw, _, err := t.Store.Get(ctx, key)
308 if err != nil {
309 return errors.Join(ErrWakeResetUncertain, err)
310 }
311 var p StopReceipt
312 if decode(raw, &p) != nil {
313 continue
314 }
315 if p.Host != r.Host || p.Incarnation != r.Incarnation || p.InstallationStamp != r.Stamp || p.WriterSession != r.Session || p.OperationID != r.Operation || p.Phase != r.Phase || p.Box != r.Attempt.Box || p.Enrollment != r.Attempt.Enrollment || p.BootID != r.Attempt.BootID || p.ControllerID != r.Attempt.ControllerID || p.ClaimETag != r.ClaimETag || p.ClaimGen != r.ClaimGen {
316 continue
317 }
318 revision, err := strconv.ParseUint(p.DirectoryRevision, 10, 64)
319 if err != nil || revision == 0 || revision > roster.Roster.Revision || p.Region != actor.Region || p.V != 1 || !safeSegment(p.ID) || key != prefix+p.ID+".json" || p.Kind != "stopped" || p.ObligationID != "" || p.FenceID == "" || p.RequestLimitNS != int64(hostbox.RequestLimit) || p.EvidenceAt.IsZero() || p.EvidenceAt.After(t.Now().Add(5*time.Second)) || !digestOK(p.InventorySHA256) || !digestOK(p.LocalJournalSHA256) || !digestOK(p.ProducerDrainDigest) || len(p.CgroupIdentities) == 0 {
320 continue
321 }
322 sig, _, err := t.Store.Get(ctx, key+".sig")
323 if err != nil {
324 continue
325 }
326 if hostevidence.Verify(actor.CensusPublicKey, key, raw, sig) == nil {
327 return nil
328 }
329 }
330 return ErrWakeResetUncertain
331}
332func (t WakeResetTransport) Wait(ctx context.Context, r WakeResetRequest, every time.Duration) error {
333 if every <= 0 {
334 return ErrContract
335 }
336 ticker := time.NewTicker(every)
337 defer ticker.Stop()
338 for {
339 if err := ctx.Err(); err != nil {
340 return errors.Join(ErrWakeResetUncertain, err)
341 }
342 if err := t.VerifyReceipt(ctx, r); err == nil {
343 return nil
344 }
345 if !t.Now().Before(r.Expires) {
346 return ErrWakeResetUncertain
347 }
348 select {
349 case <-ctx.Done():
350 return errors.Join(ErrWakeResetUncertain, ctx.Err())
351 case <-ticker.C:
352 }
353 }
354}
355
356// RequestWakeReset is the sole console write door. Only a live exact reset
357// registration can issue it; raw bucket handles never escape ControlWriter.
358func (w *ControlWriter) RequestWakeReset(ctx context.Context, t WakeResetTransport, signer release.Signer, console, enrollment, namespace string, o hostclaim.Observed, attempt hostclaim.WakeRetryAttempt) (WakeResetRequest, error) {
359 var r WakeResetRequest
360 if w == nil || w.ready() != nil || o.Claim == nil || o.Claim.Holder == nil || o.Claim.Holder.WakeRetry == nil {
361 return r, ErrContract
362 }
363 grants, _ := ctx.Value(lifecycleGrantKey{}).([]lifecycleGrant)
364 allowed := false
365 h := o.Claim.Holder
366 var binding WakeResetAdmission
367 for _, g := range grants {
368 if g.w != w || g.registration != RegistrationWakeRetryReset || g.incarnation != h.Incarnation || g.account != h.Account || g.app != h.App || g.owner.Console != console || g.owner.Enrollment != enrollment {
369 continue
370 }
371 live, err := grantLive(ctx, w.raw, g)
372 if err != nil {
373 return r, err
374 }
375 allowed = allowed || live
376 if live {
377 binding = WakeResetAdmission{Account: g.account, App: g.app, AppID: g.appID, Registration: g.operation, Epoch: g.epoch, Owner: g.owner}
378 }
379 }
380 if !allowed {
381 return r, ErrIdentity
382 }
383 r = WakeResetRequest{Admission: binding, V: 1, Bucket: t.Directory.Bucket, Console: console, ConsoleEnrollment: enrollment, Namespace: namespace, Host: o.Host, ClaimETag: o.ETag, ClaimGen: o.Claim.Gen, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Operation: o.Claim.Op.ID, Phase: o.Claim.Op.Phase, Episode: h.WakeRetry.EpisodeID, Attempt: attempt}
384 // Stable identity binds the observed attempt, issuer, claim and round,
385 // independent of retries or a console restart. Existing immutable bytes
386 // retain their expiry. An archived round stays closed; a later issue for
387 // the same claim ETag moves to the next round instead of pinning it.
388 var receipt error
389 checked := false
390 // Rounds only accumulate one per expired lifetime, so the scan stays short.
391 for round := uint64(0); ; round++ {
392 if err := ctx.Err(); err != nil {
393 return r, errors.Join(ErrWakeResetUncertain, receipt, err)
394 }
395 r.Round = round
396 r.ID = wakeResetID(r)
397 archived, err := t.readArchive(ctx, r)
398 if err == nil {
399 // Expiry closed dispatch, not evidence: a root receipt for the
400 // tuple is still proof. Without one the outcome stays uncertain and
401 // only a fresh round may dispatch again.
402 if !checked {
403 if receipt = t.VerifyReceipt(ctx, archived); receipt == nil {
404 return archived, nil
405 }
406 checked = true
407 }
408 continue
409 }
410 if !errors.Is(err, os.ErrNotExist) {
411 return r, err
412 }
413 existing, err := t.Read(ctx, r.Key())
414 if err == nil {
415 return existing, nil
416 }
417 if !errors.Is(err, os.ErrNotExist) {
418 return r, err
419 }
420 r.IssuedAt = w.now()
421 r.Expires = r.IssuedAt.Add(WakeResetLifetime)
422 if err = t.Authorize(ctx, r); err != nil {
423 return r, err
424 }
425 wire, err := release.SignRootEnvelope(signer, r)
426 if err != nil {
427 return r, err
428 }
429 if err = release.CreateExact(ctx, w.raw, r.Key(), wire); err != nil {
430 saved, readErr := t.Read(ctx, r.Key())
431 if readErr == nil && saved.ID == r.ID {
432 return saved, nil
433 }
434 return r, errors.Join(ErrWakeResetUncertain, err, readErr)
435 }
436 return r, nil
437 }
438}
439
440// readArchive returns an archived request only if it is the signed request
441// for exactly r's id; anything else at the archive key is an identity error.
442func (t WakeResetTransport) readArchive(ctx context.Context, r WakeResetRequest) (WakeResetRequest, error) {
443 var a WakeResetRequest
444 raw, _, err := t.Store.Get(ctx, wakeResetArchiveKey(r))
445 if err != nil {
446 return a, err
447 }
448 if len(raw) > 32<<10 || release.VerifyRootEnvelope(t.Directory.PublicKey, raw, &a) != nil || a.validate() != nil || a.ID != wakeResetID(a) || a.ID != r.ID {
449 return a, ErrIdentity
450 }
451 return a, nil
452}
453func (w *ControlWriter) CancelWakeReset(ctx context.Context, t WakeResetTransport, signer release.Signer, r WakeResetRequest) error {
454 if w == nil || w.ready() != nil || t.Store == nil || r.validate() != nil {
455 return ErrContract
456 }
457 if !w.now().Before(r.Expires) {
458 return ErrIdentity
459 }
460 if _, _, err := t.Store.Get(ctx, wakeResetArchiveKey(r)); err == nil {
461 return ErrIdentity
462 } else if !errors.Is(err, os.ErrNotExist) {
463 return err
464 }
465 request, done := context.WithDeadline(ctx, minResetDeadline(w.now(), r.Expires))
466 defer done()
467 if err := t.Authority(request, r.Console, r.ConsoleEnrollment, r.Namespace); err != nil {
468 return err
469 }
470 raw, _ := json.Marshal(r)
471 wire, err := release.SignRootEnvelope(signer, WakeResetCancellation{V: 1, RequestSHA256: evidenceDigest(raw)})
472 if err != nil {
473 return err
474 }
475 return release.CreateExact(request, w.raw, r.Key()+".cancel", wire)
476}
477
478func minResetDeadline(now, expires time.Time) time.Time {
479 limit := now.Add(hostbox.RequestLimit / 4)
480 if expires.Before(limit) {
481 return expires
482 }
483 return limit
484}
485func wakeResetArchiveKey(r WakeResetRequest) string {
486 return WakeResetArchivePrefix + r.Attempt.Box + "/" + r.ID + ".json"
487}
488
489// SweepWakeResets archives signed expired requests before conditionally removing
490// them from dispatch. The immutable archive prevents deterministic-ID reuse;
491// it is never a stop proof. Root evidence remains independently recoverable.
492func (w *ControlWriter) SweepWakeResets(ctx context.Context, t WakeResetTransport) error {
493 if w == nil || w.ready() != nil || t.Store == nil || t.Now == nil {
494 return ErrContract
495 }
496 keys, err := t.Store.List(ctx, WakeResetPrefix)
497 if err != nil {
498 return err
499 }
500 var errs []error
501 for _, key := range keys {
502 if !strings.HasSuffix(key, ".json") {
503 continue
504 }
505 r, err := t.Read(ctx, key)
506 if err != nil {
507 errs = append(errs, err)
508 continue
509 }
510 if !w.now().After(r.Expires.Add(hostbox.RequestLimit)) {
511 continue
512 }
513 raw, etag, err := w.raw.Get(ctx, key)
514 if err != nil {
515 errs = append(errs, err)
516 continue
517 }
518 var again WakeResetRequest
519 if release.VerifyRootEnvelope(w.pub, raw, &again) != nil || again.ID != r.ID || again.ID != wakeResetID(again) || again.validate() != nil || !again.Expires.Equal(r.Expires) {
520 errs = append(errs, ErrIdentity)
521 continue
522 }
523 if err = release.CreateExact(ctx, w.raw, wakeResetArchiveKey(r), raw); err != nil {
524 errs = append(errs, err)
525 continue
526 }
527 // Keep the request discoverable if cancellation cleanup fails. Archival
528 // closes issuance first; cancellation is removed before dispatch.
529 if cancelled, cancelErr := t.cancelled(ctx, r); cancelErr != nil {
530 errs = append(errs, cancelErr)
531 continue
532 } else if cancelled {
533 _, tag, cancelErr := w.raw.Get(ctx, key+".cancel")
534 if cancelErr == nil {
535 cancelErr = release.DeleteIfMatch(ctx, w.raw, key+".cancel", tag)
536 }
537 if cancelErr != nil {
538 errs = append(errs, cancelErr)
539 continue
540 }
541 }
542 if err = release.DeleteIfMatch(ctx, w.raw, key, etag); err != nil {
543 errs = append(errs, err)
544 }
545 }
546 return errors.Join(errs...)
547}
548