carlos / platform Public

Clone
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package host
2
3import (
4 "context"
5 "encoding/json"
6 "errors"
7 "fmt"
8 "github.com/carlosframework/platform/internal/hostactors"
9 execlease "github.com/carlosframework/platform/internal/hostexec"
10 "github.com/carlosframework/platform/internal/hostfloor"
11 "github.com/carlosframework/platform/internal/release"
12 "github.com/carlosframework/platform/internal/rootparticipation"
13 "os"
14 "path/filepath"
15 "slices"
16 "time"
17)
18
19type guardTransaction struct {
20 Roster json.RawMessage `json:"roster"`
21 ID string `json:"id"`
22 Actor string `json:"actor"`
23 Enrollment string `json:"enrollment"`
24}
25
26func (a *systemApplier) guarded() bool { return a.floor != nil }
27func (a *systemApplier) activeFloor() (bool, error) {
28 if !a.guarded() {
29 return false, nil
30 }
31 p, _, err := a.floor.Load()
32 return p.Mode != "unpublished", err
33}
34func (a *systemApplier) guardImage(path string) error {
35 _, err := a.guardImageHash(path)
36 return err
37}
38
39// guardImageHash is guardImage, returning the verified image's SHA-256
40// ("" when no floor guards this box).
41func (a *systemApplier) guardImageHash(path string) (string, error) {
42 if !a.guarded() {
43 return "", nil
44 }
45 open := hostfloor.OpenProtected
46 if a.guardOpen != nil {
47 open = a.guardOpen
48 }
49 f, err := open(path)
50 if err != nil {
51 return "", err
52 }
53 defer f.Close()
54 hash, err := a.floor.VerifyFile(f, a.platform)
55 if err != nil {
56 return "", err
57 }
58 p, _, err := a.floor.Load()
59 if err != nil || p.Mode == "unpublished" {
60 return hash, err
61 }
62 tx, err := a.guardTransaction()
63 if err != nil {
64 if os.IsNotExist(err) {
65 if j, e := readSystemJournal(a.optDir); e == nil && j == nil && path == systemBinaryPath(a.optDir) {
66 return hash, nil
67 }
68 }
69 return "", err
70 }
71 var roster hostactors.Roster
72 if release.VerifyRootEnvelope(a.floor.PublicKey, tx.Roster, &roster) != nil || roster.Validate(a.floor.Bucket) != nil {
73 return "", hostfloor.ErrFloor
74 }
75 reservation, ok := roster.Activations[tx.ID]
76 _, floor, err := a.floor.Load()
77 if err != nil {
78 return "", err
79 }
80 if !ok || reservation.FloorSHA256 != floor.Digest || reservation.Actors[tx.Actor] != tx.Enrollment || !slices.Contains(reservation.Hashes[tx.Actor], hash) {
81 return "", hostfloor.ErrFloor
82 }
83 return hash, nil
84}
85
86// guardTarget is the transaction's target image: the middle of the three
87// hashes prepareGuard selected the reservation by, read from the signed
88// roster root persisted with the transaction.
89func (a *systemApplier) guardTarget(tx guardTransaction) (string, error) {
90 var roster hostactors.Roster
91 if release.VerifyRootEnvelope(a.floor.PublicKey, tx.Roster, &roster) != nil || roster.Validate(a.floor.Bucket) != nil {
92 return "", hostfloor.ErrFloor
93 }
94 hashes := roster.Activations[tx.ID].Hashes[tx.Actor]
95 if len(hashes) != 3 || hashes[1] == "" {
96 return "", hostfloor.ErrFloor
97 }
98 return hashes[1], nil
99}
100
101// guardFinished reports whether a transaction with no journal open has
102// finished: the verified image on disk is its target, or root's marker
103// records that it rolled back (Opus re-review M1-4). A transaction that is
104// neither never changed the box: it was refused or deferred before its
105// first destructive step, or abandoned at the rename.
106func (a *systemApplier) guardFinished(tx guardTransaction) (bool, error) {
107 hash, err := a.guardImageHash(systemBinaryPath(a.optDir))
108 if err != nil {
109 return false, err
110 }
111 begun, err := hostfloor.ReadBegun(a.floor.Dir)
112 if err == nil {
113 if begun.ID != tx.ID {
114 return false, hostfloor.ErrFloor
115 }
116 if begun.RolledBack {
117 return true, nil
118 }
119 } else if !errors.Is(err, os.ErrNotExist) {
120 return false, err
121 }
122 target, err := a.guardTarget(tx)
123 if err != nil {
124 return false, err
125 }
126 return hash == target, nil
127}
128
129// settleGuard closes a transaction with no journal open. A finished one is
130// completed, once no controller is active, as before. One that never
131// changed the box records no completion: root withdraws its own marker and
132// transaction, so neither wedges the box nor reports the old image as the
133// result, and the console's reservation stays held for the agent's retry
134// or an operator. The withdrawal does not wait for controllers: it gives up
135// only root's intent, on a box it did not change.
136func (a *systemApplier) settleGuard(ctx context.Context, tx guardTransaction, beforeCompletion func() error) error {
137 finished, err := a.guardFinished(tx)
138 if err != nil {
139 return err
140 }
141 if !finished {
142 // The marker goes inside the settlement, after root's census, so a
143 // census that refuses leaves it with the rest of the transaction.
144 return a.floor.EndParticipation(ctx, tx.ID, func() error {
145 if err := hostfloor.RemoveDurable(filepath.Join(a.floor.Dir, "update-begun.json")); err != nil {
146 return err
147 }
148 return hostfloor.RemoveDurable(filepath.Join(a.floor.Dir, "activation.json"))
149 })
150 }
151 if beforeCompletion != nil {
152 if err := beforeCompletion(); err != nil {
153 return err
154 }
155 }
156 // Completion is durable before the marker disappears. Replaying either
157 // write is harmless; readers never report completion with a marker present.
158 if err = hostfloor.Durable(filepath.Join(a.floor.Dir, "completed.json"), []byte(tx.ID), 0644); err != nil {
159 return err
160 }
161 return a.floor.EndParticipation(ctx, tx.ID, func() error { return hostfloor.RemoveDurable(filepath.Join(a.floor.Dir, "update-begun.json")) })
162}
163func (a *systemApplier) guardTransaction() (guardTransaction, error) {
164 raw, err := os.ReadFile(filepath.Join(a.floor.Dir, "activation.json"))
165 if err != nil {
166 return guardTransaction{}, err
167 }
168 var tx guardTransaction
169 if json.Unmarshal(raw, &tx) != nil || tx.ID == "" || tx.Enrollment == "" || tx.Actor == "" {
170 return guardTransaction{}, hostfloor.ErrFloor
171 }
172 return tx, nil
173}
174func (a *systemApplier) beginGuard(ctx context.Context, target *release.SystemPointer) error {
175 if !a.guarded() {
176 return nil
177 }
178 p, _, err := a.floor.Load()
179 if err != nil {
180 return err
181 }
182 if p.Mode == "unpublished" {
183 return nil
184 }
185 if err := a.prepareGuard(ctx, target); err != nil {
186 return err
187 }
188 tx, err := a.guardTransaction()
189 if err != nil {
190 return err
191 }
192 // The signed reservation is persisted by the root approval step, never by
193 // the worker's request. A new transaction must still find it in the roster.
194 store, pub, err := a.openStore(ctx)
195 if err != nil {
196 return err
197 }
198 _, local, err := a.floor.Load()
199 if err != nil {
200 return err
201 }
202 remote, err := hostfloor.Read(ctx, store, pub, a.floor.Bucket)
203 if err != nil || remote.Digest != local.Digest {
204 return hostfloor.ErrFloor
205 }
206 d := hostactors.Directory{Store: store, PublicKey: pub, Bucket: a.floor.Bucket}
207 live, err := d.Read(ctx)
208 if err != nil {
209 return err
210 }
211 h, ok := live.Roster.Activations[tx.ID]
212 if !ok || h.Actors[tx.Actor] != tx.Enrollment {
213 return hostfloor.ErrFloor
214 }
215 var saved hostactors.Roster
216 if release.VerifyRootEnvelope(pub, tx.Roster, &saved) != nil {
217 return hostfloor.ErrFloor
218 }
219 x, _ := json.Marshal(h)
220 y, _ := json.Marshal(saved.Activations[tx.ID])
221 if string(x) != string(y) {
222 return hostfloor.ErrFloor
223 }
224 return a.floor.Begin(ctx, tx.ID, a.dataDir)
225}
226
227// checkGuardRecovery runs with the apply lock held. No-journal markers can
228// only precede the first destructive step or follow durable journal removal.
229// Both are terminal after the selected current image has been verified.
230func (a *systemApplier) checkGuardRecovery(ctx context.Context) error {
231 if !a.guarded() {
232 return a.checkUnarmedParticipationRecovery(ctx)
233 }
234 p, _, err := a.floor.Load()
235 if err != nil {
236 return err
237 }
238 if p.Mode == "unpublished" {
239 return a.checkUnarmedParticipationRecovery(ctx)
240 }
241 j, err := readSystemJournal(a.optDir)
242 if err != nil {
243 return err
244 }
245 marker := filepath.Join(a.floor.Dir, "update-begun.json")
246 raw, err := os.ReadFile(marker)
247 if errors.Is(err, os.ErrNotExist) && j == nil {
248 tx, txErr := a.guardTransaction()
249 if errors.Is(txErr, os.ErrNotExist) {
250 return nil
251 }
252 if txErr != nil {
253 return txErr
254 }
255 completed, readErr := os.ReadFile(filepath.Join(a.floor.Dir, "completed.json"))
256 if readErr == nil && string(completed) == tx.ID {
257 // Completion recorded and root's marker gone. With root's update
258 // record gone too the transaction is settled: a replay has
259 // nothing left to close, and runs no census.
260 open, err := a.updateRecordOpen(ctx)
261 if err != nil || !open {
262 return err
263 }
264 return a.settleWithoutJournal(ctx, tx.ID, a.floor.EndParticipation(ctx, tx.ID, nil))
265 }
266 if readErr != nil && !errors.Is(readErr, os.ErrNotExist) {
267 return readErr
268 }
269 return a.settleWithoutJournal(ctx, tx.ID, a.settleGuard(ctx, tx, func() error { return a.controllersRecovered(ctx) }))
270 }
271 if err != nil {
272 return err
273 }
274 tx, err := a.guardTransaction()
275 if err != nil {
276 return err
277 }
278 var begun struct {
279 ID string `json:"id"`
280 }
281 if json.Unmarshal(raw, &begun) != nil || begun.ID != tx.ID {
282 return hostfloor.ErrFloor
283 }
284 if j == nil {
285 return a.settleWithoutJournal(ctx, tx.ID, a.settleGuard(ctx, tx, func() error { return a.controllersRecovered(ctx) }))
286 }
287 if err := a.controllersRecovered(ctx); err != nil {
288 return err
289 }
290 if j.Phase == systemPhaseBackingUp {
291 return a.guardImage(systemBinaryPath(a.optDir))
292 }
293 return a.guardImage(systemPrevPath(a.optDir))
294}
295
296// errSettlementDeferred marks a settlement that root's generation census
297// refused with no journal open. No apply is half done and the files are
298// whole, so there is nothing for recovery to restore and no reason to keep
299// the edge down (its drop-in Requires= the recover unit). The transaction
300// evidence stays as it was: the reservation, root's marker and update
301// record, so every new claim acquisition stays refused, and the next apply,
302// or recover on the next edge start, settles it. Run under --recover logs
303// it and exits 0; the apply returns it as before. beginExecutableChange
304// treats a refused census as a deferral too (systemparticipation.go).
305var errSettlementDeferred = errors.New("participation settlement deferred")
306
307// errAcquisitionBarrier marks a deferral that could not hold claim
308// acquisitions refused (Ledger.HoldUpdate failed). It is never swallowed:
309// recover must not exit 0 with acquisitions open (Astra rounds 2 and 3).
310var errAcquisitionBarrier = errors.New("acquisition barrier")
311
312// settleWithoutJournal is the result of one settlement made with no
313// journal open: the pending-controller recovery, the completion or
314// withdrawal, and the end of root's update record, whose census runs again
315// inside EndParticipation; and, after an apply's journal is durably
316// removed, the settlement that follows it (closeGuardJournal). A result
317// that is only a census refusal, on a box with root participation, is
318// wrapped in errSettlementDeferred, once acquisitions are held refused:
319// root's update record is kept, or, where the refused census was the one
320// that would have begun it, written under barrier's id (Ledger.HoldUpdate).
321// Anything else is returned unchanged: an image that does not verify,
322// malformed state, a live or unreadable controller, a durable write that
323// failed, a barrier that could not be written, and every refusal on a box
324// without a census, which sees exactly main's recovery (ruling 6).
325func (a *systemApplier) settleWithoutJournal(ctx context.Context, barrier string, err error) error {
326 if err == nil || !onlyCensusRefusal(err) || ctx.Err() != nil {
327 return err
328 }
329 required, checkErr := a.participationDisk().ParticipationRequired(ctx)
330 if checkErr != nil {
331 // Whether a barrier is needed could not be established (the store
332 // did not open, a retained census did not read): that is a barrier
333 // failure too, never a plain refusal a caller may log (Astra
334 // round 4). Both errors are kept.
335 return errors.Join(err, fmt.Errorf("%w: participation required: %w", errAcquisitionBarrier, checkErr))
336 }
337 if !required {
338 return err
339 }
340 if holdErr := a.participationDisk().WithParticipation(ctx, func(ctx context.Context, l rootparticipation.Ledger) error {
341 return l.HoldUpdate(ctx, barrier)
342 }, func() error { return rootparticipation.ErrUnavailable }); holdErr != nil {
343 return errors.Join(err, fmt.Errorf("%w: %w", errAcquisitionBarrier, holdErr))
344 }
345 return fmt.Errorf("%w: no journal is open, so the files are whole; root's update record holds claim acquisitions refused until the settlement passes: %w", errSettlementDeferred, err)
346}
347
348// updateRecordOpen reports whether root's update record is there, on a box
349// with root participation. Without one it is false: main's legacy path.
350func (a *systemApplier) updateRecordOpen(ctx context.Context) (bool, error) {
351 open := false
352 err := a.participationDisk().WithParticipation(ctx, func(_ context.Context, l rootparticipation.Ledger) error {
353 _, err := l.UpdateID()
354 if errors.Is(err, os.ErrNotExist) {
355 return nil
356 }
357 open = err == nil
358 return err
359 }, func() error { open = true; return nil })
360 return open, err
361}
362
363// deferSettlement reports whether Run, under --recover, leaves err's
364// settlement to a later run instead of failing: only an errSettlementDeferred.
365// It logs why.
366func (a *systemApplier) deferSettlement(recover bool, err error) bool {
367 if !recover || !errors.Is(err, errSettlementDeferred) {
368 return false
369 }
370 a.ops.logf("system apply: recover: %v; the edge may start, and the next apply or recover settles it", err)
371 return true
372}
373
374// settleMarker settles the update-begun marker at the end of a run
375// (settleUpdateBegun) and decides what its outcome means for the run: a
376// deferral under --recover is logged and left to a later run; a barrier
377// that could not be written joins the run's error, whatever the mode,
378// because acquisitions would otherwise be open with the settlement
379// unmade; a deferral under the apply is logged, as every marker failure
380// was on main, with acquisitions held refused by the barrier.
381func (a *systemApplier) settleMarker(ctx context.Context, recover bool, retErr *error) {
382 err := a.settleUpdateBegun(ctx)
383 if err == nil || a.deferSettlement(recover, err) {
384 return
385 }
386 if errors.Is(err, errAcquisitionBarrier) {
387 *retErr = errors.Join(*retErr, err)
388 return
389 }
390 a.ops.logf("system apply: %v", err)
391}
392
393func (a *systemApplier) endGuard(ctx context.Context) error {
394 if !a.guarded() {
395 return nil
396 }
397 p, _, err := a.floor.Load()
398 if err != nil || p.Mode == "unpublished" {
399 return err
400 }
401 tx, err := a.guardTransaction()
402 if err != nil {
403 return err
404 }
405 if j, err := readSystemJournal(a.optDir); err != nil {
406 return err
407 } else if j != nil {
408 return hostfloor.ErrFloor
409 }
410 return a.settleGuard(ctx, tx, nil)
411}
412
413func (a *systemApplier) controllersIdle(ctx context.Context) error {
414 return a.participationDisk().ControllersIdle(ctx, a.dataDir)
415}
416
417var recoverPendingControllers = recoverRootExecution
418
419// controllersRecovered lets root recover a provably dead pending controller
420// before refusing, whether or not a journal is open; a live or unreadable one
421// still refuses.
422func (a *systemApplier) controllersRecovered(ctx context.Context) error {
423 err := a.controllersIdle(ctx)
424 if err == nil {
425 return nil
426 }
427 guardDir := systemGuardDir(a.optDir)
428 if a.floor != nil {
429 guardDir = a.floor.Dir
430 }
431 required, checkErr := a.participationDisk().ParticipationRequired(ctx)
432 if checkErr != nil {
433 return errors.Join(err, checkErr)
434 }
435 recover := recoverPendingControllers
436 if !required {
437 recover = execlease.RecoverPending
438 }
439 if recoveryErr := recover(ctx, a.dataDir, guardDir); recoveryErr != nil {
440 return errors.Join(err, recoveryErr)
441 }
442 return a.controllersIdle(ctx)
443}
444func (a *systemApplier) closeGuardJournal(ctx context.Context) error {
445 required, err := a.participationDisk().ParticipationRequired(ctx)
446 if err != nil {
447 return err
448 }
449 if !required {
450 if !a.guarded() {
451 a.removeJournal()
452 return nil
453 }
454 if err := hostfloor.RemoveDurable(a.file("journal.json")); err != nil {
455 return err
456 }
457 return a.endGuard(ctx)
458 }
459 active, err := a.activeFloor()
460 if err != nil {
461 return err
462 }
463 if err := hostfloor.RemoveDurable(a.file("journal.json")); err != nil {
464 return err
465 }
466 // The journal is gone and the files are whole: from here a settlement
467 // the census refuses is deferred as with no journal open.
468 if !active {
469 return a.settleWithoutJournal(ctx, "recover-legacy", a.clearUpdateBegun(ctx))
470 }
471 err = a.endGuard(ctx)
472 if err == nil || !onlyCensusRefusal(err) {
473 return err
474 }
475 tx, txErr := a.guardTransaction()
476 if txErr != nil {
477 return errors.Join(err, txErr)
478 }
479 return a.settleWithoutJournal(ctx, tx.ID, err)
480}
481func (a *systemApplier) guardRun(ctx context.Context, wait time.Duration, name string, args ...string) (string, error) {
482 if id, required, err := a.rootMoves.read(); err != nil {
483 return "", err
484 } else if required {
485 binary, err := readRootFloorBinary(a.optDir, name)
486 if err != nil {
487 return "", err
488 }
489 floor, err := readEffectiveMovesFloor(a.rootMoves, a.optDir, a.dataDir)
490 if err != nil {
491 return "", err
492 }
493 if !binary.supports(floor) {
494 repair := name == systemBinaryPath(a.optDir) || name == filepath.Join(systemGuardDir(a.optDir), "carlos")
495 if name == systemPrevPath(a.optDir) && rootFloorAllowsPrevIdentity(a.optDir, id) == nil {
496 repair = true
497 }
498 if !binary.supports(nil) || !repair {
499 return "", errRootFloorHistory
500 }
501 rootFloorAlert()
502 }
503 }
504
505 if !a.guarded() {
506 return a.ops.run(ctx, wait, name, args...)
507 }
508 if err := a.guardImage(name); err != nil {
509 return "", err
510 }
511 bounded, cancel := context.WithTimeout(ctx, wait)
512 defer cancel()
513 return a.floor.Run(bounded, name, a.platform, args...)
514}
515func installFloorHook(a *systemApplier) error {
516 dir := systemGuardDir(a.optDir)
517 fi, err := os.Lstat(dir)
518 if errors.Is(err, os.ErrNotExist) {
519 return nil
520 }
521 if err != nil {
522 return err
523 }
524 if !fi.IsDir() {
525 return hostfloor.ErrFloor
526 }
527 entries, err := os.ReadDir(dir)
528 if err != nil {
529 return err
530 }
531 // Main's participation marker predates floor arming, is written through
532 // update-begun.json.tmp, and leaves an empty directory after completion.
533 // A torn or leftover write of it is still legacy state; only a marker
534 // naming a floor transaction ("id") or a floor artifact needs a policy.
535 if legacyGuardEntries(dir, entries) {
536 return nil
537 }
538 d, err := floorDisk(a.optDir)
539 if err != nil {
540 return err
541 }
542 a.floor = &d
543 _, _, err = d.Load()
544 return err
545}
546
547func (a *systemApplier) prepareGuard(ctx context.Context, target *release.SystemPointer) error {
548 enrollment, err := hostfloor.ReadEnrollment(filepath.Join(a.etcDir, "carlos", "host-actor.json"))
549 if err != nil {
550 return err
551 }
552 store, pub, err := a.openStore(ctx)
553 if err != nil {
554 return err
555 }
556 raw, _, err := store.Get(ctx, hostactors.Key)
557 if err != nil {
558 return err
559 }
560 var roster hostactors.Roster
561 if release.VerifyRootEnvelope(pub, raw, &roster) != nil || roster.Validate(a.floor.Bucket) != nil {
562 return hostfloor.ErrFloor
563 }
564 b, ok := roster.Boxes[enrollment.Box]
565 if !ok || b.Enrollment != enrollment.Enrollment || b.Cloud.InstanceID != enrollment.InstanceID {
566 return hostfloor.ErrFloor
567 }
568 actor := "box:" + enrollment.Box
569 id := ""
570 for name, h := range roster.Activations {
571 if h.Publication != nil && target.ActivationID != name {
572 continue
573 }
574 hashes := h.Hashes[actor]
575 if h.Actors[actor] == enrollment.Enrollment && len(hashes) == 3 && hashes[1] == target.Current.Artifacts[a.platform].SHA256 {
576 if id != "" {
577 return hostfloor.ErrFloor
578 }
579 id = name
580 }
581 }
582 if id == "" {
583 return hostfloor.ErrFloor
584 }
585 identityKey, err := release.IdentityKey(target.Current.ReleaseTag)
586 if err != nil {
587 return err
588 }
589 identity, _, err := store.Get(ctx, identityKey)
590 if err != nil {
591 return err
592 }
593 _, floor, err := a.floor.Load()
594 if err != nil {
595 return err
596 }
597 if err = hostfloor.CheckImage(pub, floor, identity, target.Current.Version, a.platform, target.Current.Artifacts[a.platform].SHA256); err != nil {
598 return err
599 }
600 if err = hostfloor.Durable(filepath.Join(a.floor.Dir, "identities", target.Current.ReleaseTag+".json"), identity, 0644); err != nil {
601 return err
602 }
603 txRaw, _ := json.Marshal(guardTransaction{raw, id, actor, enrollment.Enrollment})
604 return hostfloor.Durable(filepath.Join(a.floor.Dir, "activation.json"), txRaw, 0644)
605}
606
607// legacyGuardEntries reports whether the guard directory holds nothing but
608// main's region participation marker, its temporary file, or both, with no
609// floor transaction ID in the marker.
610func legacyGuardEntries(dir string, entries []os.DirEntry) bool {
611 for _, e := range entries {
612 if !e.Type().IsRegular() || (e.Name() != updateBegunFile && e.Name() != updateBegunFile+".tmp") {
613 return false
614 }
615 if e.Name() == updateBegunFile {
616 raw, err := os.ReadFile(filepath.Join(dir, e.Name()))
617 if err != nil {
618 return false
619 }
620 var marker map[string]json.RawMessage
621 if json.Unmarshal(raw, &marker) == nil {
622 if _, ok := marker["id"]; ok {
623 return false
624 }
625 }
626 }
627 }
628 return true
629}
630