| index fd2856e2e..cb5ac6357 100644 |
| --- a/internal/host/systemapply.go |
| +++ b/internal/host/systemapply.go |
| @@ -225,7 +225,7 @@ type systemApplier struct { |
| // store is the deploy store verifyPointer opened, kept for armFloor. |
| // Nil on --recover, which reads no store. |
| store release.Store |
| - // beforeSwap, when set, runs inside guardActivation after the rule has |
| + // beforeSwap, when set, runs inside activationGuard after the rule has |
| // allowed and just before the step it guards: a test's schedule hook. |
| // Nil in production. |
| beforeSwap func() |
| @@ -445,15 +445,16 @@ func (a *systemApplier) install(ctx context.Context, req systemRequest, p *relea |
| // passes are idempotent. Which of their triggers were active is kept, so |
| // a refused rename can put them back. |
| prior := a.quiesce() |
| - // The activation rule again, at the step it guards (moves spec §11; fix |
| - // round 1): the check before staging can have gone stale, since the |
| - // agent may have raised the floor or held a row since. Here the floor |
| - // and the holds are held still across the rename. |
| - if err := a.guardActivation(target, running, func() error { |
| + // The activation rule again, at the step it guards (moves spec §11): the |
| + // check before staging can have gone stale, since the agent may have |
| + // raised the floor or held a row since. activationGuard holds the floor |
| + // and the holds still across the rename. |
| + renamed := false |
| + if err := a.activationGuard(target, running, func() error { |
| + renamed = true |
| return os.Rename(staged, systemBinaryPath(a.optDir)) |
| }); err != nil { |
| - var refusal activationRefused |
| - if errors.As(err, &refusal) { |
| + if !renamed { |
| return a.abandonInstall(req, err, prior) |
| } |
| return fail("install", err) |
| @@ -487,6 +488,11 @@ func (a *systemApplier) install(ctx context.Context, req systemRequest, p *relea |
| if err := a.journal(j); err != nil { |
| return fail("journal", err) |
| } |
| + // Outside activationGuard, deliberately: the floor protects activation, |
| + // and the guarded rename above was this install's activation. A floor |
| + // raised since applies to the next one. (Recovery's forward finish is |
| + // different: it has no rename of its own, so the finish is its |
| + // activation, and it is guarded.) |
| return a.finish(req, target, newPlan) |
| } |
| |
| @@ -505,16 +511,23 @@ func (a *systemApplier) install(ctx context.Context, req systemRequest, p *relea |
| // interrupted install and restores itself over a binary that never changed. |
| // Only when even the retry state cannot be written does the journal stay. |
| func (a *systemApplier) abandonInstall(req systemRequest, refusal error, prior []string) error { |
| + // A guard that could not get the floor lock is deferred (retried at the |
| + // next run); every other stop is a refusal. |
| + kind := errSystemRefused |
| + var deferred activationDeferred |
| + if errors.As(refusal, &deferred) { |
| + kind = errSystemDeferred |
| + } |
| rerr := a.rearm(prior) |
| if rerr == nil { |
| a.removeJournal() |
| - return a.record(req, fmt.Errorf("%w: %v", errSystemRefused, refusal)) |
| + return a.record(req, fmt.Errorf("%w: %v", kind, refusal)) |
| } |
| reason := fmt.Errorf("%w: %v; the root passes paused for the install could not be started again: %v. the next apply or recovery starts them", |
| - errSystemRefused, refusal, rerr) |
| + kind, refusal, rerr) |
| if err := a.notePendingRearm(prior); err != nil { |
| reason = fmt.Errorf("%w: %v; the root passes paused for the install could not be started again: %v, and the retry could not be recorded: %v. the journal stays for the next recovery", |
| - errSystemRefused, refusal, rerr, err) |
| + kind, refusal, rerr, err) |
| } else { |
| a.removeJournal() |
| } |
| @@ -778,7 +791,7 @@ func (a *systemApplier) armFloor(ctx context.Context) error { |
| // |
| // This is the EARLY check, made before anything is staged so a refused |
| // build is never copied in or run. It can go stale by the time the build is |
| -// put in place, so the step that puts it there runs under guardActivation, |
| +// put in place, so the step that puts it there runs under activationGuard, |
| // which applies the same rule again with its inputs held still. |
| // |
| // How the hold half relates to systemparticipation.go (region HA H14): that |
| @@ -791,12 +804,11 @@ func (a *systemApplier) armFloor(ctx context.Context) error { |
| // (this check before staging, H14's marker just before the journal) and |
| // either refusal stands. Neither reads or relaxes the other. |
| func (a *systemApplier) activationAllowed(ctx context.Context, version string) error { |
| - return a.activationRule(version, func() string { return a.onDiskBuild(ctx) }, nil) |
| + return a.activationRule(version, func() string { return a.onDiskBuild(ctx) }) |
| } |
| |
| // activationRefused is a refusal by the activation rule, told apart from a |
| -// failure of the step it guards: the install abandons on one and rolls |
| -// back on the other. |
| +// failure of the step it guards. |
| type activationRefused struct{ error } |
| |
| func (e activationRefused) Unwrap() error { return e.error } |
| @@ -805,93 +817,144 @@ func refuse(format string, args ...any) error { |
| return activationRefused{fmt.Errorf(format, args...)} |
| } |
| |
| -// guardActivation runs swap, the step that makes version this box's |
| -// executable, with the rule's inputs held still across it (fix round 1): |
| -// the floor files under the floor lock, which every floor writer and every |
| -// judgement of carlos.prev also takes, and, when the rule needs them, the |
| -// held rows under the registry's write lock (WithHoldsFrozen: a SetHold |
| -// waits). running is the release on disk, read by the caller before any |
| -// lock, since reading it runs a binary. A refusal is an activationRefused; |
| -// swap's own error comes back as it is. |
| -func (a *systemApplier) guardActivation(version, running string, swap func() error) error { |
| - guarded := func() error { |
| +// activationDeferred is the guard unable to get floor.lock because another |
| +// process holds it: a writer is active and may be about to publish a floor. |
| +// The step is put off to the next run, never taken unlocked (fix round 3). |
| +type activationDeferred struct{ error } |
| + |
| +func (e activationDeferred) Unwrap() error { return e.error } |
| + |
| +// activationGuard is the ONE guard for every step that makes an executable |
| +// this box's (moves spec §11; fix round 3): the install's rename, the |
| +// restore's rename, and each finish of a journal that has no rename of its |
| +// own (recovery forward and backward, and rollback's). Nothing else renames |
| +// an executable into place or finishes a journal; the scan in |
| +// activationguard_test.go fails if something does. |
| +// |
| +// It always takes floor.lock first, creating .moves for it on a box with |
| +// no floor yet, so no writer can arm or raise a floor until fn has run. |
| +// Then, with a floor armed, it always freezes the held rows |
| +// (registry.WithHoldsFrozen), whether or not target is the running release, |
| +// and checks the floor and the hold rule inside, immediately before fn. |
| +// With no floor armed, under the lock none can appear, the hold rule does |
| +// not apply, and the registry is not consulted (index D13: an unarmed |
| +// deployment behaves as main). |
| +// |
| +// Lock failures: |
| +// - contention (the wait ran out): activationDeferred. Never unlocked: a |
| +// writer holding the lock may not have published yet. |
| +// - the lock cannot exist (.moves or floor.lock cannot be created: |
| +// read-only, full or unwritable data dir) and no floor is armed: one |
| +// log line and fn runs as main's step would (F15). No writer can arm a |
| +// floor on such a box either. |
| +// - anything else, or a floor armed: a refusal. |
| +// |
| +// running is the release on disk, read by the caller before any lock, |
| +// since reading it runs a binary. A refusal or deferral comes back typed; |
| +// fn's own error comes back as it is. |
| +func (a *systemApplier) activationGuard(target, running string, fn func() error) error { |
| + step := func() error { |
| if a.beforeSwap != nil { |
| a.beforeSwap() |
| } |
| - return swap() |
| + return fn() |
| } |
| - ran := false |
| - // create: the lock is taken even on a box with no floor yet, so an |
| - // agent arming the first floor waits for this step rather than landing |
| - // between the rule and the swap (fix round 2). |
| + locked := false |
| err := withFloorLock(a.dataDir, true, func(*movesDir) error { |
| - ran = true |
| - return a.activationRule(version, func() string { return running }, guarded) |
| + locked = true |
| + floor, err := readLocalFloor(a.dataDir) |
| + if err != nil { |
| + return refuse("moves protocol floor: the floor on this box cannot be read: %v", err) |
| + } |
| + if floor == nil { |
| + return step() |
| + } |
| + if err := floorRefusal(target, floor); err != nil { |
| + return err |
| + } |
| + frozen := false |
| + err = a.freezeHolds(func(held []registry.Route) error { |
| + frozen = true |
| + if err := holdRefusal(target, running, held); err != nil { |
| + return err |
| + } |
| + return step() |
| + }) |
| + if !frozen { |
| + return refuse("a participant in a held operation does not downgrade, and the registry cannot say whether this box is one: %v", err) |
| + } |
| + return err |
| }) |
| - if ran { |
| + if locked { |
| return err |
| } |
| - // The lock could not be had. With a floor armed (or one that cannot be |
| - // read) that is a refusal. With none, the F15 dormancy rule: one log |
| - // line and the step goes ahead as main's would; an agent that cannot |
| - // create the lock cannot arm a floor either. |
| - if a.floorArmed() { |
| - return refuse("moves protocol floor: the floor on this box cannot be read: %v", err) |
| + if errors.Is(err, errFloorLockBusy) { |
| + return activationDeferred{fmt.Errorf("moves protocol floor: %v; this waits for the next run", errFloorLockBusy)} |
| + } |
| + if floorLockCannotExist(err) && !a.floorArmed() { |
| + a.ops.logf("system apply: moves protocol floor lock: %v (no floor is armed, and none can be on this box; continuing)", err) |
| + return step() |
| } |
| - a.ops.logf("system apply: moves protocol floor lock: %v (no floor is armed; continuing)", err) |
| - return guarded() |
| + return refuse("moves protocol floor: the floor on this box cannot be read: %v", err) |
| } |
| |
| -// activationRule is the rule itself. swap, when not nil, runs only once the |
| -// rule allows, inside whatever the rule had to hold to decide: guardActivation |
| -// passes the step, and the early check (activationAllowed) passes nil. |
| -func (a *systemApplier) activationRule(version string, running func() string, swap func() error) error { |
| - early := swap == nil |
| - if early { |
| - swap = func() error { return nil } |
| +// floorRefusal is the floor half of the rule. |
| +func floorRefusal(version string, floor *moves.Floor) error { |
| + if version == "" { |
| + return refuse("moves protocol floor: carlos.prev records no version, so it is below %s", floor.MinRelease) |
| } |
| + if ok, _ := moves.ReleaseAtOrAbove(version, floor.MinRelease); !ok { |
| + return refuse("moves protocol floor: %s is below %s", version, floor.MinRelease) |
| + } |
| + return nil |
| +} |
| + |
| +// holdRefusal is the hold half: while any row is held, nothing below the |
| +// running release. The same release, or a later one, is not a downgrade. |
| +func holdRefusal(version, running string, held []registry.Route) error { |
| + if len(held) == 0 || version == running { |
| + return nil |
| + } |
| + if ok, _ := moves.ReleaseAtOrAbove(version, running); ok { |
| + return nil |
| + } |
| + if running == "" { |
| + return refuse("a participant in a held operation does not downgrade, and the running release is unknown") |
| + } |
| + return refuse("a participant in a held operation does not downgrade: %s is below the running %s", version, running) |
| +} |
| + |
| +// activationRule is the early, unlocked check (activationAllowed, and |
| +// restoreFiles before its quiesce): the same two halves, read once, so a |
| +// refused build is never staged and a refused restore never quiesces. It |
| +// guards nothing by itself; the step it precedes runs under |
| +// activationGuard. |
| +func (a *systemApplier) activationRule(version string, running func() string) error { |
| floor, err := readLocalFloor(a.dataDir) |
| if err != nil { |
| return refuse("moves protocol floor: the floor on this box cannot be read: %v", err) |
| } |
| if floor == nil { |
| - if !early { |
| - return swap() |
| - } |
| a.ops.logf("system apply: no moves protocol floor is armed on this box; %q is not checked against one", version) |
| - return swap() |
| - } |
| - if version == "" { |
| - return refuse("moves protocol floor: carlos.prev records no version, so it is below %s", floor.MinRelease) |
| + return nil |
| } |
| - if ok, _ := moves.ReleaseAtOrAbove(version, floor.MinRelease); !ok { |
| - return refuse("moves protocol floor: %s is below %s", version, floor.MinRelease) |
| + if err := floorRefusal(version, floor); err != nil { |
| + return err |
| } |
| run := running() |
| if version == run { |
| - return swap() |
| + return nil |
| } |
| if ok, _ := moves.ReleaseAtOrAbove(version, run); ok { |
| - return swap() // not a downgrade: the held rows do not matter |
| + return nil |
| } |
| - // A downgrade, or a running release that cannot be compared: allowed |
| - // only while no row is held, and the rows stay as they were read until |
| - // swap has run. |
| + var held []registry.Route |
| called := false |
| - err = a.freezeHolds(func(held []registry.Route) error { |
| - called = true |
| - if len(held) == 0 { |
| - return swap() |
| - } |
| - if run == "" { |
| - return refuse("a participant in a held operation does not downgrade, and the running release is unknown") |
| - } |
| - return refuse("a participant in a held operation does not downgrade: %s is below the running %s", version, run) |
| - }) |
| + err = a.freezeHolds(func(h []registry.Route) error { called = true; held = h; return nil }) |
| if !called { |
| return refuse("a participant in a held operation does not downgrade, and the registry cannot say whether this box is one: %v", err) |
| } |
| - return err |
| + return holdRefusal(version, run, held) |
| } |
| |
| // floorAllowsForward is the floor half alone, for recovery's forward path: |
| @@ -1306,22 +1369,35 @@ func (a *systemApplier) rollback(ctx context.Context, req systemRequest, reason |
| if err != nil { |
| return failed("restore", err) // journal kept: the restore is half done |
| } |
| + // carlos.prev, now on disk: what this rollback's journal finish makes |
| + // current, checked again under activationGuard at that finish. |
| + restored, _ := readLine(prevVersionPath(a.optDir)) |
| if err := a.restartEdge(); err != nil { |
| return failed("restart", err) // journal kept: the edge is still to come back |
| } |
| a.restartRelay() |
| if err := a.gate(ctx, j.PrevBuild); err != nil { |
| err := failed("previous build failed its gate too", err) |
| - a.removeJournal() |
| + if gerr := a.activationGuard(restored, restored, func() error { |
| + a.removeJournal() |
| + return nil |
| + }); gerr != nil { |
| + return fmt.Errorf("%v; the journal stays: %v", err, gerr) |
| + } |
| return err |
| } |
| if err := a.arm(plan); err != nil { |
| a.ops.logf("system apply: arm previous plan: %v", err) |
| } |
| - if err := a.writeResult(req, systemOutcomeRolledBack, reason); err != nil { |
| - return err |
| + if err := a.activationGuard(restored, restored, func() error { |
| + if err := a.writeResult(req, systemOutcomeRolledBack, reason); err != nil { |
| + return err |
| + } |
| + a.removeJournal() |
| + return nil |
| + }); err != nil { |
| + return failed("finish", err) // journal kept |
| } |
| - a.removeJournal() |
| return fmt.Errorf("system apply: %s rolled back: %s", j.Target, reason) |
| } |
| |
| @@ -1342,7 +1418,7 @@ func (a *systemApplier) restoreFiles(ctx context.Context, j *systemJournal) (boo |
| // box stays on the build it is on, and the caller reports it. |
| prevVersion, _ := readLine(prevVersionPath(a.optDir)) |
| running := a.onDiskBuild(ctx) |
| - if err := a.activationRule(prevVersion, func() string { return running }, nil); err != nil { |
| + if err := a.activationRule(prevVersion, func() string { return running }); err != nil { |
| return bootstrapPlan{}, err |
| } |
| // The same quiesce the install did: a root pass that is mid-flight |
| @@ -1355,20 +1431,27 @@ func (a *systemApplier) restoreFiles(ctx context.Context, j *systemJournal) (boo |
| if err := copyFileDurable(prev, restore, 0o755); err != nil { |
| return bootstrapPlan{}, err |
| } |
| - if err := a.guardActivation(prevVersion, running, func() error { |
| + renamed := false |
| + if err := a.activationGuard(prevVersion, running, func() error { |
| + renamed = true |
| if err := os.Rename(restore, systemBinaryPath(a.optDir)); err != nil { |
| return err |
| } |
| return syncDir(a.optDir) |
| }); err != nil { |
| os.Remove(restore) |
| - // Refused before the rename (the floor or a hold changed during the |
| - // copy): nothing was restored, so the paused triggers go back to how |
| - // they were. The caller keeps the journal. |
| - var refusal activationRefused |
| - if errors.As(err, &refusal) { |
| + // Stopped before the rename (refused or deferred: the floor or a hold |
| + // changed during the copy, or a writer holds the floor lock): nothing |
| + // was restored, so the paused triggers go back to how they were. If |
| + // they cannot be started, they go to rearm-pending.json, captured as |
| + // they were before the quiesce, and every run starts them first, |
| + // before any early refusal. The caller keeps the journal. |
| + if !renamed { |
| if rerr := a.rearm(prior); rerr != nil { |
| err = fmt.Errorf("%w; the root passes paused for the restore could not be started again: %v", err, rerr) |
| + if nerr := a.notePendingRearm(prior); nerr != nil { |
| + err = fmt.Errorf("%w, and the retry could not be recorded: %v", err, nerr) |
| + } |
| } |
| } |
| return bootstrapPlan{}, err |
| @@ -1500,11 +1583,18 @@ func (a *systemApplier) recoverIfNeeded(ctx context.Context) (bool, error) { |
| // Finishing the journal is the forward path's activation: the |
| // rule again, under the guard, at the step itself, since the |
| // floor can rise while the plan is staged (fix round 2). |
| - err = a.guardActivation(j.Target, j.Target, func() error { return a.finish(req, j.Target, plan) }) |
| - var refusal activationRefused |
| - if !errors.As(err, &refusal) { |
| + finished := false |
| + err = a.activationGuard(j.Target, j.Target, func() error { |
| + finished = true |
| + return a.finish(req, j.Target, plan) |
| + }) |
| + if finished { |
| return true, err |
| } |
| + var deferred activationDeferred |
| + if errors.As(err, &deferred) { |
| + return false, fmt.Errorf("system apply: recovery: %w", err) // journal kept |
| + } |
| a.ops.logf("system apply: recovery: %s passed its gate but %v; rolling back instead", j.Target, err) |
| } |
| } |
| @@ -1512,6 +1602,8 @@ func (a *systemApplier) recoverIfNeeded(ctx context.Context) (bool, error) { |
| if err != nil { |
| return false, fmt.Errorf("system apply: recovery: %w", err) // journal kept: the next run recovers again |
| } |
| + // carlos.prev, now on disk: what the finish below makes current. |
| + restored, _ := readLine(prevVersionPath(a.optDir)) |
| if a.ops.edgeStartedThisBoot() { |
| a.ops.systemctl("reset-failed", systemEdgeUnit) |
| // --no-block: this can be running as the edge's own OnFailure= |
| @@ -1530,10 +1622,17 @@ func (a *systemApplier) recoverIfNeeded(ctx context.Context) (bool, error) { |
| a.ops.systemctl(append([]string{"start"}, armed...)...) |
| } |
| } |
| - if err := a.writeResult(req, systemOutcomeRolledBack, "interrupted at "+j.Phase); err != nil { |
| - return false, err |
| + // The backward finish is guarded like every other: a floor raised since |
| + // the restore's rename keeps the journal, and the next run judges again. |
| + if err := a.activationGuard(restored, restored, func() error { |
| + if err := a.writeResult(req, systemOutcomeRolledBack, "interrupted at "+j.Phase); err != nil { |
| + return err |
| + } |
| + a.removeJournal() |
| + return nil |
| + }); err != nil { |
| + return false, fmt.Errorf("system apply: recovery: %w", err) // journal kept |
| } |
| - a.removeJournal() |
| return true, nil |
| } |
| |
amadan