amadan

carlos / platform Public

system: with region HA on, root marks an update begun and refuses it while the box takes part in a claim operation

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Paul Campbell pushed by claude 5352ae6d52567fd9679c0d007d94067bcb13107f parent 089decb
3 files changed, +356 −0
  • internal/host/systemapply.go +17 −0
  • internal/host/systemparticipation.go +134 −0
  • internal/host/systemparticipation_test.go +205 −0
diff --git a/internal/host/systemapply.go b/internal/host/systemapply.go
index da0441136..fa34ad671 100644
--- a/internal/host/systemapply.go
+++ b/internal/host/systemapply.go
@@ -108,6 +108,7 @@ func RunSystemApply(args []string) {
ops: productionSysOps(), openStore: openSystemStore, channel: channel,
optDir: release.OptDir(), dataDir: dataDir(), etcDir: "/etc", build: version.Build,
platform: runtime.GOOS + "-" + runtime.GOARCH, gateWait: systemGateWait, soak: systemGateSoak,
+ regionHA: regionHAEnabled(os.Getenv),
}
if err := a.Run(ctx, *recover); err != nil {
log.Fatalf("%v", err)
@@ -216,6 +217,10 @@ type systemApplier struct {
build string // version.Build of THIS process
platform string // "<goos>-<goarch>"; the caller sets it from runtime
gateWait, soak time.Duration
+ // regionHA is CARLOS_REGION_HA: root writes its update-begun marker and
+ // refuses an update while this box takes part in a claim operation
+ // (region HA spec 1 §5.10; systemparticipation.go).
+ regionHA bool
}
func (a *systemApplier) rootDir() string { return systemRootDir(a.optDir) }
@@ -259,6 +264,9 @@ func (a *systemApplier) Run(ctx context.Context, recover bool) error {
return nil
}
defer os.RemoveAll(a.stagingDir())
+ // Whatever this run does, a finished or recovered update leaves no
+ // update-begun marker behind (spec §5.10).
+ defer a.settleUpdateBegun()
recovered, err := a.recoverIfNeeded(ctx)
if err != nil {
@@ -387,6 +395,15 @@ func (a *systemApplier) install(ctx context.Context, req systemRequest, p *relea
if err != nil {
return a.record(req, fmt.Errorf("%w: carlos.prev bootstrap: %v", errSystemRefused, err))
}
+ // Region HA (spec 1 §5.10): the marker, then the participation check,
+ // before the first step that changes the box. Either failing defers the
+ // update, which the agent retries; the box is left exactly as it was.
+ if err := a.beginExecutableChange(target, req.PromotedAt); err != nil {
+ if !errors.Is(err, errSystemDeferred) {
+ err = fmt.Errorf("%w: update-begun marker: %v", errSystemDeferred, err)
+ }
+ return a.record(req, err)
+ }
// PromotedAt rides along for the whole transaction: it is half of the
// tuple a recovery has to answer for.
j := systemJournal{Phase: systemPhaseInstalling, Target: target, PromotedAt: req.PromotedAt, PrevBuild: a.prevBuild(ctx),
diff --git a/internal/host/systemparticipation.go b/internal/host/systemparticipation.go
new file mode 100644
index 000000000..25740ef6d
--- /dev/null
+++ b/internal/host/systemparticipation.go
@@ -0,0 +1,134 @@
+package host
+
+import (
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+ "time"
+)
+
+// Region HA spec 1 §5.10 (H14), root's half. A box that takes part in a
+// running claim operation never takes an executable update. Each side
+// records its own intent durably, then checks the other's, so of two that
+// race at least one sees the other:
+//
+// - root writes its update-begun marker, then scans <DataDir>/.hostexec/
+// and refuses the update while any controller record is there, in any
+// state, or while it cannot tell;
+// - a participant writes its .hostexec record (claim D6), then reads the
+// marker and refuses to acknowledge while one exists. That half waits
+// for the claim's D6 controller and is not built here.
+//
+// The marker stays until the update finishes or is recovered. Dormant
+// unless CARLOS_REGION_HA is on.
+
+const (
+ updateBegunFile = "update-begun.json"
+ hostexecDir = ".hostexec"
+)
+
+// systemGuardDir is /opt/carlos/system/guard, beside root's update state.
+func systemGuardDir(optDir string) string { return filepath.Join(optDir, "system", "guard") }
+
+func updateBegunPath(optDir string) string {
+ return filepath.Join(systemGuardDir(optDir), updateBegunFile)
+}
+
+type updateBegun struct {
+ Target string `json:"target"`
+ PromotedAt time.Time `json:"promoted_at"`
+ At time.Time `json:"at"`
+}
+
+// hostexecParticipation reports why this box may still be a participant
+// in a claim operation, or nil when it certainly is not. The directory
+// absent is the only "not". Anything in it, whatever its name or state,
+// and anything root cannot list or that is not a plain directory, is a
+// refusal (spec §5.10, B1-5). Only names are read: root never opens a
+// record here.
+func hostexecParticipation(dataDir string) error {
+ dir := filepath.Join(dataDir, hostexecDir)
+ fi, err := os.Lstat(dir)
+ if errors.Is(err, os.ErrNotExist) {
+ return nil
+ }
+ if err != nil {
+ return fmt.Errorf("%s cannot be read: %v", dir, err)
+ }
+ if !fi.IsDir() {
+ return fmt.Errorf("%s is not a directory", dir)
+ }
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ return fmt.Errorf("%s cannot be listed: %v", dir, err)
+ }
+ if len(entries) == 0 {
+ return nil
+ }
+ names := make([]string, 0, len(entries))
+ for _, e := range entries {
+ names = append(names, e.Name())
+ }
+ sort.Strings(names)
+ if len(names) > 3 {
+ names = append(names[:3], fmt.Sprintf("and %d more", len(entries)-3))
+ }
+ return fmt.Errorf("this box takes part in a running claim operation (%s holds %s)", dir, strings.Join(names, ", "))
+}
+
+// beginExecutableChange is §5.10's root side, run under the apply lock
+// just before the first box-changing step: write the marker durably, then
+// look for participation. On a refusal the marker is removed again and the
+// error, wrapping errSystemDeferred, says why. The caller records it as
+// "deferred", never "refused": a refusal is terminal for that promote, and
+// this one must be retried once the operation ends (spec §5.10: the
+// update is refused and retried later). The agent's leg re-requests a
+// deferred apply on its backoff ladder, at most every five minutes.
+func (a *systemApplier) beginExecutableChange(target string, promotedAt time.Time) error {
+ if !a.regionHA {
+ return nil
+ }
+ if err := os.MkdirAll(systemGuardDir(a.optDir), 0o755); err != nil {
+ return err
+ }
+ if err := writeJSONDurable(updateBegunPath(a.optDir), updateBegun{Target: target, PromotedAt: promotedAt, At: a.ops.now().UTC()}, 0o644); err != nil {
+ return err
+ }
+ if reason := hostexecParticipation(a.dataDir); reason != nil {
+ if err := a.clearUpdateBegun(); err != nil {
+ a.ops.logf("system apply: %v", err)
+ }
+ return fmt.Errorf("%w: %v", errSystemDeferred, reason)
+ }
+ return nil
+}
+
+// clearUpdateBegun removes the marker durably. A marker that is not there
+// is already cleared.
+func (a *systemApplier) clearUpdateBegun() error {
+ err := os.Remove(updateBegunPath(a.optDir))
+ if errors.Is(err, os.ErrNotExist) {
+ return nil
+ }
+ if err != nil {
+ return fmt.Errorf("remove %s: %w", updateBegunPath(a.optDir), err)
+ }
+ return syncDir(systemGuardDir(a.optDir))
+}
+
+// settleUpdateBegun clears the marker once no update is in progress: no
+// journal means the update finished, rolled back, was recovered, or never
+// got past its first step. A kept journal (a rollback that could not
+// finish) keeps the marker with it. Run with or without CARLOS_REGION_HA,
+// so a marker left behind after the flag is turned off still goes.
+func (a *systemApplier) settleUpdateBegun() {
+ if j, err := readSystemJournal(a.optDir); err != nil || j != nil {
+ return
+ }
+ if err := a.clearUpdateBegun(); err != nil {
+ a.ops.logf("system apply: %v", err)
+ }
+}
diff --git a/internal/host/systemparticipation_test.go b/internal/host/systemparticipation_test.go
new file mode 100644
index 000000000..90731379a
--- /dev/null
+++ b/internal/host/systemparticipation_test.go
@@ -0,0 +1,205 @@
+package host
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+ "runtime"
+ "strings"
+ "testing"
+)
+
+func hostexec(t *testing.T, f *applyFixture) string {
+ t.Helper()
+ dir := filepath.Join(f.a.dataDir, hostexecDir)
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ return dir
+}
+
+func assertNotUpdated(t *testing.T, f *applyFixture, why string) {
+ t.Helper()
+ r := f.result()
+ if r == nil || r.Outcome != systemOutcomeDeferred || !strings.Contains(r.Reason, why) {
+ t.Fatalf("result = %+v, want deferred naming %q", r, why)
+ }
+ if r.terminal() {
+ t.Fatal("the agent would never retry this promote")
+ }
+ if bin, _ := os.ReadFile(systemBinaryPath(f.a.optDir)); !strings.Contains(string(bin), "carlos v1") {
+ t.Fatal("the binary changed under a refusal")
+ }
+ if j, _ := readSystemJournal(f.a.optDir); j != nil {
+ t.Fatalf("a refusal opened a journal: %+v", j)
+ }
+ if _, err := os.Stat(updateBegunPath(f.a.optDir)); !os.IsNotExist(err) {
+ t.Fatalf("the update-begun marker outlived the refusal: %v", err)
+ }
+ if f.called("systemctl stop") || f.called("systemctl restart") {
+ t.Fatal("a refused update touched a unit")
+ }
+}
+
+// Spec §5.10: any controller record, in any state, refuses the update.
+func TestApplyRefusesWhileThisBoxTakesPartInAClaimOperation(t *testing.T) {
+ for _, state := range []string{"pending", "active", "lost"} {
+ t.Run(state, func(t *testing.T) {
+ f, _ := applyHappy(t)
+ f.a.regionHA = true
+ os.WriteFile(filepath.Join(hostexec(t, f), "shop.bab.test.json"), []byte(`{"state":"`+state+`"}`), 0o600)
+ if err := f.a.Run(context.Background(), false); err != nil {
+ t.Fatal(err)
+ }
+ assertNotUpdated(t, f, "takes part in a running claim operation")
+ })
+ }
+}
+
+// Review Focus 4: what root cannot read is a refusal, never a pass.
+func TestApplyRefusesWhenItCannotTellWhetherThisBoxTakesPart(t *testing.T) {
+ cases := map[string]func(t *testing.T, f *applyFixture){
+ "a temporary file": func(t *testing.T, f *applyFixture) {
+ os.WriteFile(filepath.Join(hostexec(t, f), ".shop.bab.test.json.tmp"), nil, 0o600)
+ },
+ "garbage": func(t *testing.T, f *applyFixture) {
+ os.WriteFile(filepath.Join(hostexec(t, f), "shop.bab.test.json"), []byte("{half"), 0o600)
+ },
+ "a symlink in place of the directory": func(t *testing.T, f *applyFixture) {
+ elsewhere := t.TempDir()
+ if err := os.Symlink(elsewhere, filepath.Join(f.a.dataDir, hostexecDir)); err != nil {
+ t.Skip("no symlinks here:", err)
+ }
+ },
+ "a file in place of the directory": func(t *testing.T, f *applyFixture) {
+ os.WriteFile(filepath.Join(f.a.dataDir, hostexecDir), nil, 0o600)
+ },
+ }
+ for name, setup := range cases {
+ t.Run(name, func(t *testing.T) {
+ f, _ := applyHappy(t)
+ f.a.regionHA = true
+ setup(t, f)
+ if err := f.a.Run(context.Background(), false); err != nil {
+ t.Fatal(err)
+ }
+ assertNotUpdated(t, f, "hostexec")
+ })
+ }
+}
+
+func TestApplyRefusesWhenItCannotListTheControllerDirectory(t *testing.T) {
+ if runtime.GOOS == "windows" || os.Geteuid() == 0 {
+ t.Skip("permission bits do not stop this user")
+ }
+ f, _ := applyHappy(t)
+ f.a.regionHA = true
+ dir := hostexec(t, f)
+ os.Chmod(dir, 0)
+ t.Cleanup(func() { os.Chmod(dir, 0o755) })
+ if err := f.a.Run(context.Background(), false); err != nil {
+ t.Fatal(err)
+ }
+ assertNotUpdated(t, f, "cannot be listed")
+}
+
+// The marker is written before the box changes and is gone once the
+// update is done; an empty controller directory is no participation.
+func TestApplyHoldsTheUpdateBegunMarkerAcrossTheInstall(t *testing.T) {
+ f, req := applyHappy(t)
+ f.a.regionHA = true
+ hostexec(t, f) // present and empty
+ seen := false
+ orig := f.a.ops.systemctl
+ f.a.ops.systemctl = func(args ...string) (string, error) {
+ if args[0] == "restart" && args[len(args)-1] == systemEdgeUnit {
+ _, err := os.Stat(updateBegunPath(f.a.optDir))
+ seen = err == nil
+ }
+ return orig(args...)
+ }
+ if err := f.a.Run(context.Background(), false); err != nil {
+ t.Fatal(err)
+ }
+ if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied || !r.matches("stable", "v2", req.PromotedAt) {
+ t.Fatalf("result = %+v", r)
+ }
+ if !seen {
+ t.Fatal("no update-begun marker while the edge restarted onto the new binary")
+ }
+ if _, err := os.Stat(updateBegunPath(f.a.optDir)); !os.IsNotExist(err) {
+ t.Fatalf("marker left after the update finished: %v", err)
+ }
+}
+
+func TestApplyWithoutRegionHAWritesNoMarkerAndIgnoresControllerRecords(t *testing.T) {
+ f, _ := applyHappy(t)
+ os.WriteFile(filepath.Join(hostexec(t, f), "shop.bab.test.json"), []byte(`{}`), 0o600)
+ if err := f.a.Run(context.Background(), false); err != nil {
+ t.Fatal(err)
+ }
+ if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied {
+ t.Fatalf("result = %+v: the flag is off, so nothing changes", r)
+ }
+ if _, err := os.Stat(systemGuardDir(f.a.optDir)); !os.IsNotExist(err) {
+ t.Fatalf("the guard directory was created with the flag off: %v", err)
+ }
+}
+
+// A crash between the marker and the journal leaves a marker with no
+// update behind it; the next run, --recover included, clears it.
+func TestAStaleUpdateBegunMarkerIsClearedByTheNextRun(t *testing.T) {
+ f := newApplyFixture(t, "stable", "v1")
+ os.MkdirAll(systemGuardDir(f.a.optDir), 0o755)
+ os.WriteFile(updateBegunPath(f.a.optDir), []byte(`{"target":"v2"}`), 0o644)
+ if err := f.a.Run(context.Background(), true); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := os.Stat(updateBegunPath(f.a.optDir)); !os.IsNotExist(err) {
+ t.Fatalf("stale marker kept: %v", err)
+ }
+}
+
+// A rollback that could not finish keeps its journal for the next run, and
+// the marker with it: the update is not over.
+func TestTheMarkerStaysWhileAJournalIsOpen(t *testing.T) {
+ f := newApplyFixture(t, "stable", "v1")
+ f.a.regionHA = true
+ os.MkdirAll(systemGuardDir(f.a.optDir), 0o755)
+ os.WriteFile(updateBegunPath(f.a.optDir), []byte(`{"target":"v2"}`), 0o644)
+ // A journal recovery cannot act on: no carlos.prev to restore.
+ if err := writeJSONDurable(f.a.file("journal.json"), systemJournal{Phase: systemPhaseInstalled, Target: "v2"}, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ _ = f.a.Run(context.Background(), true)
+ if j, _ := readSystemJournal(f.a.optDir); j == nil {
+ t.Fatal("recovery closed the journal; this test needs one it cannot finish")
+ }
+ if _, err := os.Stat(updateBegunPath(f.a.optDir)); err != nil {
+ t.Fatalf("marker cleared while the journal is still open: %v", err)
+ }
+}
+
+// Astra plan R1-3: participation defers, it does not refuse. Once the
+// operation ends and its record is cleared, the same promote applies. The
+// agent's side, re-requesting a deferred apply, is
+// TestSystemLegReusesAVerifiedCacheOnARetry.
+func TestApplyDeferredForParticipationAppliesOnceItEnds(t *testing.T) {
+ f, req := applyHappy(t)
+ f.a.regionHA = true
+ rec := filepath.Join(hostexec(t, f), "shop.bab.test.json")
+ os.WriteFile(rec, []byte(`{"state":"active"}`), 0o600)
+ if err := f.a.Run(context.Background(), false); err != nil {
+ t.Fatal(err)
+ }
+ if r := f.result(); r == nil || r.Outcome != systemOutcomeDeferred || r.terminal() {
+ t.Fatalf("result = %+v, want deferred, which the agent retries", r)
+ }
+ os.Remove(rec)
+ if err := f.a.Run(context.Background(), false); err != nil {
+ t.Fatal(err)
+ }
+ if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied || !r.matches("stable", "v2", req.PromotedAt) {
+ t.Fatalf("result after the operation ended = %+v, want applied", r)
+ }
+}