amadan

carlos / platform Public

host: the self-update relay, apply and recover units, and a JSON arming plan

bootstrap -root <dir> -plan-json prints this binary's enabledUnits and
inert list, so an apply arms what the NEW binary's bootstrap would arm.
Both oneshots exec carlos.prev when a journal is open.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015QCUCpfxbCyEAVQ2m51dH5
Paul Campbell pushed by paul@keymail.dev ff892461148a5ced679c1a914468fdabdfbc4052 parent 9c62d09
6 files changed, +232 −10
  • internal/host/bootstrap.go +54 −9
  • internal/host/host_test.go +73 −1
  • internal/host/probes.go +14 −0
  • internal/host/units/carlos-system-recover.service +36 −0
  • internal/host/units/carlos-system-update.path +13 −0
  • internal/host/units/carlos-system-update.service +42 −0
diff --git a/internal/host/bootstrap.go b/internal/host/bootstrap.go
index a34b66e08..7834b2553 100644
--- a/internal/host/bootstrap.go
+++ b/internal/host/bootstrap.go
@@ -13,8 +13,10 @@ import (
"github.com/carlosframework/platform/internal/cliflag"
"embed"
+ "encoding/json"
"flag"
"fmt"
+ "io"
"log"
"os"
"os/exec"
@@ -60,16 +62,40 @@ const tmpfilesConf = "d /run/carlos 2750 " + serviceUser + " " + socketsGroup +
"d /run/carlos/apps 0750 " + serviceUser + " " + serviceUser + " -\n"
// enabledUnits are what a live bootstrap turns on: the always-on front door,
-// the hourly adoption timer, and the three path units that let the sandboxed
+// the hourly adoption timer, and the path units that let the sandboxed
// agent ask a root pass for the halves it cannot do itself: carlos-adopt.path
// (config fast convergence, issue #44), carlos-stage.path (release staging
-// — the agent sees a promotion in seconds but cannot write /opt) and
+// — the agent sees a promotion in seconds but cannot write /opt),
// carlos-restart.path (a member's `carlos restart`, issue #113 — the agent
-// bounces its own exec children but cannot cycle a unit). The litestream + restore-verify units install
+// bounces its own exec children but cannot cycle a unit) and
+// carlos-system-update.path (the platform self-update relay, spec §4 — the
+// agent can fetch and verify a candidate but cannot write /opt or /etc).
+// carlos-system-recover.service is enabled (not armed by a path — it also
+// runs Before=carlos-edge.service on every boot, and on the update oneshot's
+// own OnFailure=) so recovery from an interrupted self-update never depends
+// on a marker file surviving a crash. The litestream + restore-verify units install
// everywhere but stay inert until an operator stages /data/litestream.env and
// generates /etc/litestream.yml (`carlos ops litestream`) — a box with no
// replication bucket is unaffected.
-var enabledUnits = []string{"carlos-edge.service", "carlos-adopt.timer", "carlos-adopt.path", "carlos-stage.path", "carlos-restart.path"}
+var enabledUnits = []string{"carlos-edge.service", "carlos-adopt.timer", "carlos-adopt.path", "carlos-stage.path", "carlos-restart.path",
+ "carlos-system-update.path", "carlos-system-recover.service"}
+
+// inertUnits are installed everywhere and deliberately never armed by
+// bootstrap or by a self-update apply: restore-verify until an operator
+// stages litestream, hibernate-verify by long-standing choice.
+var inertUnits = []string{"carlos-restore-verify.timer", "carlos-hibernate-verify.timer"}
+
+// bootstrapPlan is what `bootstrap -root <dir> -plan-json` prints: the
+// arming list of THIS binary, so a self-update apply arms what the new
+// binary's bootstrap would arm and never a list an older binary remembers.
+type bootstrapPlan struct {
+ Enabled []string `json:"enabled"`
+ Inert []string `json:"inert"`
+}
+
+func currentBootstrapPlan() bootstrapPlan {
+ return bootstrapPlan{Enabled: append([]string(nil), enabledUnits...), Inert: append([]string(nil), inertUnits...)}
+}
// systemctlPlan is the exact systemctl sequence a live bootstrap runs — and
// the same sequence a staging run (-root <dir>) PRINTS instead of running, so
@@ -116,17 +142,21 @@ func RunBootstrap(args []string) {
fs := flag.NewFlagSet("bootstrap", flag.ExitOnError)
root := fs.String("root", "/", "install root; a staging dir writes files only (no user creation, no systemctl)")
offcloud := fs.Bool("offcloud", false, "off-cloud box: require a staged 0600 credentials file instead of an EC2 instance profile")
+ planJSON := fs.Bool("plan-json", false, "with -root: print the arming plan as JSON on stdout instead of the human summary")
var consumers repeatableFlag
fs.Var(&consumers, "socket-consumer", "host-app user to enroll in the "+socketsGroup+" group (repeatable); members may dial every instance socket on the box")
fs.Usage = cliflag.Usage(fs, "")
_ = fs.Parse(args)
+ if *planJSON && filepath.Clean(*root) == "/" {
+ log.Fatal("bootstrap: -plan-json is for a staging root")
+ }
if *offcloud {
if err := bootstrapOffcloud(*root, consumers); err != nil {
log.Fatalf("bootstrap: %v", err)
}
return
}
- if err := bootstrap(*root, consumers); err != nil {
+ if err := bootstrapWith(*root, consumers, *planJSON); err != nil {
log.Fatalf("bootstrap: %v", err)
}
}
@@ -179,7 +209,19 @@ func bootstrapOffcloud(root string, consumers []string) error {
// A non-"/" root is the staging/test path: files only, with the privileged
// steps printed instead of run.
func bootstrap(root string, consumers []string) error {
+ return bootstrapWith(root, consumers, false)
+}
+
+// bootstrapWith is bootstrap's body, plus the -plan-json staging path: when
+// planJSON is set (staging root only — RunBootstrap refuses it with -root /),
+// every "wrote"/"created" line below is redirected to stderr so the JSON
+// plan currentBootstrapPlan() prints is alone on stdout.
+func bootstrapWith(root string, consumers []string, planJSON bool) error {
live := filepath.Clean(root) == "/"
+ out := io.Writer(os.Stdout)
+ if planJSON {
+ out = os.Stderr
+ }
if live {
// Validate consumer users BEFORE any mutation (the offcloud
// precondition's discipline): a typo in -socket-consumer must leave
@@ -270,7 +312,7 @@ func bootstrap(root string, consumers []string) error {
if err := f.Close(); err != nil {
return fmt.Errorf("create %s: %v", regPath, err)
}
- fmt.Printf(" created %s\n", regPath)
+ fmt.Fprintf(out, " created %s\n", regPath)
}
names, err := unitNames()
@@ -286,7 +328,7 @@ func bootstrap(root string, consumers []string) error {
if err := os.WriteFile(path, content, 0o644); err != nil {
return fmt.Errorf("write %s: %v", path, err)
}
- fmt.Printf(" wrote %s\n", path)
+ fmt.Fprintf(out, " wrote %s\n", path)
}
// On a live root the privileged plan runs BEFORE the tmpfiles conf is
@@ -316,15 +358,18 @@ func bootstrap(root string, consumers []string) error {
if err := os.WriteFile(sysusersPath, []byte("g "+socketsGroup+" -\n"), 0o644); err != nil {
return fmt.Errorf("write %s: %v", sysusersPath, err)
}
- fmt.Printf(" wrote %s\n", sysusersPath)
+ fmt.Fprintf(out, " wrote %s\n", sysusersPath)
tmpfilesPath := filepath.Join(root, "etc", "tmpfiles.d", "carlos.conf")
if err := os.WriteFile(tmpfilesPath, []byte(tmpfilesConf), 0o644); err != nil {
return fmt.Errorf("write %s: %v", tmpfilesPath, err)
}
- fmt.Printf(" wrote %s\n", tmpfilesPath)
+ fmt.Fprintf(out, " wrote %s\n", tmpfilesPath)
if !live {
+ if planJSON {
+ return json.NewEncoder(os.Stdout).Encode(currentBootstrapPlan())
+ }
fmt.Printf("bootstrap: staged under %s — on a real box (--root /) it would also:\n", root)
// Printed, never run: a staging root is how a LIVE box refreshes its
// unit files (pinfra's update-platform-binary.sh stages into
diff --git a/internal/host/host_test.go b/internal/host/host_test.go
index 77a958504..d954032d5 100644
--- a/internal/host/host_test.go
+++ b/internal/host/host_test.go
@@ -43,6 +43,9 @@ var expectedUnits = []string{
"carlos-restore-verify.timer",
"carlos-stage.path",
"carlos-stage.service",
+ "carlos-system-recover.service",
+ "carlos-system-update.path",
+ "carlos-system-update.service",
}
// requireUnixSockets skips a test when the sandbox forbids unix-socket
@@ -1127,7 +1130,7 @@ func TestParseLoad1(t *testing.T) {
// confident false — so a typo here would report every box in the fleet as
// broken, and a renamed unit would report a live box as dead.
func TestProbedUnitsAreTheOnesBootstrapArms(t *testing.T) {
- for _, unit := range []string{adoptTimerUnit, configPathUnit} {
+ for _, unit := range []string{adoptTimerUnit, configPathUnit, systemUpdatePathUnit} {
if !slices.Contains(enabledUnits, unit) {
t.Errorf("%s: probed but not in enabledUnits — bootstrap never arms it", unit)
}
@@ -1137,6 +1140,75 @@ func TestProbedUnitsAreTheOnesBootstrapArms(t *testing.T) {
}
}
+func TestBootstrapPlanJSONNamesWhatALiveRunArms(t *testing.T) {
+ p := currentBootstrapPlan()
+ want := map[string]bool{"carlos-edge.service": true, "carlos-adopt.timer": true, "carlos-adopt.path": true,
+ "carlos-stage.path": true, "carlos-restart.path": true, "carlos-system-update.path": true, "carlos-system-recover.service": true}
+ got := map[string]bool{}
+ for _, u := range p.Enabled {
+ got[u] = true
+ }
+ for u := range want {
+ if !got[u] {
+ t.Errorf("plan lacks %s", u)
+ }
+ }
+ if got["carlos-system-update.service"] {
+ t.Error("the path-triggered oneshot must not be enabled: it is started by its .path")
+ }
+ inert := map[string]bool{}
+ for _, u := range p.Inert {
+ inert[u] = true
+ }
+ if !inert["carlos-restore-verify.timer"] || !inert["carlos-hibernate-verify.timer"] {
+ t.Errorf("inert = %v", p.Inert)
+ }
+ // And systemctlPlan never starts a service, whatever is enabled.
+ for _, args := range systemctlPlan(p.Enabled) {
+ if args[0] == "start" {
+ for _, u := range args[1:] {
+ if strings.HasSuffix(u, ".service") {
+ t.Errorf("plan starts a service: %v", args)
+ }
+ }
+ }
+ }
+}
+
+func TestSystemUnitsAreShapedLikeTheStageRelay(t *testing.T) {
+ read := func(name string) string {
+ b, err := unitsFS.ReadFile("units/" + name)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return string(b)
+ }
+ p := read("carlos-system-update.path")
+ for _, want := range []string{"PathChanged=/data/system-update-requested", "Unit=carlos-system-update.service", "WantedBy=multi-user.target"} {
+ if !strings.Contains(p, want) {
+ t.Errorf("path unit lacks %q", want)
+ }
+ }
+ u := read("carlos-system-update.service")
+ for _, want := range []string{"Type=oneshot", "StartLimitIntervalSec=0", "TimeoutStartSec=15min", "OnFailure=carlos-system-recover.service",
+ "EnvironmentFile=-/etc/carlos/host.env", "system apply", "carlos.prev", "MemoryMax=256M"} {
+ if !strings.Contains(u, want) {
+ t.Errorf("update unit lacks %q", want)
+ }
+ }
+ for _, never := range []string{"ProtectSystem", "User="} {
+ if strings.Contains(u, never) {
+ t.Errorf("update unit must run as root with /opt writable; found %q", never)
+ }
+ }
+ r := read("carlos-system-recover.service")
+ for _, want := range []string{"Type=oneshot", "Before=carlos-edge.service", "WantedBy=multi-user.target", "TimeoutStartSec=10min", "system apply --recover", "carlos.prev"} {
+ if !strings.Contains(r, want) {
+ t.Errorf("recover unit lacks %q", want)
+ }
+ }
+}
+
// TestStatusPublishNeverReadsMetricsPrefixes is the privacy tripwire the
// monitoring spec commits to: operator-grade needles live under metrics/ and
// alerts/, and nothing the public status publisher writes may ever be
diff --git a/internal/host/probes.go b/internal/host/probes.go
index 481f376ee..c17681403 100644
--- a/internal/host/probes.go
+++ b/internal/host/probes.go
@@ -83,6 +83,20 @@ func restartPathActive() *bool {
return parseIsActive(string(out))
}
+// systemUpdatePathUnit is the unit whose liveness IS "an agent's fetched
+// platform candidate reaches a root apply" (spec §4): the path unit watching
+// /data/system-update-requested. Installed and armed by bootstrap alongside
+// the other path units (see enabledUnits).
+const systemUpdatePathUnit = "carlos-system-update.path"
+
+// systemUpdatePathActive asks the same question restartPathActive asks, about
+// the self-update relay: an unarmed path here means an agent that fetched and
+// verified a new build touches its marker and nothing ever picks it up.
+func systemUpdatePathActive() *bool {
+ out, _ := exec.Command("systemctl", "is-active", systemUpdatePathUnit).Output()
+ return parseIsActive(string(out))
+}
+
// parseIsActive maps systemd's is-active vocabulary onto the tri-state the
// status page publishes. "unknown" — systemd's answer when it has no such unit
// loaded — is a genuine false: a box with no adoption timer adopts nothing.
diff --git a/internal/host/units/carlos-system-recover.service b/internal/host/units/carlos-system-recover.service
new file mode 100644
index 000000000..8e74c0e20
--- /dev/null
+++ b/internal/host/units/carlos-system-recover.service
@@ -0,0 +1,36 @@
+# Recovery of an interrupted platform self-update (spec §6, "every run
+# begins with recovery"). Runs once per boot, BEFORE the edge, and on the
+# update oneshot's failure. It restores files only — carlos.prev over
+# carlos, the previous unit set — and never restarts or gates the edge
+# from inside a unit the edge is ordered after: at boot the edge's own
+# start IS the restart. At runtime (edge already started this boot) it
+# issues a non-blocking restart and exits.
+[Unit]
+Description=carlos platform self-update recovery
+Before=carlos-edge.service
+DefaultDependencies=yes
+
+[Service]
+Type=oneshot
+TimeoutStartSec=10min
+EnvironmentFile=-/etc/carlos/host.env
+ExecStart=/bin/sh -c 'if [ -f /opt/carlos/system/router/journal.json ] && [ -x /opt/carlos/carlos.prev ]; then exec /opt/carlos/carlos.prev system apply --recover; fi; exec /opt/carlos/carlos system apply --recover'
+
+ProtectHome=yes
+PrivateTmp=yes
+PrivateDevices=yes
+ProtectClock=yes
+ProtectHostname=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
+RestrictNamespaces=yes
+RestrictRealtime=yes
+LockPersonality=yes
+MemoryDenyWriteExecute=yes
+MemoryMax=256M
+TasksMax=64
+
+[Install]
+WantedBy=multi-user.target
diff --git a/internal/host/units/carlos-system-update.path b/internal/host/units/carlos-system-update.path
new file mode 100644
index 000000000..ed24c2d8b
--- /dev/null
+++ b/internal/host/units/carlos-system-update.path
@@ -0,0 +1,13 @@
+# The relay from the sandboxed agent to a root apply of a new platform
+# binary (spec §4). The agent (User=carlos, ProtectSystem=strict) can fetch
+# and verify the artifact but cannot write /opt or /etc; it touches this
+# marker and root does the rest. Same shape as carlos-stage.path.
+[Unit]
+Description=CARLOS platform self-update on agent request
+
+[Path]
+PathChanged=/data/system-update-requested
+Unit=carlos-system-update.service
+
+[Install]
+WantedBy=multi-user.target
diff --git a/internal/host/units/carlos-system-update.service b/internal/host/units/carlos-system-update.service
new file mode 100644
index 000000000..a2271f826
--- /dev/null
+++ b/internal/host/units/carlos-system-update.service
@@ -0,0 +1,42 @@
+# Root apply of a platform self-update (spec §6): re-verify the pointer,
+# copy the candidate into a root-owned dir, journal, swap the binary and
+# units, restart the edge, gate it, roll back to carlos.prev on failure.
+#
+# ExecStart goes through one line of shell so that when a journal is open
+# (a previous apply was cut off) and carlos.prev exists, the RECOVERY runs
+# from carlos.prev rather than from the candidate that did not work.
+#
+# Deliberately no filesystem-sandboxing directive and no dedicated run
+# account: it must write /opt/carlos and /etc/systemd/system, which only
+# root can do. StartLimitIntervalSec=0 because the path unit can fire in
+# bursts. TimeoutStartSec bounds a hung apply; OnFailure= makes a killed or
+# timed-out apply recover without waiting for a touch.
+[Unit]
+Description=carlos platform self-update (root apply)
+After=network-online.target
+Wants=network-online.target
+StartLimitIntervalSec=0
+OnFailure=carlos-system-recover.service
+
+[Service]
+Type=oneshot
+TimeoutStartSec=15min
+EnvironmentFile=-/etc/carlos/host.env
+ExecStart=/bin/sh -c 'if [ -f /opt/carlos/system/router/journal.json ] && [ -x /opt/carlos/carlos.prev ]; then exec /opt/carlos/carlos.prev system apply; fi; exec /opt/carlos/carlos system apply'
+
+# --- hardening (the carlos-stage.service set) ---
+ProtectHome=yes
+PrivateTmp=yes
+PrivateDevices=yes
+ProtectClock=yes
+ProtectHostname=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
+RestrictNamespaces=yes
+RestrictRealtime=yes
+LockPersonality=yes
+MemoryDenyWriteExecute=yes
+MemoryMax=256M
+TasksMax=64