carlos / platform Public

relay: the unit, bootstrap arming and the adopt restart (spec §3)

carlos-relay.service is installed everywhere and, unlike every other
service, STARTED by bootstrap. That is safe because the unit carries
ConditionPathExists=/etc/carlos/relay-masters: on a box the operator has
not armed with a region master the start is a no-op systemd records as a
failed condition, so the fleet-wide arming costs nothing and an armed box
does not wait for a reboot to listen.

The unit is the edge's hardening cut down — no CAP_NET_BIND_SERVICE (3478
is unprivileged), ReadWritePaths=/data/relay /run/carlos, MemoryMax=256M,
MemoryHigh=160M and OOMScoreAdjust=1000, because the relay shares the box
with the edge and must lose first.

A platform update restarts it beside the edge, with try-restart rather
than restart and the error logged rather than returned: an unarmed box's
inactive unit must not fail an update that is otherwise fine.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Paul Campbell pushed by paul@keymail.dev 1 month ago e5ecdacc03af86e93e3181c6958b44a77720701e parent d4653dc

4 files changed, +127 −13

internal/host/bootstrap.go +28 −9
internal/host/host_test.go +31 −4
internal/host/systemapply.go +18 −0
internal/host/units/carlos-relay.service +50 −0

diff --git a/internal/host/bootstrap.go b/internal/host/bootstrap.go

index 7834b2553..e7e907956 100644

--- a/internal/host/bootstrap.go

+++ b/internal/host/bootstrap.go

@@ -76,8 +76,16 @@ const tmpfilesConf = "d /run/carlos 2750 " + serviceUser + " " + socketsGroup +

// on a marker file surviving a crash. The litestream + restore-verify units install

// everywhere but stay inert until an operator stages /data/litestream.env and

// generates /etc/litestream.yml (`carlos ops litestream`) — a box with no

-// replication bucket is unaffected.

-var enabledUnits = []string{"carlos-edge.service", "carlos-adopt.timer", "carlos-adopt.path", "carlos-stage.path", "carlos-restart.path",

+// replication bucket is unaffected. carlos-relay.service (relay spec §3) is

+// enabled AND started fleet-wide, unlike the edge: its

+// ConditionPathExists=/etc/carlos/relay-masters makes the start a no-op on

+// every box the operator has not armed with a region master.

+//

+// relayUnitName is spelled once because two places must agree on it: this

+// list and the systemctl plan's start set.

+const relayUnitName = "carlos-relay.service"

+var enabledUnits = []string{"carlos-edge.service", relayUnitName, "carlos-adopt.timer", "carlos-adopt.path", "carlos-stage.path", "carlos-restart.path",

"carlos-system-update.path", "carlos-system-recover.service"}

// inertUnits are installed everywhere and deliberately never armed by

@@ -113,12 +121,19 @@ func currentBootstrapPlan() bootstrapPlan {

// already-running timer or path unit is a no-op, which keeps bootstrap

// idempotent.

-// Services are deliberately NOT started. carlos-edge needs its binary at

-// /opt/carlos/carlos and /etc/carlos/host.env staged first — neither of which

-// bootstrap can do for the operator, which is why it ends by telling them to.

-// Starting timers is safe in a way starting services is not: a timer that

-// fires before the binary lands just fails one adoption pass and tries again

-// on the next tick.

+// Services are deliberately NOT started, with one exception. carlos-edge

+// needs its binary at /opt/carlos/carlos and /etc/carlos/host.env staged

+// first — neither of which bootstrap can do for the operator, which is why

+// it ends by telling them to. Starting timers is safe in a way starting

+// services is not: a timer that fires before the binary lands just fails one

+// adoption pass and tries again on the next tick.

+//

+// carlos-relay.service is the exception (relay spec §3). It carries

+// ConditionPathExists=/etc/carlos/relay-masters, so on a box the operator

+// has not armed with a region master the start is a no-op systemd records as

+// a failed condition, not a failure — which is what makes starting it safe

+// fleet-wide. On a box that IS armed, starting it here is what saves a

+// reboot.

func systemctlPlan(units []string) [][]string {

plan := [][]string{

{"daemon-reload"},

@@ -126,7 +141,7 @@ func systemctlPlan(units []string) [][]string {

}

var armed []string

for _, u := range units {

- if strings.HasSuffix(u, ".timer") || strings.HasSuffix(u, ".path") {

+ if strings.HasSuffix(u, ".timer") || strings.HasSuffix(u, ".path") || u == relayUnitName {

armed = append(armed, u)

}

@@ -244,6 +259,10 @@ func bootstrapWith(root string, consumers []string, planJSON bool) error {

dirs := []dir{

{dataDir, 0o750},

{filepath.Join(dataDir, "acme"), 0o750},

+ // The relay's one writable state dir (carlos-relay.service's

+ // ReadWritePaths). Created on every box, armed or not, so the unit

+ // never fails for want of a directory the moment a master lands.

+ {filepath.Join(dataDir, "relay"), 0o750},

// Setgid: sockets bound inside inherit the dir's group — the

// consumers group, once a live run (or the tmpfiles line) sets it.

// See socketsGroup/tmpfilesConf.

diff --git a/internal/host/host_test.go b/internal/host/host_test.go

index d954032d5..2fd411efa 100644

--- a/internal/host/host_test.go

+++ b/internal/host/host_test.go

@@ -37,6 +37,7 @@ var expectedUnits = []string{

"carlos-hibernate-verify.service",

"carlos-hibernate-verify.timer",

"carlos-litestream.service",

+ "carlos-relay.service",

"carlos-restart.path",

"carlos-restart.service",

"carlos-restore-verify.service",

@@ -83,6 +84,27 @@ func TestUnitsEmbedComplete(t *testing.T) {

}

+// TestBootstrapStartsRelayUnit pins the one service bootstrap starts and the

+// one it must not. The relay is safe to start fleet-wide because its

+// ConditionPathExists gates it on a masters file; the edge is not, and a

+// change that quietly promoted every service into the start line would arm

+// half-staged boxes.

+func TestBootstrapStartsRelayUnit(t *testing.T) {

+ plan := systemctlPlan(enabledUnits)

+ var started []string

+ for _, step := range plan {

+ if step[0] == "start" {

+ started = step[1:]

+ }

+ if !slices.Contains(started, "carlos-relay.service") {

+ t.Fatalf("relay not started by plan: %v", plan)

+ }

+ if slices.Contains(started, "carlos-edge.service") {

+ t.Fatalf("edge must stay start-by-hand: %v", plan)

+ }

func TestBootstrapWritesUnitsToStagingDir(t *testing.T) {

root := t.TempDir()

if err := bootstrap(root, nil); err != nil {

@@ -889,7 +911,10 @@ func TestIsCanaryRoute(t *testing.T) {

// deploy quietly needed a hand-run `systemctl start carlos-adopt.service`.

// Every timer bootstrap enables must therefore also be started, and no

// SERVICE may be (carlos-edge needs its binary and host.env staged first —

-// bootstrap's own closing line tells the operator to start it).

+// bootstrap's own closing line tells the operator to start it). The one

+// exemption is carlos-relay.service, whose ConditionPathExists makes the

+// start a no-op on any box the operator has not armed with a region master

+// (relay spec §3); TestBootstrapStartsRelayUnit pins that it IS started.

// PATH units are the same trap with the same fix: an enabled-but-unstarted

// carlos-adopt.path watches nothing until the next reboot, so the agent's

@@ -920,7 +945,7 @@ func TestSystemctlPlanArmsTimers(t *testing.T) {

if (strings.HasSuffix(u, ".timer") || strings.HasSuffix(u, ".path")) && !started[u] {

t.Errorf("%s: enabled but never started — that is exactly the bug (enable only symlinks; the unit arms at the next boot)", u)

}

- if strings.HasSuffix(u, ".service") && started[u] {

+ if strings.HasSuffix(u, ".service") && u != relayUnitName && started[u] {

t.Errorf("%s: bootstrap must not start a service — the binary and /etc/carlos/host.env may not be staged yet", u)

}

@@ -1163,11 +1188,13 @@ func TestBootstrapPlanJSONNamesWhatALiveRunArms(t *testing.T) {

if !inert["carlos-restore-verify.timer"] || !inert["carlos-hibernate-verify.timer"] {

t.Errorf("inert = %v", p.Inert)

}

- // And systemctlPlan never starts a service, whatever is enabled.

+ // And systemctlPlan starts no service but the relay, whatever is

+ // enabled — the relay is safe because its ConditionPathExists gates it

+ // on a masters file the operator stages (relay spec §3).

for _, args := range systemctlPlan(p.Enabled) {

if args[0] == "start" {

for _, u := range args[1:] {

- if strings.HasSuffix(u, ".service") {

+ if strings.HasSuffix(u, ".service") && u != relayUnitName {

t.Errorf("plan starts a service: %v", args)

}

diff --git a/internal/host/systemapply.go b/internal/host/systemapply.go

index f37f4f537..2cb0605ba 100644

--- a/internal/host/systemapply.go

+++ b/internal/host/systemapply.go

@@ -31,6 +31,7 @@ import (

const (

systemEdgeUnit = "carlos-edge"

+ systemRelayUnit = "carlos-relay"

systemLockFile = "lock"

systemMaxRequest = 4 << 10

systemGateWait = 20 * time.Second

@@ -382,6 +383,7 @@ func (a *systemApplier) install(ctx context.Context, req systemRequest, p *relea

if err := a.restartEdge(); err != nil {

return fail("restart", err)

}

+ a.restartRelay()

j.Phase = systemPhaseRestarted

if err := a.journal(j); err != nil {

return fail("journal", err)

@@ -571,6 +573,21 @@ func (a *systemApplier) restartEdge() error {

return err

}

+// restartRelay mirrors restartEdge for the relay (relay spec §3), with one

+// difference that decides its shape: the relay is optional. A box with no

+// /etc/carlos/relay-masters has the unit installed but inactive, because its

+// ConditionPathExists failed, and `try-restart` on it is a no-op that may

+// still report an error. That is not a failure of the update — the edge, the

+// binary and the gate are all fine — so the error is logged and swallowed,

+// and try-restart (not restart) is what keeps an unarmed box from being

+// handed a relay it never asked for.

+func (a *systemApplier) restartRelay() {

+ a.ops.systemctl("reset-failed", systemRelayUnit)

+ if _, err := a.ops.systemctl("try-restart", systemRelayUnit); err != nil {

+ a.ops.logf("system apply: relay try-restart: %v", err)

+ }

// gate: within gateWait the edge is active, answers on loopback, and the

// installed binary reports the target; then a soak across which NRestarts

// stays 0 — step 9's reset-failed zeroed it, so anything above 0 is a

@@ -692,6 +709,7 @@ func (a *systemApplier) rollback(ctx context.Context, req systemRequest, reason

if err := a.restartEdge(); err != nil {

return failed("restart", err) // journal kept: the edge is still to come back

}

+ a.restartRelay()

if err := a.gate(ctx, j.PrevBuild); err != nil {

err := failed("previous build failed its gate too", err)

a.removeJournal()

diff --git a/internal/host/units/carlos-relay.service b/internal/host/units/carlos-relay.service

new file mode 100644

index 000000000..dc03507b6

--- /dev/null

+++ b/internal/host/units/carlos-relay.service

@@ -0,0 +1,50 @@

+# The platform STUN/TURN relay (media roadmap stage 4a). Armed only on a

+# box that carries /etc/carlos/relay-masters; ConditionPathExists keeps a

+# box without a master from ever listening. Shares the box with the edge and

+# must lose first: bounded memory, PSI-driven shedding in the process, and

+# the highest OOM adjustment there is.

+[Unit]

+Description=carlos relay (STUN and TURN)

+After=network-online.target

+Wants=network-online.target

+ConditionPathExists=/etc/carlos/relay-masters

+[Service]

+EnvironmentFile=-/etc/carlos/host.env

+ExecStart=/opt/carlos/carlos relay serve

+Restart=always

+RestartSec=1

+User=carlos

+Group=carlos

+NoNewPrivileges=yes

+ProtectSystem=strict

+ProtectHome=yes

+ReadWritePaths=/data/relay /run/carlos

+PrivateTmp=yes

+PrivateDevices=yes

+ProtectClock=yes

+ProtectHostname=yes

+ProtectKernelTunables=yes

+ProtectKernelModules=yes

+ProtectKernelLogs=yes

+ProtectControlGroups=yes

+ProtectProc=invisible

+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX

+RestrictNamespaces=yes

+RestrictRealtime=yes

+RestrictSUIDSGID=yes

+LockPersonality=yes

+MemoryDenyWriteExecute=yes

+SystemCallFilter=@system-service

+SystemCallErrorNumber=EPERM

+SystemCallArchitectures=native

+MemoryHigh=160M

+MemoryMax=256M

+OOMScoreAdjust=1000

+TasksMax=512

+Nice=5

+CPUWeight=50

+[Install]

+WantedBy=multi-user.target