amadan
Fresh boxes: keep the registry owned by the user the edge runs as
Discussion #8: all three region boxes brought up on 2026-08-31 were left
unstartable by the same cause, and two needed a second manual repair
because it is a race.
The registry is written by two users. `carlos agent` (carlos-edge.service)
runs as `carlos`; four units — carlos-adopt, -stage, -restart,
-config-sync — run `carlos adopt` as root, because a pass writes
/opt/carlos and cycles units via systemctl. SQLite creates the database
and its -wal/-shm owned by whoever opens it first. On an established box
that is the edge and root writes are fine; on a fresh box `carlos
bootstrap` arms adopt's .path and .timer before the edge has ever
started, so adopt can create the file as root and the edge then dies with
agent: open registry /data/registry.db: attempt to write a readonly database (8)
after which systemd gives up in three seconds.
The report's suggested fix — `User=carlos` on carlos-adopt.service — is
not available: the pass genuinely needs root for /opt/carlos and
systemctl, which is why the unit says so out loud. So the ownership is
converged instead, at both ends of the race:
- bootstrap creates /data/registry.db itself, before it arms anything,
inside the tree privilegedPlan already chowns to the service user. An
empty file is a valid empty SQLite database; the first real opener
migrates into it. An existing registry is left untouched — bootstrap
re-runs over live boxes whose edge holds the file open.
- every root adopt pass chowns the database and its two sidecars back to
the service user. Once by hand is not enough and the report says so:
the armed units fire again and re-root the sidecars in between, and a
carlos-owned database with a root-owned -wal is just as unwritable.
Chown failures are logged, never fatal: an adoption pass that can read
the registry has releases to stage, and refusing that over a chown would
turn a cosmetic problem into an outage.
Also the cheap fence the report asked for. "attempt to write a readonly
database (8)" is SQLite's words, and they sent the operator to disk space
and mount options before file ownership; both registry openers now add a
hint naming the file's actual owner, the user carlos-edge.service runs
as, and the chown that fixes it. Other SQLite errors still read as
themselves.
fileOwner is build-tagged: internal/host is linked into the CLI, which
people run on Macs and Windows, and syscall.Stat_t is not portable.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01QVvK41AUKXndpcy5KkUZRS
6 files changed,
+372
−0
internal/host/agent.go+11 −0internal/host/bootstrap.go+29 −0internal/host/fileowner_other.go+10 −0internal/host/fileowner_unix.go+20 −0internal/host/registryowner.go+123 −0internal/host/registryowner_test.go+179 −0
diff --git a/internal/host/agent.go b/internal/host/agent.go| index 395fa43ab..1713beadf 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -198,6 +198,9 @@ func RunAgent(args []string) { |
| // one feeds the activator's due-time index and the adopt loop. |
| reg, err := registry.Open(regPath) |
| if err != nil { |
| + if hint := registryOpenHint(regPath, err); hint != "" { |
| + log.Fatalf("agent: open registry %s: %v\nagent: %s", regPath, err, hint) |
| + } |
| log.Fatalf("agent: open registry %s: %v", regPath, err) |
| } |
| @@ -2813,8 +2816,16 @@ func RunAdopt(args []string) { |
| _ = fs.Parse(args) |
| reg, err := registry.Open(*regPath) |
| if err != nil { |
| + if hint := registryOpenHint(*regPath, err); hint != "" { |
| + log.Fatalf("adopt: open registry %s: %v\nadopt: %s", *regPath, err, hint) |
| + } |
| log.Fatalf("adopt: open registry %s: %v", *regPath, err) |
| } |
| + // This pass runs as root (carlos-adopt/-stage/-restart/-config-sync have |
| + // no User=), and opening the registry is what creates its -wal/-shm. Hand |
| + // them back to the user the edge runs as, on every pass — see |
| + // registryowner.go for why once is not enough. |
| + chownRegistryToService(*regPath) |
| ctx := context.Background() |
| store, err := openDeployStore(ctx) |
| if err != nil { |
diff --git a/internal/host/bootstrap.go b/internal/host/bootstrap.go| index 7611bde29..d59aed6f9 100644 |
| --- a/internal/host/bootstrap.go |
| +++ b/internal/host/bootstrap.go |
| @@ -234,6 +234,35 @@ func bootstrap(root string, consumers []string) error { |
| } |
| } |
| + // The registry file, created here and by nothing else on a fresh box. |
| + // |
| + // It has to exist BEFORE the units below are armed, because four of them |
| + // (carlos-adopt, -stage, -restart, -config-sync) run `carlos adopt` as |
| + // root and open this same file — and on a fresh box adopt's .path and |
| + // .timer can fire before the edge has ever started. Whoever creates the |
| + // file owns it, and an edge running as serviceUser cannot write a |
| + // root-owned registry: that is discussion #8, three region boxes left |
| + // unstartable on 2026-08-31. Creating it here puts it inside the |
| + // `chown -R /data` that privilegedPlan already runs, so it is |
| + // serviceUser's before anything can race for it. |
| + // |
| + // An empty file is a valid empty SQLite database — the first opener |
| + // (edge or adopt) runs the migrations into it. Deliberately NOT opened |
| + // as a registry here: on a re-run over a live box this must not touch |
| + // the schema of a database the running edge holds open, and "the file |
| + // exists, owned by the right user" is the whole job. |
| + regPath := filepath.Join(dataDir, "registry.db") |
| + if _, err := os.Stat(regPath); os.IsNotExist(err) { |
| + f, err := os.OpenFile(regPath, os.O_CREATE|os.O_WRONLY, 0o640) |
| + if err != nil { |
| + return fmt.Errorf("create %s: %v", regPath, err) |
| + } |
| + if err := f.Close(); err != nil { |
| + return fmt.Errorf("create %s: %v", regPath, err) |
| + } |
| + fmt.Printf(" created %s\n", regPath) |
| + } |
| + |
| names, err := unitNames() |
| if err != nil { |
| return err |
diff --git a/internal/host/fileowner_other.go b/internal/host/fileowner_other.go| new file mode 100644 |
| index 000000000..9ab9347f4 |
| --- /dev/null |
| +++ b/internal/host/fileowner_other.go |
| @@ -0,0 +1,10 @@ |
| +//go:build !unix |
| + |
| +package host |
| + |
| +import "io/fs" |
| + |
| +// fileOwner has no answer off unix. Every caller treats !ok as "say nothing |
| +// about ownership" rather than as a failure, so the CLI still builds and runs |
| +// on Windows; the box side this serves is Linux-only anyway. |
| +func fileOwner(fs.FileInfo) (uid, gid int, ok bool) { return 0, 0, false } |
diff --git a/internal/host/fileowner_unix.go b/internal/host/fileowner_unix.go| new file mode 100644 |
| index 000000000..1a8067ba6 |
| --- /dev/null |
| +++ b/internal/host/fileowner_unix.go |
| @@ -0,0 +1,20 @@ |
| +//go:build unix |
| + |
| +package host |
| + |
| +import ( |
| + "io/fs" |
| + "syscall" |
| +) |
| + |
| +// fileOwner reports a file's owning uid and gid. Split by build tag because |
| +// internal/host is linked into the CLI, which people run on Macs and Windows, |
| +// and syscall.Stat_t does not exist everywhere — the gate cross-compiles all |
| +// five targets for exactly this reason. |
| +func fileOwner(fi fs.FileInfo) (uid, gid int, ok bool) { |
| + st, ok := fi.Sys().(*syscall.Stat_t) |
| + if !ok { |
| + return 0, 0, false |
| + } |
| + return int(st.Uid), int(st.Gid), true |
| +} |
diff --git a/internal/host/registryowner.go b/internal/host/registryowner.go| new file mode 100644 |
| index 000000000..e704afd48 |
| --- /dev/null |
| +++ b/internal/host/registryowner.go |
| @@ -0,0 +1,123 @@ |
| +package host |
| + |
| +import ( |
| + "fmt" |
| + "log" |
| + "os" |
| + "os/user" |
| + "strconv" |
| + "strings" |
| +) |
| + |
| +// The registry is written by TWO users. `carlos agent` (carlos-edge.service) |
| +// runs as serviceUser; every adoption-shaped pass — carlos-adopt, |
| +// carlos-stage, carlos-restart, carlos-config-sync — runs as root, because a |
| +// pass writes /opt/carlos and cycles units via systemctl. Both open |
| +// /data/registry.db, and SQLite creates the database and its -wal/-shm |
| +// sidecars owned by whoever got there first. |
| +// |
| +// On an established box the edge got there first, the files are carlos-owned, |
| +// and root writing to them is fine. On a FRESH box it is a race that root can |
| +// win: `carlos bootstrap` arms adopt's .path and .timer before the edge has |
| +// ever started, so adopt can create the registry as root and leave the edge |
| +// unable to start at all: |
| +// |
| +// agent: open registry /data/registry.db: attempt to write a readonly database (8) |
| +// |
| +// systemd gives up after five restarts in three seconds, so the box then sits |
| +// failed with no further attempts. All three region boxes brought up on |
| +// 2026-08-31 landed exactly there (discussion #8). |
| +// |
| +// Two fences, because one is not enough: |
| +// |
| +// 1. bootstrap creates the file itself, before the units that could race for |
| +// it are armed (see bootstrap.go) — so root never gets to create it. |
| +// 2. every root pass converges the ownership afterwards, below. Chowning by |
| +// hand once is NOT enough on a live box and the report says so: the armed |
| +// path/timer units fire again and re-create the sidecars as root in |
| +// between. Two of the three boxes needed the chown twice. |
| + |
| +// chownRegistryToService leaves the registry and its WAL sidecars owned by |
| +// serviceUser. It is a no-op unless this process is root: a pass running as |
| +// the service user already creates everything with the right owner. |
| +// |
| +// Failures are logged, never fatal. An adoption pass that can read the |
| +// registry has real work to do — releases to stage, units to cycle — and |
| +// refusing to do it because a chown failed would turn a cosmetic problem into |
| +// an outage. The next pass tries again. |
| +func chownRegistryToService(path string) { |
| + // Geteuid reports -1 on platforms without the concept (the CLI |
| + // cross-compiles to windows), which takes the same early exit. |
| + if os.Geteuid() != 0 { |
| + return |
| + } |
| + u, err := user.Lookup(serviceUser) |
| + if err != nil { |
| + log.Printf("adopt: registry ownership: no %s user: %v", serviceUser, err) |
| + return |
| + } |
| + uid, err := strconv.Atoi(u.Uid) |
| + if err != nil { |
| + return |
| + } |
| + gid, err := strconv.Atoi(u.Gid) |
| + if err != nil { |
| + return |
| + } |
| + for _, p := range registryFiles(path) { |
| + fi, err := os.Stat(p) |
| + if err != nil { |
| + // The -wal/-shm sidecars only exist while some connection is |
| + // open. Their absence is the normal state, not a problem. |
| + continue |
| + } |
| + if fuid, fgid, ok := fileOwner(fi); ok && fuid == uid && fgid == gid { |
| + continue |
| + } |
| + if err := os.Chown(p, uid, gid); err != nil { |
| + log.Printf("adopt: chown %s to %s: %v", p, serviceUser, err) |
| + continue |
| + } |
| + // Worth a line every time it fires: on a healthy box it never does, |
| + // so a run of these is the box telling you something armed root is |
| + // creating the registry ahead of the edge. |
| + log.Printf("adopt: %s was not owned by %s — fixed; the edge runs as %s and could not have written it", p, serviceUser, serviceUser) |
| + } |
| +} |
| + |
| +// registryFiles is the database and the two sidecars SQLite keeps beside it |
| +// in WAL mode. Ownership has to cover all three: a carlos-owned database with |
| +// a root-owned -wal is just as unwritable as a root-owned database. |
| +func registryFiles(path string) []string { |
| + return []string{path, path + "-wal", path + "-shm"} |
| +} |
| + |
| +// registryOpenHint turns SQLite's words into the box's words. |
| +// |
| +// "attempt to write a readonly database (8)" is what the edge printed on all |
| +// three boxes, and it sent the operator looking at disk space and mount |
| +// options before file ownership — which is the actual cause every time so far, |
| +// because the registry is the one file on the box two different users write. |
| +// The hint names the owner it found, the owner it needs, and the command that |
| +// fixes it. It returns "" for any other error, so a genuinely full disk still |
| +// reads as itself. |
| +func registryOpenHint(path string, err error) string { |
| + if err == nil || !strings.Contains(strings.ToLower(err.Error()), "readonly database") { |
| + return "" |
| + } |
| + var b strings.Builder |
| + fmt.Fprintf(&b, "the registry is not writable by this process (uid %d).", os.Geteuid()) |
| + if fi, statErr := os.Stat(path); statErr == nil { |
| + if uid, gid, ok := fileOwner(fi); ok { |
| + owner := strconv.Itoa(uid) |
| + if u, lookupErr := user.LookupId(owner); lookupErr == nil { |
| + owner = u.Username |
| + } |
| + fmt.Fprintf(&b, " %s is owned by %s (uid %d, gid %d)", path, owner, uid, gid) |
| + } |
| + } |
| + fmt.Fprintf(&b, " — it and its -wal/-shm sidecars must be owned by %s, the user carlos-edge.service runs as."+ |
| + " A root `carlos adopt` pass that reached the file before the edge did is the usual cause."+ |
| + " Fix: chown %s:%s %s*", serviceUser, serviceUser, serviceUser, path) |
| + return b.String() |
| +} |
diff --git a/internal/host/registryowner_test.go b/internal/host/registryowner_test.go| new file mode 100644 |
| index 000000000..48e7d22a3 |
| --- /dev/null |
| +++ b/internal/host/registryowner_test.go |
| @@ -0,0 +1,179 @@ |
| +package host |
| + |
| +import ( |
| + "errors" |
| + "os" |
| + "path/filepath" |
| + "strings" |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/registry" |
| +) |
| + |
| +// TestBootstrapCreatesRegistryBeforeArmingUnits is the fence for discussion |
| +// #8. Four of the units bootstrap arms run `carlos adopt` as ROOT and open |
| +// /data/registry.db; the edge that also opens it runs as the service user. |
| +// Whoever creates the file owns it, so on a fresh box the file must exist |
| +// before anything is armed — created inside the /data tree that |
| +// privilegedPlan then chowns to the service user. |
| +func TestBootstrapCreatesRegistryBeforeArmingUnits(t *testing.T) { |
| + root := t.TempDir() |
| + if err := bootstrap(root, nil); err != nil { |
| + t.Fatalf("bootstrap: %v", err) |
| + } |
| + reg := filepath.Join(root, "data", "registry.db") |
| + fi, err := os.Stat(reg) |
| + if err != nil { |
| + t.Fatalf("bootstrap did not create the registry: %v", err) |
| + } |
| + if fi.IsDir() { |
| + t.Fatalf("%s is a directory", reg) |
| + } |
| + // It is inside the tree privilegedPlan hands to the service user — the |
| + // step that makes creating it here worth anything at all. |
| + var chowned bool |
| + for _, args := range privilegedPlan(nil) { |
| + if args[0] == "chown" && args[len(args)-1] == "/data" { |
| + chowned = true |
| + } |
| + } |
| + if !chowned { |
| + t.Error("privilegedPlan no longer chowns /data — the registry bootstrap creates would stay root-owned") |
| + } |
| +} |
| + |
| +// TestBootstrapRegistryIsAnOpenableRegistry: the file bootstrap leaves behind |
| +// is an empty SQLite database, not a placeholder the first real opener has to |
| +// delete. Zero length is a valid empty database; migrations run into it. |
| +func TestBootstrapRegistryIsAnOpenableRegistry(t *testing.T) { |
| + root := t.TempDir() |
| + if err := bootstrap(root, nil); err != nil { |
| + t.Fatalf("bootstrap: %v", err) |
| + } |
| + reg, err := registry.Open(filepath.Join(root, "data", "registry.db")) |
| + if err != nil { |
| + t.Fatalf("open the registry bootstrap created: %v", err) |
| + } |
| + defer reg.Close() |
| + if _, err := reg.List(); err != nil { |
| + t.Fatalf("list routes: %v", err) |
| + } |
| +} |
| + |
| +// TestBootstrapKeepsAnExistingRegistry: bootstrap re-runs over live boxes |
| +// (that is how a unit refresh happens), and the running edge holds this file |
| +// open. A re-run must not truncate it or touch its schema. |
| +func TestBootstrapKeepsAnExistingRegistry(t *testing.T) { |
| + root := t.TempDir() |
| + if err := bootstrap(root, nil); err != nil { |
| + t.Fatalf("first bootstrap: %v", err) |
| + } |
| + regPath := filepath.Join(root, "data", "registry.db") |
| + reg, err := registry.Open(regPath) |
| + if err != nil { |
| + t.Fatalf("open: %v", err) |
| + } |
| + if err := reg.Upsert(registry.Route{Host: "app.example.com", Kind: "static"}); err != nil { |
| + t.Fatalf("upsert: %v", err) |
| + } |
| + if err := reg.Close(); err != nil { |
| + t.Fatalf("close: %v", err) |
| + } |
| + |
| + if err := bootstrap(root, nil); err != nil { |
| + t.Fatalf("second bootstrap: %v", err) |
| + } |
| + |
| + reg2, err := registry.Open(regPath) |
| + if err != nil { |
| + t.Fatalf("reopen: %v", err) |
| + } |
| + defer reg2.Close() |
| + routes, err := reg2.List() |
| + if err != nil { |
| + t.Fatalf("list: %v", err) |
| + } |
| + if len(routes) != 1 || routes[0].Host != "app.example.com" { |
| + t.Fatalf("re-running bootstrap lost the registry contents: %+v", routes) |
| + } |
| +} |
| + |
| +// TestChownRegistryToServiceIsANoopOffRoot: the helper runs on every adoption |
| +// pass, and the tests (and any developer box) are not root. It must do |
| +// nothing at all there rather than log or fail. |
| +func TestChownRegistryToServiceIsANoopOffRoot(t *testing.T) { |
| + if os.Geteuid() == 0 { |
| + t.Skip("running as root: this test pins the non-root path") |
| + } |
| + path := filepath.Join(t.TempDir(), "registry.db") |
| + if err := os.WriteFile(path, []byte("x"), 0o640); err != nil { |
| + t.Fatal(err) |
| + } |
| + before, err := os.Stat(path) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + chownRegistryToService(path) // must not panic, must not change anything |
| + after, err := os.Stat(path) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + buid, bgid, ok := fileOwner(before) |
| + auid, agid, ok2 := fileOwner(after) |
| + if ok != ok2 || buid != auid || bgid != agid { |
| + t.Errorf("ownership moved off root: %d:%d -> %d:%d", buid, bgid, auid, agid) |
| + } |
| +} |
| + |
| +// TestRegistryFilesCoversTheWALSidecars: a carlos-owned database with a |
| +// root-owned -wal is exactly as unwritable as a root-owned database, and the |
| +// sidecars are what a root pass re-creates after a hand chown. Two of the |
| +// three boxes in #8 needed the chown twice for this reason. |
| +func TestRegistryFilesCoversTheWALSidecars(t *testing.T) { |
| + got := registryFiles("/data/registry.db") |
| + want := []string{"/data/registry.db", "/data/registry.db-wal", "/data/registry.db-shm"} |
| + if len(got) != len(want) { |
| + t.Fatalf("registryFiles = %v, want %v", got, want) |
| + } |
| + for i := range want { |
| + if got[i] != want[i] { |
| + t.Errorf("registryFiles[%d] = %q, want %q", i, got[i], want[i]) |
| + } |
| + } |
| +} |
| + |
| +// TestRegistryOpenHintNamesOwnership is the cheap fence #8 asked for: |
| +// "attempt to write a readonly database (8)" is SQLite's words, and they sent |
| +// the operator looking at disk space and mount options. The hint has to name |
| +// ownership, the service user, and the command that fixes it. |
| +func TestRegistryOpenHintNamesOwnership(t *testing.T) { |
| + path := filepath.Join(t.TempDir(), "registry.db") |
| + if err := os.WriteFile(path, nil, 0o640); err != nil { |
| + t.Fatal(err) |
| + } |
| + hint := registryOpenHint(path, errors.New("attempt to write a readonly database (8)")) |
| + if hint == "" { |
| + t.Fatal("no hint for the readonly-database error") |
| + } |
| + for _, want := range []string{serviceUser, "chown", path, "owned by"} { |
| + if !strings.Contains(hint, want) { |
| + t.Errorf("hint does not mention %q:\n%s", want, hint) |
| + } |
| + } |
| +} |
| + |
| +// TestRegistryOpenHintSilentOnOtherErrors: a genuinely full disk or a missing |
| +// directory must still read as itself. The hint is only for the one error it |
| +// can actually explain. |
| +func TestRegistryOpenHintSilentOnOtherErrors(t *testing.T) { |
| + path := filepath.Join(t.TempDir(), "registry.db") |
| + for _, err := range []error{ |
| + nil, |
| + errors.New("database or disk is full (13)"), |
| + errors.New("unable to open database file"), |
| + } { |
| + if hint := registryOpenHint(path, err); hint != "" { |
| + t.Errorf("registryOpenHint(%v) = %q, want no hint", err, hint) |
| + } |
| + } |
| +} |