amadan

carlos / platform Public

layout: ownership and socket-length preflight, destination check, raw directories

Preflight refuses a config-env two routes own and a nested socket whose
control twin overflows a unix socket address. DestinationCheck refuses a
database or litestream shadow already at the destination and moves an
env file or quarantine aside. EnsureDirs builds the raw instance and run
directories from symlink-free directory fds and refuses a planted link.
The raw-directory grep test matches the path literal only.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 6534a486d844fc6b13343c96e7f6c56850e3d976 parent 16f31d1
3 files changed, +447 −0
  • internal/layout/preflight.go +152 −0
  • internal/layout/preflight_test.go +186 −0
  • internal/layout/rawdir_test.go +109 −0
diff --git a/internal/layout/preflight.go b/internal/layout/preflight.go
new file mode 100644
index 000000000..0cdeaa2db
--- /dev/null
+++ b/internal/layout/preflight.go
@@ -0,0 +1,152 @@
+package layout
+
+import (
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "github.com/carlosframework/platform/internal/instancefs"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+// Preflight is the ownership check of design §8.2 step 4, plus the socket
+// length check of §3.1. Under the flat rules <host>.config.env is host's
+// rendered bundle UNLESS a route named <host>.config exists, in which case
+// it is that route's instance env (SyncExec's own collision rule). Moving
+// it with host would take another route's file; reverting onto it would
+// overwrite one. Either is a refusal naming both owners.
+func Preflight(p *Plan, routes []registry.Route) error {
+ hosts := map[string]bool{}
+ for _, rt := range routes {
+ hosts[rt.Host] = true
+ }
+ if other := p.Host + ".config"; hosts[other] {
+ return fmt.Errorf("layout: %s: %s.config.env is owned by both %s (its bundle) and %s (its instance env); refusing to move it", p.Host, p.Host, p.Host, other)
+ }
+ if strings.HasSuffix(p.Host, ".config") {
+ if owner := strings.TrimSuffix(p.Host, ".config"); hosts[owner] {
+ return fmt.Errorf("layout: %s: its env file %s.env is owned by both %s (its instance env) and %s (its bundle); refusing to move it", p.Host, p.Host, p.Host, owner)
+ }
+ }
+ // The activator binds a control socket beside every exec listener at
+ // the listener's path plus ".control". A nested address that leaves no
+ // room for it would make the host unwakeable after the move, so it is
+ // refused before anything moves.
+ if p.Mode == ModeNested {
+ if n := len(p.To.Socket) + len(".control"); n > provision.MaxUnixPath {
+ return fmt.Errorf("layout: %s: control socket %s.control is %d bytes, over the %d a unix socket address holds; refusing to move it", p.Host, p.To.Socket, n, provision.MaxUnixPath)
+ }
+ }
+ return nil
+}
+
+// DestinationCheck is step 5, run before the journal so that at the moment
+// the journal is written nothing the pass will move has anything at its
+// destination. A database, or any file of one, already at the destination
+// is a refusal (two databases; refusing to choose), and so is a litestream
+// shadow (two replication states for one database). An env file already
+// there, which an older binary running between a migration and a revert
+// can have minted fresh, is moved aside as <name>.aside-<unix> so the
+// source, the file the instance ran with for longer, takes its place; a
+// quarantined restore is kept the same way. The rendered bundle is simply
+// cleared; sync re-renders it anyway.
+//
+// Every change goes through a directory fd that follows no symlink, so a
+// planted link at a destination directory cannot redirect it.
+func DestinationCheck(p *Plan, now func() time.Time) (aside []string, err error) {
+ for _, a := range p.Artifacts {
+ if a.Name == "socket" {
+ continue
+ }
+ present, err := lstatPresent(a.Dst)
+ if err != nil {
+ return aside, err
+ }
+ if !present {
+ continue
+ }
+ dir, name := filepath.Dir(a.Dst), filepath.Base(a.Dst)
+ switch a.Name {
+ case "db", "db-wal", "db-shm":
+ return aside, fmt.Errorf("layout: %s: two databases (%s exists at the destination); refusing to choose", p.Host, a.Dst)
+ case "litestream-shadow":
+ return aside, fmt.Errorf("layout: %s: two replication states (%s exists at the destination); refusing to choose", p.Host, a.Dst)
+ case "env", "sidecar-env", "corrupt":
+ to := fmt.Sprintf("%s.aside-%d", name, now().Unix())
+ taken, err := lstatPresent(filepath.Join(dir, to))
+ if err != nil {
+ return aside, err
+ }
+ if taken {
+ return aside, fmt.Errorf("layout: %s: %s and %s both exist; refusing to replace a kept copy", p.Host, a.Dst, filepath.Join(dir, to))
+ }
+ if err := instancefs.RenameBetween(dir, name, dir, to); err != nil {
+ return aside, fmt.Errorf("layout: %s: move %s aside: %w", p.Host, a.Dst, err)
+ }
+ aside = append(aside, filepath.Join(dir, to))
+ case "config-env":
+ if err := instancefs.RemoveBeneath(dir, name); err != nil {
+ return aside, fmt.Errorf("layout: %s: clear %s: %w", p.Host, a.Dst, err)
+ }
+ default:
+ return aside, fmt.Errorf("layout: %s: no destination rule for artifact %q", p.Host, a.Name)
+ }
+ }
+ return aside, nil
+}
+
+// EnsureDirs creates the raw directories the bind mounts show: the data
+// directory 0700 (the tenant's own ground) and the run directory 2750 with
+// the carlos-sockets group, so an enrolled consumer can reach app.sock. The
+// raw parents get the modes bootstrap's tmpfiles lines give them, in case
+// the box has not run those yet. The directories must exist before the row
+// flips, because Restore and prepare both assume them.
+//
+// Each directory is made from an fd on its parent that followed no symlink,
+// and is then required to be a real directory: a same-uid tenant can plant
+// a symlink at <data>/.instances/<iid> before the pass arrives, and a move
+// into it would land the database wherever the link points.
+func EnsureDirs(dataDir, socketDir, host string, socketsGID int) error {
+ data := rawInstanceDir(dataDir, host)
+ run := rawRunDir(socketDir, host)
+ for _, d := range []struct {
+ path string
+ perm os.FileMode
+ gid int
+ }{
+ {filepath.Dir(data), 0o700, -1},
+ {data, 0o700, -1},
+ {filepath.Dir(run), 0o750 | os.ModeSetgid, socketsGID},
+ {run, 0o750 | os.ModeSetgid, socketsGID},
+ } {
+ parent, name := filepath.Dir(d.path), filepath.Base(d.path)
+ if err := instancefs.MkdirBeneath(parent, name, d.perm); err != nil {
+ return fmt.Errorf("layout: %s: %w", host, err)
+ }
+ if d.gid >= 0 {
+ if err := instancefs.ChownBeneath(parent, name, -1, d.gid); err != nil {
+ return fmt.Errorf("layout: %s: %w", host, err)
+ }
+ }
+ fi, err := os.Lstat(d.path)
+ if err != nil {
+ return fmt.Errorf("layout: %s: %w", host, err)
+ }
+ if !fi.IsDir() {
+ return fmt.Errorf("layout: %s: %s is not a directory (%v)", host, d.path, fi.Mode().Type())
+ }
+ }
+ return nil
+}
+
+func lstatPresent(path string) (bool, error) {
+ _, err := os.Lstat(path)
+ if errors.Is(err, os.ErrNotExist) {
+ return false, nil
+ }
+ return err == nil, err
+}
diff --git a/internal/layout/preflight_test.go b/internal/layout/preflight_test.go
new file mode 100644
index 000000000..dcd4d4ddb
--- /dev/null
+++ b/internal/layout/preflight_test.go
@@ -0,0 +1,186 @@
+//go:build linux
+
+package layout
+
+import (
+ "errors"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/instancefs"
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+func TestPreflightRefusesAConfigEnvOwnedByAnotherRoute(t *testing.T) {
+ p, _ := PlanFor("/data", "/run/carlos", "x.example.com", ModeNested)
+ routes := []registry.Route{{Host: "x.example.com"}, {Host: "x.example.com.config"}}
+ err := Preflight(p, routes)
+ if err == nil || !strings.Contains(err.Error(), "x.example.com.config") {
+ t.Fatalf("ambiguous config-env accepted: %v", err)
+ }
+ if err := Preflight(p, []registry.Route{{Host: "x.example.com"}}); err != nil {
+ t.Fatalf("unambiguous plan refused: %v", err)
+ }
+ // Reverting applies the same table: a flat destination another route owns is a refusal.
+ r, _ := PlanFor("/data", "/run/carlos", "x.example.com", ModeFlat)
+ if err := Preflight(r, routes); err == nil {
+ t.Fatal("revert onto another route's file accepted")
+ }
+ // The other direction of the same rule: a host named <owner>.config
+ // has an env file that is also <owner>'s rendered bundle.
+ c, _ := PlanFor("/data", "/run/carlos", "x.example.com.config", ModeNested)
+ if err := Preflight(c, routes); err == nil || !strings.Contains(err.Error(), "x.example.com") {
+ t.Fatalf("an env file that is another route's bundle accepted: %v", err)
+ }
+}
+
+// A nested socket's control twin must fit a unix socket address. The
+// socket dir here is long enough that the nested address overflows it.
+func TestPreflightRefusesANestedSocketTooLongForItsControlTwin(t *testing.T) {
+ long := "/" + strings.Repeat("s", 60)
+ host := strings.Repeat("h", 30) + ".example.com"
+ p, _ := PlanFor("/data", long, host, ModeNested)
+ want := len(p.To.Socket) + len(".control")
+ err := Preflight(p, []registry.Route{{Host: host}})
+ if err == nil || !strings.Contains(err.Error(), "107") || !strings.Contains(err.Error(), strconv.Itoa(want)) {
+ t.Fatalf("overlong socket accepted or not measured: %v (want %d named)", err, want)
+ }
+ // Reverting to flat does not create the nested address, so it is not
+ // measured.
+ r, _ := PlanFor("/data", long, host, ModeFlat)
+ if err := Preflight(r, []registry.Route{{Host: host}}); err != nil {
+ t.Fatalf("revert refused over a nested length: %v", err)
+ }
+}
+
+func TestDestinationCheckRefusesTwoDatabasesAndMovesEnvAside(t *testing.T) {
+ dd := t.TempDir()
+ p, _ := PlanFor(dd, filepath.Join(dd, "run"), "h.example.com", ModeNested)
+ if err := os.MkdirAll(rawInstanceDir(dd, "h.example.com"), 0o700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(p.Artifacts[0].Dst, []byte("x"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := DestinationCheck(p, time.Now); err == nil || !strings.Contains(err.Error(), "two databases") {
+ t.Fatalf("two databases not refused: %v", err)
+ }
+ os.Remove(p.Artifacts[0].Dst)
+ if err := os.WriteFile(p.Artifacts[3].Dst, []byte("fresh env minted by an older binary"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ aside, err := DestinationCheck(p, func() time.Time { return time.Unix(1700000000, 0) })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(aside) != 1 || !strings.HasSuffix(aside[0], ".env.aside-1700000000") {
+ t.Fatalf("aside = %v", aside)
+ }
+ if _, err := os.Stat(p.Artifacts[3].Dst); !os.IsNotExist(err) {
+ t.Error("env destination still occupied")
+ }
+ if b, err := os.ReadFile(aside[0]); err != nil || string(b) != "fresh env minted by an older binary" {
+ t.Errorf("aside copy: %q %v", b, err)
+ }
+}
+
+func artifactNamed(t *testing.T, p *Plan, name string) Artifact {
+ t.Helper()
+ for _, a := range p.Artifacts {
+ if a.Name == name {
+ return a
+ }
+ }
+ t.Fatalf("no artifact %q", name)
+ return Artifact{}
+}
+
+// A litestream shadow already at the destination is a second replication
+// state for one database: refused like a second database. A quarantine
+// already there is moved aside like an env file, and the rendered bundle
+// is cleared so the source takes its name.
+func TestDestinationCheckShadowRefusesCorruptAsideConfigCleared(t *testing.T) {
+ dd := t.TempDir()
+ host := "h.example.com"
+ p, _ := PlanFor(dd, filepath.Join(dd, "run"), host, ModeNested)
+ if err := os.MkdirAll(rawInstanceDir(dd, host), 0o700); err != nil {
+ t.Fatal(err)
+ }
+ shadow := artifactNamed(t, p, "litestream-shadow").Dst
+ if err := os.Mkdir(shadow, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := DestinationCheck(p, time.Now); err == nil || !strings.Contains(err.Error(), "two replication states") {
+ t.Fatalf("destination shadow not refused: %v", err)
+ }
+ os.Remove(shadow)
+ corrupt := artifactNamed(t, p, "corrupt").Dst
+ config := artifactNamed(t, p, "config-env").Dst
+ for _, f := range []string{corrupt, config} {
+ if err := os.WriteFile(f, []byte("old"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ }
+ aside, err := DestinationCheck(p, func() time.Time { return time.Unix(1700000000, 0) })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(aside) != 1 || aside[0] != corrupt+".aside-1700000000" {
+ t.Fatalf("aside = %v", aside)
+ }
+ for _, f := range []string{corrupt, config} {
+ if _, err := os.Lstat(f); !errors.Is(err, os.ErrNotExist) {
+ t.Errorf("%s still occupied", f)
+ }
+ }
+}
+
+// The reverse direction checks the flat side the same way.
+func TestDestinationCheckRefusesADatabaseAtTheFlatPathOnRevert(t *testing.T) {
+ dd := t.TempDir()
+ p, _ := PlanFor(dd, filepath.Join(dd, "run"), "h.example.com", ModeFlat)
+ if err := os.WriteFile(filepath.Join(dd, "h.example.com.db-wal"), []byte("w"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := DestinationCheck(p, time.Now); err == nil || !strings.Contains(err.Error(), "two databases") {
+ t.Fatalf("a flat wal on revert not refused: %v", err)
+ }
+}
+
+func TestEnsureDirsCreatesTheRawDirectoriesWithTheRightModes(t *testing.T) {
+ dd, rd := t.TempDir(), t.TempDir()
+ if err := EnsureDirs(dd, rd, "h.example.com", -1); err != nil {
+ t.Fatal(err)
+ }
+ fi, err := os.Lstat(rawInstanceDir(dd, "h.example.com"))
+ if err != nil || !fi.IsDir() || fi.Mode().Perm() != 0o700 {
+ t.Errorf("data dir: %v %v", fi, err)
+ }
+ fi, err = os.Lstat(rawRunDir(rd, "h.example.com"))
+ if err != nil || !fi.IsDir() || fi.Mode().Perm() != 0o750 || fi.Mode()&os.ModeSetgid == 0 {
+ t.Errorf("run dir: %v %v", fi, err)
+ }
+ // Idempotent: the pass calls it on every move.
+ if err := EnsureDirs(dd, rd, "h.example.com", -1); err != nil {
+ t.Fatalf("second call: %v", err)
+ }
+}
+
+func TestEnsureDirsRefusesASymlinkAtTheInstanceDirectory(t *testing.T) {
+ dd, rd, elsewhere := t.TempDir(), t.TempDir(), t.TempDir()
+ host := "h.example.com"
+ raw := rawInstanceDir(dd, host)
+ if err := os.MkdirAll(filepath.Dir(raw), 0o700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(elsewhere, raw); err != nil {
+ t.Fatal(err)
+ }
+ if err := EnsureDirs(dd, rd, host, -1); !errors.Is(err, instancefs.ErrSymlink) {
+ t.Fatalf("err = %v, want ErrSymlink", err)
+ }
+}
diff --git a/internal/layout/rawdir_test.go b/internal/layout/rawdir_test.go
new file mode 100644
index 000000000..fa22b40a8
--- /dev/null
+++ b/internal/layout/rawdir_test.go
@@ -0,0 +1,109 @@
+package layout
+
+import (
+ "io/fs"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "regexp"
+ "strings"
+ "testing"
+)
+
+// rawDirLiteral matches the raw directory as a path literal: a quote or a
+// slash, then ".instances" as a whole word. The bare substring would also
+// match ordinary field names such as s.cli.instances.
+var rawDirLiteral = regexp.MustCompile(`["/]\.instances\b`)
+
+// The raw directories are named in exactly one package, plus bootstrap's
+// tmpfiles and fixture lines that create them: a second caller would be a
+// second way to bypass the bind mount's nosymfollow (design §3.2).
+func TestRawInstancesDirectoryIsNamedOnlyHere(t *testing.T) {
+ root := filepath.Join("..", "..")
+ var files []string
+ err := filepath.WalkDir(filepath.Join(root, "internal"), func(path string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ if !d.IsDir() && strings.HasSuffix(path, ".go") {
+ files = append(files, path)
+ }
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ top, err := filepath.Glob(filepath.Join(root, "*.go"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ files = append(files, top...)
+ if len(top) == 0 {
+ t.Fatal("no .go files at the repository root; the walk is looking in the wrong place")
+ }
+ for _, path := range files {
+ if strings.HasSuffix(path, "_test.go") {
+ continue
+ }
+ slash := filepath.ToSlash(path)
+ if strings.Contains(slash, "internal/layout/") || strings.HasSuffix(slash, "internal/host/bootstrap.go") {
+ continue
+ }
+ b, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if loc := rawDirLiteral.FindIndex(b); loc != nil {
+ line := 1 + strings.Count(string(b[:loc[0]]), "\n")
+ t.Errorf("%s:%d names the raw .instances directory; only internal/layout (and bootstrap's tmpfiles lines) may", slash, line)
+ }
+ }
+}
+
+// The pattern must catch the literal forms and nothing else.
+func TestRawDirLiteralPattern(t *testing.T) {
+ for _, s := range []string{`filepath.Join(d, ".instances")`, `"/data/.instances/x"`, `"/run/carlos/.instances"`} {
+ if !rawDirLiteral.MatchString(s) {
+ t.Errorf("missed %s", s)
+ }
+ }
+ for _, s := range []string{`s.cli.instances.allow`, `"/data/instances"`, `".instancesX"`} {
+ if rawDirLiteral.MatchString(s) {
+ t.Errorf("matched %s", s)
+ }
+ }
+}
+
+// With a real bind mount, a rename THROUGH the mount answers EXDEV and the
+// pass's rename through the raw directory succeeds. Needs CAP_SYS_ADMIN;
+// skipped where it cannot mount.
+func TestRenameThroughTheRawDirectoryAvoidsEXDEV(t *testing.T) {
+ if os.Geteuid() != 0 {
+ t.Skip("needs root to bind mount")
+ }
+ base := t.TempDir()
+ raw, view := filepath.Join(base, ".instances"), filepath.Join(base, "instances")
+ if err := os.MkdirAll(raw, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.MkdirAll(view, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ if out, err := exec.Command("mount", "--bind", "-o", "nosymfollow", raw, view).CombinedOutput(); err != nil {
+ t.Skipf("cannot bind mount here: %v %s", err, out)
+ }
+ defer exec.Command("umount", view).Run()
+ src := filepath.Join(base, "h.db")
+ if err := os.WriteFile(src, []byte("x"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Rename(src, filepath.Join(view, "h.db")); err == nil {
+ t.Fatal("rename through the bind mount succeeded; the EXDEV premise of §3.2 does not hold here")
+ }
+ if err := os.Rename(src, filepath.Join(raw, "h.db")); err != nil {
+ t.Fatalf("rename through the raw directory: %v", err)
+ }
+ if _, err := os.Stat(filepath.Join(view, "h.db")); err != nil {
+ t.Fatalf("the view does not show the renamed file: %v", err)
+ }
+}