amadan

carlos / platform Public

instanceexec: the readiness probe with injectable checks and the EPERM rule for a packet filter (plan 2, task 4)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01YZ1ZvWj7naf73GjTX8MBv5
Paul Campbell pushed by paul@keymail.dev cd0d0dfc7e8d9a00b25f5174319ff1ec55804040 parent f55b80a
4 files changed, +448 −0
  • internal/instanceexec/probe.go +213 −0
  • internal/instanceexec/probe_linux.go +53 −0
  • internal/instanceexec/probe_other.go +12 −0
  • internal/instanceexec/probe_test.go +170 −0
diff --git a/internal/instanceexec/probe.go b/internal/instanceexec/probe.go
new file mode 100644
index 000000000..a8edba65f
--- /dev/null
+++ b/internal/instanceexec/probe.go
@@ -0,0 +1,213 @@
+package instanceexec
+
+import (
+ "encoding/json"
+ "errors"
+ "flag"
+ "fmt"
+ "os"
+ "path/filepath"
+ "sort"
+ "strconv"
+ "syscall"
+ "time"
+
+ "github.com/carlosframework/platform/internal/cliflag"
+ "github.com/carlosframework/platform/internal/instancectl"
+ "github.com/carlosframework/platform/internal/instancefs"
+)
+
+// probeEnv is everything the checks touch, so the verdict logic is testable
+// against a fixture tree and fake syscalls while production runs against
+// the real namespace.
+type probeEnv struct {
+ dataRoot string // /data as the probe sees it
+ runRoot string // /run/carlos as the probe sees it
+ varCache string // /var/cache
+ varLib string // /var/lib
+ etcCarlos string // /etc/carlos
+ procRoot string // /proc
+ cgroupProcs string // a cgroupfs file to test writability on
+ pid func() int
+ writable func(path string) error
+ sendUDP func(network, addr string) error
+ connectTCP func(addr string, timeout time.Duration) error
+ mountNoSym func(mountpoint string) (bool, error)
+ // symlinkCheck plants a symlink under dir and reports whether it was
+ // refused. Production wires this to realSymlinkCheck; the test fixture
+ // stubs it because the fixture's tree is a plain temp dir with no
+ // nosymfollow mount, so the real check would fail there for a reason
+ // unrelated to the property under test in the other cases.
+ symlinkCheck func(dir string) error
+}
+
+var errNotELOOP = errors.New("symlink was followed")
+
+// RunProbe is `carlos instance-probe`: it runs inside the probe unit's
+// namespace (the same text as the tenant templates) and prints a
+// ProbeReport on stdout, which the unit's StandardOutput=truncate: lands in
+// /run/carlos-control/probe.out for the control service to read (design
+// §8.5). It exits 0 whatever the verdict: the verdict is the output, and a
+// failed unit would hide it.
+func RunProbe(args []string) {
+ fs := flag.NewFlagSet("instance-probe", flag.ExitOnError)
+ fs.Usage = cliflag.Usage(fs, "usage: carlos instance-probe")
+ _ = fs.Parse(args)
+ r := runChecks(productionProbeEnv())
+ enc := json.NewEncoder(os.Stdout)
+ enc.SetIndent("", " ")
+ _ = enc.Encode(r)
+}
+
+func productionProbeEnv() probeEnv {
+ return probeEnv{
+ dataRoot: "/data", runRoot: "/run/carlos", varCache: "/var/cache", varLib: "/var/lib",
+ etcCarlos: "/etc/carlos", procRoot: "/proc", cgroupProcs: "/sys/fs/cgroup/cgroup.procs",
+ pid: os.Getpid,
+ writable: accessWritable,
+ sendUDP: sendOneDatagram,
+ connectTCP: connectWithTimeout,
+ mountNoSym: instancefs.MountNoSymfollow,
+ symlinkCheck: realSymlinkCheck,
+ }
+}
+
+func listNames(dir string) ([]string, error) {
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ return nil, err
+ }
+ names := make([]string, 0, len(entries))
+ for _, e := range entries {
+ names = append(names, e.Name())
+ }
+ sort.Strings(names)
+ return names, nil
+}
+
+func exactly(dir string, want ...string) (string, bool) {
+ got, err := listNames(dir)
+ if err != nil {
+ return err.Error(), false
+ }
+ sort.Strings(want)
+ if fmt.Sprint(got) != fmt.Sprint(want) {
+ return fmt.Sprintf("%s lists %v, want %v", dir, got, want), false
+ }
+ return "", true
+}
+
+// runChecks is the verdict. Each line names the property it checks; "fail"
+// blocks readiness, "unverified" does not, "ok" is what a ready box shows
+// on every line.
+func runChecks(e probeEnv) instancectl.ProbeReport {
+ var r instancectl.ProbeReport
+ add := func(check, verdict, detail string) {
+ r.Lines = append(r.Lines, instancectl.VerdictLine{Check: check, Verdict: verdict, Detail: detail})
+ }
+ pass := func(check string, detail string, ok bool) {
+ if ok {
+ add(check, "ok", "")
+ } else {
+ add(check, "fail", detail)
+ }
+ }
+
+ // The two trees hold exactly the probe's own directory.
+ d1, ok1 := exactly(e.dataRoot, "instances")
+ d2, ok2 := exactly(filepath.Join(e.dataRoot, "instances"), instancectl.ProbeDirName)
+ pass("data-tree", d1+d2, ok1 && ok2)
+ r1, ok1 := exactly(e.runRoot, "instances")
+ r2, ok2 := exactly(filepath.Join(e.runRoot, "instances"), instancectl.ProbeDirName)
+ pass("run-tree", r1+r2, ok1 && ok2)
+
+ // The two trees other carlos-uid units write are empty.
+ v1, ok1 := exactly(e.varCache)
+ v2, ok2 := exactly(e.varLib)
+ pass("var-empty", v1+v2, ok1 && ok2)
+
+ // A private pid namespace: this process is pid 1 and /proc lists no
+ // other pid.
+ pids, err := listNames(e.procRoot)
+ others := ""
+ for _, n := range pids {
+ if v, err := strconv.Atoi(n); err == nil && v != 1 {
+ others += n + " "
+ }
+ }
+ pid := e.pid()
+ pass("pid1", fmt.Sprintf("pid=%d others=[%s] err=%v", pid, others, err), err == nil && pid == 1 && others == "")
+
+ // cgroupfs is read-only: access(W_OK) answers EROFS without writing.
+ werr := e.writable(e.cgroupProcs)
+ pass("cgroup-ro", fmt.Sprintf("access(W_OK) = %v", werr), errors.Is(werr, syscall.EROFS))
+
+ // /etc/carlos cannot be opened and its fixture cannot be read.
+ _, derr := os.ReadDir(e.etcCarlos)
+ _, ferr := os.ReadFile(filepath.Join(e.etcCarlos, "probe-secret"))
+ pass("etc-carlos", fmt.Sprintf("readdir=%v readfile=%v", derr, ferr), derr != nil && ferr != nil)
+
+ // A symlink planted in the probe's own directory is refused with ELOOP.
+ serr := e.symlinkCheck(filepath.Join(e.dataRoot, "instances", instancectl.ProbeDirName))
+ pass("symlink-eloop", fmt.Sprint(serr), serr == nil)
+
+ // Both bound directories are on nosymfollow mounts.
+ n1, err1 := e.mountNoSym(filepath.Join(e.dataRoot, "instances", instancectl.ProbeDirName))
+ n2, err2 := e.mountNoSym(filepath.Join(e.runRoot, "instances", instancectl.ProbeDirName))
+ pass("nosymfollow", fmt.Sprintf("data=%v/%v run=%v/%v", n1, err1, n2, err2), n1 && n2 && err1 == nil && err2 == nil)
+
+ // IMDS, for a packet filter: a datagram's verdict comes straight back
+ // from sendmsg. v4 must be EPERM. v6 is EPERM (enforced) or ENETUNREACH
+ // (no route to that address: unverified, not blocking). The loopback
+ // control proves the stack is live. A TCP connect must not succeed.
+ v4 := e.sendUDP("udp4", "169.254.169.254:80")
+ pass("imds-v4", fmt.Sprintf("sendto = %v", v4), errors.Is(v4, syscall.EPERM))
+ v6 := e.sendUDP("udp6", "[fd00:ec2::254]:80")
+ switch {
+ case errors.Is(v6, syscall.EPERM):
+ add("imds-v6", "ok", "")
+ case errors.Is(v6, syscall.ENETUNREACH):
+ add("imds-v6", "unverified", "no route to the address; deny not exercised")
+ default:
+ add("imds-v6", "fail", fmt.Sprintf("sendto = %v", v6))
+ }
+ lo := e.sendUDP("udp4", "127.0.0.1:9")
+ pass("loopback", fmt.Sprintf("sendto = %v", lo), lo == nil)
+ tcp := e.connectTCP("169.254.169.254:80", 2*time.Second)
+ pass("imds-tcp", fmt.Sprintf("connect = %v", tcp), tcp != nil)
+
+ r.Ready = true
+ for _, l := range r.Lines {
+ if l.Verdict == "fail" {
+ r.Ready = false
+ }
+ }
+ return r
+}
+
+// realSymlinkCheck plants a symlink inside dir and tries to open it through
+// its name; on a nosymfollow mount the open fails with ELOOP. It returns nil
+// for ELOOP, errNotELOOP if the link was followed, and any other error.
+func realSymlinkCheck(dir string) error {
+ target := filepath.Join(dir, "probe-target")
+ link := filepath.Join(dir, "probe-link")
+ os.Remove(link)
+ os.Remove(target)
+ if err := os.WriteFile(target, []byte("x"), 0o600); err != nil {
+ return err
+ }
+ defer os.Remove(target)
+ if err := os.Symlink(target, link); err != nil {
+ return err
+ }
+ defer os.Remove(link)
+ f, err := os.Open(link)
+ if err == nil {
+ f.Close()
+ return errNotELOOP
+ }
+ if errors.Is(err, syscall.ELOOP) {
+ return nil
+ }
+ return err
+}
diff --git a/internal/instanceexec/probe_linux.go b/internal/instanceexec/probe_linux.go
new file mode 100644
index 000000000..7083f6cf9
--- /dev/null
+++ b/internal/instanceexec/probe_linux.go
@@ -0,0 +1,53 @@
+//go:build linux
+
+package instanceexec
+
+import (
+ "errors"
+ "net"
+ "syscall"
+ "time"
+
+ "golang.org/x/sys/unix"
+)
+
+// accessWritable asks the kernel whether path could be opened for writing
+// without opening it: on a read-only mount access(2) answers EROFS, which
+// is exactly the property ProtectControlGroups= must give a tenant.
+func accessWritable(path string) error {
+ return unix.Access(path, unix.W_OK)
+}
+
+// sendOneDatagram sends one empty UDP datagram. With a cgroup BPF egress
+// filter (IPAddressDeny=) the filter's verdict is sendmsg's return value,
+// EPERM; with no route it is ENETUNREACH; on a live path it is nil.
+func sendOneDatagram(network, addr string) error {
+ c, err := net.Dial(network, addr)
+ if err != nil {
+ return unwrapErrno(err)
+ }
+ defer c.Close()
+ _, err = c.Write([]byte{})
+ return unwrapErrno(err)
+}
+
+func connectWithTimeout(addr string, timeout time.Duration) error {
+ c, err := net.DialTimeout("tcp4", addr, timeout)
+ if err != nil {
+ return unwrapErrno(err)
+ }
+ c.Close()
+ return nil
+}
+
+// unwrapErrno returns the syscall.Errno inside err so callers can
+// errors.Is against EPERM and ENETUNREACH; net.OpError and os.SyscallError
+// both unwrap to the Errno, so errors.As finds it. Anything else is
+// returned as is. (unix.Errno is an alias of syscall.Errno.)
+func unwrapErrno(err error) error {
+ var errno syscall.Errno
+ if errors.As(err, &errno) {
+ return errno
+ }
+ return err
+}
diff --git a/internal/instanceexec/probe_other.go b/internal/instanceexec/probe_other.go
new file mode 100644
index 000000000..7aa0490f5
--- /dev/null
+++ b/internal/instanceexec/probe_other.go
@@ -0,0 +1,12 @@
+//go:build !linux
+
+package instanceexec
+
+import (
+ "errors"
+ "time"
+)
+
+func accessWritable(string) error { return errors.ErrUnsupported }
+func sendOneDatagram(string, string) error { return errors.ErrUnsupported }
+func connectWithTimeout(string, time.Duration) error { return errors.ErrUnsupported }
diff --git a/internal/instanceexec/probe_test.go b/internal/instanceexec/probe_test.go
new file mode 100644
index 000000000..89062a4ea
--- /dev/null
+++ b/internal/instanceexec/probe_test.go
@@ -0,0 +1,170 @@
+//go:build linux
+
+package instanceexec
+
+import (
+ "errors"
+ "os"
+ "path/filepath"
+ "syscall"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/instancectl"
+)
+
+// goodEnv is a fixture that passes every check: the trees an isolated
+// instance sees, pid 1, a read-only cgroupfs, an unopenable /etc/carlos, a
+// nosymfollow mount, EPERM on both metadata addresses, a live loopback and
+// a TCP connect that never succeeds.
+func goodEnv(t *testing.T) probeEnv {
+ t.Helper()
+ root := t.TempDir()
+ mk := func(p string) string {
+ if err := os.MkdirAll(filepath.Join(root, p), 0o700); err != nil {
+ t.Fatal(err)
+ }
+ return filepath.Join(root, p)
+ }
+ data := mk("data")
+ mk("data/instances/_probe")
+ run := mk("run/carlos")
+ mk("run/carlos/instances/_probe")
+ varCache := mk("var/cache")
+ varLib := mk("var/lib")
+ etc := filepath.Join(root, "etc", "carlos") // deliberately absent: opening it fails
+ proc := mk("proc")
+ mk("proc/1")
+ mk("proc/self")
+ return probeEnv{
+ dataRoot: data, runRoot: run, varCache: varCache, varLib: varLib, etcCarlos: etc, procRoot: proc,
+ cgroupProcs: filepath.Join(root, "sys", "cgroup.procs"),
+ pid: func() int { return 1 },
+ writable: func(string) error { return syscall.EROFS },
+ sendUDP: func(network, addr string) error {
+ switch addr {
+ case "169.254.169.254:80", "[fd00:ec2::254]:80":
+ return syscall.EPERM
+ }
+ return nil
+ },
+ connectTCP: func(string, time.Duration) error { return syscall.ETIMEDOUT },
+ mountNoSym: func(string) (bool, error) { return true, nil },
+ symlinkCheck: func(string) error { return nil },
+ }
+}
+
+func verdict(r instancectl.ProbeReport, check string) string {
+ for _, l := range r.Lines {
+ if l.Check == check {
+ return l.Verdict
+ }
+ }
+ return "<missing>"
+}
+
+func TestProbePassesOnAnIsolatedNamespace(t *testing.T) {
+ r := runChecks(goodEnv(t))
+ if !r.Ready {
+ t.Fatalf("not ready: %+v", r.Lines)
+ }
+ for _, c := range []string{"data-tree", "run-tree", "var-empty", "pid1", "cgroup-ro", "etc-carlos", "symlink-eloop", "nosymfollow", "imds-v4", "imds-v6", "loopback", "imds-tcp"} {
+ if v := verdict(r, c); v != "ok" {
+ t.Errorf("%s = %s", c, v)
+ }
+ }
+}
+
+func TestProbeFailsWhenASiblingIsVisible(t *testing.T) {
+ e := goodEnv(t)
+ if err := os.MkdirAll(filepath.Join(e.dataRoot, "instances", "other.example.com"), 0o700); err != nil {
+ t.Fatal(err)
+ }
+ r := runChecks(e)
+ if r.Ready || verdict(r, "data-tree") != "fail" {
+ t.Errorf("sibling visible yet ready=%v data-tree=%s", r.Ready, verdict(r, "data-tree"))
+ }
+}
+
+func TestProbeFailsWhenCgroupfsIsWritable(t *testing.T) {
+ e := goodEnv(t)
+ e.writable = func(string) error { return nil }
+ r := runChecks(e)
+ if r.Ready || verdict(r, "cgroup-ro") != "fail" {
+ t.Errorf("writable cgroupfs yet ready=%v", r.Ready)
+ }
+}
+
+func TestProbeFailsWhenEtcCarlosIsReadable(t *testing.T) {
+ e := goodEnv(t)
+ if err := os.MkdirAll(e.etcCarlos, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(e.etcCarlos, "probe-secret"), []byte("probe"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ r := runChecks(e)
+ if r.Ready || verdict(r, "etc-carlos") != "fail" {
+ t.Errorf("readable /etc/carlos yet ready=%v", r.Ready)
+ }
+}
+
+func TestProbeIMDSVerdicts(t *testing.T) {
+ // v4 must be EPERM; anything else is a failure, including a timeout.
+ e := goodEnv(t)
+ e.sendUDP = func(_, addr string) error {
+ if addr == "169.254.169.254:80" {
+ return syscall.ETIMEDOUT
+ }
+ return syscall.EPERM
+ }
+ if r := runChecks(e); r.Ready || verdict(r, "imds-v4") != "fail" {
+ t.Errorf("v4 timeout: ready=%v v4=%s", r.Ready, verdict(r, "imds-v4"))
+ }
+ // v6 ENETUNREACH is unverified and does not block readiness.
+ e = goodEnv(t)
+ e.sendUDP = func(_, addr string) error {
+ if addr == "[fd00:ec2::254]:80" {
+ return syscall.ENETUNREACH
+ }
+ if addr == "169.254.169.254:80" {
+ return syscall.EPERM
+ }
+ return nil
+ }
+ if r := runChecks(e); !r.Ready || verdict(r, "imds-v6") != "unverified" {
+ t.Errorf("v6 no route: ready=%v v6=%s", r.Ready, verdict(r, "imds-v6"))
+ }
+ // A successful send to either address is a failure.
+ e = goodEnv(t)
+ e.sendUDP = func(string, string) error { return nil }
+ if r := runChecks(e); r.Ready || verdict(r, "imds-v4") != "fail" {
+ t.Errorf("v4 success: ready=%v", r.Ready)
+ }
+ // A dead loopback means the stack, not the filter, said no.
+ e = goodEnv(t)
+ e.sendUDP = func(_, addr string) error {
+ if addr == "127.0.0.1:9" {
+ return syscall.ENETUNREACH
+ }
+ return syscall.EPERM
+ }
+ if r := runChecks(e); r.Ready || verdict(r, "loopback") != "fail" {
+ t.Errorf("dead loopback: ready=%v", r.Ready)
+ }
+ // A TCP connect that succeeds is a failure.
+ e = goodEnv(t)
+ e.connectTCP = func(string, time.Duration) error { return nil }
+ if r := runChecks(e); r.Ready || verdict(r, "imds-tcp") != "fail" {
+ t.Errorf("tcp success: ready=%v", r.Ready)
+ }
+}
+
+func TestProbeSymlinkCheckUsesARealSymlink(t *testing.T) {
+ // The fixture's data tree is a plain temp dir with no nosymfollow, so
+ // the real symlink check FAILS here: that proves the check is real.
+ e := goodEnv(t)
+ if !errors.Is(realSymlinkCheck(filepath.Join(e.dataRoot, "instances", "_probe")), errNotELOOP) {
+ t.Error("a followable symlink was not reported")
+ }
+}