amadan

oficina / home Public

Add native durable storage and signed floor history primitives

Paul Campbell pushed by paul@keymail.dev 8402a9675339c9ba2e43c655a0ca77889a314319 parent 39aaed1
4 files changed, +586 −0
  • internal/provisioner/floor_linux.go +211 −0
  • internal/provisioner/floor_linux_test.go +113 −0
  • internal/provisioner/storage_linux.go +121 −0
  • internal/provisioner/storage_linux_test.go +141 −0

CI failed — run details

Intent

full ledger: feat-home-operator-core

diff --git a/internal/provisioner/floor_linux.go b/internal/provisioner/floor_linux.go
new file mode 100644
index 0000000..e4a39b9
--- /dev/null
+++ b/internal/provisioner/floor_linux.go
@@ -0,0 +1,211 @@
+//go:build linux
+
+package provisioner
+
+import (
+ "bytes"
+ "crypto/elliptic"
+ "crypto/sha256"
+ "encoding/base64"
+ "fmt"
+ "io"
+ "os"
+ "sort"
+
+ family "amadan.net/rastrillo/rastrillo/crypto"
+ "github.com/go-json-experiment/json"
+ "github.com/go-json-experiment/json/jsontext"
+)
+
+const floorContext = "oficina/provisioner/floor/v1"
+const maxRecords = 10000
+
+type floorIdentity struct {
+ Registry string `json:"registry"`
+ Device string `json:"device"`
+ Public string `json:"public_key"`
+}
+type floorPoint struct {
+ Sequence int64 `json:"sequence"`
+ Head string `json:"head"`
+}
+type floorRecord struct {
+ Version int `json:"version"`
+ Identity floorIdentity `json:"identity"`
+ Point floorPoint `json:"point"`
+ Previous string `json:"previous"`
+}
+type floorEnvelope struct {
+ Record floorRecord `json:"record"`
+ Signature string `json:"signature"`
+}
+type nativeFloor struct {
+ dir *privateDirectory
+ identity floorIdentity
+ public []byte
+}
+
+func tokenBytes(s string, n int) ([]byte, bool) {
+ b, e := base64.RawURLEncoding.DecodeString(s)
+ return b, e == nil && len(b) == n && base64.RawURLEncoding.EncodeToString(b) == s
+}
+func canonicalJSON(v any) ([]byte, error) {
+ b, e := json.Marshal(v)
+ if e != nil {
+ return nil, ErrRefused
+ }
+ j := jsontext.Value(b)
+ if j.Canonicalize() != nil {
+ return nil, ErrRefused
+ }
+ return j, nil
+}
+func closedJSON(b []byte, v any) error {
+ if json.Unmarshal(b, v, json.RejectUnknownMembers(true)) != nil {
+ return ErrRefused
+ }
+ canonical, e := canonicalJSON(v)
+ if e != nil || !bytes.Equal(canonical, b) {
+ return ErrRefused
+ }
+ return nil
+}
+func hash(b []byte) string { h := sha256.Sum256(b); return base64.RawURLEncoding.EncodeToString(h[:]) }
+func floorName(sequence int64) string { return fmt.Sprintf("%020d.json", sequence) }
+func newNativeFloor(dir *privateDirectory, id floorIdentity) (*nativeFloor, error) {
+ _, a := tokenBytes(id.Registry, 32)
+ _, b := tokenBytes(id.Device, 32)
+ pub, c := tokenBytes(id.Public, 65)
+ x, y := elliptic.Unmarshal(elliptic.P256(), pub)
+ if dir == nil || !a || !b || !c || x == nil || y == nil {
+ return nil, ErrRefused
+ }
+ return &nativeFloor{dir, id, pub}, nil
+}
+func (f *nativeFloor) signed(k *family.Keypair, p floorPoint, previous string) ([]byte, error) {
+ if k == nil || k.SignPriv == nil || !bytes.Equal(k.SignPub(), f.public) {
+ return nil, ErrRefused
+ }
+ r := floorRecord{1, f.identity, p, previous}
+ body, e := canonicalJSON(r)
+ if e != nil {
+ return nil, e
+ }
+ sig, e := family.Sign(k, floorContext, body)
+ if e != nil {
+ return nil, ErrUnavailable
+ }
+ return canonicalJSON(floorEnvelope{r, base64.RawURLEncoding.EncodeToString(sig)})
+}
+func (f *nativeFloor) initialize(k *family.Keypair) error {
+ files, e := f.names()
+ if e != nil {
+ return e
+ }
+ if len(files) != 0 {
+ return ErrRefused
+ }
+ raw, e := f.signed(k, floorPoint{}, "")
+ if e != nil {
+ return e
+ }
+ return f.dir.create(floorName(0), raw)
+}
+func (f *nativeFloor) names() ([]string, error) {
+ // Open a fresh directory description: a retained Readdir offset must never
+ // hide files appended since a previous verification.
+ fd, e := openPrivateDirectory(f.dir.file.Name())
+ if e != nil {
+ return nil, e
+ }
+ defer fd.close()
+ oldInfo, e := f.dir.file.Stat()
+ if e != nil {
+ return nil, ErrUnavailable
+ }
+ newInfo, e := fd.file.Stat()
+ if e != nil || !os.SameFile(oldInfo, newInfo) {
+ return nil, ErrRefused
+ }
+ entries, e := fd.file.Readdirnames(maxRecords + 3)
+ if e != nil && e != io.EOF {
+ return nil, ErrUnavailable
+ }
+ if len(entries) > maxRecords+2 {
+ return nil, ErrRefused
+ }
+ names := make([]string, 0, len(entries))
+ for _, name := range entries {
+ if name == "lock" {
+ continue
+ }
+ if len(name) != 25 || name[20:] != ".json" {
+ return nil, ErrRefused
+ }
+ names = append(names, name)
+ }
+ sort.Strings(names)
+ return names, nil
+}
+func (f *nativeFloor) current() (floorPoint, string, error) {
+ names, e := f.names()
+ if e != nil {
+ return floorPoint{}, "", e
+ }
+ if len(names) == 0 || len(names) > maxRecords+1 {
+ return floorPoint{}, "", ErrRefused
+ }
+ var last floorPoint
+ var previous string
+ for i, name := range names {
+ if name != floorName(int64(i)) {
+ return floorPoint{}, "", ErrRefused
+ }
+ raw, e := f.dir.read(name, 8192)
+ if e != nil {
+ return floorPoint{}, "", e
+ }
+ var env floorEnvelope
+ if closedJSON(raw, &env) != nil || env.Record.Version != 1 || env.Record.Identity != f.identity || env.Record.Point.Sequence != int64(i) || env.Record.Previous != previous {
+ return floorPoint{}, "", ErrRefused
+ }
+ r := env.Record
+ if i == 0 {
+ if r.Point.Head != "" {
+ return floorPoint{}, "", ErrRefused
+ }
+ } else {
+ if _, ok := tokenBytes(r.Point.Head, 32); !ok {
+ return floorPoint{}, "", ErrRefused
+ }
+ }
+ sig, ok := tokenBytes(env.Signature, 64)
+ body, e := canonicalJSON(r)
+ if !ok || e != nil || !family.Verify(f.public, floorContext, body, sig) {
+ return floorPoint{}, "", ErrRefused
+ }
+ previous = hash(raw)
+ last = r.Point
+ }
+ return last, previous, nil
+}
+
+// advance requires the caller to hold the device process lock. It checks every
+// immutable floor record again, then exclusively creates the adjacent record.
+func (f *nativeFloor) advance(k *family.Keypair, expected, next floorPoint) error {
+ current, previous, e := f.current()
+ if e != nil {
+ return e
+ }
+ if current != expected || next.Sequence != expected.Sequence+1 || next.Sequence > maxRecords {
+ return ErrRefused
+ }
+ if _, ok := tokenBytes(next.Head, 32); !ok {
+ return ErrRefused
+ }
+ raw, e := f.signed(k, next, previous)
+ if e != nil {
+ return e
+ }
+ return f.dir.create(floorName(next.Sequence), raw)
+}
diff --git a/internal/provisioner/floor_linux_test.go b/internal/provisioner/floor_linux_test.go
new file mode 100644
index 0000000..9ad72a0
--- /dev/null
+++ b/internal/provisioner/floor_linux_test.go
@@ -0,0 +1,113 @@
+//go:build linux
+
+package provisioner
+
+import (
+ "encoding/base64"
+ "os"
+ "path/filepath"
+ "testing"
+
+ family "amadan.net/rastrillo/rastrillo/crypto"
+)
+
+func testToken(n byte) string {
+ b := make([]byte, 32)
+ for i := range b {
+ b[i] = n
+ }
+ return base64.RawURLEncoding.EncodeToString(b)
+}
+func floorFixture(t *testing.T) (*nativeFloor, *family.Keypair) {
+ t.Helper()
+ path := persistentDirectory(t)
+ dir, e := openPrivateDirectory(path)
+ if e != nil {
+ t.Fatal(e)
+ }
+ t.Cleanup(func() { dir.close() })
+ key, e := family.Generate()
+ if e != nil {
+ t.Fatal(e)
+ }
+ floor, e := newNativeFloor(dir, floorIdentity{testToken(1), testToken(2), base64.RawURLEncoding.EncodeToString(key.SignPub())})
+ if e != nil {
+ t.Fatal(e)
+ }
+ return floor, key
+}
+func TestFloorDurableCompareAndSwap(t *testing.T) {
+ f, k := floorFixture(t)
+ if _, _, e := f.current(); e != ErrRefused {
+ t.Fatal("absent floor accepted", e)
+ }
+ if e := f.initialize(k); e != nil {
+ t.Fatal(e)
+ }
+ if e := f.initialize(k); e != ErrRefused {
+ t.Fatal("floor reinitialized", e)
+ }
+ zero, _, e := f.current()
+ if e != nil || zero != (floorPoint{}) {
+ t.Fatal(e)
+ }
+ one := floorPoint{1, testToken(3)}
+ if e = f.advance(k, zero, one); e != nil {
+ t.Fatal(e)
+ }
+ if e = f.advance(k, zero, floorPoint{1, testToken(4)}); e != ErrRefused {
+ t.Fatal("stale registry rolled floor back", e)
+ }
+ again, _, e := f.current()
+ if e != nil || again != one {
+ t.Fatal("floor failed durable scan", e)
+ }
+ // Reopening with a competing device or key never creates a fresh genesis.
+ wrong, e := newNativeFloor(f.dir, floorIdentity{f.identity.Registry, testToken(9), f.identity.Public})
+ if e != nil {
+ t.Fatal(e)
+ }
+ if _, _, e = wrong.current(); e != ErrRefused {
+ t.Fatal("competing device accepted", e)
+ }
+}
+func TestFloorRefusesPartialTamperedAndMissingHistory(t *testing.T) {
+ for _, which := range []string{"partial", "tamper", "gap"} {
+ t.Run(which, func(t *testing.T) {
+ f, k := floorFixture(t)
+ if e := f.initialize(k); e != nil {
+ t.Fatal(e)
+ }
+ one := floorPoint{1, testToken(3)}
+ if e := f.advance(k, floorPoint{}, one); e != nil {
+ t.Fatal(e)
+ }
+ switch which {
+ case "partial":
+ if e := f.dir.create(floorName(2), []byte("{")); e != nil {
+ t.Fatal(e)
+ }
+ case "tamper":
+ path := filepath.Join(f.dir.file.Name(), floorName(1))
+ raw, e := os.ReadFile(path)
+ if e != nil {
+ t.Fatal(e)
+ }
+ raw[len(raw)-5] ^= 1
+ if e = os.WriteFile(path, raw, 0600); e != nil {
+ t.Fatal(e)
+ }
+ case "gap":
+ if e := f.advance(k, one, floorPoint{2, testToken(4)}); e != nil {
+ t.Fatal(e)
+ }
+ if e := os.Remove(filepath.Join(f.dir.file.Name(), floorName(1))); e != nil {
+ t.Fatal(e)
+ }
+ }
+ if _, _, e := f.current(); e != ErrRefused {
+ t.Fatal("damaged history accepted", e)
+ }
+ })
+ }
+}
diff --git a/internal/provisioner/storage_linux.go b/internal/provisioner/storage_linux.go
new file mode 100644
index 0000000..4d544e1
--- /dev/null
+++ b/internal/provisioner/storage_linux.go
@@ -0,0 +1,121 @@
+//go:build linux
+
+package provisioner
+
+import (
+ "errors"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "golang.org/x/sys/unix"
+)
+
+var ErrRefused = errors.New("provisioner: refused")
+var ErrUnavailable = errors.New("provisioner: unavailable")
+
+// privateDirectory retains an opened directory so later operations cannot follow
+// a replacement path or a symlink supplied in a file name.
+type privateDirectory struct{ file *os.File }
+
+func openPrivateDirectory(path string) (*privateDirectory, error) {
+ if !filepath.IsAbs(path) || filepath.Clean(path) != path || path == "/" {
+ return nil, ErrRefused
+ }
+ fd, err := unix.Open("/", unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC, 0)
+ if err != nil {
+ return nil, ErrUnavailable
+ }
+ for _, part := range strings.Split(strings.TrimPrefix(path, "/"), "/") {
+ next, e := unix.Openat(fd, part, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0)
+ unix.Close(fd)
+ if e != nil {
+ return nil, ErrRefused
+ }
+ fd = next
+ }
+ var stat unix.Stat_t
+ var fs unix.Statfs_t
+ if unix.Fstat(fd, &stat) != nil || unix.Fstatfs(fd, &fs) != nil {
+ unix.Close(fd)
+ return nil, ErrUnavailable
+ }
+ if stat.Uid != uint32(os.Getuid()) || stat.Mode&0777 != 0700 || (fs.Type != unix.EXT4_SUPER_MAGIC && fs.Type != unix.XFS_SUPER_MAGIC) {
+ unix.Close(fd)
+ return nil, ErrRefused
+ }
+ return &privateDirectory{os.NewFile(uintptr(fd), path)}, nil
+}
+func (d *privateDirectory) close() error { return d.file.Close() }
+func leaf(name string) bool {
+ return name != "" && name != "." && name != ".." && len(name) <= 128 && !strings.ContainsAny(name, "/\\\x00")
+}
+func (d *privateDirectory) create(name string, body []byte) error {
+ if !leaf(name) || len(body) > 128<<10 {
+ return ErrRefused
+ }
+ fd, e := unix.Openat(int(d.file.Fd()), name, unix.O_WRONLY|unix.O_CREAT|unix.O_EXCL|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0600)
+ if e != nil {
+ return ErrRefused
+ }
+ f := os.NewFile(uintptr(fd), name)
+ // A failed exclusive write remains present: recovery must validate or refuse
+ // the partial record instead of replacing evidence of an interrupted write.
+ _, e = f.Write(body)
+ if e == nil {
+ e = f.Sync()
+ }
+ closeErr := f.Close()
+ if e != nil || closeErr != nil || d.file.Sync() != nil {
+ return ErrUnavailable
+ }
+ return nil
+}
+func (d *privateDirectory) read(name string, limit int) ([]byte, error) {
+ if !leaf(name) || limit <= 0 || limit > 128<<10 {
+ return nil, ErrRefused
+ }
+ fd, e := unix.Openat(int(d.file.Fd()), name, unix.O_RDONLY|unix.O_NOFOLLOW|unix.O_NONBLOCK|unix.O_CLOEXEC, 0)
+ if e != nil {
+ return nil, ErrRefused
+ }
+ f := os.NewFile(uintptr(fd), name)
+ defer f.Close()
+ var s unix.Stat_t
+ if unix.Fstat(fd, &s) != nil {
+ return nil, ErrUnavailable
+ }
+ if s.Mode&unix.S_IFMT != unix.S_IFREG || s.Mode&0777 != 0600 || s.Uid != uint32(os.Getuid()) || s.Nlink != 1 || s.Size > int64(limit) {
+ return nil, ErrRefused
+ }
+ body, e := io.ReadAll(io.LimitReader(f, int64(limit)+1))
+ if e != nil {
+ return nil, ErrUnavailable
+ }
+ if len(body) > limit {
+ return nil, ErrRefused
+ }
+ return body, nil
+}
+func (d *privateDirectory) lock() (*os.File, error) {
+ fd, e := unix.Openat(int(d.file.Fd()), "lock", unix.O_RDWR|unix.O_CREAT|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0600)
+ if e != nil {
+ return nil, ErrRefused
+ }
+ f := os.NewFile(uintptr(fd), "lock")
+ var s unix.Stat_t
+ if unix.Fstat(fd, &s) != nil || s.Mode&unix.S_IFMT != unix.S_IFREG || s.Mode&0777 != 0600 || s.Uid != uint32(os.Getuid()) || s.Nlink != 1 {
+ f.Close()
+ return nil, ErrRefused
+ }
+ if unix.Flock(fd, unix.LOCK_EX|unix.LOCK_NB) != nil {
+ f.Close()
+ return nil, ErrRefused
+ }
+ if f.Sync() != nil || d.file.Sync() != nil {
+ f.Close()
+ return nil, ErrUnavailable
+ }
+ return f, nil
+}
diff --git a/internal/provisioner/storage_linux_test.go b/internal/provisioner/storage_linux_test.go
new file mode 100644
index 0000000..a8bb52c
--- /dev/null
+++ b/internal/provisioner/storage_linux_test.go
@@ -0,0 +1,141 @@
+//go:build linux
+
+package provisioner
+
+import (
+ "bytes"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "testing"
+)
+
+func persistentDirectory(t *testing.T) string {
+ t.Helper()
+ // The native backend deliberately rejects /tmp when it is tmpfs. Exercise
+ // durable operations on this checkout's persistent filesystem instead.
+ path, err := os.MkdirTemp(".", ".provisioner-test-")
+ if err != nil {
+ t.Fatal(err)
+ }
+ path, err = filepath.Abs(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { os.RemoveAll(path) })
+ if err = os.Chmod(path, 0700); err != nil {
+ t.Fatal(err)
+ }
+ return path
+}
+func TestNativeStorageExclusiveDurableRecords(t *testing.T) {
+ path := persistentDirectory(t)
+ d, err := openPrivateDirectory(path)
+ if err != nil {
+ t.Fatal("native filesystem required", err)
+ }
+ body := []byte("synthetic signed record")
+ if err = d.create("00000001", body); err != nil {
+ t.Fatal(err)
+ }
+ if err = d.create("00000001", []byte("replacement")); err != ErrRefused {
+ t.Fatal("immutable record replaced", err)
+ }
+ d.close()
+ d, err = openPrivateDirectory(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer d.close()
+ got, err := d.read("00000001", 128)
+ if err != nil || !bytes.Equal(got, body) {
+ t.Fatal("durable reopen", err)
+ }
+ if _, err = d.read("00000001", 2); err != ErrRefused {
+ t.Fatal("unbounded read", err)
+ }
+ if err = os.Link(filepath.Join(path, "00000001"), filepath.Join(path, "alias")); err != nil {
+ t.Fatal(err)
+ }
+ if _, err = d.read("00000001", 128); err != ErrRefused {
+ t.Fatal("hardlink accepted", err)
+ }
+}
+func TestNativeStorageRefusesSymlinksAndPermissions(t *testing.T) {
+ path := persistentDirectory(t)
+ real := filepath.Join(path, "real")
+ if err := os.Mkdir(real, 0700); err != nil {
+ t.Fatal(err)
+ }
+ alias := filepath.Join(path, "alias")
+ if err := os.Symlink(real, alias); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := openPrivateDirectory(alias); err != ErrRefused {
+ t.Fatal("symlink directory", err)
+ }
+ if err := os.Mkdir(filepath.Join(real, "nested"), 0700); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := openPrivateDirectory(filepath.Join(alias, "nested")); err != ErrRefused {
+ t.Fatal("symlink ancestor", err)
+ }
+ d, err := openPrivateDirectory(real)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer d.close()
+ if err = os.Symlink("target", filepath.Join(real, "record")); err != nil {
+ t.Fatal(err)
+ }
+ if err = d.create("record", []byte("x")); err != ErrRefused {
+ t.Fatal("followed record symlink", err)
+ }
+ if _, err = d.read("record", 128); err != ErrRefused {
+ t.Fatal("read record symlink", err)
+ }
+ if err = os.Chmod(real, 0755); err != nil {
+ t.Fatal(err)
+ }
+ if _, err = openPrivateDirectory(real); err != ErrRefused {
+ t.Fatal("public directory", err)
+ }
+}
+func TestNativeStorageProcessLock(t *testing.T) {
+ if path := os.Getenv("OFICINA_TEST_LOCK_PATH"); path != "" {
+ d, err := openPrivateDirectory(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer d.close()
+ lock, err := d.lock()
+ if lock != nil {
+ lock.Close()
+ }
+ if err != ErrRefused {
+ t.Fatal("second process acquired lock", err)
+ }
+ return
+ }
+ path := persistentDirectory(t)
+ d, err := openPrivateDirectory(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer d.close()
+ lock, err := d.lock()
+ if err != nil {
+ t.Fatal(err)
+ }
+ cmd := exec.Command(os.Args[0], "-test.run=^TestNativeStorageProcessLock$")
+ cmd.Env = append(os.Environ(), "OFICINA_TEST_LOCK_PATH="+path)
+ if output, err := cmd.CombinedOutput(); err != nil {
+ t.Fatalf("child: %s: %v", output, err)
+ }
+ lock.Close()
+ lock, err = d.lock()
+ if err != nil {
+ t.Fatal("lock did not release", err)
+ }
+ lock.Close()
+}