oficina / home Public

Journal private operations with durable floor recovery

Paul Campbell pushed by paul@keymail.dev a1d76186b60a597991fd3f0bb051aeb51dc2b024 parent 643ff56
3 files changed, +548 −0
  • internal/provisioner/journal_linux.go +339 −0
  • internal/provisioner/journal_linux_test.go +193 −0
  • internal/provisioner/storage_linux.go +16 −0

CI passed — run details

diff --git a/internal/provisioner/journal_linux.go b/internal/provisioner/journal_linux.go
new file mode 100644
index 0000000..b5b7cee
--- /dev/null
+++ b/internal/provisioner/journal_linux.go
@@ -0,0 +1,339 @@
+//go:build linux
+
+package provisioner
+
+import (
+ "bytes"
+ "context"
+ "database/sql"
+ "encoding/base64"
+ "errors"
+ "net/url"
+ "os"
+ "path/filepath"
+ "sync"
+ "time"
+
+ family "amadan.net/rastrillo/rastrillo/crypto"
+ _ "modernc.org/sqlite"
+)
+
+const recordContext = "oficina/provisioner/record/v1"
+const payloadContext = "oficina/provisioner/private-record/v1"
+
+type journalRecord struct {
+ Version int `json:"version"`
+ Identity floorIdentity `json:"identity"`
+ Installation string `json:"installation"`
+ Sequence int64 `json:"sequence"`
+ Previous string `json:"previous"`
+ Operation string `json:"operation"`
+ Action string `json:"action"`
+ Expected string `json:"expected_state"`
+ Created int64 `json:"created_at"`
+ CiphertextDigest string `json:"ciphertext_digest"`
+}
+type signedRecord struct {
+ Record journalRecord `json:"record"`
+ Signature string `json:"signature"`
+}
+type journal struct {
+ mu sync.Mutex
+ locks []*os.File
+ db *sql.DB
+ dir *privateDirectory
+ floor *nativeFloor
+ key *family.Keypair
+ now func() time.Time
+ boundary func(string) error
+}
+
+func openJournal(dir *privateDirectory, floor *nativeFloor, key *family.Keypair, create bool) (*journal, error) {
+ if dir == nil || floor == nil || key == nil || key.SignPriv == nil || key.BoxPriv == nil || !bytes.Equal(key.SignPub(), floor.public) {
+ return nil, ErrRefused
+ }
+ stateInfo, e := dir.file.Stat()
+ floorInfo, floorErr := floor.dir.file.Stat()
+ if e != nil || floorErr != nil || os.SameFile(stateInfo, floorInfo) {
+ return nil, ErrRefused
+ }
+ stateLock, e := dir.lock()
+ if e != nil {
+ return nil, e
+ }
+ floorLock, e := floor.dir.lock()
+ if e != nil {
+ stateLock.Close()
+ return nil, e
+ }
+ keepLocks := false
+ defer func() {
+ if !keepLocks {
+ floorLock.Close()
+ stateLock.Close()
+ }
+ }()
+ if create {
+ if e := dir.create("registry.db", nil); e != nil {
+ return nil, e
+ }
+ }
+ if _, e := dir.read("registry.db", 128<<10); e != nil {
+ // Registry size is bounded by record count and private payload limits, not
+ // the small-file reader; inspect its type without reading all ciphertext.
+ if e = checkRegistryFile(dir); e != nil {
+ return nil, e
+ }
+ }
+ u := url.URL{Scheme: "file", Path: filepath.Join(dir.file.Name(), "registry.db")}
+ q := u.Query()
+ q.Set("mode", "rw")
+ u.RawQuery = q.Encode()
+ db, e := sql.Open("sqlite", u.String())
+ if e != nil {
+ return nil, ErrUnavailable
+ }
+ db.SetMaxOpenConns(1)
+ j := &journal{db: db, dir: dir, floor: floor, key: key, now: time.Now, locks: []*os.File{stateLock, floorLock}}
+ fail := func() (*journal, error) { db.Close(); return nil, ErrUnavailable }
+ for _, statement := range []string{"PRAGMA foreign_keys=ON", "PRAGMA secure_delete=ON", "PRAGMA synchronous=FULL", "PRAGMA busy_timeout=1000", "PRAGMA journal_mode=WAL"} {
+ if _, e = db.Exec(statement); e != nil {
+ return fail()
+ }
+ }
+ var fk, secure, synchronous int
+ if db.QueryRow("PRAGMA foreign_keys").Scan(&fk) != nil || db.QueryRow("PRAGMA secure_delete").Scan(&secure) != nil || db.QueryRow("PRAGMA synchronous").Scan(&synchronous) != nil || fk != 1 || secure != 1 || synchronous != 2 {
+ return fail()
+ }
+ if create {
+ point, _, e := floor.current()
+ if e != nil || point != (floorPoint{}) {
+ db.Close()
+ return nil, ErrRefused
+ }
+ _, e = db.Exec(`BEGIN IMMEDIATE;
+CREATE TABLE identity(singleton INTEGER PRIMARY KEY CHECK(singleton=1), registry TEXT NOT NULL, device TEXT NOT NULL, public_key TEXT NOT NULL, sequence INTEGER NOT NULL, head TEXT NOT NULL);
+CREATE TABLE records(sequence INTEGER PRIMARY KEY CHECK(sequence>0), operation TEXT NOT NULL UNIQUE, signed_record BLOB NOT NULL, sealed_payload BLOB NOT NULL, state TEXT NOT NULL CHECK(state IN ('prepared','finalized')));
+INSERT INTO identity VALUES(1,?,?,?,0,''); COMMIT;`, floor.identity.Registry, floor.identity.Device, floor.identity.Public)
+ if e != nil {
+ return fail()
+ }
+ if e = dir.file.Sync(); e != nil {
+ return fail()
+ }
+ }
+ if e = j.recover(); e != nil {
+ db.Close()
+ return nil, e
+ }
+ keepLocks = true
+ return j, nil
+}
+func (j *journal) close() error {
+ j.mu.Lock()
+ defer j.mu.Unlock()
+ err := j.db.Close()
+ for _, lock := range j.locks {
+ lock.Close()
+ }
+ j.locks = nil
+ return err
+}
+func (j *journal) stop(stage string) error {
+ if j.boundary != nil {
+ return j.boundary(stage)
+ }
+ return nil
+}
+func (j *journal) validate(raw, ciphertext []byte) (journalRecord, error) {
+ var signed signedRecord
+ if len(raw) > 8192 || len(ciphertext) > 96<<10 || closedJSON(raw, &signed) != nil {
+ return journalRecord{}, ErrRefused
+ }
+ r := signed.Record
+ _, installation := tokenBytes(r.Installation, 32)
+ _, operation := tokenBytes(r.Operation, 32)
+ if r.Version != 1 || r.Identity != j.floor.identity || !installation || !operation || r.Sequence <= 0 || r.Sequence > maxRecords || r.Created <= 0 || r.Action == "" || len(r.Action) > 64 || len(r.Expected) > 64 || r.CiphertextDigest != hash(ciphertext) {
+ return journalRecord{}, ErrRefused
+ }
+ body, e := canonicalJSON(r)
+ sig, ok := tokenBytes(signed.Signature, 64)
+ if e != nil || !ok || !family.Verify(j.floor.public, recordContext, body, sig) {
+ return journalRecord{}, ErrRefused
+ }
+ return r, nil
+}
+func (j *journal) recover() error {
+ var id floorIdentity
+ var accepted floorPoint
+ if e := j.db.QueryRow("SELECT registry,device,public_key,sequence,head FROM identity WHERE singleton=1").Scan(&id.Registry, &id.Device, &id.Public, &accepted.Sequence, &accepted.Head); e != nil {
+ return ErrUnavailable
+ }
+ if id != j.floor.identity {
+ return ErrRefused
+ }
+ rows, e := j.db.Query("SELECT sequence,operation,signed_record,sealed_payload,state FROM records ORDER BY sequence")
+ if e != nil {
+ return ErrUnavailable
+ }
+ var latest floorPoint
+ var pending *floorPoint
+ for rows.Next() {
+ var seq int64
+ var op, state string
+ var raw, sealed []byte
+ if rows.Scan(&seq, &op, &raw, &sealed, &state) != nil {
+ rows.Close()
+ return ErrUnavailable
+ }
+ r, e := j.validate(raw, sealed)
+ if e != nil || pending != nil || seq != latest.Sequence+1 || r.Sequence != seq || r.Operation != op || r.Previous != latest.Head {
+ rows.Close()
+ return ErrRefused
+ }
+ point := floorPoint{seq, hash(raw)}
+ if state == "prepared" {
+ if latest != accepted {
+ rows.Close()
+ return ErrRefused
+ }
+ pending = &point
+ } else if state != "finalized" {
+ rows.Close()
+ return ErrRefused
+ }
+ latest = point
+ }
+ e = rows.Err()
+ rows.Close()
+ if e != nil {
+ return ErrUnavailable
+ }
+ if pending == nil && latest != accepted {
+ return ErrRefused
+ }
+ current, _, e := j.floor.current()
+ if e != nil {
+ return e
+ }
+ if pending == nil {
+ if current != accepted {
+ return ErrRefused
+ }
+ return nil
+ }
+ if current == accepted {
+ if e = j.floor.advance(j.key, accepted, *pending); e != nil {
+ return e
+ }
+ } else if current != *pending {
+ return ErrRefused
+ }
+ return j.finalize(accepted, *pending)
+}
+func (j *journal) finalize(old, next floorPoint) error {
+ tx, e := j.db.Begin()
+ if e != nil {
+ return ErrUnavailable
+ }
+ defer tx.Rollback()
+ result, e := tx.Exec("UPDATE identity SET sequence=?,head=? WHERE singleton=1 AND sequence=? AND head=?", next.Sequence, next.Head, old.Sequence, old.Head)
+ if e != nil {
+ return ErrUnavailable
+ }
+ n, e := result.RowsAffected()
+ if e != nil || n != 1 {
+ return ErrRefused
+ }
+ result, e = tx.Exec("UPDATE records SET state='finalized' WHERE sequence=? AND state='prepared'", next.Sequence)
+ if e != nil {
+ return ErrUnavailable
+ }
+ n, e = result.RowsAffected()
+ if e != nil || n != 1 {
+ return ErrRefused
+ }
+ if tx.Commit() != nil {
+ return ErrUnavailable
+ }
+ if _, e = j.db.Exec("PRAGMA wal_checkpoint(FULL)"); e != nil || j.dir.file.Sync() != nil {
+ return ErrUnavailable
+ }
+ return nil
+}
+func (j *journal) append(ctx context.Context, installation, operation, action, expected string, payload []byte) ([]byte, error) {
+ if j == nil || len(payload) == 0 || len(payload) > 64<<10 {
+ return nil, ErrRefused
+ }
+ j.mu.Lock()
+ defer j.mu.Unlock()
+ if len(j.locks) != 2 {
+ return nil, ErrRefused
+ }
+ if e := j.recover(); e != nil {
+ return nil, e
+ }
+ var oldRaw, oldSealed []byte
+ e := j.db.QueryRowContext(ctx, "SELECT signed_record,sealed_payload FROM records WHERE operation=?", operation).Scan(&oldRaw, &oldSealed)
+ if e == nil {
+ r, e := j.validate(oldRaw, oldSealed)
+ if e != nil || r.Installation != installation || r.Action != action || r.Expected != expected {
+ return nil, ErrRefused
+ }
+ plain, e := family.Open(j.key, payloadContext+"/"+operation, oldSealed)
+ if e != nil || !bytes.Equal(plain, payload) {
+ return nil, ErrRefused
+ }
+ return oldRaw, nil
+ }
+ if !errors.Is(e, sql.ErrNoRows) {
+ return nil, ErrUnavailable
+ }
+ old, _, e := j.floor.current()
+ if e != nil {
+ return nil, e
+ }
+ if old.Sequence >= maxRecords {
+ return nil, ErrRefused
+ }
+ sealed, e := family.Seal(j.key.BoxPub(), payloadContext+"/"+operation, payload)
+ if e != nil {
+ return nil, ErrUnavailable
+ }
+ r := journalRecord{1, j.floor.identity, installation, old.Sequence + 1, old.Head, operation, action, expected, j.now().Unix(), hash(sealed)}
+ body, e := canonicalJSON(r)
+ if e != nil {
+ return nil, e
+ }
+ sig, e := family.Sign(j.key, recordContext, body)
+ if e != nil {
+ return nil, ErrUnavailable
+ }
+ raw, e := canonicalJSON(signedRecord{r, base64.RawURLEncoding.EncodeToString(sig)})
+ if e != nil {
+ return nil, e
+ }
+ if _, e = j.validate(raw, sealed); e != nil {
+ return nil, e
+ }
+ if _, e = j.db.ExecContext(ctx, "INSERT INTO records VALUES(?,?,?,?,'prepared')", r.Sequence, operation, raw, sealed); e != nil {
+ return nil, ErrRefused
+ }
+ if e = j.stop("prepared"); e != nil {
+ return nil, e
+ }
+ next := floorPoint{r.Sequence, hash(raw)}
+ if e = j.floor.advance(j.key, old, next); e != nil {
+ return nil, e
+ }
+ if e = j.stop("floor"); e != nil {
+ return nil, e
+ }
+ if e = j.finalize(old, next); e != nil {
+ return nil, e
+ }
+ if e = j.stop("finalized"); e != nil {
+ return nil, e
+ }
+ return raw, nil
+}
diff --git a/internal/provisioner/journal_linux_test.go b/internal/provisioner/journal_linux_test.go
new file mode 100644
index 0000000..452d547
--- /dev/null
+++ b/internal/provisioner/journal_linux_test.go
@@ -0,0 +1,193 @@
+//go:build linux
+
+package provisioner
+
+import (
+ family "amadan.net/rastrillo/rastrillo/crypto"
+ "bytes"
+ "context"
+ "errors"
+ "io"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "testing"
+)
+
+func journalFixture(t *testing.T) *journal {
+ t.Helper()
+ f, k := floorFixture(t)
+ if e := f.initialize(k); e != nil {
+ t.Fatal(e)
+ }
+ d, e := openPrivateDirectory(persistentDirectory(t))
+ if e != nil {
+ t.Fatal(e)
+ }
+ t.Cleanup(func() { d.close() })
+ j, e := openJournal(d, f, k, true)
+ if e != nil {
+ t.Fatal(e)
+ }
+ t.Cleanup(func() { j.close() })
+ return j
+}
+func TestJournalCrashRecoveryAndExactReplay(t *testing.T) {
+ for _, boundary := range []string{"prepared", "floor", "finalized"} {
+ t.Run(boundary, func(t *testing.T) {
+ j := journalFixture(t)
+ synthetic := []byte("private-owner-481@example.invalid")
+ stop := errors.New("synthetic interruption")
+ j.boundary = func(stage string) error {
+ if stage == boundary {
+ return stop
+ }
+ return nil
+ }
+ if _, e := j.append(context.Background(), testToken(5), testToken(6), "directory.seed-team", "unseeded", synthetic); e != stop {
+ t.Fatal("boundary not reached", e)
+ }
+ j.close()
+ reopened, e := openJournal(j.dir, j.floor, j.key, false)
+ if e != nil {
+ t.Fatal("resume", e)
+ }
+ defer reopened.close()
+ first, e := reopened.append(context.Background(), testToken(5), testToken(6), "directory.seed-team", "unseeded", synthetic)
+ if e != nil {
+ t.Fatal(e)
+ }
+ repeated, e := reopened.append(context.Background(), testToken(5), testToken(6), "directory.seed-team", "unseeded", synthetic)
+ if e != nil || !bytes.Equal(first, repeated) {
+ t.Fatal("replay changed record", e)
+ }
+ if _, e = reopened.append(context.Background(), testToken(5), testToken(6), "directory.seed-team", "unseeded", []byte("changed")); e != ErrRefused {
+ t.Fatal("changed replay", e)
+ }
+ var n int
+ if e = reopened.db.QueryRow("SELECT count(*) FROM records WHERE state='finalized'").Scan(&n); e != nil || n != 1 {
+ t.Fatal("non-atomic journal", e)
+ }
+ for _, suffix := range []string{"", "-wal", "-shm"} {
+ data, e := os.ReadFile(filepath.Join(j.dir.file.Name(), "registry.db") + suffix)
+ if e != nil && !errors.Is(e, os.ErrNotExist) {
+ t.Fatal(e)
+ }
+ if bytes.Contains(data, synthetic) {
+ t.Fatal("plaintext journal payload")
+ }
+ }
+ })
+ }
+}
+func TestJournalSurvivingFloorRefusesOldBackup(t *testing.T) {
+ j := journalFixture(t)
+ if _, e := j.append(context.Background(), testToken(5), testToken(6), "directory.initialize", "absent", []byte("first private input")); e != nil {
+ t.Fatal(e)
+ }
+ if _, e := j.db.Exec("PRAGMA wal_checkpoint(TRUNCATE)"); e != nil {
+ t.Fatal(e)
+ }
+ backup, e := os.ReadFile(filepath.Join(j.dir.file.Name(), "registry.db"))
+ if e != nil {
+ t.Fatal(e)
+ }
+ if _, e = j.append(context.Background(), testToken(5), testToken(7), "directory.seed-team", "empty", []byte("second private input")); e != nil {
+ t.Fatal(e)
+ }
+ j.close()
+ if e = os.WriteFile(filepath.Join(j.dir.file.Name(), "registry.db"), backup, 0600); e != nil {
+ t.Fatal(e)
+ }
+ if restored, e := openJournal(j.dir, j.floor, j.key, false); e != ErrRefused {
+ if restored != nil {
+ restored.close()
+ }
+ t.Fatal("stale backup accepted", e)
+ }
+}
+
+func TestJournalProcessDeathRecovery(t *testing.T) {
+ type childInput struct {
+ State, Floor, Boundary string
+ Identity floorIdentity
+ Key []byte
+ }
+ if os.Getenv("OFICINA_JOURNAL_CHILD") == "1" {
+ raw, e := io.ReadAll(io.LimitReader(os.Stdin, 16385))
+ if e != nil || len(raw) > 16384 {
+ t.Fatal("private child input")
+ }
+ var in childInput
+ if closedJSON(raw, &in) != nil {
+ t.Fatal("child shape")
+ }
+ key, e := family.UnmarshalKeypair(in.Key)
+ if e != nil {
+ t.Fatal("child key")
+ }
+ dir, e := openPrivateDirectory(in.State)
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer dir.close()
+ floorDir, e := openPrivateDirectory(in.Floor)
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer floorDir.close()
+ floor, e := newNativeFloor(floorDir, in.Identity)
+ if e != nil {
+ t.Fatal(e)
+ }
+ j, e := openJournal(dir, floor, key, false)
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer j.close()
+ j.boundary = func(stage string) error {
+ if stage == in.Boundary {
+ os.Exit(72)
+ }
+ return nil
+ }
+ if _, e = j.append(context.Background(), testToken(5), testToken(6), "directory.seed-team", "empty", []byte("synthetic process-death payload")); e != nil {
+ t.Fatal(e)
+ }
+ t.Fatal("death boundary not reached")
+ }
+ for _, stage := range []string{"prepared", "floor", "finalized"} {
+ t.Run(stage, func(t *testing.T) {
+ j := journalFixture(t)
+ j.close()
+ key, e := family.MarshalKeypair(j.key)
+ if e != nil {
+ t.Fatal(e)
+ }
+ input, e := canonicalJSON(childInput{j.dir.file.Name(), j.floor.dir.file.Name(), stage, j.floor.identity, key})
+ if e != nil {
+ t.Fatal(e)
+ }
+ cmd := exec.Command(os.Args[0], "-test.run=^TestJournalProcessDeathRecovery$")
+ cmd.Env = append(os.Environ(), "OFICINA_JOURNAL_CHILD=1")
+ cmd.Stdin = bytes.NewReader(input)
+ output, e := cmd.CombinedOutput()
+ var exit *exec.ExitError
+ if !errors.As(e, &exit) || exit.ExitCode() != 72 {
+ t.Fatalf("child did not stop at durable boundary: %v %s", e, output)
+ }
+ reopened, e := openJournal(j.dir, j.floor, j.key, false)
+ if e != nil {
+ t.Fatal("process death resume", e)
+ }
+ defer reopened.close()
+ if _, e = reopened.append(context.Background(), testToken(5), testToken(6), "directory.seed-team", "empty", []byte("synthetic process-death payload")); e != nil {
+ t.Fatal(e)
+ }
+ point, _, e := j.floor.current()
+ if e != nil || point.Sequence != 1 {
+ t.Fatal("sequence reused", e)
+ }
+ })
+ }
+}
diff --git a/internal/provisioner/storage_linux.go b/internal/provisioner/storage_linux.go
index 4d544e1..9381f24 100644
--- a/internal/provisioner/storage_linux.go
+++ b/internal/provisioner/storage_linux.go
@@ -119,3 +119,19 @@ func (d *privateDirectory) lock() (*os.File, error) {
}
return f, nil
}
+
+func checkRegistryFile(d *privateDirectory) error {
+ fd, e := unix.Openat(int(d.file.Fd()), "registry.db", unix.O_RDONLY|unix.O_NOFOLLOW|unix.O_NONBLOCK|unix.O_CLOEXEC, 0)
+ if e != nil {
+ return ErrRefused
+ }
+ defer unix.Close(fd)
+ var s unix.Stat_t
+ if unix.Fstat(fd, &s) != nil {
+ return ErrUnavailable
+ }
+ if s.Mode&unix.S_IFMT != unix.S_IFREG || s.Mode&0777 != 0600 || s.Uid != uint32(os.Getuid()) || s.Nlink != 1 || s.Size > int64(maxRecords)*(128<<10) {
+ return ErrRefused
+ }
+ return nil
+}