| 1 | //go:build linux |
| 2 | |
| 3 | package operator |
| 4 | |
| 5 | import ( |
| 6 | "bytes" |
| 7 | "context" |
| 8 | "database/sql" |
| 9 | "encoding/json" |
| 10 | "errors" |
| 11 | "io" |
| 12 | "os" |
| 13 | "os/exec" |
| 14 | "path/filepath" |
| 15 | "testing" |
| 16 | "time" |
| 17 | |
| 18 | sharedseal "amadan.net/oficina/home/handoff/seal" |
| 19 | "golang.org/x/sys/unix" |
| 20 | "modernc.org/sqlite" |
| 21 | ) |
| 22 | |
| 23 | type openProbe struct { |
| 24 | Path, Other, Mode string |
| 25 | Device, Inode uint64 |
| 26 | } |
| 27 | |
| 28 | func seedOpenDB(t *testing.T, path string) { |
| 29 | t.Helper() |
| 30 | db, e := sql.Open("sqlite", "file:"+path) |
| 31 | if e != nil { |
| 32 | t.Fatal(e) |
| 33 | } |
| 34 | if _, e = db.Exec("PRAGMA journal_mode=WAL; CREATE TABLE changes(value INTEGER)"); e != nil { |
| 35 | t.Fatal(e) |
| 36 | } |
| 37 | if e = db.Close(); e != nil { |
| 38 | t.Fatal(e) |
| 39 | } |
| 40 | } |
| 41 | func TestBoundDatabaseSubprocess(t *testing.T) { |
| 42 | if os.Getenv("OFICINA_OPEN_PROBE") == "1" { |
| 43 | raw, e := io.ReadAll(io.LimitReader(os.Stdin, 16384)) |
| 44 | if e != nil { |
| 45 | t.Fatal(e) |
| 46 | } |
| 47 | var p openProbe |
| 48 | if json.Unmarshal(raw, &p) != nil { |
| 49 | t.Fatal("probe input") |
| 50 | } |
| 51 | if p.Mode == "contender" { |
| 52 | db, err := sql.Open("sqlite", "file:"+p.Path+"?mode=rw") |
| 53 | if err != nil { |
| 54 | t.Fatal(err) |
| 55 | } |
| 56 | defer db.Close() |
| 57 | _, err = db.Exec("BEGIN IMMEDIATE") |
| 58 | var sqliteErr *sqlite.Error |
| 59 | if !errors.As(err, &sqliteErr) || sqliteErr.Code() != 5 { |
| 60 | t.Fatal("expected SQLITE_BUSY while owner holds transaction", err) |
| 61 | } |
| 62 | return |
| 63 | } |
| 64 | saved := p.Path + ".saved" |
| 65 | swap := func() { |
| 66 | t.Helper() |
| 67 | if e := os.Rename(p.Path, saved); e != nil { |
| 68 | t.Fatal(e) |
| 69 | } |
| 70 | if e := os.Rename(p.Other, p.Path); e != nil { |
| 71 | t.Fatal(e) |
| 72 | } |
| 73 | } |
| 74 | restore := func() { |
| 75 | t.Helper() |
| 76 | if e := os.Rename(p.Path, p.Other); e != nil { |
| 77 | t.Fatal(e) |
| 78 | } |
| 79 | if e := os.Rename(saved, p.Path); e != nil { |
| 80 | t.Fatal(e) |
| 81 | } |
| 82 | } |
| 83 | if p.Mode == "symlink" { |
| 84 | if e = os.Rename(p.Path, saved); e != nil { |
| 85 | t.Fatal(e) |
| 86 | } |
| 87 | if e = os.Symlink(p.Other, p.Path); e != nil { |
| 88 | t.Fatal(e) |
| 89 | } |
| 90 | } |
| 91 | if p.Mode == "symlink-ancestor" { |
| 92 | link := filepath.Join(filepath.Dir(p.Path), "alias") |
| 93 | if e = os.Symlink(filepath.Dir(p.Path), link); e != nil { |
| 94 | t.Fatal(e) |
| 95 | } |
| 96 | p.Path = filepath.Join(link, filepath.Base(p.Path)) |
| 97 | } |
| 98 | if p.Mode == "missing" { |
| 99 | if e = os.Remove(p.Path); e != nil { |
| 100 | t.Fatal(e) |
| 101 | } |
| 102 | } |
| 103 | if p.Mode == "hardlink" { |
| 104 | if e = os.Link(p.Path, saved); e != nil { |
| 105 | t.Fatal(e) |
| 106 | } |
| 107 | } |
| 108 | if p.Mode == "inherited-writeonly" || p.Mode == "inherited-readwrite" { |
| 109 | flag := os.O_WRONLY |
| 110 | if p.Mode == "inherited-readwrite" { |
| 111 | flag = os.O_RDWR |
| 112 | } |
| 113 | f, e := os.OpenFile(p.Other, flag, 0) |
| 114 | if e != nil { |
| 115 | t.Fatal(e) |
| 116 | } |
| 117 | defer f.Close() |
| 118 | } |
| 119 | deadline := time.Now().Add(time.Minute) |
| 120 | if p.Mode == "expired" { |
| 121 | deadline = time.Now().Add(-time.Second) |
| 122 | } |
| 123 | bound, e := newBoundDatabase(p.Path, p.Device, p.Inode, deadline) |
| 124 | early := map[string]bool{"symlink": true, "symlink-ancestor": true, "missing": true, "hardlink": true, "expired": true} |
| 125 | if early[p.Mode] { |
| 126 | if e != ErrRefused { |
| 127 | if bound != nil { |
| 128 | bound.close() |
| 129 | } |
| 130 | t.Fatal("pin refusal", e) |
| 131 | } |
| 132 | return |
| 133 | } |
| 134 | if e != nil { |
| 135 | t.Fatal(e) |
| 136 | } |
| 137 | defer bound.close() |
| 138 | if p.Mode == "replace-restore" { |
| 139 | bound.connector.hook = func(stage string) { |
| 140 | if stage == "before-driver" { |
| 141 | swap() |
| 142 | } |
| 143 | if stage == "after-open" { |
| 144 | restore() |
| 145 | } |
| 146 | } |
| 147 | } |
| 148 | if p.Mode == "replace-after" { |
| 149 | bound.connector.hook = func(stage string) { |
| 150 | if stage == "after-open" { |
| 151 | swap() |
| 152 | } |
| 153 | } |
| 154 | } |
| 155 | var extra *os.File |
| 156 | defer func() { |
| 157 | if extra != nil { |
| 158 | extra.Close() |
| 159 | } |
| 160 | }() |
| 161 | if p.Mode == "symlink-after" || p.Mode == "ambiguous" { |
| 162 | bound.connector.hook = func(stage string) { |
| 163 | if stage != "after-open" { |
| 164 | return |
| 165 | } |
| 166 | if p.Mode == "ambiguous" { |
| 167 | extra, e = os.OpenFile(p.Path, os.O_RDWR, 0) |
| 168 | if e != nil { |
| 169 | t.Fatal(e) |
| 170 | } |
| 171 | } else { |
| 172 | if e = os.Rename(p.Path, saved); e != nil { |
| 173 | t.Fatal(e) |
| 174 | } |
| 175 | if e = os.Symlink(saved, p.Path); e != nil { |
| 176 | t.Fatal(e) |
| 177 | } |
| 178 | } |
| 179 | } |
| 180 | } |
| 181 | e = bound.db.PingContext(context.Background()) |
| 182 | refused := p.Mode == "symlink-after" || p.Mode == "ambiguous" || p.Mode == "replace-restore" || p.Mode == "replace-after" || p.Mode == "inherited-writeonly" || p.Mode == "inherited-readwrite" |
| 183 | if refused { |
| 184 | if e != ErrRefused { |
| 185 | t.Fatal("connection refusal", e) |
| 186 | } |
| 187 | return |
| 188 | } |
| 189 | if e != nil { |
| 190 | t.Fatal("ordinary open", e) |
| 191 | } |
| 192 | if p.Mode == "locks" || p.Mode == "reconnect" { |
| 193 | if p.Mode == "reconnect" { |
| 194 | bound.db.SetMaxIdleConns(0) |
| 195 | if err := bound.db.Ping(); err != nil { |
| 196 | t.Fatal("reconnect", err) |
| 197 | } |
| 198 | bound.db.SetMaxIdleConns(1) |
| 199 | } |
| 200 | conn, err := bound.db.Conn(context.Background()) |
| 201 | if err != nil { |
| 202 | t.Fatal(err) |
| 203 | } |
| 204 | defer conn.Close() |
| 205 | if _, err = conn.ExecContext(context.Background(), "BEGIN IMMEDIATE"); err != nil { |
| 206 | t.Fatal(err) |
| 207 | } |
| 208 | defer conn.ExecContext(context.Background(), "ROLLBACK") |
| 209 | if _, err = conn.ExecContext(context.Background(), "INSERT INTO changes VALUES(1)"); err != nil { |
| 210 | t.Fatal(err) |
| 211 | } |
| 212 | p.Mode = "contender" |
| 213 | raw, _ := json.Marshal(p) |
| 214 | cmd := exec.Command(os.Args[0], "-test.run=^TestBoundDatabaseSubprocess$") |
| 215 | cmd.Env = append(os.Environ(), "OFICINA_OPEN_PROBE=1") |
| 216 | cmd.Stdin = bytes.NewReader(raw) |
| 217 | if output, err := cmd.CombinedOutput(); err != nil { |
| 218 | t.Fatalf("lock contender: %v %s", err, output) |
| 219 | } |
| 220 | return |
| 221 | } |
| 222 | if p.Mode == "wal-replaced" || p.Mode == "shm-symlink" || p.Mode == "foreign-descriptor" || p.Mode == "unlinked-temp" || p.Mode == "expired-connected" { |
| 223 | conn, err := bound.db.Conn(context.Background()) |
| 224 | if err != nil { |
| 225 | t.Fatal(err) |
| 226 | } |
| 227 | defer conn.Close() |
| 228 | var count int |
| 229 | if err = conn.QueryRowContext(context.Background(), "SELECT count(*) FROM changes").Scan(&count); err != nil { |
| 230 | t.Fatal(err) |
| 231 | } |
| 232 | if p.Mode == "wal-replaced" || p.Mode == "shm-symlink" { |
| 233 | suffix := "-wal" |
| 234 | if p.Mode == "shm-symlink" { |
| 235 | suffix = "-shm" |
| 236 | } |
| 237 | if err = os.Rename(p.Path+suffix, p.Path+suffix+".saved"); err != nil { |
| 238 | t.Fatal(err) |
| 239 | } |
| 240 | if p.Mode == "shm-symlink" { |
| 241 | err = os.Symlink(p.Path+suffix+".saved", p.Path+suffix) |
| 242 | } else { |
| 243 | err = os.WriteFile(p.Path+suffix, nil, 0600) |
| 244 | } |
| 245 | if err != nil { |
| 246 | t.Fatal(err) |
| 247 | } |
| 248 | defer func() { |
| 249 | os.Remove(p.Path + suffix) |
| 250 | os.Rename(p.Path+suffix+".saved", p.Path+suffix) |
| 251 | }() |
| 252 | } else if p.Mode == "expired-connected" { |
| 253 | bound.connector.deadline = time.Now().Add(-time.Second) |
| 254 | } else { |
| 255 | f, err := os.OpenFile(p.Other, os.O_RDWR, 0) |
| 256 | if err != nil { |
| 257 | t.Fatal(err) |
| 258 | } |
| 259 | defer f.Close() |
| 260 | if p.Mode == "unlinked-temp" { |
| 261 | if err = os.Remove(p.Other); err != nil { |
| 262 | t.Fatal(err) |
| 263 | } |
| 264 | } |
| 265 | } |
| 266 | if _, err = conn.ExecContext(context.Background(), "INSERT INTO changes VALUES(1)"); err != ErrRefused { |
| 267 | t.Fatal("boundary refusal", err) |
| 268 | } |
| 269 | return |
| 270 | } |
| 271 | if p.Mode == "post-commit" { |
| 272 | bound.connector.hook = func(stage string) { |
| 273 | if stage == "after-exec" { |
| 274 | swap() |
| 275 | } |
| 276 | } |
| 277 | } |
| 278 | _, e = bound.db.ExecContext(context.Background(), "INSERT INTO changes VALUES(1)") |
| 279 | if p.Mode == "post-commit" { |
| 280 | if e != ErrUnavailable || errors.Is(e, sql.ErrConnDone) { |
| 281 | t.Fatal("commit ambiguity", e) |
| 282 | } |
| 283 | restore() |
| 284 | bound.connector.hook = nil |
| 285 | return |
| 286 | } |
| 287 | if e != nil { |
| 288 | t.Fatal("ordinary write", e) |
| 289 | } |
| 290 | return |
| 291 | } |
| 292 | modes := []string{"symlink-after", "symlink-ancestor", "ambiguous", "wal-replaced", "shm-symlink", "foreign-descriptor", "unlinked-temp", "expired-connected", "reconnect", "locks", "ordinary", "replace-restore", "replace-after", "symlink", "missing", "hardlink", "expired", "inherited-writeonly", "inherited-readwrite", "post-commit"} |
| 293 | for _, mode := range modes { |
| 294 | t.Run(mode, func(t *testing.T) { |
| 295 | dir := t.TempDir() |
| 296 | path := filepath.Join(dir, "home.db") |
| 297 | other := filepath.Join(dir, "other.db") |
| 298 | seedOpenDB(t, path) |
| 299 | seedOpenDB(t, other) |
| 300 | var stat unix.Stat_t |
| 301 | if e := unix.Stat(path, &stat); e != nil { |
| 302 | t.Fatal(e) |
| 303 | } |
| 304 | raw, e := json.Marshal(openProbe{path, other, mode, uint64(stat.Dev), stat.Ino}) |
| 305 | if e != nil { |
| 306 | t.Fatal(e) |
| 307 | } |
| 308 | cmd := exec.Command(os.Args[0], "-test.run=^TestBoundDatabaseSubprocess$") |
| 309 | cmd.Env = append(os.Environ(), "OFICINA_OPEN_PROBE=1") |
| 310 | cmd.Stdin = bytes.NewReader(raw) |
| 311 | if output, e := cmd.CombinedOutput(); e != nil { |
| 312 | t.Fatalf("probe failed: %v %s", e, output) |
| 313 | } |
| 314 | if mode == "missing" { |
| 315 | if _, e := os.Stat(path); !os.IsNotExist(e) { |
| 316 | t.Fatal("missing DB created") |
| 317 | } |
| 318 | return |
| 319 | } |
| 320 | readPath := path |
| 321 | if mode == "replace-after" || mode == "symlink" || mode == "symlink-after" { |
| 322 | readPath = path + ".saved" |
| 323 | } |
| 324 | db, e := sql.Open("sqlite", "file:"+readPath+"?mode=rw") |
| 325 | if e != nil { |
| 326 | t.Fatal(e) |
| 327 | } |
| 328 | defer db.Close() |
| 329 | var count int |
| 330 | if e = db.QueryRow("SELECT count(*) FROM changes").Scan(&count); e != nil { |
| 331 | t.Fatal(e) |
| 332 | } |
| 333 | expected := 0 |
| 334 | if mode == "ordinary" || mode == "post-commit" { |
| 335 | expected = 1 |
| 336 | } |
| 337 | if count != expected { |
| 338 | t.Fatal("wrong mutation count", count, expected) |
| 339 | } |
| 340 | }) |
| 341 | } |
| 342 | } |
| 343 | |
| 344 | func TestBoundDatabaseEngineSubprocess(t *testing.T) { |
| 345 | if os.Getenv("OFICINA_ENGINE_OPEN_PROBE") == "1" { |
| 346 | var p openProbe |
| 347 | if json.NewDecoder(io.LimitReader(os.Stdin, 16384)).Decode(&p) != nil { |
| 348 | t.Fatal("probe input") |
| 349 | } |
| 350 | bound, err := newBoundDatabase(p.Path, p.Device, p.Inode, time.Now().Add(time.Minute)) |
| 351 | if err != nil { |
| 352 | t.Fatal(err) |
| 353 | } |
| 354 | defer bound.close() |
| 355 | k, verifier, auth := fixture(t) |
| 356 | sealer, err := sharedseal.New(bytes.Repeat([]byte{6}, 32), "oficina/operator") |
| 357 | if err != nil { |
| 358 | t.Fatal(err) |
| 359 | } |
| 360 | now := time.Unix(auth.Issued, 0) |
| 361 | engine, err := NewEngine(bound.db, verifier, sealer, bytes.Repeat([]byte{7}, 32), func() time.Time { return now }) |
| 362 | if err != nil { |
| 363 | t.Fatal(err) |
| 364 | } |
| 365 | raw := signed(t, k, auth) |
| 366 | out, err := engine.Execute(context.Background(), raw, insertOne) |
| 367 | if err != nil || !bytes.Equal(out, privateResult) { |
| 368 | t.Fatal("bound engine", err) |
| 369 | } |
| 370 | now = now.Add(301 * time.Second) |
| 371 | out, err = engine.Execute(context.Background(), raw, func(context.Context, *sql.Conn, Request) ([]byte, error) { |
| 372 | t.Fatal("replay mutated") |
| 373 | return nil, nil |
| 374 | }) |
| 375 | if err != nil || !bytes.Equal(out, privateResult) { |
| 376 | t.Fatal("bound recovery", err) |
| 377 | } |
| 378 | return |
| 379 | } |
| 380 | f := newEngineFixture(t) |
| 381 | if err := f.db.Close(); err != nil { |
| 382 | t.Fatal(err) |
| 383 | } |
| 384 | var stat unix.Stat_t |
| 385 | if err := unix.Stat(f.path, &stat); err != nil { |
| 386 | t.Fatal(err) |
| 387 | } |
| 388 | raw, _ := json.Marshal(openProbe{Path: f.path, Device: uint64(stat.Dev), Inode: stat.Ino}) |
| 389 | cmd := exec.Command(os.Args[0], "-test.run=^TestBoundDatabaseEngineSubprocess$") |
| 390 | cmd.Env = append(os.Environ(), "OFICINA_ENGINE_OPEN_PROBE=1") |
| 391 | cmd.Stdin = bytes.NewReader(raw) |
| 392 | if out, err := cmd.CombinedOutput(); err != nil { |
| 393 | t.Fatalf("engine probe: %v %s", err, out) |
| 394 | } |
| 395 | db, err := sql.Open("sqlite", "file:"+f.path+"?mode=rw") |
| 396 | if err != nil { |
| 397 | t.Fatal(err) |
| 398 | } |
| 399 | defer db.Close() |
| 400 | if countRows(t, db, "test_items") != 1 || countRows(t, db, "directory_operator_results") != 1 { |
| 401 | t.Fatal("bound atomic recovery") |
| 402 | } |
| 403 | } |
| 404 | |
amadan