| 1 | //go:build linux |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "bytes" |
| 7 | "context" |
| 8 | "errors" |
| 9 | "os" |
| 10 | "path/filepath" |
| 11 | "strings" |
| 12 | "sync/atomic" |
| 13 | "syscall" |
| 14 | "testing" |
| 15 | "time" |
| 16 | |
| 17 | "github.com/carlosframework/platform/internal/hostclaim" |
| 18 | "github.com/carlosframework/platform/internal/hostfiles" |
| 19 | "github.com/carlosframework/platform/internal/instancefs" |
| 20 | "github.com/carlosframework/platform/internal/instancekey" |
| 21 | "github.com/carlosframework/platform/internal/registry" |
| 22 | "github.com/carlosframework/platform/internal/release" |
| 23 | ) |
| 24 | |
| 25 | // What a tenant can leave at a name in its own instance directory, and the |
| 26 | // error main's safe read (instancefs) answers it with. |
| 27 | type tenantPlant struct { |
| 28 | name string |
| 29 | want error |
| 30 | } |
| 31 | |
| 32 | var tenantPlants = []tenantPlant{ |
| 33 | {"a symlink to the right bytes", instancefs.ErrSymlink}, |
| 34 | {"a FIFO", instancefs.ErrNotRegular}, |
| 35 | {"an oversized file", instancefs.ErrTooLarge}, |
| 36 | {"a symlinked parent directory", instancefs.ErrSymlink}, |
| 37 | } |
| 38 | |
| 39 | // plantAt replaces the file at path with what p names. body is the file's |
| 40 | // legitimate bytes: the symlink points at a copy of them, so a read that |
| 41 | // followed it would see nothing wrong; the oversized file is body padded |
| 42 | // past limit with what a reader of the file would ignore (pad), so a read |
| 43 | // that took it whole would see nothing wrong either. |
| 44 | func plantAt(t *testing.T, p tenantPlant, path string, body []byte, limit int64, pad byte) { |
| 45 | t.Helper() |
| 46 | switch p.name { |
| 47 | case "a symlink to the right bytes": |
| 48 | real := filepath.Join(t.TempDir(), "real") |
| 49 | if err := os.WriteFile(real, body, 0o600); err != nil { |
| 50 | t.Fatal(err) |
| 51 | } |
| 52 | if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { |
| 53 | t.Fatal(err) |
| 54 | } |
| 55 | if err := os.Symlink(real, path); err != nil { |
| 56 | t.Fatal(err) |
| 57 | } |
| 58 | case "a FIFO": |
| 59 | if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { |
| 60 | t.Fatal(err) |
| 61 | } |
| 62 | if err := syscall.Mkfifo(path, 0o600); err != nil { |
| 63 | t.Fatal(err) |
| 64 | } |
| 65 | case "an oversized file": |
| 66 | big := append(append([]byte(nil), body...), bytes.Repeat([]byte{pad}, int(limit)+1-len(body))...) |
| 67 | if err := os.WriteFile(path, big, 0o600); err != nil { |
| 68 | t.Fatal(err) |
| 69 | } |
| 70 | case "a symlinked parent directory": |
| 71 | dir := filepath.Dir(path) |
| 72 | moved := dir + ".moved" |
| 73 | if err := os.Rename(dir, moved); err != nil { |
| 74 | t.Fatal(err) |
| 75 | } |
| 76 | if err := os.Symlink(moved, dir); err != nil { |
| 77 | t.Fatal(err) |
| 78 | } |
| 79 | default: |
| 80 | t.Fatalf("unknown plant %q", p.name) |
| 81 | } |
| 82 | } |
| 83 | |
| 84 | // bounded runs fn and fails the test if it does not return in time: a read |
| 85 | // that opened a planted FIFO without O_NONBLOCK would block for ever. |
| 86 | func bounded(t *testing.T, fn func() error) error { |
| 87 | t.Helper() |
| 88 | done := make(chan error, 1) |
| 89 | go func() { done <- fn() }() |
| 90 | select { |
| 91 | case err := <-done: |
| 92 | return err |
| 93 | case <-time.After(20 * time.Second): |
| 94 | t.Fatal("the read did not return: it blocked on what the tenant planted") |
| 95 | return nil |
| 96 | } |
| 97 | } |
| 98 | |
| 99 | // readInstanceFile is main's safe read for tenant-writable ground, and |
| 100 | // absence only on positive evidence. |
| 101 | func TestReadInstanceFileRefusesWhatTheTenantCanPlant(t *testing.T) { |
| 102 | const max = 64 |
| 103 | body := []byte("KEY=v\n") |
| 104 | for _, p := range tenantPlants { |
| 105 | t.Run(p.name, func(t *testing.T) { |
| 106 | dir := filepath.Join(t.TempDir(), "inst") |
| 107 | if err := os.Mkdir(dir, 0o700); err != nil { |
| 108 | t.Fatal(err) |
| 109 | } |
| 110 | path := filepath.Join(dir, "shop.example.env") |
| 111 | if err := os.WriteFile(path, body, 0o600); err != nil { |
| 112 | t.Fatal(err) |
| 113 | } |
| 114 | plantAt(t, p, path, body, max, '\n') |
| 115 | var raw []byte |
| 116 | var absent bool |
| 117 | err := bounded(t, func() (err error) { |
| 118 | raw, absent, err = readInstanceFile(dir, "shop.example.env", max) |
| 119 | return err |
| 120 | }) |
| 121 | if !errors.Is(err, p.want) || absent || raw != nil { |
| 122 | t.Fatalf("got %q absent=%v %v; want %v", raw, absent, err, p.want) |
| 123 | } |
| 124 | }) |
| 125 | } |
| 126 | t.Run("the cap is inclusive", func(t *testing.T) { |
| 127 | dir := t.TempDir() |
| 128 | full := bytes.Repeat([]byte("x"), max) |
| 129 | if err := os.WriteFile(filepath.Join(dir, "f"), full, 0o600); err != nil { |
| 130 | t.Fatal(err) |
| 131 | } |
| 132 | raw, absent, err := readInstanceFile(dir, "f", max) |
| 133 | if err != nil || absent || !bytes.Equal(raw, full) { |
| 134 | t.Fatalf("a file at the cap: %d bytes absent=%v %v", len(raw), absent, err) |
| 135 | } |
| 136 | }) |
| 137 | t.Run("absent in a directory that opened", func(t *testing.T) { |
| 138 | raw, absent, err := readInstanceFile(t.TempDir(), "f", max) |
| 139 | if err != nil || !absent || raw != nil { |
| 140 | t.Fatalf("got %q absent=%v %v", raw, absent, err) |
| 141 | } |
| 142 | }) |
| 143 | t.Run("a missing directory is not evidence", func(t *testing.T) { |
| 144 | raw, absent, err := readInstanceFile(filepath.Join(t.TempDir(), "gone"), "f", max) |
| 145 | if err == nil || absent || raw != nil { |
| 146 | t.Fatalf("got %q absent=%v %v", raw, absent, err) |
| 147 | } |
| 148 | }) |
| 149 | } |
| 150 | |
| 151 | // flushFixture is one host's release flush over an env file and the |
| 152 | // releasing writer's adoption record in envDir, and files in filesDir. |
| 153 | type flushFixture struct { |
| 154 | data release.Store |
| 155 | f ReleaseFlushFiles |
| 156 | obs hostclaim.Observed |
| 157 | envDir, filesDir string |
| 158 | env, record []byte |
| 159 | envPath, recPath string |
| 160 | prefix string |
| 161 | filesRoot, secret string |
| 162 | } |
| 163 | |
| 164 | func newFlushFixture(t *testing.T) *flushFixture { |
| 165 | t.Helper() |
| 166 | const host = "shop.example" |
| 167 | data := release.NewDirStore(t.TempDir()) |
| 168 | reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 169 | if err != nil { |
| 170 | t.Fatal(err) |
| 171 | } |
| 172 | t.Cleanup(func() { reg.Close() }) |
| 173 | if err := reg.Upsert(registry.Route{Host: host, Kind: "service", Addr: "10.0.0.1:3000", Status: "ready", Hibernate: true}); err != nil { |
| 174 | t.Fatal(err) |
| 175 | } |
| 176 | inst := filepath.Join(t.TempDir(), "inst") |
| 177 | files := filepath.Join(inst, host) |
| 178 | if err := os.MkdirAll(files, 0o700); err != nil { |
| 179 | t.Fatal(err) |
| 180 | } |
| 181 | if err := os.WriteFile(filepath.Join(files, "a.txt"), []byte("hello"), 0o600); err != nil { |
| 182 | t.Fatal(err) |
| 183 | } |
| 184 | writer := instancekey.Binding{Host: host, Incarnation: "inc-1", Session: 6, Generation: "g6"} |
| 185 | x := &flushFixture{ |
| 186 | data: data, envDir: inst, filesDir: files, |
| 187 | env: []byte("CARLOS_INSTANCE_KEY=k1\n"), |
| 188 | record: instancekey.AdoptionRecord(writer, instancekey.SourcePlatform, instancekey.Digest("k1")), |
| 189 | envPath: filepath.Join(inst, host+".env"), recPath: filepath.Join(inst, host+instancekey.AdoptionSuffix), |
| 190 | prefix: "instances/" + host + "/inc/files/g-g6/", |
| 191 | } |
| 192 | if err := os.WriteFile(x.envPath, x.env, 0o600); err != nil { |
| 193 | t.Fatal(err) |
| 194 | } |
| 195 | if err := os.WriteFile(x.recPath, x.record, 0o600); err != nil { |
| 196 | t.Fatal(err) |
| 197 | } |
| 198 | // A directory the tenant must never get published as its files: the |
| 199 | // box's own data, say. |
| 200 | x.filesRoot = t.TempDir() |
| 201 | x.secret = "bucket-credentials" |
| 202 | if err := os.WriteFile(filepath.Join(x.filesRoot, "litestream.env"), []byte(x.secret), 0o600); err != nil { |
| 203 | t.Fatal(err) |
| 204 | } |
| 205 | x.f = ReleaseFlushFiles{Data: data, Registry: reg, FilesDir: func(string) string { return x.filesDir }, EnvPath: func(string) string { return x.envPath }} |
| 206 | x.obs = hostclaim.Observed{Host: host, Claim: &hostclaim.Claim{Host: host, Holder: &hostclaim.Holder{Location: "instances/" + host + "/inc", Incarnation: "inc-1", Session: 6, Generation: "g6"}}} |
| 207 | return x |
| 208 | } |
| 209 | |
| 210 | func (x *flushFixture) flush(t *testing.T) (int64, error) { |
| 211 | t.Helper() |
| 212 | var seq int64 |
| 213 | err := bounded(t, func() (err error) { |
| 214 | seq, err = x.f.Flush(context.Background(), x.obs) |
| 215 | return err |
| 216 | }) |
| 217 | return seq, err |
| 218 | } |
| 219 | |
| 220 | func (x *flushFixture) refused(t *testing.T, err, want error) { |
| 221 | t.Helper() |
| 222 | if !errors.Is(err, ErrClaimStopUnavailable) || !errors.Is(err, want) { |
| 223 | t.Fatalf("flush: %v; want %v", err, want) |
| 224 | } |
| 225 | if _, ok, err := hostfiles.Load(context.Background(), x.data, x.prefix); ok || err != nil { |
| 226 | t.Fatalf("a refused flush committed a manifest: %v %v", ok, err) |
| 227 | } |
| 228 | } |
| 229 | |
| 230 | // The release flush reads <host>.env and the adoption record through main's |
| 231 | // safe read: a symlink (at the name or a parent), a FIFO or an oversized |
| 232 | // file planted at either refuses the flush, nothing is committed, and the |
| 233 | // flush never blocks. |
| 234 | func TestReleaseFlushRefusesWhatTheTenantPlantsAtTheEnvAndAdoptionRecord(t *testing.T) { |
| 235 | t.Parallel() |
| 236 | for _, target := range []string{"env", "adoption"} { |
| 237 | for _, p := range tenantPlants { |
| 238 | if target == "adoption" && p.name == "a symlinked parent directory" { |
| 239 | continue // the env file's directory: the env case covers it |
| 240 | } |
| 241 | t.Run(target+"/"+p.name, func(t *testing.T) { |
| 242 | x := newFlushFixture(t) |
| 243 | path, body, limit, pad := x.envPath, x.env, int64(instancekey.EnvReadLimit), byte('\n') |
| 244 | if target == "adoption" { |
| 245 | path, body, limit, pad = x.recPath, x.record, instancekey.AdoptionReadLimit, ' ' |
| 246 | } |
| 247 | plantAt(t, p, path, body, limit, pad) |
| 248 | _, err := x.flush(t) |
| 249 | x.refused(t, err, p.want) |
| 250 | }) |
| 251 | } |
| 252 | } |
| 253 | } |
| 254 | |
| 255 | // SyncDir follows a symlink at its root, so the flush opens the files |
| 256 | // directory (the tenant's to rename) as a real directory with no link at |
| 257 | // any component first: a link there would publish the directory it names. |
| 258 | func TestReleaseFlushRefusesALinkedFilesDirectory(t *testing.T) { |
| 259 | t.Parallel() |
| 260 | for _, tc := range []struct { |
| 261 | name string |
| 262 | plant func(t *testing.T, x *flushFixture) |
| 263 | want error |
| 264 | }{ |
| 265 | {"a symlink at the files directory", func(t *testing.T, x *flushFixture) { |
| 266 | if err := os.RemoveAll(x.filesDir); err != nil { |
| 267 | t.Fatal(err) |
| 268 | } |
| 269 | if err := os.Symlink(x.filesRoot, x.filesDir); err != nil { |
| 270 | t.Fatal(err) |
| 271 | } |
| 272 | }, instancefs.ErrSymlink}, |
| 273 | {"a symlinked parent directory", func(t *testing.T, x *flushFixture) { |
| 274 | // The files directory lives under a linked parent of its own; the |
| 275 | // env directory stays real, so only the files root is at issue. |
| 276 | parent := filepath.Join(t.TempDir(), "linked") |
| 277 | if err := os.Symlink(filepath.Dir(x.filesRoot), parent); err != nil { |
| 278 | t.Fatal(err) |
| 279 | } |
| 280 | x.filesDir = filepath.Join(parent, filepath.Base(x.filesRoot)) |
| 281 | }, instancefs.ErrSymlink}, |
| 282 | {"a FIFO", func(t *testing.T, x *flushFixture) { |
| 283 | if err := os.RemoveAll(x.filesDir); err != nil { |
| 284 | t.Fatal(err) |
| 285 | } |
| 286 | if err := syscall.Mkfifo(x.filesDir, 0o600); err != nil { |
| 287 | t.Fatal(err) |
| 288 | } |
| 289 | }, syscall.ENOTDIR}, |
| 290 | } { |
| 291 | t.Run(tc.name, func(t *testing.T) { |
| 292 | x := newFlushFixture(t) |
| 293 | tc.plant(t, x) |
| 294 | _, err := x.flush(t) |
| 295 | x.refused(t, err, tc.want) |
| 296 | }) |
| 297 | } |
| 298 | } |
| 299 | |
| 300 | // Absence stays absence only on positive evidence: a missing env file or |
| 301 | // record in a directory that opened stages their deletion as before; a |
| 302 | // missing directory refuses. |
| 303 | func TestReleaseFlushReadsAbsenceOnlyFromADirectoryThatOpened(t *testing.T) { |
| 304 | t.Parallel() |
| 305 | x := newFlushFixture(t) |
| 306 | for _, p := range []string{x.envPath, x.recPath} { |
| 307 | if err := os.Remove(p); err != nil { |
| 308 | t.Fatal(err) |
| 309 | } |
| 310 | } |
| 311 | seq, err := x.flush(t) |
| 312 | if err != nil { |
| 313 | t.Fatal(err) |
| 314 | } |
| 315 | m, ok, err := hostfiles.Load(context.Background(), x.data, x.prefix) |
| 316 | if err != nil || !ok || m.Seq != seq { |
| 317 | t.Fatalf("manifest: %v %v", ok, err) |
| 318 | } |
| 319 | for _, item := range []string{hostfiles.EnvItem, instancekey.AdoptionItem} { |
| 320 | if _, ok := m.Platform[item]; ok { |
| 321 | t.Fatalf("%s published for a file that is not there", item) |
| 322 | } |
| 323 | } |
| 324 | |
| 325 | y := newFlushFixture(t) |
| 326 | y.envPath = filepath.Join(t.TempDir(), "gone", "shop.example.env") |
| 327 | _, err = y.flush(t) |
| 328 | y.refused(t, err, os.ErrNotExist) |
| 329 | } |
| 330 | |
| 331 | // The race a check-then-open left: a symlink renamed in at the files |
| 332 | // directory after the flush opened it is never followed. The flush |
| 333 | // publishes the directory it opened, never the one the link names. |
| 334 | func TestReleaseFlushPublishesTheDirectoryItOpenedNotALinkSwappedInAfter(t *testing.T) { |
| 335 | x := newFlushFixture(t) |
| 336 | swapOnce(t, "opened", func() { |
| 337 | if err := os.Rename(x.filesDir, x.filesDir+".moved"); err != nil { |
| 338 | t.Fatal(err) |
| 339 | } |
| 340 | if err := os.Symlink(x.filesRoot, x.filesDir); err != nil { |
| 341 | t.Fatal(err) |
| 342 | } |
| 343 | }) |
| 344 | seq, err := x.flush(t) |
| 345 | if err != nil { |
| 346 | t.Fatal(err) |
| 347 | } |
| 348 | m, ok, err := hostfiles.Load(context.Background(), x.data, x.prefix) |
| 349 | if err != nil || !ok || m.Seq != seq { |
| 350 | t.Fatalf("manifest: %v %v", ok, err) |
| 351 | } |
| 352 | if _, ok := m.Entries["litestream.env"]; ok { |
| 353 | t.Fatal("the flush published the directory a swapped-in link names") |
| 354 | } |
| 355 | if _, ok := m.Entries["a.txt"]; !ok { |
| 356 | t.Fatalf("the flush did not publish the directory it opened: %v", m.Entries) |
| 357 | } |
| 358 | } |
| 359 | |
| 360 | // Astra P2 / Opus M4: a directory renamed away and back while |
| 361 | // readInstanceFile looks is an error or the file, never "absent": a flush |
| 362 | // must never stage the deletion of an env file that is there. |
| 363 | func TestReadInstanceFileNeverReadsAVanishingDirectoryAsAbsent(t *testing.T) { |
| 364 | t.Parallel() |
| 365 | dir := filepath.Join(t.TempDir(), "inst") |
| 366 | if err := os.Mkdir(dir, 0o700); err != nil { |
| 367 | t.Fatal(err) |
| 368 | } |
| 369 | if err := os.WriteFile(filepath.Join(dir, "shop.example.env"), []byte("K=v\n"), 0o600); err != nil { |
| 370 | t.Fatal(err) |
| 371 | } |
| 372 | var stop atomic.Bool |
| 373 | flapped := make(chan struct{}) |
| 374 | go func() { |
| 375 | defer close(flapped) |
| 376 | for !stop.Load() { |
| 377 | os.Rename(dir, dir+".away") |
| 378 | os.Rename(dir+".away", dir) |
| 379 | } |
| 380 | }() |
| 381 | defer func() { stop.Store(true); <-flapped }() |
| 382 | deadline := time.Now().Add(1500 * time.Millisecond) |
| 383 | for time.Now().Before(deadline) { |
| 384 | if _, absent, _ := readInstanceFile(dir, "shop.example.env", 64); absent { |
| 385 | t.Fatal("a directory that vanished between the steps read as an absent file") |
| 386 | } |
| 387 | } |
| 388 | } |
| 389 | |
| 390 | // Opus I1: a files tree past the walk's caps refuses the flush whole; |
| 391 | // nothing is committed. |
| 392 | func TestReleaseFlushRefusesATreePastTheWalksCaps(t *testing.T) { |
| 393 | t.Parallel() |
| 394 | x := newFlushFixture(t) |
| 395 | root, err := os.OpenRoot(x.filesDir) |
| 396 | if err != nil { |
| 397 | t.Fatal(err) |
| 398 | } |
| 399 | defer root.Close() |
| 400 | if err := root.MkdirAll(strings.TrimSuffix(strings.Repeat("d/", hostfiles.MaxPathBytes/2+2), "/"), 0o700); err != nil { |
| 401 | t.Fatal(err) |
| 402 | } |
| 403 | _, err = x.flush(t) |
| 404 | if !errors.Is(err, hostfiles.ErrTooDeep) { |
| 405 | t.Fatalf("flush of a tree past the caps: %v", err) |
| 406 | } |
| 407 | if _, ok, err := hostfiles.Load(context.Background(), x.data, x.prefix); ok || err != nil { |
| 408 | t.Fatalf("a refused flush committed a manifest: %v %v", ok, err) |
| 409 | } |
| 410 | } |
| 411 | |
amadan