amadan

carlos / platform Public

console: the seed scratch reservation pool and its limits (Plan 5 Task 3)

Start-up cleanup removes only the pool's own from-* entries inside the
configured directory, refuses a relative path or /, and leaves the
directory and anything else in it alone.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude f9ffec4688a04b0f748ca7dcecea58ab17bf06f7 parent 10c960a
6 files changed, +347 −2
  • approved_seeded_copy_test.go +2 −2
  • internal/console/seedscratch.go +139 −0
  • internal/console/seedscratch_test.go +175 −0
  • internal/console/seedscratch_unix.go +16 −0
  • internal/console/seedscratch_windows.go +9 −0
  • internal/console/server.go +6 −0
diff --git a/approved_seeded_copy_test.go b/approved_seeded_copy_test.go
index d1c4d105d..fcaa131aa 100644
--- a/approved_seeded_copy_test.go
+++ b/approved_seeded_copy_test.go
@@ -33,8 +33,8 @@ var seededCopyIDs = func() []string {
// seededWiredIDs grows as the plans wire rows. Plan 1 wires none; Plan 3
// Task 3 wires SD-N1, Task 5 SD-N2 to SD-N8 and SD-N113, Task 6 SD-N9 to SD-N13,
-// SD-N111 and SD-N112; Plan 4 Task 3 SD-N14 to SD-N19.
-var seededWiredIDs = []string{"SD-N1", "SD-N2", "SD-N3", "SD-N4", "SD-N5", "SD-N6", "SD-N7", "SD-N8", "SD-N113", "SD-N9", "SD-N10", "SD-N11", "SD-N12", "SD-N13", "SD-N111", "SD-N112", "SD-N14", "SD-N15", "SD-N16", "SD-N17", "SD-N18", "SD-N19"}
+// SD-N111 and SD-N112; Plan 4 Task 3 SD-N14 to SD-N19; Plan 5 Task 3 SD-N27 to SD-N29.
+var seededWiredIDs = []string{"SD-N1", "SD-N2", "SD-N3", "SD-N4", "SD-N5", "SD-N6", "SD-N7", "SD-N8", "SD-N113", "SD-N9", "SD-N10", "SD-N11", "SD-N12", "SD-N13", "SD-N111", "SD-N112", "SD-N14", "SD-N15", "SD-N16", "SD-N17", "SD-N18", "SD-N19", "SD-N27", "SD-N28", "SD-N29"}
func mustParseSeededRecord(t *testing.T) []copyRow {
t.Helper()
diff --git a/internal/console/seedscratch.go b/internal/console/seedscratch.go
new file mode 100644
index 000000000..eea255b8b
--- /dev/null
+++ b/internal/console/seedscratch.go
@@ -0,0 +1,139 @@
+package console
+
+import (
+ "errors"
+ "fmt"
+ "log"
+ "net/http"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+)
+
+// seedScratch is the console's one scratch reservation pool, shared by
+// --from restores and import spools (seeded-instances spec §4 step 5,
+// §5.2): an in-process counter over <console data>/scratch/seeds/, and the
+// per-app and deployment limits on how many run at once.
+type seedScratch struct {
+ root string
+ free func(string) (int64, error)
+
+ mu sync.Mutex
+ capacity int64
+ used int64
+ running int
+ perApp map[string]bool
+}
+
+const (
+ seedScratchSlack = 1 << 30 // capacity is free space less this
+ seedScratchAllApps = 2 // reservations across the deployment; one per app
+
+ // seedScratchPrefix starts the name of every entry the pool's users
+ // create directly inside the scratch directory. Start-up cleanup removes
+ // entries with this prefix and nothing else.
+ seedScratchPrefix = "from-"
+)
+
+func newSeedScratch(root string, free func(string) (int64, error)) *seedScratch {
+ p := &seedScratch{root: root, free: free, perApp: map[string]bool{}}
+ p.readCapacity()
+ return p
+}
+
+// startSeedScratch is what NewServer calls: it clears the pool's leftovers
+// and builds the pool. An empty directory, or one the cleanup refuses,
+// leaves a pool with no capacity, so every reservation is a 507.
+func startSeedScratch(dir string) *seedScratch {
+ usable := dir != ""
+ if usable {
+ if err := clearSeedScratch(dir); err != nil {
+ log.Printf("console: seed scratch %s: %v (the seed doors will refuse)", dir, err)
+ usable = false
+ }
+ }
+ return newSeedScratch(dir, func(d string) (int64, error) {
+ if !usable {
+ return 0, errors.New("no usable seed scratch directory")
+ }
+ return scratchFree(d)
+ })
+}
+
+func (p *seedScratch) dir() string { return p.root }
+
+// readCapacity is called with p.mu held, or before the pool is shared.
+func (p *seedScratch) readCapacity() {
+ n, err := p.free(p.root)
+ if err != nil {
+ p.capacity = 0
+ return
+ }
+ p.capacity = max(n-seedScratchSlack, 0)
+}
+
+// reserve takes bytes for appKey (account/app), or answers why not. The
+// returned release is safe to call more than once.
+func (p *seedScratch) reserve(appKey string, bytes int64) (func(), *doorRefusal) {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ if p.perApp[appKey] {
+ app := appKey[strings.LastIndex(appKey, "/")+1:]
+ return nil, &doorRefusal{http.StatusTooManyRequests, fmt.Sprintf("another copy or upload is running for %s. Try again when it finishes.", app)}
+ }
+ if p.running >= seedScratchAllApps {
+ return nil, &doorRefusal{http.StatusTooManyRequests, "the console is already running two copies or uploads. Try again in a few minutes."}
+ }
+ if p.used == 0 {
+ p.readCapacity()
+ }
+ if bytes < 0 || bytes > p.capacity-p.used {
+ return nil, &doorRefusal{http.StatusInsufficientStorage, "the console has no room for this right now. Try again later."}
+ }
+ p.used += bytes
+ p.running++
+ p.perApp[appKey] = true
+ var once sync.Once
+ return func() {
+ once.Do(func() {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ p.used -= bytes
+ p.running--
+ delete(p.perApp, appKey)
+ })
+ }, nil
+}
+
+// clearSeedScratch removes what a crashed restore or spool left at console
+// start (spec §4 "Cleanup", §5.2): the entries named seedScratchPrefix* that
+// sit directly inside dir. Anything else in dir stays, and dir itself is
+// never removed. A relative path or "/" is refused; an empty dir does
+// nothing. A symlink entry is removed as a link, never followed.
+func clearSeedScratch(dir string) error {
+ if dir == "" {
+ return nil
+ }
+ if !filepath.IsAbs(dir) || filepath.Clean(dir) == string(filepath.Separator) {
+ return fmt.Errorf("refusing to clear %q: it must be an absolute path other than /", dir)
+ }
+ dir = filepath.Clean(dir)
+ if err := os.MkdirAll(dir, 0o700); err != nil {
+ return err
+ }
+ ents, err := os.ReadDir(dir)
+ if err != nil {
+ return err
+ }
+ var errs []error
+ for _, e := range ents {
+ if !strings.HasPrefix(e.Name(), seedScratchPrefix) {
+ continue
+ }
+ if err := os.RemoveAll(filepath.Join(dir, e.Name())); err != nil {
+ errs = append(errs, err)
+ }
+ }
+ return errors.Join(errs...)
+}
diff --git a/internal/console/seedscratch_test.go b/internal/console/seedscratch_test.go
new file mode 100644
index 000000000..d134e7fc6
--- /dev/null
+++ b/internal/console/seedscratch_test.go
@@ -0,0 +1,175 @@
+package console
+
+import (
+ "net/http"
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+func TestScratchLimitsPerAppAndDeployment(t *testing.T) {
+ p := newSeedScratch(t.TempDir(), func(string) (int64, error) { return 100 << 30, nil })
+ rel1, ref := p.reserve("acct/shop", 1<<30)
+ if ref != nil {
+ t.Fatalf("first: %+v", ref)
+ }
+ if _, ref := p.reserve("acct/shop", 1<<30); ref == nil || ref.Status != http.StatusTooManyRequests {
+ t.Fatalf("second for the same app: %+v, want 429", ref)
+ } else if want := "another copy or upload is running for shop. Try again when it finishes."; ref.Msg != want {
+ t.Fatalf("per-app message %q, want %q", ref.Msg, want)
+ }
+ rel2, ref := p.reserve("acct/kass", 1<<30)
+ if ref != nil {
+ t.Fatalf("another app: %+v", ref)
+ }
+ if _, ref := p.reserve("acct/blog", 1<<30); ref == nil || ref.Status != http.StatusTooManyRequests {
+ t.Fatalf("a third across the deployment: %+v, want 429", ref)
+ } else if want := "the console is already running two copies or uploads. Try again in a few minutes."; ref.Msg != want {
+ t.Fatalf("deployment message %q, want %q", ref.Msg, want)
+ }
+ rel1()
+ rel1() // releasing twice is harmless
+ if rel3, ref := p.reserve("acct/blog", 1<<30); ref != nil {
+ t.Fatalf("after a release: %+v", ref)
+ } else {
+ rel3()
+ }
+ rel2()
+}
+
+func TestScratchCapacityIsFreeSpaceLessAGiB(t *testing.T) {
+ free := int64(5 << 30)
+ reads := 0
+ p := newSeedScratch(t.TempDir(), func(string) (int64, error) { reads++; return free, nil })
+ if _, ref := p.reserve("a/x", 4<<30+1); ref == nil || ref.Status != http.StatusInsufficientStorage {
+ t.Fatalf("over capacity: %+v, want 507", ref)
+ } else if want := "the console has no room for this right now. Try again later."; ref.Msg != want {
+ t.Fatalf("capacity message %q, want %q", ref.Msg, want)
+ }
+ rel, ref := p.reserve("a/x", 4<<30)
+ if ref != nil {
+ t.Fatalf("at capacity: %+v", ref)
+ }
+ if _, ref := p.reserve("a/y", 1); ref == nil || ref.Status != http.StatusInsufficientStorage {
+ t.Fatalf("a full pool: %+v, want 507", ref)
+ }
+ rel()
+ free = 2 << 30 // the disk filled from elsewhere; an empty pool re-reads it
+ if _, ref := p.reserve("a/x", 2<<30); ref == nil || ref.Status != http.StatusInsufficientStorage {
+ t.Fatalf("after a re-read: %+v, want 507", ref)
+ }
+ if reads < 2 {
+ t.Fatalf("free space read %d times, want a re-read when the pool emptied", reads)
+ }
+}
+
+func TestSeedScratchReleaseIsExactlyOnce(t *testing.T) {
+ p := newSeedScratch(t.TempDir(), func(string) (int64, error) { return 10 << 30, nil })
+ relA, _ := p.reserve("acct/shop", 1<<30)
+ relB, _ := p.reserve("acct/blog", 1<<30)
+ relA()
+ relA() // must not give back a second reservation's bytes or slot
+ if _, ref := p.reserve("acct/kass", 1<<30); ref != nil {
+ t.Fatalf("one slot is free: %+v", ref)
+ }
+ if _, ref := p.reserve("acct/more", 1<<30); ref == nil || ref.Status != http.StatusTooManyRequests {
+ t.Fatalf("a double release freed a second slot: %+v", ref)
+ }
+ relB()
+}
+
+func TestSeedScratchWithoutADirectoryIsUnavailable(t *testing.T) {
+ p := startSeedScratch("")
+ if _, ref := p.reserve("acct/shop", 1); ref == nil || ref.Status != http.StatusInsufficientStorage {
+ t.Fatalf("no directory: %+v, want 507", ref)
+ }
+}
+
+func TestClearSeedScratch(t *testing.T) {
+ dir := filepath.Join(t.TempDir(), "scratch", "seeds")
+ if err := os.MkdirAll(filepath.Join(dir, "from-123"), 0o700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(dir, "from-123", "restore.db"), []byte("spool"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(dir, "from-456"), []byte("spool"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := clearSeedScratch(dir); err != nil {
+ t.Fatal(err)
+ }
+ ents, err := os.ReadDir(dir)
+ if err != nil || len(ents) != 0 {
+ t.Fatalf("after clear: %v, %v", ents, err)
+ }
+ st, _ := os.Stat(dir)
+ if st.Mode().Perm() != 0o700 {
+ t.Fatalf("mode %v, want 0700", st.Mode().Perm())
+ }
+}
+
+func TestSeedScratchClearKeepsWhatThePoolDidNotCreate(t *testing.T) {
+ dir := t.TempDir()
+ keep := filepath.Join(dir, "shop.example.com.db")
+ keepDir := filepath.Join(dir, "notes")
+ if err := os.WriteFile(keep, []byte("not ours"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.MkdirAll(keepDir, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.MkdirAll(filepath.Join(dir, "from-9"), 0o700); err != nil {
+ t.Fatal(err)
+ }
+ p := startSeedScratch(dir)
+ if _, err := os.Stat(keep); err != nil {
+ t.Fatalf("a file the pool did not create was removed: %v", err)
+ }
+ if _, err := os.Stat(keepDir); err != nil {
+ t.Fatalf("a directory the pool did not create was removed: %v", err)
+ }
+ if _, err := os.Stat(filepath.Join(dir, "from-9")); !os.IsNotExist(err) {
+ t.Fatalf("the pool's own leftover survived: %v", err)
+ }
+ if p.dir() != dir {
+ t.Fatalf("dir() = %q, want %q", p.dir(), dir)
+ }
+}
+
+func TestSeedScratchClearRefusesUnsafePaths(t *testing.T) {
+ for _, dir := range []string{"/", "scratch/seeds", ".", "../x"} {
+ if err := clearSeedScratch(dir); err == nil {
+ t.Errorf("clearSeedScratch(%q) = nil, want a refusal", dir)
+ }
+ }
+ if err := clearSeedScratch(""); err != nil {
+ t.Errorf("empty directory: %v, want nil", err)
+ }
+ // A refused path leaves the pool unavailable rather than counting space.
+ p := startSeedScratch("relative/dir")
+ if _, ref := p.reserve("acct/shop", 1); ref == nil || ref.Status != http.StatusInsufficientStorage {
+ t.Fatalf("refused directory: %+v, want 507", ref)
+ }
+}
+
+func TestSeedScratchClearDoesNotFollowSymlinks(t *testing.T) {
+ dir := t.TempDir()
+ outside := t.TempDir()
+ victim := filepath.Join(outside, "victim.db")
+ if err := os.WriteFile(victim, []byte("keep"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(outside, filepath.Join(dir, "from-link")); err != nil {
+ t.Skip("no symlinks here")
+ }
+ if err := clearSeedScratch(dir); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := os.Stat(victim); err != nil {
+ t.Fatalf("clearing followed a symlink out of the directory: %v", err)
+ }
+ if _, err := os.Lstat(filepath.Join(dir, "from-link")); !os.IsNotExist(err) {
+ t.Fatalf("the link itself survived: %v", err)
+ }
+}
diff --git a/internal/console/seedscratch_unix.go b/internal/console/seedscratch_unix.go
new file mode 100644
index 000000000..40cd21540
--- /dev/null
+++ b/internal/console/seedscratch_unix.go
@@ -0,0 +1,16 @@
+//go:build !windows
+
+package console
+
+import "syscall"
+
+// scratchFree is the bytes available to the console on dir's filesystem.
+// Bavail times Bsize, multiplied once here (Bsize is int64 on Linux and
+// uint32 on darwin, internal/host/statfs_unix.go's note).
+func scratchFree(dir string) (int64, error) {
+ var st syscall.Statfs_t
+ if err := syscall.Statfs(dir, &st); err != nil {
+ return 0, err
+ }
+ return int64(uint64(st.Bavail) * uint64(st.Bsize)), nil
+}
diff --git a/internal/console/seedscratch_windows.go b/internal/console/seedscratch_windows.go
new file mode 100644
index 000000000..712efcc9c
--- /dev/null
+++ b/internal/console/seedscratch_windows.go
@@ -0,0 +1,9 @@
+//go:build windows
+
+package console
+
+import "errors"
+
+// scratchFree: the console does not run on Windows; the pool then has no
+// capacity and every seed door answers 507.
+func scratchFree(string) (int64, error) { return 0, errors.New("no free-space reading on windows") }
diff --git a/internal/console/server.go b/internal/console/server.go
index a0827041b..828b00994 100644
--- a/internal/console/server.go
+++ b/internal/console/server.go
@@ -479,6 +479,10 @@ type Config struct {
HostClaim bool
WakeRetryReset *WakeRetryResetDoor
+ // SeedScratchDir is <console data>/scratch/seeds: --from restores and
+ // import spools. Start-up removes the pool's own leftovers from it.
+ // Empty: the seed doors answer 507.
+ SeedScratchDir string
DBVersions lease.VersionedStore
Origin string // the console's public origin — the OAuth client_id
Keymail string // keymail base URL
@@ -828,6 +832,7 @@ type Server struct {
claimEntries claimInventory
claimWorkers map[string]func(context.Context)
dbHandleKey []byte
+ seedScratch *seedScratch
cfg Config
store *Store
// appRegionOf and beforeClaimInsert are TEST SEAMS (nil in
@@ -1255,6 +1260,7 @@ func NewServer(cfg Config, st *Store) (*Server, error) {
// each other apart (relayops.go).
relayHolder: relayHolderID(),
}
+ s.seedScratch = startSeedScratch(cfg.SeedScratchDir)
if s.lookupTXT == nil {
s.lookupTXT = net.DefaultResolver.LookupTXT
}