amadan
server: one free space per account, and a quota-blocked create reclaims your own dead weight (#838)
Default LCHAT_CLOUD_QUOTA 3 → 1: one free space is the taste of the product; wanting a second is the upgrade moment (paid spaces never counted). Every free space is standing liability — a ledger slot against the box cap, disk, an S3 prefix, and a chance to leak awake — with no revenue signal at 3. With one slot, a fumbled create must not lock the account out for the reaper's 14 days. So on quota_exceeded the create path now frees the requester's own abandoned allocation first: a stale pending row (a create that died between allocate and finalize — previously held its slot forever), or a provisioned space passing the reaper's exact never-used guards (never set up, nobody joined, zero content), age-gated to 1h so a space mid-setup in another tab is never touched. Reclaimed instances archive to /data/trash via the normal controld delete. Anything in real use keeps the 409 (message now pluralises at quota 1 and mentions delete as the way out). Existing accounts holding 2-3 free spaces are naturally grandfathered — the check runs at create time only.
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
6 files changed,
+274
−5
cloud.go+20 −2cloud_db.go+26 −0cloud_test.go+144 −0docs/cloud-provisioner.md+23 −1home/pow.go+2 −2reaper.go+59 −0
diff --git a/cloud.go b/cloud.go| index d71dc36..ec3ee81 100644 |
| --- a/cloud.go |
| +++ b/cloud.go |
| @@ -170,8 +170,26 @@ func (s *cloudServer) createSpaceHandler(w http.ResponseWriter, r *http.Request) |
| return |
| } |
| if overQuota { |
| + // Before refusing, try to free the slot ourselves: the requester's own |
| + // abandoned space (a create that died mid-flight, or one they never set |
| + // up) shouldn't block their fresh start for the reaper's 14 days. Only |
| + // ever reclaims under the reaper's own never-used guards + an age gate, |
| + // so a space mid-setup or in real use is never eaten. |
| + if s.reclaimOwnSpace(callerID, req.Requester) { |
| + id, overQuota, err = s.db.allocateSpace(callerID, req.Requester, s.quota) |
| + if err != nil { |
| + writeControlErr(w, http.StatusInternalServerError, "create_failed", "db error") |
| + return |
| + } |
| + } |
| + } |
| + if overQuota { |
| + noun := "spaces" |
| + if s.quota == 1 { |
| + noun = "space" |
| + } |
| writeControlErr(w, http.StatusConflict, "quota_exceeded", |
| - fmt.Sprintf("this account already has %d free spaces (the limit) — upgrading a space frees its slot", s.quota)) |
| + fmt.Sprintf("this account already has %d free %s (the limit) — upgrading a space frees its slot, and so does deleting one you no longer use", s.quota, noun)) |
| return |
| } |
| @@ -388,7 +406,7 @@ func runCloud(args []string) { |
| socketPath := fs.String("socket", envOr("LCHAT_SOCKET", ""), "unix socket to serve on instead of -addr") |
| dbPath := fs.String("db", envOr("LCHAT_CLOUD_DB", "cloud.db"), "provisioner SQLite path") |
| baseDomain := fs.String("base-domain", os.Getenv("LCHAT_CLOUD_BASE_DOMAIN"), "domain spaces are created under (required; needs wildcard DNS at this box)") |
| - quota := fs.Int("quota", envInt("LCHAT_CLOUD_QUOTA", 3), "max spaces per caller+requester") |
| + quota := fs.Int("quota", envInt("LCHAT_CLOUD_QUOTA", 1), "max free spaces per caller+requester (paid spaces never count)") |
| controlSocket := fs.String("control-socket", envOr("LCHAT_CONTROL_SOCKET", "/run/lchat/control.sock"), "the control daemon's unix socket") |
| reapAfterDays := fs.Int("reap-after-days", envInt("LCHAT_CLOUD_REAP_AFTER_DAYS", 0), "reclaim never-claimed (zero-member) spaces older than this many days; 0 disables the reaper") |
| reapIntervalHours := fs.Int("reap-interval-hours", envInt("LCHAT_CLOUD_REAP_INTERVAL_HOURS", 6), "how often the reaper sweeps") |
diff --git a/cloud_db.go b/cloud_db.go| index 7b394be..c8b04a4 100644 |
| --- a/cloud_db.go |
| +++ b/cloud_db.go |
| @@ -349,6 +349,32 @@ type reapCandidate struct { |
| CreatedAt int64 |
| } |
| +// reclaimCandidates returns one account's own free-entitlement rows created |
| +// at/before cutoff, oldest first — the pool the create path may self-reclaim |
| +// from when the quota is hit. Unlike spacesOlderThan this INCLUDES pending |
| +// rows (NULL host, returned as ""): a create that died between allocate and |
| +// finalize holds a quota slot forever otherwise, which at quota 1 would brick |
| +// the account. Paid rows are excluded at the source, same as the reaper. |
| +func (db *cloudDB) reclaimCandidates(callerID, requester string, cutoff int64) ([]reapCandidate, error) { |
| + rows, err := db.sql.Query( |
| + `SELECT id, COALESCE(host, ''), created_at FROM spaces |
| + WHERE caller_id = ? AND requester = ? AND entitlement = 'free' AND created_at <= ? |
| + ORDER BY created_at`, callerID, requester, cutoff) |
| + if err != nil { |
| + return nil, err |
| + } |
| + defer rows.Close() |
| + var out []reapCandidate |
| + for rows.Next() { |
| + var c reapCandidate |
| + if err := rows.Scan(&c.ID, &c.Host, &c.CreatedAt); err != nil { |
| + return nil, err |
| + } |
| + out = append(out, c) |
| + } |
| + return out, rows.Err() |
| +} |
| + |
| // spacesOlderThan returns finalized spaces created at/before cutoff (unix secs), |
| // oldest first — the reaper's candidate set. A NULL host (create in flight or |
| // rolled back) is never a candidate, and neither is a paid space: someone paying |
diff --git a/cloud_test.go b/cloud_test.go| index 9b15201..529ed3d 100644 |
| --- a/cloud_test.go |
| +++ b/cloud_test.go |
| @@ -10,7 +10,9 @@ import ( |
| "os" |
| "path/filepath" |
| "strings" |
| + "sync" |
| "testing" |
| + "time" |
| ) |
| // fakeControl serves a stand-in control daemon on a unix socket, so the cloud |
| @@ -318,6 +320,148 @@ func mustCallerID(t *testing.T, s *cloudServer, label string) string { |
| return "" |
| } |
| +// reclaimTracker scripts the fake control daemon for the self-reclaim tests |
| +// and records what got deleted. |
| +type reclaimTracker struct { |
| + mu sync.Mutex |
| + stats instanceStats |
| + deleted []string |
| +} |
| + |
| +func (tr *reclaimTracker) count() int { |
| + tr.mu.Lock() |
| + defer tr.mu.Unlock() |
| + return len(tr.deleted) |
| +} |
| + |
| +// reclaimControl answers creates with 201, stats from tr.stats, and records deletes. |
| +func reclaimControl(t *testing.T, tr *reclaimTracker) string { |
| + return fakeControl(t, func(w http.ResponseWriter, r *http.Request) { |
| + switch { |
| + case r.Method == "GET" && strings.HasSuffix(r.URL.Path, "/stats"): |
| + tr.mu.Lock() |
| + st := tr.stats |
| + tr.mu.Unlock() |
| + writeJSON(w, http.StatusOK, st) |
| + case r.Method == "DELETE": |
| + tr.mu.Lock() |
| + tr.deleted = append(tr.deleted, strings.TrimPrefix(r.URL.Path, "/instances/")) |
| + tr.mu.Unlock() |
| + writeJSON(w, http.StatusOK, map[string]any{"ok": true}) |
| + default: |
| + var req controlCreateReq |
| + json.NewDecoder(r.Body).Decode(&req) |
| + writeJSON(w, http.StatusCreated, map[string]any{ |
| + "host": req.Host, "url": "https://" + req.Host, "setup_link": "https://" + req.Host + "/join/setup#S", |
| + }) |
| + } |
| + }) |
| +} |
| + |
| +// A quota-blocked create reclaims the account's own abandoned space (never set |
| +// up, past the age gate) instead of failing: the old instance archives via |
| +// controld and the fresh one takes its slot. A space younger than the age gate |
| +// is never touched — not even consulted. |
| +func TestCloudQuotaSelfReclaim(t *testing.T) { |
| + tr := &reclaimTracker{} // zero-value stats = never set up, nobody joined, no content |
| + srv, s := newTestCloud(t, reclaimControl(t, tr)) |
| + s.quota = 1 |
| + tok := mintCaller(t, srv, "home") |
| + |
| + resp := createSpace(t, srv, tok, "user-1") |
| + if resp.StatusCode != http.StatusCreated { |
| + t.Fatalf("first create: %d", resp.StatusCode) |
| + } |
| + first, _ := decodeBody(t, resp)["host"].(string) |
| + |
| + // Fresh space → the age gate holds: 409, and controld is never asked. |
| + if r := createSpace(t, srv, tok, "user-1"); r.StatusCode != http.StatusConflict { |
| + t.Fatalf("fresh over-quota create: %d, want 409 (age gate)", r.StatusCode) |
| + } |
| + if n := tr.count(); n != 0 { |
| + t.Fatalf("age-gated create deleted %d space(s) — must not touch a fresh space", n) |
| + } |
| + |
| + // Age it past the gate → the next create reclaims it and succeeds. |
| + if _, err := s.db.sql.Exec(`UPDATE spaces SET created_at = created_at - 7200`); err != nil { |
| + t.Fatal(err) |
| + } |
| + resp = createSpace(t, srv, tok, "user-1") |
| + if resp.StatusCode != http.StatusCreated { |
| + t.Fatalf("reclaiming create: %d, want 201", resp.StatusCode) |
| + } |
| + second, _ := decodeBody(t, resp)["host"].(string) |
| + if second == first { |
| + t.Fatalf("second host = first (%s), want a fresh instance", first) |
| + } |
| + tr.mu.Lock() |
| + deleted := append([]string(nil), tr.deleted...) |
| + tr.mu.Unlock() |
| + if len(deleted) != 1 || deleted[0] != first { |
| + t.Errorf("deleted = %v, want [%s]", deleted, first) |
| + } |
| + spaces, err := s.db.listSpaces(mustCallerID(t, s, "home"), "user-1") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(spaces) != 1 || spaces[0].Host != second { |
| + t.Errorf("ledger = %+v, want just %s", spaces, second) |
| + } |
| +} |
| + |
| +// A space in real use — set up, joined, or holding content — is never |
| +// self-reclaimed: the quota refusal stands. |
| +func TestCloudQuotaSelfReclaimKeepsUsedSpaces(t *testing.T) { |
| + tr := &reclaimTracker{stats: instanceStats{Members: 1, SetupComplete: true}} |
| + srv, s := newTestCloud(t, reclaimControl(t, tr)) |
| + s.quota = 1 |
| + tok := mintCaller(t, srv, "home") |
| + |
| + if r := createSpace(t, srv, tok, "user-1"); r.StatusCode != http.StatusCreated { |
| + t.Fatalf("create: %d", r.StatusCode) |
| + } |
| + if _, err := s.db.sql.Exec(`UPDATE spaces SET created_at = created_at - 7200`); err != nil { |
| + t.Fatal(err) |
| + } |
| + if r := createSpace(t, srv, tok, "user-1"); r.StatusCode != http.StatusConflict { |
| + t.Fatalf("over quota vs a set-up space: %d, want 409", r.StatusCode) |
| + } |
| + if n := tr.count(); n != 0 { |
| + t.Errorf("deleted %d space(s) — a set-up space must never be reclaimed", n) |
| + } |
| +} |
| + |
| +// A pending row (a create that died between allocate and finalize) past the |
| +// age gate frees on the next create instead of holding the slot forever — |
| +// ledger-only, no controld call (there is no instance behind it). |
| +func TestCloudQuotaSelfReclaimStalePending(t *testing.T) { |
| + tr := &reclaimTracker{} |
| + srv, s := newTestCloud(t, reclaimControl(t, tr)) |
| + s.quota = 1 |
| + tok := mintCaller(t, srv, "home") |
| + callerID := mustCallerID(t, s, "home") |
| + |
| + if _, err := s.db.sql.Exec( |
| + `INSERT INTO spaces(host, caller_id, requester, created_at) VALUES(NULL, ?, ?, ?)`, |
| + callerID, "user-1", time.Now().Add(-2*time.Hour).Unix()); err != nil { |
| + t.Fatal(err) |
| + } |
| + resp := createSpace(t, srv, tok, "user-1") |
| + if resp.StatusCode != http.StatusCreated { |
| + t.Fatalf("create blocked by a stale pending row: %d, want 201", resp.StatusCode) |
| + } |
| + if n := tr.count(); n != 0 { |
| + t.Errorf("controld deletes = %d, want 0 (pending rows are ledger-only)", n) |
| + } |
| + spaces, err := s.db.listSpaces(callerID, "user-1") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(spaces) != 1 { |
| + t.Errorf("ledger = %+v, want exactly the new space", spaces) |
| + } |
| +} |
| + |
| func TestCloudAdminTokenLifecycle(t *testing.T) { |
| srv, _ := newTestCloud(t, okControl(t)) |
diff --git a/docs/cloud-provisioner.md b/docs/cloud-provisioner.md| index 5e656af..7f98173 100644 |
| --- a/docs/cloud-provisioner.md |
| +++ b/docs/cloud-provisioner.md |
| @@ -159,7 +159,9 @@ subdomains land on the fallback over https too. |
| ``` |
| POST /api/instances bearer: caller token {requester: <opaque id>} |
| 201 {host, url, setup_link} |
| - 409 {error: quota_exceeded} per (caller, requester) quota, default 3 |
| + 409 {error: quota_exceeded} per (caller, requester) FREE-space quota, |
| + default 1 (LCHAT_CLOUD_QUOTA; paid spaces |
| + never count, and upgrading frees the slot) |
| 503 {error: at_capacity, message} the box is full |
| GET /api/instances?requester=… this requester's spaces (+ entitlement/tier, |
| and a top-level billing price book) |
| @@ -292,6 +294,26 @@ would otherwise sit there forever: empty, but holding a slot and a disk. The |
| Because the reaper only ever considers hosts **in the cloud ledger**, operator-run |
| instances (family.…, home, push, …) are never candidates. |
| +### Self-reclaim: a quota-blocked create frees the account's own dead weight |
| + |
| +With the free quota at 1, a fumbled create (made a space, lost the tab, never |
| +set it up) must not lock the account out until the reaper's 14-day sweep. So |
| +when a create hits `quota_exceeded`, the cloud first tries to reclaim **one of |
| +the requester's own abandoned allocations** and, if that frees a slot, proceeds |
| +with the create instead of failing: |
| + |
| +- Candidates are only the same (caller, requester)'s free-entitlement rows |
| + **older than one hour** — setup takes minutes, so the age gate cleanly |
| + separates "abandoned, retrying" from "setting it up in another tab right |
| + now" (the home page keeps the setup link for continuing inside the window). |
| +- A **pending row** (a create that died between allocate and finalize — it |
| + would otherwise hold the slot forever) is cleared ledger-only. |
| +- A **provisioned space** must clear the reaper's exact three guards above |
| + (never set up, nobody joined, zero content); it archives to `/data/trash` |
| + through the same controld delete. A stats error is a skip, never a reclaim. |
| +- Anything in real use stays untouched and the 409 stands — the way out then |
| + is upgrading a space (paid spaces don't count) or deleting one from home. |
| + |
| **Hibernating spaces never wake for the reaper.** Stats always come from local |
| state, not from the instance: an *asleep* space is read from its on-disk DB (the |
| warm cache the activator left at its last sleep — on one box, the newest truth), |
diff --git a/home/pow.go b/home/pow.go| index e419038..64f48b8 100644 |
| --- a/home/pow.go |
| +++ b/home/pow.go |
| @@ -1,8 +1,8 @@ |
| package main |
| // Hashcash at space creation (#561): the client burns a few seconds of CPU |
| -// per space, making the Sybil loop (mint invites → fresh accounts → 3 more |
| -// free spaces each) cost compute instead of nothing. The burn runs during the |
| +// per space, making the Sybil loop (mint invites → fresh accounts → another |
| +// free space each) cost compute instead of nothing. The burn runs during the |
| // fader's "Hold on…" build phase — the cost IS the animation. |
| // |
| // Classic hashcash: the server hands out a sealed, expiring, single-use |
diff --git a/reaper.go b/reaper.go| index 0c26645..cd5c35b 100644 |
| --- a/reaper.go |
| +++ b/reaper.go |
| @@ -254,3 +254,62 @@ func (s *cloudServer) reapOnce(now time.Time) (reaped, kept int) { |
| } |
| return reaped, kept |
| } |
| + |
| +// ---- self-reclaim (the create path's escape hatch) ------------------------------ |
| + |
| +// reclaimAge is how old one of the account's own spaces must be before a |
| +// quota-blocked create may reclaim it. Setup takes minutes, so an hour cleanly |
| +// separates "abandoned yesterday, retrying now" (reclaim) from "setting it up |
| +// in another tab right now" (never touch) — and the home page keeps the setup |
| +// link for continuing inside the window. |
| +const reclaimAge = time.Hour |
| + |
| +// reclaimOwnSpace frees at most one quota slot for (caller, requester) by |
| +// reclaiming their own oldest abandoned allocation: a pending row whose create |
| +// died between allocate and finalize, or a provisioned space that clears the |
| +// reaper's never-used guards (never set up, nobody joined, zero content) — |
| +// plus ownership and the age gate on top. Data still archives to /data/trash |
| +// via controld, same as the reaper. A stats error is a skip, never a reclaim. |
| +// Returns whether a slot was freed. |
| +func (s *cloudServer) reclaimOwnSpace(callerID, requester string) bool { |
| + cands, err := s.db.reclaimCandidates(callerID, requester, time.Now().Add(-reclaimAge).Unix()) |
| + if err != nil { |
| + log.Printf("cloud: self-reclaim: ledger query: %v", err) |
| + return false |
| + } |
| + for _, c := range cands { |
| + if c.Host == "" { |
| + // A pending row this old is a dead create (real ones finalize in |
| + // seconds); there is no instance behind it — ledger-only cleanup. |
| + if err := s.db.deleteSpaceByID(c.ID); err != nil { |
| + log.Printf("cloud: self-reclaim: clear stale pending row (id %d): %v", c.ID, err) |
| + continue |
| + } |
| + log.Printf("cloud: self-reclaim: cleared stale pending allocation (id %d, requester of caller %.8s…)", c.ID, callerID) |
| + return true |
| + } |
| + st, err := s.controlStats(c.Host) |
| + if err != nil { |
| + log.Printf("cloud: self-reclaim: skip %s (stats unavailable): %v", c.Host, err) |
| + continue |
| + } |
| + if st.SetupComplete || st.Members > 1 || st.HasContent { |
| + continue // in real use — the quota refusal stands |
| + } |
| + status, body, err := s.controlDelete(c.Host) |
| + if err != nil { |
| + log.Printf("cloud: self-reclaim: delete %s failed: %v", c.Host, err) |
| + continue |
| + } |
| + if status != http.StatusOK && body["error"] != "unknown_host" { |
| + log.Printf("cloud: self-reclaim: delete %s: HTTP %d", c.Host, status) |
| + continue |
| + } |
| + if err := s.db.deleteSpaceByID(c.ID); err != nil { |
| + log.Printf("cloud: self-reclaim: ledger delete %s (id %d): %v", c.Host, c.ID, err) |
| + } |
| + log.Printf("cloud: self-reclaim: reclaimed %s (id %d) — owner's own never-set-up space, making room for their new create", c.Host, c.ID) |
| + return true |
| + } |
| + return false |
| +} |