| 1 | package main |
| 2 | |
| 3 | // The storage meter (#668 slice 1) and its enforcement (#561, insurance-grade): |
| 4 | // how many bytes each cloud-provisioned space occupies, measured server-side |
| 5 | // and shown to members — plus, when caps are configured, per-tier storage |
| 6 | // ceilings pushed into each instance's own LCHAT_BLOB_QUOTA_BYTES guard. |
| 7 | // |
| 8 | // Metering only what ciphertext exposes: an instance's bytes are its DB files |
| 9 | // plus its attachment blobs, all opaque. Nothing here reads content, and |
| 10 | // nothing here ever WAKES a space — measurement is pure filesystem metadata, |
| 11 | // so a sweep over a fleet of hibernating instances stays free. |
| 12 | // |
| 13 | // Split like the reaper: the measuring half runs in controld (root, owns |
| 14 | // /data), the sweeping half in the cloud service (unprivileged, owns the |
| 15 | // ledger), joined by the existing stats endpoint. |
| 16 | |
| 17 | import ( |
| 18 | "bytes" |
| 19 | "encoding/json" |
| 20 | "fmt" |
| 21 | "log" |
| 22 | "net/http" |
| 23 | "os" |
| 24 | "path/filepath" |
| 25 | "strconv" |
| 26 | "strings" |
| 27 | "sync" |
| 28 | "time" |
| 29 | ) |
| 30 | |
| 31 | // ---- measuring (controld-side, root) ------------------------------------------- |
| 32 | |
| 33 | // storageBytesForDB sums the on-disk footprint of an instance rooted at its DB |
| 34 | // path: the SQLite files (db, -wal, -shm) plus the attachment blob dir |
| 35 | // (<db-sans-ext>-blobs/, the disk blobStore's layout — db.go). Pure but for |
| 36 | // the filesystem, so it's directly testable. |
| 37 | // |
| 38 | // Slice-1 honesty: a hibernating space whose local copy has been cleaned away |
| 39 | // measures as only what remains on disk; measuring the S3 prefix instead is |
| 40 | // follow-up work in #668. Today the activator keeps local copies, so this is |
| 41 | // the fleet's truth in practice. |
| 42 | func storageBytesForDB(dbPath string) int64 { |
| 43 | var total int64 |
| 44 | for _, suffix := range []string{"", "-wal", "-shm"} { |
| 45 | if info, err := os.Stat(dbPath + suffix); err == nil { |
| 46 | total += info.Size() |
| 47 | } |
| 48 | } |
| 49 | blobDir := strings.TrimSuffix(dbPath, filepath.Ext(dbPath)) + "-blobs" |
| 50 | _ = filepath.WalkDir(blobDir, func(_ string, d os.DirEntry, err error) error { |
| 51 | if err != nil || d.IsDir() { |
| 52 | return nil // unreadable entries just don't count; never fail a sweep |
| 53 | } |
| 54 | if info, err := d.Info(); err == nil { |
| 55 | total += info.Size() |
| 56 | } |
| 57 | return nil |
| 58 | }) |
| 59 | return total |
| 60 | } |
| 61 | |
| 62 | // instanceStorageBytes measures one hosted instance. |
| 63 | func instanceStorageBytes(host string) int64 { |
| 64 | return storageBytesForDB(filepath.Join(instanceDataDir, host+".db")) |
| 65 | } |
| 66 | |
| 67 | // ---- sweeping (cloud-side) ------------------------------------------------------ |
| 68 | |
| 69 | // meterLoop refreshes every ledger space's storage_bytes on an interval. |
| 70 | // Enabled by default (interval <= 0 disables): the meter is harmless metadata |
| 71 | // and members should see their usage without an operator opting in. |
| 72 | func (s *cloudServer) meterLoop(interval time.Duration) { |
| 73 | if interval <= 0 { |
| 74 | return |
| 75 | } |
| 76 | log.Printf("cloud: storage meter on — sweeping every %s", interval) |
| 77 | for { |
| 78 | s.meterOnce() |
| 79 | time.Sleep(interval) |
| 80 | } |
| 81 | } |
| 82 | |
| 83 | // meterOnce asks controld for each finalized space's byte count and records |
| 84 | // it. A stats error skips that space (its last reading stands, metered_at |
| 85 | // shows the staleness); the sweep never fails as a whole. When storage caps |
| 86 | // are configured, the sweep also RECONCILES quotas: any space whose pushed |
| 87 | // quota doesn't match its tier's cap gets a fresh push — the self-healing |
| 88 | // backstop behind the event-driven pushes (webhooks, creation). |
| 89 | func (s *cloudServer) meterOnce() (metered int) { |
| 90 | spaces, err := s.db.allFinalizedSpaces() |
| 91 | if err != nil { |
| 92 | log.Printf("cloud: meter: ledger query: %v", err) |
| 93 | return 0 |
| 94 | } |
| 95 | now := time.Now().Unix() |
| 96 | if s.usageURL != "" { |
| 97 | // Applied-report dedupe ids age out with the sweep (usage pipe, #857); |
| 98 | // the sidecar retry horizon is minutes, so a week is generous. |
| 99 | if err := s.db.pruneUsageReports(now - 7*24*3600); err != nil { |
| 100 | log.Printf("cloud: meter: usage-report prune: %v", err) |
| 101 | } |
| 102 | } |
| 103 | for _, sp := range spaces { |
| 104 | st, err := s.controlStats(sp.Host) |
| 105 | if err != nil { |
| 106 | log.Printf("cloud: meter: skip %s (stats unavailable): %v", sp.Host, err) |
| 107 | continue |
| 108 | } |
| 109 | if err := s.db.setStorageBytes(sp.ID, st.StorageBytes, st.SetupComplete, st.Members, now); err != nil { |
| 110 | log.Printf("cloud: meter: record %s: %v", sp.Host, err) |
| 111 | continue |
| 112 | } |
| 113 | metered++ |
| 114 | if want := s.capFor(sp.Entitlement, sp.Tier, st.People); want > 0 && want != sp.QuotaBytes { |
| 115 | s.pushQuota(sp.ID, sp.Host, sp.Entitlement, sp.Tier, st.People) |
| 116 | } |
| 117 | // The per-seat renewal true-up rides the same sweep: in the runway |
| 118 | // before a per-seat renewal, re-count and set next year's quantity. |
| 119 | s.reconcileSeats(sp, st.People, now) |
| 120 | // The usage pipe (#857): any space without a meter token gets one — |
| 121 | // how the fleet converges without a per-space backfill step. |
| 122 | if s.usageURL != "" && sp.MeterTokenHash == "" { |
| 123 | s.pushMeterToken(sp.ID, sp.Host) |
| 124 | } |
| 125 | // Replication class reconcile (cloudreplication.go): no ledger record |
| 126 | // to compare, so push each pass — controld no-ops on a matching class. |
| 127 | s.pushReplication(sp.Host, sp.Entitlement, sp.Tier) |
| 128 | } |
| 129 | if metered > 0 { |
| 130 | log.Printf("cloud: meter: refreshed %d space(s)", metered) |
| 131 | } |
| 132 | // The free-pool gauge rides the same sweep (pool.go): the storage figures |
| 133 | // it reads from the ledger were refreshed moments ago. |
| 134 | if s.pool.enabled() { |
| 135 | s.poolSweep(time.Now()) |
| 136 | } |
| 137 | return metered |
| 138 | } |
| 139 | |
| 140 | // ---- storage caps (#561 via #668): enforcement as insurance --------------------- |
| 141 | // |
| 142 | // The instance already refuses uploads past LCHAT_BLOB_QUOTA_BYTES (507, |
| 143 | // nothing deleted, text messaging never gated). Enforcement is therefore pure |
| 144 | // wiring: map each tier to a cap, and push the cap into the instance's env |
| 145 | // when its tier changes. Caps are config (LCHAT_CLOUD_STORAGE_CAPS); no |
| 146 | // config = no enforcement anywhere, and self-hosted or operator-run instances |
| 147 | // are never touched (they're not in this ledger). |
| 148 | |
| 149 | // parseByteSize reads "25GB" / "2TB" / "500MB" / plain bytes. Decimal units — |
| 150 | // matching how storage is priced and displayed. |
| 151 | func parseByteSize(v string) (int64, error) { |
| 152 | v = strings.TrimSpace(strings.ToUpper(v)) |
| 153 | mult := int64(1) |
| 154 | for _, u := range []struct { |
| 155 | suffix string |
| 156 | mult int64 |
| 157 | }{{"TB", 1e12}, {"GB", 1e9}, {"MB", 1e6}, {"KB", 1e3}} { |
| 158 | if strings.HasSuffix(v, u.suffix) { |
| 159 | v, mult = strings.TrimSpace(strings.TrimSuffix(v, u.suffix)), u.mult |
| 160 | break |
| 161 | } |
| 162 | } |
| 163 | n, err := strconv.ParseInt(v, 10, 64) |
| 164 | if err != nil || n <= 0 { |
| 165 | return 0, fmt.Errorf("bad size %q", v) |
| 166 | } |
| 167 | return n * mult, nil |
| 168 | } |
| 169 | |
| 170 | // parseStorageCaps reads LCHAT_CLOUD_STORAGE_CAPS: "free=10GB,family=25GB,…" — |
| 171 | // keys are tier keys from the price book, plus "free" for unpaid spaces. A |
| 172 | // "/seat" suffix on the size ("business=11GB/seat") makes the entry per-seat: |
| 173 | // the pushed cap is that size × the space's live head count (pooled — one |
| 174 | // shared pot, no per-member walls; docs/tier-economics.md § Per-seat). |
| 175 | func parseStorageCaps(v string) (caps, seatCaps map[string]int64, err error) { |
| 176 | if strings.TrimSpace(v) == "" { |
| 177 | return nil, nil, nil |
| 178 | } |
| 179 | caps, seatCaps = map[string]int64{}, map[string]int64{} |
| 180 | for _, pair := range strings.Split(v, ",") { |
| 181 | k, val, ok := strings.Cut(strings.TrimSpace(pair), "=") |
| 182 | if !ok || strings.TrimSpace(k) == "" { |
| 183 | return nil, nil, fmt.Errorf("bad entry %q (want key=size)", pair) |
| 184 | } |
| 185 | into := caps |
| 186 | if rest, perSeat := strings.CutSuffix(strings.TrimSpace(val), "/seat"); perSeat { |
| 187 | val, into = rest, seatCaps |
| 188 | } |
| 189 | n, err := parseByteSize(val) |
| 190 | if err != nil { |
| 191 | return nil, nil, fmt.Errorf("cap %q: %v", k, err) |
| 192 | } |
| 193 | into[strings.TrimSpace(k)] = n |
| 194 | } |
| 195 | return caps, seatCaps, nil |
| 196 | } |
| 197 | |
| 198 | // capFor maps a space's billing state to its storage cap. people is the live |
| 199 | // non-agent member count, which only per-seat entries use. 0 = no cap to push |
| 200 | // (caps unconfigured, or a paid tier with no entry — never guess a limit). |
| 201 | func (s *cloudServer) capFor(entitlement, tier string, people int) int64 { |
| 202 | if len(s.storageCaps) == 0 && len(s.seatCaps) == 0 { |
| 203 | return 0 |
| 204 | } |
| 205 | if entitlement == entPaid { |
| 206 | if per, ok := s.seatCaps[tier]; ok { |
| 207 | return per * int64(max(people, 1)) |
| 208 | } |
| 209 | return s.storageCaps[tier] |
| 210 | } |
| 211 | // Demos get their own (tiny) cap when configured, else the free one — |
| 212 | // falling through beats leaving a demo uncapped. |
| 213 | if entitlement == entDemo { |
| 214 | if cap, ok := s.storageCaps[entDemo]; ok { |
| 215 | return cap |
| 216 | } |
| 217 | } |
| 218 | return s.storageCaps["free"] |
| 219 | } |
| 220 | |
| 221 | // controlSetQuota pushes one instance's storage ceiling to controld. |
| 222 | func (s *cloudServer) controlSetQuota(host string, quota int64, entitlement string) error { |
| 223 | payload, _ := json.Marshal(map[string]any{"bytes": quota, "entitlement": entitlement}) |
| 224 | req, err := http.NewRequest("POST", "http://control/instances/"+host+"/quota", bytes.NewReader(payload)) |
| 225 | if err != nil { |
| 226 | return err |
| 227 | } |
| 228 | req.Header.Set("Authorization", "Bearer "+s.controlToken) |
| 229 | req.Header.Set("Content-Type", "application/json") |
| 230 | resp, err := s.control.Do(req) |
| 231 | if err != nil { |
| 232 | return err |
| 233 | } |
| 234 | defer resp.Body.Close() |
| 235 | if resp.StatusCode != http.StatusOK { |
| 236 | return fmt.Errorf("control quota %s: HTTP %d", host, resp.StatusCode) |
| 237 | } |
| 238 | return nil |
| 239 | } |
| 240 | |
| 241 | // pushQuota applies a space's tier cap to its instance, recording what was |
| 242 | // pushed. Best-effort by design: a failed push logs and stands down — the |
| 243 | // meter sweep reconciles it on the next pass. |
| 244 | func (s *cloudServer) pushQuota(id int64, host, entitlement, tier string, people int) { |
| 245 | cap := s.capFor(entitlement, tier, people) |
| 246 | if cap <= 0 { |
| 247 | return |
| 248 | } |
| 249 | if err := s.controlSetQuota(host, cap, entitlement); err != nil { |
| 250 | log.Printf("cloud: quota push %s: %v (sweep will retry)", host, err) |
| 251 | return |
| 252 | } |
| 253 | if err := s.db.setQuotaBytes(id, cap); err != nil { |
| 254 | log.Printf("cloud: quota record %s: %v", host, err) |
| 255 | return |
| 256 | } |
| 257 | log.Printf("cloud: quota %s → %d bytes (%s)", host, cap, entitlementLabel(entitlement, tier)) |
| 258 | } |
| 259 | |
| 260 | func entitlementLabel(entitlement, tier string) string { |
| 261 | if entitlement == entPaid { |
| 262 | return "tier " + tier |
| 263 | } |
| 264 | return "free" |
| 265 | } |
| 266 | |
| 267 | // ---- quota apply (controld-side, root) ------------------------------------------- |
| 268 | |
| 269 | // instanceEnvMu serializes every read-modify-write of instance env files. |
| 270 | // controld's quota and meter pushes arrive on separate request goroutines |
| 271 | // (a Stripe webhook and the sweep can land together) and write DIFFERENT |
| 272 | // keys to the SAME file — unserialized, the loser's read-before-write |
| 273 | // snapshot would silently drop the winner's keys, and because both callers |
| 274 | // record success in the ledger, nothing would ever retry. One process-wide |
| 275 | // mutex is plenty at this scale; per-host granularity would buy nothing. |
| 276 | var instanceEnvMu sync.Mutex |
| 277 | |
| 278 | // updateInstanceEnv applies one mutation to an env file under the lock — |
| 279 | // the only correct way to touch an instance env after the daemon is up. |
| 280 | func updateInstanceEnv(envPath string, mutate func(env map[string]string)) error { |
| 281 | instanceEnvMu.Lock() |
| 282 | defer instanceEnvMu.Unlock() |
| 283 | env, err := readEnvFile(envPath) |
| 284 | if err != nil { |
| 285 | return err |
| 286 | } |
| 287 | mutate(env) |
| 288 | return writeEnvFile(envPath, env) |
| 289 | } |
| 290 | |
| 291 | // setInstanceQuota upserts LCHAT_BLOB_QUOTA_BYTES in an instance's env file |
| 292 | // (0 removes it — back to the compiled default) and, when entitlement is |
| 293 | // non-empty, mirrors LCHAT_ENTITLEMENT ("free"/"paid"/"demo" — the UI's |
| 294 | // billing-affordance / demo-band gate) in the same write, preserving every |
| 295 | // other var. |
| 296 | func setInstanceQuota(envPath string, quota int64, entitlement string) error { |
| 297 | return updateInstanceEnv(envPath, func(env map[string]string) { |
| 298 | if quota <= 0 { |
| 299 | delete(env, "LCHAT_BLOB_QUOTA_BYTES") |
| 300 | } else { |
| 301 | env["LCHAT_BLOB_QUOTA_BYTES"] = strconv.FormatInt(quota, 10) |
| 302 | } |
| 303 | if entitlement != "" { |
| 304 | env["LCHAT_ENTITLEMENT"] = entitlement |
| 305 | } |
| 306 | }) |
| 307 | } |
| 308 | |
| 309 | // setInstanceMeter upserts LCHAT_METER_URL + LCHAT_METER_TOKEN in an |
| 310 | // instance's env file (both empty removes the pair — metering off), |
| 311 | // preserving every other var. The controld half of the meter-token push, |
| 312 | // next to its quota sibling. |
| 313 | func setInstanceMeter(envPath, url, token string) error { |
| 314 | return updateInstanceEnv(envPath, func(env map[string]string) { |
| 315 | if url == "" { |
| 316 | delete(env, "LCHAT_METER_URL") |
| 317 | delete(env, "LCHAT_METER_TOKEN") |
| 318 | } else { |
| 319 | env["LCHAT_METER_URL"] = url |
| 320 | env["LCHAT_METER_TOKEN"] = token |
| 321 | } |
| 322 | }) |
| 323 | } |
| 324 | |
| 325 | // pushQuotaByID looks a space's billing state up and applies its cap — |
| 326 | // the webhook-side entry point (events carry ids, not rows). |
| 327 | func (s *cloudServer) pushQuotaByID(id int64) { |
| 328 | if len(s.storageCaps) == 0 && len(s.seatCaps) == 0 { |
| 329 | return |
| 330 | } |
| 331 | m, err := s.db.spaceBillingByID(id) |
| 332 | if err != nil || m.Host == "" { |
| 333 | return // pending create or gone — the sweep reconciles survivors |
| 334 | } |
| 335 | // Per-seat caps want the live head count; the last metered member count |
| 336 | // (agents included — slightly generous, never tight) covers a stats |
| 337 | // outage, and the sweep reconciles either way. |
| 338 | people := m.Members |
| 339 | if _, perSeat := s.seatCaps[m.Tier]; perSeat && m.Entitlement == entPaid { |
| 340 | if st, err := s.controlStats(m.Host); err == nil { |
| 341 | people = st.People |
| 342 | } |
| 343 | } |
| 344 | s.pushQuota(id, m.Host, m.Entitlement, m.Tier, people) |
| 345 | } |
| 346 | |
| 347 | // seatTrueUpWindow is how far before a per-seat renewal the sweep starts |
| 348 | // reconciling the subscription's quantity with the live head count. A week of |
| 349 | // hourly passes absorbs a missed webhook or a Stripe blip comfortably. |
| 350 | const seatTrueUpWindow = 7 * 24 * time.Hour |
| 351 | |
| 352 | // reconcileSeats is the renewal true-up (docs/tier-economics.md § Per-seat): |
| 353 | // inside the window before a per-seat space renews, set the subscription's |
| 354 | // quantity to the live non-agent head count so the next invoice bills the |
| 355 | // real team size. proration_behavior=none inside trueUpSeats means nothing is |
| 356 | // charged or credited mid-cycle — this is the entire dropoff/growth policy. |
| 357 | // The ledger's seats update when Stripe's subscription.updated webhook lands; |
| 358 | // until then a repeat pass just re-reads the (now matching) quantity and |
| 359 | // no-ops. |
| 360 | func (s *cloudServer) reconcileSeats(sp meterRow, people int, now int64) { |
| 361 | if s.billing == nil || sp.Entitlement != entPaid || sp.SubscriptionID == "" { |
| 362 | return |
| 363 | } |
| 364 | t, ok := s.billing.tier(sp.Tier) |
| 365 | if !ok || !t.PerSeat { |
| 366 | return |
| 367 | } |
| 368 | if sp.PaidUntil <= now || sp.PaidUntil-now > int64(seatTrueUpWindow/time.Second) { |
| 369 | return |
| 370 | } |
| 371 | want := max(people, t.MinSeats, 1) |
| 372 | if want == sp.Seats { |
| 373 | return |
| 374 | } |
| 375 | changed, err := s.billing.trueUpSeats(sp.SubscriptionID, want) |
| 376 | if err != nil { |
| 377 | log.Printf("cloud: seat true-up %s: %v (sweep will retry)", sp.Host, err) |
| 378 | return |
| 379 | } |
| 380 | if changed { |
| 381 | log.Printf("cloud: seats %s → %d at renewal (was %d)", sp.Host, want, sp.Seats) |
| 382 | } |
| 383 | } |
| 384 | |
amadan