| 1 | // Package assetcache is a bounded disk store for cached asset responses. |
| 2 | // |
| 3 | // Each entry is a <id>.meta file (JSON) naming its versioned body files, |
| 4 | // <id>.<v>.body and optionally <id>.<v>.gz. The index lives in memory and |
| 5 | // is rebuilt from the .meta files by Open. |
| 6 | package assetcache |
| 7 | |
| 8 | import ( |
| 9 | "container/list" |
| 10 | "errors" |
| 11 | "fmt" |
| 12 | "math" |
| 13 | "os" |
| 14 | "path/filepath" |
| 15 | "sort" |
| 16 | "strings" |
| 17 | "sync" |
| 18 | "time" |
| 19 | ) |
| 20 | |
| 21 | // Config configures a Store. |
| 22 | type Config struct { |
| 23 | Dir string // the cache directory; created 0700 |
| 24 | Budget int64 // bytes of entries (bodies and .meta files) the store may hold |
| 25 | FS FS // free-space source; may be called with the store's lock held |
| 26 | Now func() time.Time // defaults to time.Now |
| 27 | } |
| 28 | |
| 29 | // Stats is a snapshot of the store's accounting. |
| 30 | type Stats struct { |
| 31 | Entries int |
| 32 | // Bytes is the disk the store holds: every indexed entry's bodies and |
| 33 | // .meta file, and the retired bodies counted in Retired. |
| 34 | Bytes int64 |
| 35 | Reserved int64 |
| 36 | // Retired is the bytes of replaced, expired, evicted or swept bodies |
| 37 | // that still hold disk: open Readers keep them, an unlink failed and |
| 38 | // will be retried, or a replacement could not be made durable and they |
| 39 | // are kept for the next Open. |
| 40 | Retired int64 |
| 41 | Apps int |
| 42 | } |
| 43 | |
| 44 | // entry is one indexed cache entry. |
| 45 | type entry struct { |
| 46 | id string |
| 47 | meta Meta |
| 48 | metaSize int64 // bytes of the .meta file |
| 49 | ver *version // the body files meta names |
| 50 | used uint64 // the store's use counter when indexed or last returned by Get |
| 51 | elem *list.Element // its place in its app's LRU list, front most recent; guarded by s.mu |
| 52 | } |
| 53 | |
| 54 | // appIndex is one app's indexed entries, most recently used first. |
| 55 | type appIndex struct { |
| 56 | lru *list.List // of *entry |
| 57 | bytes int64 // the sum of its entries' bytes() |
| 58 | } |
| 59 | |
| 60 | func newEntry(id string, m Meta, metaSize int64) *entry { |
| 61 | return &entry{id: id, meta: m, metaSize: metaSize, ver: &version{id: id, v: m.Version, bytes: m.Size + m.GzipSize}} |
| 62 | } |
| 63 | |
| 64 | // bytes is the disk the entry's files hold. |
| 65 | func (e *entry) bytes() int64 { return e.metaSize + e.ver.bytes } |
| 66 | |
| 67 | // version is one committed version's body files, <id>.<v>.body and |
| 68 | // <id>.<v>.gz. Its bytes stay counted in Store.used while any of indexed, |
| 69 | // leases, unlinking, stuck or kept holds: the disk is not free until the files are |
| 70 | // unlinked and the last Reader has closed them. Fields are guarded by s.mu. |
| 71 | type version struct { |
| 72 | id, v string |
| 73 | bytes int64 |
| 74 | indexed bool // an indexed entry names this version |
| 75 | leases int // open Readers |
| 76 | unlinking bool // an unlink is in progress |
| 77 | stuck bool // an unlink failed; retryStuck tries again |
| 78 | kept bool // left on disk for the next Open: the meta that replaced it may not be durable |
| 79 | counted bool // bytes are in s.used |
| 80 | } |
| 81 | |
| 82 | // idLock serializes the changes to one id's .meta file and its index slot. |
| 83 | type idLock struct { |
| 84 | mu sync.Mutex |
| 85 | refs int // guarded by Store.mu |
| 86 | } |
| 87 | |
| 88 | // Store is a bounded disk cache over one directory. |
| 89 | type Store struct { |
| 90 | cfg Config |
| 91 | |
| 92 | mu sync.Mutex |
| 93 | entries map[string]*entry // by id |
| 94 | apps map[string]*appIndex // by AppScope, for apps with entries |
| 95 | used int64 // bytes held by entries and retired versions |
| 96 | retired int64 // the part of used no indexed entry names |
| 97 | reserved int64 // bytes promised to fills in flight |
| 98 | releases uint64 // ticks whenever reserved bytes are released; see admit |
| 99 | fills int // fills in flight on this box |
| 100 | fillsByApp map[string]int // fills in flight by AppScope |
| 101 | useCount uint64 // ticks once per index and per Get hit |
| 102 | idLocks map[string]*idLock // by id, while held or awaited |
| 103 | stuck map[*version]struct{} // retired versions whose unlink failed |
| 104 | |
| 105 | afterSample func() // tests only: called between admit's unlocked free-space sample and its decision |
| 106 | } |
| 107 | |
| 108 | // Open creates the directory if needed and rebuilds the index from it. |
| 109 | // It removes whatever a crash can leave behind: temporaries, body files no |
| 110 | // parsed .meta names (orphans and superseded versions), and a .meta that |
| 111 | // does not parse or whose files are incomplete, together with its files. |
| 112 | // |
| 113 | // Recency starts from each entry's StoredAt, the oldest least recently used. |
| 114 | // If what it finds is more than cfg.Budget (the budget was lowered since the |
| 115 | // files were written), Open evicts by the fair-share rule until it fits. If |
| 116 | // files it must delete for that cannot be deleted, Open still succeeds, over |
| 117 | // its budget: it serves the entries it holds and refuses every fill with |
| 118 | // ErrNoRoom until it is back under, each Reserve retrying the deletions. |
| 119 | func Open(cfg Config) (*Store, error) { |
| 120 | if cfg.Dir == "" { |
| 121 | return nil, errors.New("assetcache: no directory") |
| 122 | } |
| 123 | if cfg.Budget <= 0 { |
| 124 | return nil, errors.New("assetcache: budget must be positive") |
| 125 | } |
| 126 | if cfg.Now == nil { |
| 127 | cfg.Now = time.Now |
| 128 | } |
| 129 | if cfg.FS == nil { |
| 130 | cfg.FS = Statfs |
| 131 | } |
| 132 | if err := os.MkdirAll(cfg.Dir, 0o700); err != nil { |
| 133 | return nil, fmt.Errorf("assetcache: %w", err) |
| 134 | } |
| 135 | s := &Store{ |
| 136 | cfg: cfg, |
| 137 | entries: make(map[string]*entry), |
| 138 | apps: make(map[string]*appIndex), |
| 139 | |
| 140 | fillsByApp: make(map[string]int), |
| 141 | idLocks: make(map[string]*idLock), |
| 142 | stuck: make(map[*version]struct{}), |
| 143 | } |
| 144 | if err := s.scan(); err != nil { |
| 145 | return nil, err |
| 146 | } |
| 147 | s.makeRoom(0) |
| 148 | return s, nil |
| 149 | } |
| 150 | |
| 151 | func (s *Store) scan() error { |
| 152 | ents, err := os.ReadDir(s.cfg.Dir) |
| 153 | if err != nil { |
| 154 | return fmt.Errorf("assetcache: %w", err) |
| 155 | } |
| 156 | var metas, files []string |
| 157 | for _, de := range ents { |
| 158 | if de.IsDir() { |
| 159 | continue |
| 160 | } |
| 161 | name := de.Name() |
| 162 | switch { |
| 163 | case strings.Contains(name, ".tmp-"): |
| 164 | if err := s.remove(name); err != nil { |
| 165 | return err |
| 166 | } |
| 167 | case strings.HasSuffix(name, ".meta"): |
| 168 | metas = append(metas, name) |
| 169 | case strings.HasSuffix(name, ".body"), strings.HasSuffix(name, ".gz"): |
| 170 | files = append(files, name) |
| 171 | } |
| 172 | } |
| 173 | |
| 174 | keep := make(map[string]bool) // body file names a parsed meta names |
| 175 | var found []*entry |
| 176 | var sum int64 // the bytes of found |
| 177 | for _, name := range metas { |
| 178 | id := strings.TrimSuffix(name, ".meta") |
| 179 | m, metaSize, valid, err := s.loadMeta(id) |
| 180 | if err != nil { |
| 181 | return err |
| 182 | } |
| 183 | if !valid { |
| 184 | if err := s.remove(name); err != nil { |
| 185 | return err |
| 186 | } |
| 187 | continue |
| 188 | } |
| 189 | keep[bodyName(id, m.Version)] = true |
| 190 | if m.GzipSize > 0 { |
| 191 | keep[gzipName(id, m.Version)] = true |
| 192 | } |
| 193 | e := newEntry(id, m, metaSize) |
| 194 | if e.ver.bytes > math.MaxInt64-metaSize || sum > math.MaxInt64-e.bytes() { |
| 195 | return errors.New("assetcache: indexed bytes overflow") |
| 196 | } |
| 197 | sum += e.bytes() |
| 198 | found = append(found, e) |
| 199 | } |
| 200 | // Index the oldest stored first, so it starts as the least recently used. |
| 201 | sort.Slice(found, func(i, j int) bool { |
| 202 | a, b := found[i], found[j] |
| 203 | if !a.meta.StoredAt.Equal(b.meta.StoredAt) { |
| 204 | return a.meta.StoredAt.Before(b.meta.StoredAt) |
| 205 | } |
| 206 | return a.id < b.id |
| 207 | }) |
| 208 | for _, e := range found { |
| 209 | s.index(e) |
| 210 | } |
| 211 | for _, name := range files { |
| 212 | if !keep[name] { |
| 213 | if err := s.remove(name); err != nil { |
| 214 | return err |
| 215 | } |
| 216 | } |
| 217 | } |
| 218 | return nil |
| 219 | } |
| 220 | |
| 221 | // loadMeta reads and checks <id>.meta. It reports valid=false for an entry |
| 222 | // that is certainly broken: a missing or malformed meta, one that belongs to |
| 223 | // another key, or one whose body files are missing or not the size it |
| 224 | // records. Any other failure to read the disk is returned as an error, so a |
| 225 | // permission or I/O fault never leads to deleting a good entry. metaSize is |
| 226 | // the .meta file's length. |
| 227 | func (s *Store) loadMeta(id string) (m Meta, metaSize int64, valid bool, err error) { |
| 228 | raw, err := os.ReadFile(filepath.Join(s.cfg.Dir, metaName(id))) |
| 229 | if errors.Is(err, os.ErrNotExist) { |
| 230 | return Meta{}, 0, false, nil |
| 231 | } |
| 232 | if err != nil { |
| 233 | return Meta{}, 0, false, fmt.Errorf("assetcache: %w", err) |
| 234 | } |
| 235 | m, err = decodeMeta(raw) |
| 236 | if err != nil || m.Key.ID() != id { |
| 237 | return Meta{}, 0, false, nil |
| 238 | } |
| 239 | ok, err := s.bodyMatches(bodyName(id, m.Version), m.Size) |
| 240 | if err != nil || !ok { |
| 241 | return Meta{}, 0, false, err |
| 242 | } |
| 243 | if m.GzipSize > 0 { |
| 244 | ok, err = s.bodyMatches(gzipName(id, m.Version), m.GzipSize) |
| 245 | if err != nil || !ok { |
| 246 | return Meta{}, 0, false, err |
| 247 | } |
| 248 | } |
| 249 | return m, int64(len(raw)), true, nil |
| 250 | } |
| 251 | |
| 252 | // bodyMatches reports whether the named file is a regular file of exactly |
| 253 | // size bytes. A missing file is false with no error. |
| 254 | func (s *Store) bodyMatches(name string, size int64) (bool, error) { |
| 255 | fi, err := os.Stat(filepath.Join(s.cfg.Dir, name)) |
| 256 | if errors.Is(err, os.ErrNotExist) { |
| 257 | return false, nil |
| 258 | } |
| 259 | if err != nil { |
| 260 | return false, fmt.Errorf("assetcache: %w", err) |
| 261 | } |
| 262 | return fi.Mode().IsRegular() && fi.Size() == size, nil |
| 263 | } |
| 264 | |
| 265 | // remove deletes one file of the store's directory by name. |
| 266 | func (s *Store) remove(name string) error { |
| 267 | if err := os.Remove(filepath.Join(s.cfg.Dir, name)); err != nil && !errors.Is(err, os.ErrNotExist) { |
| 268 | return fmt.Errorf("assetcache: %w", err) |
| 269 | } |
| 270 | return nil |
| 271 | } |
| 272 | |
| 273 | // index adds e, whose id has no indexed entry, to the in-memory index and |
| 274 | // counts its files. The caller holds s.mu or has not yet shared s. |
| 275 | func (s *Store) index(e *entry) { |
| 276 | s.entries[e.id] = e |
| 277 | scope := e.meta.Key.AppScope() |
| 278 | app := s.apps[scope] |
| 279 | if app == nil { |
| 280 | app = &appIndex{lru: list.New()} |
| 281 | s.apps[scope] = app |
| 282 | } |
| 283 | s.useCount++ |
| 284 | e.used = s.useCount |
| 285 | e.elem = app.lru.PushFront(e) |
| 286 | app.bytes += e.bytes() |
| 287 | e.ver.indexed = true |
| 288 | e.ver.counted = true |
| 289 | s.used += e.bytes() |
| 290 | } |
| 291 | |
| 292 | // unindexLocked takes e out of the index and retires its version: the .meta |
| 293 | // bytes stop counting (the caller has removed or replaced the file), the body |
| 294 | // bytes stay counted until unlink has removed them and the last lease has |
| 295 | // closed. The caller holds s.mu and, once it has released s.mu, must call |
| 296 | // s.unlink(e.ver). |
| 297 | func (s *Store) unindexLocked(e *entry) { |
| 298 | delete(s.entries, e.id) |
| 299 | scope := e.meta.Key.AppScope() |
| 300 | if app := s.apps[scope]; app != nil { |
| 301 | app.lru.Remove(e.elem) |
| 302 | app.bytes -= e.bytes() |
| 303 | if app.lru.Len() == 0 { |
| 304 | delete(s.apps, scope) |
| 305 | } |
| 306 | } |
| 307 | e.elem = nil |
| 308 | s.used -= e.metaSize |
| 309 | v := e.ver |
| 310 | v.indexed = false |
| 311 | v.unlinking = true |
| 312 | s.retired += v.bytes |
| 313 | } |
| 314 | |
| 315 | // unlink removes a retired version's body files at once, whether or not a |
| 316 | // Reader still holds them (an unlinked open file stays readable), then |
| 317 | // settles its accounting. A failed removal leaves the version stuck: its bytes |
| 318 | // stay counted and retryStuck tries again. It is called without s.mu held. |
| 319 | func (s *Store) unlink(v *version) { |
| 320 | err := errors.Join(s.remove(bodyName(v.id, v.v)), s.remove(gzipName(v.id, v.v))) |
| 321 | s.mu.Lock() |
| 322 | defer s.mu.Unlock() |
| 323 | v.unlinking = false |
| 324 | if err != nil { |
| 325 | v.stuck = true |
| 326 | s.stuck[v] = struct{}{} |
| 327 | } else if v.stuck { |
| 328 | v.stuck = false |
| 329 | delete(s.stuck, v) |
| 330 | } |
| 331 | s.settleLocked(v) |
| 332 | } |
| 333 | |
| 334 | // retryStuck retries the removal of versions whose unlink failed, and |
| 335 | // reports whether there were any to retry. It is called without s.mu held. |
| 336 | func (s *Store) retryStuck() bool { |
| 337 | s.mu.Lock() |
| 338 | var vs []*version |
| 339 | for v := range s.stuck { |
| 340 | if !v.unlinking { |
| 341 | v.unlinking = true |
| 342 | vs = append(vs, v) |
| 343 | } |
| 344 | } |
| 345 | s.mu.Unlock() |
| 346 | for _, v := range vs { |
| 347 | s.unlink(v) |
| 348 | } |
| 349 | return len(vs) > 0 |
| 350 | } |
| 351 | |
| 352 | // settleLocked stops counting v's bytes once nothing holds them. The caller |
| 353 | // holds s.mu. |
| 354 | func (s *Store) settleLocked(v *version) { |
| 355 | if v.counted && !v.indexed && v.leases == 0 && !v.unlinking && !v.stuck && !v.kept { |
| 356 | v.counted = false |
| 357 | s.used -= v.bytes |
| 358 | s.retired -= v.bytes |
| 359 | } |
| 360 | } |
| 361 | |
| 362 | // lockID takes the lock for one id's .meta file and index slot, and returns |
| 363 | // the function that releases it. Lock order: a Fill's mu, then an id lock, |
| 364 | // then s.mu. It is called without s.mu held. |
| 365 | func (s *Store) lockID(id string) (unlock func()) { |
| 366 | s.mu.Lock() |
| 367 | l := s.idLocks[id] |
| 368 | if l == nil { |
| 369 | l = &idLock{} |
| 370 | s.idLocks[id] = l |
| 371 | } |
| 372 | l.refs++ |
| 373 | s.mu.Unlock() |
| 374 | l.mu.Lock() |
| 375 | return func() { |
| 376 | l.mu.Unlock() |
| 377 | s.mu.Lock() |
| 378 | if l.refs--; l.refs == 0 { |
| 379 | delete(s.idLocks, id) |
| 380 | } |
| 381 | s.mu.Unlock() |
| 382 | } |
| 383 | } |
| 384 | |
| 385 | // drop deletes the indexed entry e: its .meta first, so a crash part way |
| 386 | // leaves only orphan bodies that Open removes, then its bodies, subject to |
| 387 | // leases. It deletes nothing and reports false when e is no longer the |
| 388 | // indexed entry for its id, or when its .meta cannot be removed (the entry |
| 389 | // then stays indexed and counted). Get's expiry, eviction and Retain use it, and |
| 390 | // the sweep is to. It is called without s.mu held. |
| 391 | func (s *Store) drop(e *entry) bool { |
| 392 | unlock := s.lockID(e.id) |
| 393 | s.mu.Lock() |
| 394 | current := s.entries[e.id] == e |
| 395 | s.mu.Unlock() |
| 396 | if !current { |
| 397 | unlock() |
| 398 | return false |
| 399 | } |
| 400 | if err := s.remove(metaName(e.id)); err != nil { |
| 401 | unlock() |
| 402 | return false |
| 403 | } |
| 404 | s.mu.Lock() |
| 405 | s.unindexLocked(e) |
| 406 | s.mu.Unlock() |
| 407 | unlock() |
| 408 | s.unlink(e.ver) |
| 409 | s.retryStuck() |
| 410 | return true |
| 411 | } |
| 412 | |
| 413 | // Entry is a cache hit: the metadata as stored, and the means to open its |
| 414 | // bodies. Its Header is the caller's own copy. |
| 415 | type Entry struct { |
| 416 | Meta Meta |
| 417 | |
| 418 | s *Store |
| 419 | ver *version |
| 420 | } |
| 421 | |
| 422 | // Get returns the entry for k and marks it recently used. An entry whose |
| 423 | // expiry has come (Now at or past Meta.Expires) is not returned: it is |
| 424 | // deleted. |
| 425 | func (s *Store) Get(k Key) (*Entry, bool) { |
| 426 | id := k.ID() |
| 427 | now := s.cfg.Now() |
| 428 | s.mu.Lock() |
| 429 | e := s.entries[id] |
| 430 | if e == nil || e.meta.Key != k { |
| 431 | s.mu.Unlock() |
| 432 | return nil, false |
| 433 | } |
| 434 | if !now.Before(e.meta.Expires) { |
| 435 | s.mu.Unlock() |
| 436 | s.drop(e) |
| 437 | return nil, false |
| 438 | } |
| 439 | s.useCount++ |
| 440 | e.used = s.useCount |
| 441 | s.apps[e.meta.Key.AppScope()].lru.MoveToFront(e.elem) |
| 442 | out := &Entry{Meta: e.meta, s: s, ver: e.ver} |
| 443 | out.Meta.Header = e.meta.Header.Clone() |
| 444 | s.mu.Unlock() |
| 445 | return out, true |
| 446 | } |
| 447 | |
| 448 | // Open opens the identity body, or the gzip copy when gzip is true. The |
| 449 | // Reader holds a lease on the entry's version: if the entry is replaced, |
| 450 | // evicted or expired while it is open, its files are unlinked at once but |
| 451 | // their bytes stay counted until the Reader closes. Open fails once the |
| 452 | // version has been retired, and for gzip when the entry has no gzip copy. |
| 453 | func (e *Entry) Open(gzip bool) (*Reader, error) { |
| 454 | if e.s == nil { |
| 455 | return nil, errors.New("assetcache: entry not from a store") |
| 456 | } |
| 457 | name := bodyName(e.ver.id, e.ver.v) |
| 458 | if gzip { |
| 459 | if e.Meta.GzipSize == 0 { |
| 460 | return nil, fmt.Errorf("assetcache: no gzip copy: %w", os.ErrNotExist) |
| 461 | } |
| 462 | name = gzipName(e.ver.id, e.ver.v) |
| 463 | } |
| 464 | s := e.s |
| 465 | s.mu.Lock() |
| 466 | if !e.ver.indexed { |
| 467 | s.mu.Unlock() |
| 468 | return nil, fmt.Errorf("assetcache: entry replaced or deleted: %w", os.ErrNotExist) |
| 469 | } |
| 470 | e.ver.leases++ // before the open, so a retirement from here on keeps the bytes counted |
| 471 | s.mu.Unlock() |
| 472 | f, err := os.Open(filepath.Join(s.cfg.Dir, name)) |
| 473 | if err != nil { |
| 474 | s.release(e.ver) |
| 475 | return nil, fmt.Errorf("assetcache: %w", err) |
| 476 | } |
| 477 | return &Reader{f: f, s: s, ver: e.ver}, nil |
| 478 | } |
| 479 | |
| 480 | // release returns one lease on v. |
| 481 | func (s *Store) release(v *version) { |
| 482 | s.mu.Lock() |
| 483 | defer s.mu.Unlock() |
| 484 | v.leases-- |
| 485 | s.settleLocked(v) |
| 486 | } |
| 487 | |
| 488 | // Reader reads one body file of an entry. It is an io.ReadSeeker, as |
| 489 | // http.ServeContent needs, and an io.ReaderAt. Close releases its lease. |
| 490 | type Reader struct { |
| 491 | f *os.File |
| 492 | s *Store |
| 493 | ver *version |
| 494 | once sync.Once |
| 495 | } |
| 496 | |
| 497 | func (r *Reader) Read(p []byte) (int, error) { return r.f.Read(p) } |
| 498 | func (r *Reader) ReadAt(p []byte, off int64) (int, error) { return r.f.ReadAt(p, off) } |
| 499 | func (r *Reader) Seek(offset int64, whence int) (int64, error) { return r.f.Seek(offset, whence) } |
| 500 | func (r *Reader) Stat() (os.FileInfo, error) { return r.f.Stat() } |
| 501 | |
| 502 | // Close closes the file and then releases the lease, exactly once however |
| 503 | // often it is called. |
| 504 | func (r *Reader) Close() error { |
| 505 | err := r.f.Close() |
| 506 | r.once.Do(func() { r.s.release(r.ver) }) |
| 507 | return err |
| 508 | } |
| 509 | |
| 510 | // Stats reports the store's current accounting. |
| 511 | func (s *Store) Stats() Stats { |
| 512 | s.mu.Lock() |
| 513 | defer s.mu.Unlock() |
| 514 | return Stats{ |
| 515 | Entries: len(s.entries), |
| 516 | Bytes: s.used, |
| 517 | Reserved: s.reserved, |
| 518 | Retired: s.retired, |
| 519 | Apps: len(s.apps), |
| 520 | } |
| 521 | } |
| 522 | |
amadan