| 1 | package assetcache |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "compress/gzip" |
| 6 | "crypto/rand" |
| 7 | "errors" |
| 8 | "fmt" |
| 9 | "io" |
| 10 | "net/http" |
| 11 | "os" |
| 12 | "path/filepath" |
| 13 | "strings" |
| 14 | "sync" |
| 15 | "testing" |
| 16 | "time" |
| 17 | ) |
| 18 | |
| 19 | // clock is a settable Now for tests. |
| 20 | type clock struct { |
| 21 | mu sync.Mutex |
| 22 | now time.Time |
| 23 | } |
| 24 | |
| 25 | func (c *clock) Now() time.Time { |
| 26 | c.mu.Lock() |
| 27 | defer c.mu.Unlock() |
| 28 | return c.now |
| 29 | } |
| 30 | |
| 31 | func (c *clock) set(t time.Time) { |
| 32 | c.mu.Lock() |
| 33 | c.now = t |
| 34 | c.mu.Unlock() |
| 35 | } |
| 36 | |
| 37 | // entryStore opens a Store over a real temporary directory with a fake FS |
| 38 | // that has plenty of room, the given budget and a settable clock. |
| 39 | func entryStore(t *testing.T, budget int64) (*Store, *clock, string) { |
| 40 | t.Helper() |
| 41 | dir := t.TempDir() |
| 42 | c := &clock{now: testNow} |
| 43 | s, err := Open(Config{Dir: dir, Budget: budget, FS: &fakeFS{total: 64 << 30, free: 64 << 30}, Now: c.Now}) |
| 44 | if err != nil { |
| 45 | t.Fatal(err) |
| 46 | } |
| 47 | return s, c, dir |
| 48 | } |
| 49 | |
| 50 | // reopen opens a second Store over dir, as a restarted process would. |
| 51 | func reopen(t *testing.T, dir string) *Store { |
| 52 | t.Helper() |
| 53 | s, err := Open(Config{Dir: dir, Budget: 4 << 30, FS: &fakeFS{total: 64 << 30, free: 64 << 30}, Now: func() time.Time { return testNow }}) |
| 54 | if err != nil { |
| 55 | t.Fatalf("Open: %v", err) |
| 56 | } |
| 57 | return s |
| 58 | } |
| 59 | |
| 60 | func cssHeader(v string) http.Header { |
| 61 | return http.Header{"Content-Type": {"text/css"}, "X-Version": {v}} |
| 62 | } |
| 63 | |
| 64 | // fill reserves for k, writes body and returns the open fill. |
| 65 | func fill(t *testing.T, s *Store, k Key, body []byte) *Fill { |
| 66 | t.Helper() |
| 67 | f, err := s.Reserve(k, int64(len(body))) |
| 68 | if err != nil { |
| 69 | t.Fatalf("Reserve: %v", err) |
| 70 | } |
| 71 | if _, err := f.Write(body); err != nil { |
| 72 | t.Fatalf("Write: %v", err) |
| 73 | } |
| 74 | return f |
| 75 | } |
| 76 | |
| 77 | // commit stores body under k with header h, valid for an hour from testNow. |
| 78 | func commit(t *testing.T, s *Store, k Key, body []byte, h http.Header, gz bool) { |
| 79 | t.Helper() |
| 80 | f := fill(t, s, k, body) |
| 81 | if err := f.Commit(Meta{Header: h, Lifetime: time.Hour, Expires: testNow.Add(time.Hour)}, gz); err != nil { |
| 82 | t.Fatalf("Commit: %v", err) |
| 83 | } |
| 84 | } |
| 85 | |
| 86 | func mustGet(t *testing.T, s *Store, k Key) *Entry { |
| 87 | t.Helper() |
| 88 | e, ok := s.Get(k) |
| 89 | if !ok { |
| 90 | t.Fatalf("Get(%s): no entry", k.Path) |
| 91 | } |
| 92 | return e |
| 93 | } |
| 94 | |
| 95 | func readAll(t *testing.T, e *Entry, gz bool) []byte { |
| 96 | t.Helper() |
| 97 | r, err := e.Open(gz) |
| 98 | if err != nil { |
| 99 | t.Fatalf("Open(gzip=%v): %v", gz, err) |
| 100 | } |
| 101 | defer r.Close() |
| 102 | b, err := io.ReadAll(r) |
| 103 | if err != nil { |
| 104 | t.Fatal(err) |
| 105 | } |
| 106 | return b |
| 107 | } |
| 108 | |
| 109 | func gunzip(t *testing.T, b []byte) []byte { |
| 110 | t.Helper() |
| 111 | zr, err := gzip.NewReader(bytes.NewReader(b)) |
| 112 | if err != nil { |
| 113 | t.Fatalf("gzip.NewReader: %v", err) |
| 114 | } |
| 115 | out, err := io.ReadAll(zr) |
| 116 | if err != nil { |
| 117 | t.Fatalf("gunzip: %v", err) |
| 118 | } |
| 119 | return out |
| 120 | } |
| 121 | |
| 122 | func fileSize(t *testing.T, path string) int64 { |
| 123 | t.Helper() |
| 124 | fi, err := os.Stat(path) |
| 125 | if err != nil { |
| 126 | t.Fatal(err) |
| 127 | } |
| 128 | return fi.Size() |
| 129 | } |
| 130 | |
| 131 | func slots(s *Store) (box, apps int) { |
| 132 | s.mu.Lock() |
| 133 | defer s.mu.Unlock() |
| 134 | return s.fills, len(s.fillsByApp) |
| 135 | } |
| 136 | |
| 137 | var cssText = []byte(strings.Repeat("body { margin: 0; padding: 0; color: #222; }\n", 200)) |
| 138 | |
| 139 | // TestCommitThenGetRoundTrips is not acceptance: it uses a fake FS. |
| 140 | func TestCommitThenGetRoundTrips(t *testing.T) { |
| 141 | s, _, dir := entryStore(t, 4<<30) |
| 142 | k := testKey("shop", "/a.css") |
| 143 | f := fill(t, s, k, cssText) |
| 144 | if err := f.Commit(Meta{Header: cssHeader("1"), Lifetime: time.Hour, Expires: testNow.Add(time.Hour)}, false); err != nil { |
| 145 | t.Fatal(err) |
| 146 | } |
| 147 | f.Abort() // after Commit: a no-op |
| 148 | |
| 149 | e := mustGet(t, s, k) |
| 150 | if got := e.Meta.Header; got.Get("Content-Type") != "text/css" || got.Get("X-Version") != "1" { |
| 151 | t.Fatalf("Header = %v", got) |
| 152 | } |
| 153 | if e.Meta.Key != k || e.Meta.Size != int64(len(cssText)) || e.Meta.GzipSize != 0 || !e.Meta.StoredAt.Equal(testNow) || !validVersion(e.Meta.Version) { |
| 154 | t.Fatalf("Meta = %+v", e.Meta) |
| 155 | } |
| 156 | if got := readAll(t, e, false); !bytes.Equal(got, cssText) { |
| 157 | t.Fatalf("body = %d bytes, want %d", len(got), len(cssText)) |
| 158 | } |
| 159 | if _, err := e.Open(true); err == nil { |
| 160 | t.Fatal("Open(gzip) succeeded with no gzip copy") |
| 161 | } |
| 162 | |
| 163 | metaSize := fileSize(t, filepath.Join(dir, k.ID()+".meta")) |
| 164 | want := Stats{Entries: 1, Bytes: int64(len(cssText)) + metaSize, Apps: 1} |
| 165 | if st := s.Stats(); st != want { |
| 166 | t.Fatalf("Stats = %+v, want %+v", st, want) |
| 167 | } |
| 168 | if box, apps := slots(s); box != 0 || apps != 0 { |
| 169 | t.Fatalf("slots after Commit: box %d, apps %d", box, apps) |
| 170 | } |
| 171 | sameNames(t, listDir(t, dir), []string{k.ID() + ".meta", bodyName(k.ID(), e.Meta.Version)}) |
| 172 | if err := f.Commit(Meta{}, false); err == nil { |
| 173 | t.Fatal("a second Commit succeeded") |
| 174 | } |
| 175 | } |
| 176 | |
| 177 | // TestReaderServesHTTP is not acceptance: it uses a fake FS. The Reader is |
| 178 | // what http.ServeContent takes: a ranged request reads through Seek. |
| 179 | func TestReaderServesHTTP(t *testing.T) { |
| 180 | s, _, _ := entryStore(t, 4<<30) |
| 181 | k := testKey("shop", "/a.css") |
| 182 | commit(t, s, k, cssText, cssHeader("1"), false) |
| 183 | r, err := mustGet(t, s, k).Open(false) |
| 184 | if err != nil { |
| 185 | t.Fatal(err) |
| 186 | } |
| 187 | defer r.Close() |
| 188 | var _ io.ReadSeeker = r |
| 189 | var _ io.ReaderAt = r |
| 190 | fi, err := r.Stat() |
| 191 | if err != nil || fi.Size() != int64(len(cssText)) { |
| 192 | t.Fatalf("Stat = %v, %v", fi, err) |
| 193 | } |
| 194 | buf := make([]byte, 4) |
| 195 | if _, err := r.ReadAt(buf, 5); err != nil || string(buf) != string(cssText[5:9]) { |
| 196 | t.Fatalf("ReadAt = %q, %v", buf, err) |
| 197 | } |
| 198 | if _, err := r.Seek(7, io.SeekStart); err != nil { |
| 199 | t.Fatal(err) |
| 200 | } |
| 201 | if _, err := io.ReadFull(r, buf); err != nil || string(buf) != string(cssText[7:11]) { |
| 202 | t.Fatalf("Read after Seek = %q, %v", buf, err) |
| 203 | } |
| 204 | if err := r.Close(); err != nil { |
| 205 | t.Fatal(err) |
| 206 | } |
| 207 | r.Close() // a second Close releases nothing more |
| 208 | s.mu.Lock() |
| 209 | leases := s.entries[k.ID()].ver.leases |
| 210 | s.mu.Unlock() |
| 211 | if leases != 0 { |
| 212 | t.Fatalf("leases = %d after Close twice, want 0", leases) |
| 213 | } |
| 214 | } |
| 215 | |
| 216 | // TestCommitThenReopenThenRead is not acceptance: it uses a fake FS. |
| 217 | func TestCommitThenReopenThenRead(t *testing.T) { |
| 218 | s, _, dir := entryStore(t, 4<<30) |
| 219 | k := testKey("shop", "/a.css") |
| 220 | commit(t, s, k, cssText, cssHeader("1"), true) |
| 221 | committed := mustGet(t, s, k).Meta |
| 222 | |
| 223 | s2 := reopen(t, dir) |
| 224 | e := mustGet(t, s2, k) |
| 225 | if e.Meta.Version != committed.Version { |
| 226 | t.Fatalf("Version after reopen = %q, want %q", e.Meta.Version, committed.Version) |
| 227 | } |
| 228 | if e.Meta.GzipSize == 0 || e.Meta.GzipSize != committed.GzipSize || e.Meta.Header.Get("X-Version") != "1" { |
| 229 | t.Fatalf("Meta after reopen = %+v", e.Meta) |
| 230 | } |
| 231 | if got := readAll(t, e, false); !bytes.Equal(got, cssText) { |
| 232 | t.Fatal("identity body differs after reopen") |
| 233 | } |
| 234 | if got := gunzip(t, readAll(t, e, true)); !bytes.Equal(got, cssText) { |
| 235 | t.Fatal("gzip copy does not decode to the identity body after reopen") |
| 236 | } |
| 237 | if st, st2 := s.Stats(), s2.Stats(); st != st2 { |
| 238 | t.Fatalf("Stats after reopen = %+v, before %+v", st2, st) |
| 239 | } |
| 240 | } |
| 241 | |
| 242 | // TestCommitWritesMetaLast is not acceptance: it uses a fake FS. A crash |
| 243 | // after the bodies are renamed but before the meta rename leaves no entry. |
| 244 | func TestCommitWritesMetaLast(t *testing.T) { |
| 245 | s, _, dir := entryStore(t, 4<<30) |
| 246 | k := testKey("shop", "/a.css") |
| 247 | f := fill(t, s, k, cssText) |
| 248 | f.crashAt = crashBeforeMetaRename |
| 249 | if err := f.Commit(Meta{Header: cssHeader("1"), Lifetime: time.Hour, Expires: testNow.Add(time.Hour)}, true); !errors.Is(err, errSimulatedCrash) { |
| 250 | t.Fatalf("Commit = %v, want the simulated crash", err) |
| 251 | } |
| 252 | var bodies int |
| 253 | for _, name := range listDir(t, dir) { |
| 254 | if name == k.ID()+".meta" { |
| 255 | t.Fatalf("<id>.meta exists before the meta rename: %v", listDir(t, dir)) |
| 256 | } |
| 257 | if strings.HasSuffix(name, ".body") || strings.HasSuffix(name, ".gz") { |
| 258 | bodies++ |
| 259 | } |
| 260 | } |
| 261 | if bodies != 2 { |
| 262 | t.Fatalf("files at the crash: %v, want both versioned bodies renamed", listDir(t, dir)) |
| 263 | } |
| 264 | |
| 265 | s2 := reopen(t, dir) |
| 266 | if _, ok := s2.Get(k); ok { |
| 267 | t.Fatal("an entry survived a crash before its meta rename") |
| 268 | } |
| 269 | sameNames(t, listDir(t, dir), nil) |
| 270 | } |
| 271 | |
| 272 | // TestReplacementCrashKeepsTheOldEntry is not acceptance: it uses a fake FS. |
| 273 | func TestReplacementCrashKeepsTheOldEntry(t *testing.T) { |
| 274 | s, _, dir := entryStore(t, 4<<30) |
| 275 | k := testKey("shop", "/a.css") |
| 276 | commit(t, s, k, []byte("old bytes"), cssHeader("old"), false) |
| 277 | old := mustGet(t, s, k).Meta |
| 278 | oldFiles := listDir(t, dir) |
| 279 | |
| 280 | f := fill(t, s, k, cssText) |
| 281 | f.crashAt = crashBeforeMetaRename |
| 282 | if err := f.Commit(Meta{Header: cssHeader("new"), Lifetime: time.Hour, Expires: testNow.Add(time.Hour)}, true); !errors.Is(err, errSimulatedCrash) { |
| 283 | t.Fatalf("Commit = %v, want the simulated crash", err) |
| 284 | } |
| 285 | |
| 286 | s2 := reopen(t, dir) |
| 287 | e := mustGet(t, s2, k) |
| 288 | if e.Meta.Version != old.Version || e.Meta.Header.Get("X-Version") != "old" { |
| 289 | t.Fatalf("Meta after the crash = %+v, want the old entry", e.Meta) |
| 290 | } |
| 291 | if got := readAll(t, e, false); string(got) != "old bytes" { |
| 292 | t.Fatalf("body after the crash = %q, want the old bytes", got) |
| 293 | } |
| 294 | sameNames(t, listDir(t, dir), oldFiles) |
| 295 | } |
| 296 | |
| 297 | // TestOversizedMetaIsRefused is not acceptance: it uses a fake FS. |
| 298 | func TestOversizedMetaIsRefused(t *testing.T) { |
| 299 | big := strings.Repeat("x", 70<<10) |
| 300 | for _, tc := range []struct { |
| 301 | name string |
| 302 | key Key |
| 303 | hdr http.Header |
| 304 | }{ |
| 305 | {"header", testKey("shop", "/a.css"), http.Header{"X-Big": {big}}}, |
| 306 | {"query", Key{Account: "abc", App: "shop", Gen: "g1", Host: "shop.example.com", Path: "/a.css", RawQuery: "q=" + big}, cssHeader("1")}, |
| 307 | } { |
| 308 | t.Run(tc.name, func(t *testing.T) { |
| 309 | s, _, dir := entryStore(t, 4<<30) |
| 310 | f := fill(t, s, tc.key, cssText) |
| 311 | err := f.Commit(Meta{Header: tc.hdr, Lifetime: time.Hour, Expires: testNow.Add(time.Hour)}, true) |
| 312 | if !errors.Is(err, ErrTooLarge) { |
| 313 | t.Fatalf("Commit = %v, want ErrTooLarge", err) |
| 314 | } |
| 315 | f.Abort() |
| 316 | if got := listDir(t, dir); len(got) != 0 { |
| 317 | t.Fatalf("files left: %v", got) |
| 318 | } |
| 319 | if st := s.Stats(); st != (Stats{}) { |
| 320 | t.Fatalf("Stats = %+v, want nothing stored or reserved", st) |
| 321 | } |
| 322 | if box, apps := slots(s); box != 0 || apps != 0 { |
| 323 | t.Fatalf("slots: box %d, apps %d", box, apps) |
| 324 | } |
| 325 | if _, ok := s.Get(tc.key); ok { |
| 326 | t.Fatal("refused entry is indexed") |
| 327 | } |
| 328 | }) |
| 329 | } |
| 330 | } |
| 331 | |
| 332 | // TestCommitOfAFailedFillAborts is not acceptance: it uses a fake FS. |
| 333 | func TestCommitOfAFailedFillAborts(t *testing.T) { |
| 334 | s, _, dir := entryStore(t, 4<<30) |
| 335 | k := testKey("shop", "/a.css") |
| 336 | f, err := s.Reserve(k, 4) |
| 337 | if err != nil { |
| 338 | t.Fatal(err) |
| 339 | } |
| 340 | if _, err := f.Write([]byte("too long")); !errors.Is(err, ErrTooLarge) { |
| 341 | t.Fatalf("Write = %v", err) |
| 342 | } |
| 343 | if err := f.Commit(Meta{Expires: testNow.Add(time.Hour)}, false); !errors.Is(err, ErrTooLarge) { |
| 344 | t.Fatalf("Commit = %v, want ErrTooLarge", err) |
| 345 | } |
| 346 | if st := s.Stats(); st != (Stats{}) { |
| 347 | t.Fatalf("Stats = %+v", st) |
| 348 | } |
| 349 | if got := listDir(t, dir); len(got) != 0 { |
| 350 | t.Fatalf("files left: %v", got) |
| 351 | } |
| 352 | } |
| 353 | |
| 354 | // heldBytesRefuse checks that, with a reader holding retired bytes, a fill |
| 355 | // that fits only by counting them as free is refused without evicting |
| 356 | // anything (no eviction can free held bytes), and that it is admitted once |
| 357 | // the reader closes. |
| 358 | func heldBytesRefuse(t *testing.T, s *Store, held int64, r *Reader, wantBody []byte) { |
| 359 | t.Helper() |
| 360 | st := s.Stats() |
| 361 | if st.Retired != held { |
| 362 | t.Fatalf("Retired = %d, want %d", st.Retired, held) |
| 363 | } |
| 364 | // The smallest fill that does not fit while the held bytes are counted, |
| 365 | // even after eviction has freed every other entry. |
| 366 | n := (s.cfg.Budget-held-st.Reserved-metaReserve)/2 + 1 |
| 367 | if n < 0 || 2*n+metaReserve > s.cfg.Budget-st.Reserved { |
| 368 | t.Fatalf("test budget leaves no room for the probe: Stats %+v", st) |
| 369 | } |
| 370 | probe := testKey("blog", "/probe") |
| 371 | if _, err := s.Reserve(probe, n); !errors.Is(err, ErrNoRoom) { |
| 372 | t.Fatalf("Reserve(%d) with %d bytes held = %v, want ErrNoRoom", n, held, err) |
| 373 | } |
| 374 | // The refused fill evicted nothing; the held bytes remain. |
| 375 | if st1 := s.Stats(); st1 != st { |
| 376 | t.Fatalf("Stats after the refusal = %+v, want %+v unchanged", st1, st) |
| 377 | } |
| 378 | // The reader still reads the unlinked bytes. |
| 379 | got, err := io.ReadAll(r) |
| 380 | if err != nil || !bytes.Equal(got, wantBody) { |
| 381 | t.Fatalf("held reader read %d bytes, %v", len(got), err) |
| 382 | } |
| 383 | if err := r.Close(); err != nil { |
| 384 | t.Fatal(err) |
| 385 | } |
| 386 | if st2 := s.Stats(); st2.Retired != 0 || st2.Bytes != st.Bytes-held { |
| 387 | t.Fatalf("Stats after Close = %+v, want nothing held", st2) |
| 388 | } |
| 389 | f, err := s.Reserve(probe, n) |
| 390 | if err != nil { |
| 391 | t.Fatalf("Reserve(%d) after the reader closed: %v", n, err) |
| 392 | } |
| 393 | f.Abort() |
| 394 | } |
| 395 | |
| 396 | // TestEvictedBodyStaysCountedWhileAReaderHoldsIt is not acceptance: it uses a |
| 397 | // fake FS. It drives drop, the function eviction calls. |
| 398 | func TestEvictedBodyStaysCountedWhileAReaderHoldsIt(t *testing.T) { |
| 399 | s, _, dir := entryStore(t, 70000) |
| 400 | k := testKey("shop", "/a.css") |
| 401 | body := cssText[:1000] |
| 402 | commit(t, s, k, body, cssHeader("1"), false) |
| 403 | e := mustGet(t, s, k) |
| 404 | r, err := e.Open(false) |
| 405 | if err != nil { |
| 406 | t.Fatal(err) |
| 407 | } |
| 408 | s.mu.Lock() |
| 409 | ie := s.entries[k.ID()] |
| 410 | s.mu.Unlock() |
| 411 | if !s.drop(ie) { |
| 412 | t.Fatal("drop refused the current entry") |
| 413 | } |
| 414 | if got := listDir(t, dir); len(got) != 0 { |
| 415 | t.Fatalf("files after eviction: %v, want all unlinked at once", got) |
| 416 | } |
| 417 | if st := s.Stats(); st.Entries != 0 || st.Bytes != int64(len(body)) { |
| 418 | t.Fatalf("Stats = %+v, want no entries and the held body counted", st) |
| 419 | } |
| 420 | if _, err := e.Open(false); err == nil { |
| 421 | t.Fatal("Open of an evicted entry succeeded") |
| 422 | } |
| 423 | heldBytesRefuse(t, s, int64(len(body)), r, body) |
| 424 | } |
| 425 | |
| 426 | // TestReplacedBodyStaysCountedWhileAReaderHoldsIt is not acceptance: it uses a |
| 427 | // fake FS. |
| 428 | func TestReplacedBodyStaysCountedWhileAReaderHoldsIt(t *testing.T) { |
| 429 | s, _, dir := entryStore(t, 70000) |
| 430 | k := testKey("shop", "/a.css") |
| 431 | old := cssText[:1000] |
| 432 | commit(t, s, k, old, cssHeader("old"), false) |
| 433 | oldEntry := mustGet(t, s, k) |
| 434 | r, err := oldEntry.Open(false) |
| 435 | if err != nil { |
| 436 | t.Fatal(err) |
| 437 | } |
| 438 | commit(t, s, k, []byte("new"), cssHeader("new"), false) |
| 439 | e := mustGet(t, s, k) |
| 440 | if got := readAll(t, e, false); string(got) != "new" { |
| 441 | t.Fatalf("Get after replacement = %q", got) |
| 442 | } |
| 443 | sameNames(t, listDir(t, dir), []string{k.ID() + ".meta", bodyName(k.ID(), e.Meta.Version)}) |
| 444 | metaSize := fileSize(t, filepath.Join(dir, k.ID()+".meta")) |
| 445 | if st := s.Stats(); st.Entries != 1 || st.Bytes != int64(len(old))+3+metaSize { |
| 446 | t.Fatalf("Stats = %+v, want the new entry and the held old body", st) |
| 447 | } |
| 448 | heldBytesRefuse(t, s, int64(len(old)), r, old) |
| 449 | } |
| 450 | |
| 451 | // TestGzipCopyIsAbandonedWhenLarger is not acceptance: it uses a fake FS. |
| 452 | func TestGzipCopyIsAbandonedWhenLarger(t *testing.T) { |
| 453 | s, _, dir := entryStore(t, 4<<30) |
| 454 | k := testKey("shop", "/noise.bin") |
| 455 | body := make([]byte, 4096) |
| 456 | if _, err := rand.Read(body); err != nil { |
| 457 | t.Fatal(err) |
| 458 | } |
| 459 | commit(t, s, k, body, cssHeader("1"), true) |
| 460 | e := mustGet(t, s, k) |
| 461 | if e.Meta.GzipSize != 0 { |
| 462 | t.Fatalf("GzipSize = %d, want the copy abandoned", e.Meta.GzipSize) |
| 463 | } |
| 464 | if _, err := e.Open(true); err == nil { |
| 465 | t.Fatal("Open(gzip) succeeded with the copy abandoned") |
| 466 | } |
| 467 | sameNames(t, listDir(t, dir), []string{k.ID() + ".meta", bodyName(k.ID(), e.Meta.Version)}) |
| 468 | if st := s.Stats(); st.Reserved != 0 { |
| 469 | t.Fatalf("Reserved = %d", st.Reserved) |
| 470 | } |
| 471 | } |
| 472 | |
| 473 | // TestGzipCopyIsKept is not acceptance: it uses a fake FS. |
| 474 | func TestGzipCopyIsKept(t *testing.T) { |
| 475 | s, _, dir := entryStore(t, 4<<30) |
| 476 | k := testKey("shop", "/a.css") |
| 477 | commit(t, s, k, cssText, cssHeader("1"), true) |
| 478 | e := mustGet(t, s, k) |
| 479 | if e.Meta.GzipSize <= 0 || e.Meta.GzipSize >= e.Meta.Size { |
| 480 | t.Fatalf("GzipSize = %d for %d bytes of CSS", e.Meta.GzipSize, e.Meta.Size) |
| 481 | } |
| 482 | if got := fileSize(t, filepath.Join(dir, gzipName(k.ID(), e.Meta.Version))); got != e.Meta.GzipSize { |
| 483 | t.Fatalf(".gz is %d bytes, meta says %d", got, e.Meta.GzipSize) |
| 484 | } |
| 485 | if got := gunzip(t, readAll(t, e, true)); !bytes.Equal(got, cssText) { |
| 486 | t.Fatal("gzip copy does not decode to the identity body") |
| 487 | } |
| 488 | metaSize := fileSize(t, filepath.Join(dir, k.ID()+".meta")) |
| 489 | if st := s.Stats(); st.Bytes != e.Meta.Size+e.Meta.GzipSize+metaSize { |
| 490 | t.Fatalf("Bytes = %d, want both bodies and the meta", st.Bytes) |
| 491 | } |
| 492 | } |
| 493 | |
| 494 | // TestGetExpires is not acceptance: it uses a fake FS. |
| 495 | func TestGetExpires(t *testing.T) { |
| 496 | s, c, dir := entryStore(t, 4<<30) |
| 497 | k := testKey("shop", "/a.css") |
| 498 | commit(t, s, k, cssText, cssHeader("1"), true) |
| 499 | |
| 500 | c.set(testNow.Add(time.Hour - time.Nanosecond)) |
| 501 | if _, ok := s.Get(k); !ok { |
| 502 | t.Fatal("entry gone before it expires") |
| 503 | } |
| 504 | c.set(testNow.Add(time.Hour)) // exactly Expires |
| 505 | if _, ok := s.Get(k); ok { |
| 506 | t.Fatal("Get returned an entry at its expiry") |
| 507 | } |
| 508 | if got := listDir(t, dir); len(got) != 0 { |
| 509 | t.Fatalf("files after expiry: %v", got) |
| 510 | } |
| 511 | if st := s.Stats(); st != (Stats{}) { |
| 512 | t.Fatalf("Stats after expiry = %+v", st) |
| 513 | } |
| 514 | } |
| 515 | |
| 516 | // TestCommitReplacesAnEntry is not acceptance: it uses a fake FS. |
| 517 | func TestCommitReplacesAnEntry(t *testing.T) { |
| 518 | s, _, dir := entryStore(t, 4<<30) |
| 519 | k := testKey("shop", "/a.css") |
| 520 | commit(t, s, k, cssText, cssHeader("old"), true) |
| 521 | old := mustGet(t, s, k).Meta |
| 522 | commit(t, s, k, []byte("new body"), cssHeader("new"), false) |
| 523 | |
| 524 | e := mustGet(t, s, k) |
| 525 | if e.Meta.Version == old.Version || e.Meta.Header.Get("X-Version") != "new" || e.Meta.GzipSize != 0 { |
| 526 | t.Fatalf("Meta = %+v, want the replacement", e.Meta) |
| 527 | } |
| 528 | if got := readAll(t, e, false); string(got) != "new body" { |
| 529 | t.Fatalf("body = %q", got) |
| 530 | } |
| 531 | sameNames(t, listDir(t, dir), []string{k.ID() + ".meta", bodyName(k.ID(), e.Meta.Version)}) |
| 532 | metaSize := fileSize(t, filepath.Join(dir, k.ID()+".meta")) |
| 533 | want := Stats{Entries: 1, Bytes: int64(len("new body")) + metaSize, Apps: 1} |
| 534 | if st := s.Stats(); st != want { |
| 535 | t.Fatalf("Stats = %+v, want %+v", st, want) |
| 536 | } |
| 537 | } |
| 538 | |
| 539 | // TestAgeNeverNegative is not acceptance: it uses a fake FS. An entry stored |
| 540 | // "in the future" after the clock steps back stays valid until its stored |
| 541 | // expiry and reports age zero. |
| 542 | func TestAgeNeverNegative(t *testing.T) { |
| 543 | m := Meta{Lifetime: time.Hour, Expires: testNow.Add(time.Hour)} |
| 544 | for _, tc := range []struct { |
| 545 | now time.Time |
| 546 | want time.Duration |
| 547 | }{ |
| 548 | {testNow, 0}, |
| 549 | {testNow.Add(10 * time.Minute), 10 * time.Minute}, |
| 550 | {testNow.Add(-5 * time.Minute), 0}, |
| 551 | {testNow.Add(-48 * time.Hour), 0}, |
| 552 | } { |
| 553 | if got := m.Age(tc.now); got != tc.want { |
| 554 | t.Errorf("Age(%v) = %v, want %v", tc.now, got, tc.want) |
| 555 | } |
| 556 | } |
| 557 | |
| 558 | s, c, _ := entryStore(t, 4<<30) |
| 559 | k := testKey("shop", "/a.css") |
| 560 | commit(t, s, k, cssText, cssHeader("1"), false) |
| 561 | c.set(testNow.Add(-2 * time.Hour)) // the clock steps back |
| 562 | e, ok := s.Get(k) |
| 563 | if !ok { |
| 564 | t.Fatal("entry gone after the clock stepped back") |
| 565 | } |
| 566 | if got := e.Meta.Age(c.Now()); got != 0 { |
| 567 | t.Fatalf("Age = %v, want 0", got) |
| 568 | } |
| 569 | } |
| 570 | |
| 571 | // TestCommitGetAndReadConcurrently is not acceptance: it uses a fake FS. Run |
| 572 | // with -race: replacements, reads and expiry of the same keys interleave, and |
| 573 | // every counter comes back to what the directory holds. |
| 574 | func TestCommitGetAndReadConcurrently(t *testing.T) { |
| 575 | s, _, dir := entryStore(t, 4<<30) |
| 576 | keys := []Key{testKey("shop", "/a.css"), testKey("blog", "/b.css")} |
| 577 | var wg sync.WaitGroup |
| 578 | for g := 0; g < 6; g++ { |
| 579 | wg.Add(1) |
| 580 | go func(g int) { |
| 581 | defer wg.Done() |
| 582 | for i := 0; i < 30; i++ { |
| 583 | k := keys[(g+i)%2] |
| 584 | switch g % 3 { |
| 585 | case 0: |
| 586 | body := []byte(fmt.Sprintf("%s %d %d", cssText[:200], g, i)) |
| 587 | f, err := s.Reserve(k, int64(len(body))) |
| 588 | if errors.Is(err, ErrBusy) { |
| 589 | continue |
| 590 | } |
| 591 | if err != nil { |
| 592 | t.Errorf("Reserve: %v", err) |
| 593 | return |
| 594 | } |
| 595 | if _, err := f.Write(body); err != nil { |
| 596 | t.Errorf("Write: %v", err) |
| 597 | } |
| 598 | if err := f.Commit(Meta{Lifetime: time.Hour, Expires: testNow.Add(time.Hour)}, i%2 == 0); err != nil { |
| 599 | t.Errorf("Commit: %v", err) |
| 600 | } |
| 601 | case 1: |
| 602 | e, ok := s.Get(k) |
| 603 | if !ok { |
| 604 | continue |
| 605 | } |
| 606 | for _, gz := range []bool{false, true} { |
| 607 | if gz && e.Meta.GzipSize == 0 { |
| 608 | continue |
| 609 | } |
| 610 | r, err := e.Open(gz) |
| 611 | if errors.Is(err, os.ErrNotExist) { |
| 612 | continue // retired between Get and Open |
| 613 | } |
| 614 | if err != nil { |
| 615 | t.Errorf("Open(gzip=%v): %v", gz, err) |
| 616 | continue |
| 617 | } |
| 618 | got, err := io.ReadAll(r) |
| 619 | r.Close() |
| 620 | if err != nil { |
| 621 | t.Errorf("read (gzip=%v): %v", gz, err) |
| 622 | continue |
| 623 | } |
| 624 | if gz { |
| 625 | zr, err := gzip.NewReader(bytes.NewReader(got)) |
| 626 | if err == nil { |
| 627 | got, err = io.ReadAll(zr) |
| 628 | } |
| 629 | if err != nil { |
| 630 | t.Errorf("gunzip: %v", err) |
| 631 | continue |
| 632 | } |
| 633 | } |
| 634 | if int64(len(got)) != e.Meta.Size || !bytes.HasPrefix(got, cssText[:200]) { |
| 635 | t.Errorf("read %d bytes (gzip=%v), want the %d-byte body of a commit", len(got), gz, e.Meta.Size) |
| 636 | } |
| 637 | } |
| 638 | case 2: |
| 639 | s.mu.Lock() |
| 640 | e := s.entries[k.ID()] |
| 641 | s.mu.Unlock() |
| 642 | if e != nil { |
| 643 | s.drop(e) |
| 644 | } |
| 645 | } |
| 646 | } |
| 647 | }(g) |
| 648 | } |
| 649 | wg.Wait() |
| 650 | |
| 651 | st := s.Stats() |
| 652 | if st.Reserved != 0 || st.Retired != 0 { |
| 653 | t.Fatalf("Stats = %+v, want nothing reserved or retired", st) |
| 654 | } |
| 655 | var onDisk int64 |
| 656 | for _, name := range listDir(t, dir) { |
| 657 | if strings.Contains(name, ".tmp-") { |
| 658 | t.Fatalf("temporary left: %s", name) |
| 659 | } |
| 660 | onDisk += fileSize(t, filepath.Join(dir, name)) |
| 661 | } |
| 662 | if st.Bytes != onDisk { |
| 663 | t.Fatalf("Bytes = %d, directory holds %d", st.Bytes, onDisk) |
| 664 | } |
| 665 | if st2 := reopen(t, dir).Stats(); st2.Entries != st.Entries || st2.Bytes != st.Bytes { |
| 666 | t.Fatalf("Stats after reopen = %+v, before %+v", st2, st) |
| 667 | } |
| 668 | } |
| 669 | |
| 670 | // TestCommittedHeaderIsTheStoresOwn is not acceptance: it uses a fake FS. |
| 671 | // Changing the caller's header after Commit changes neither later hits nor |
| 672 | // what a restart reads. |
| 673 | func TestCommittedHeaderIsTheStoresOwn(t *testing.T) { |
| 674 | s, _, dir := entryStore(t, 4<<30) |
| 675 | k := testKey("shop", "/a.css") |
| 676 | h := cssHeader("1") |
| 677 | commit(t, s, k, cssText, h, false) |
| 678 | h.Set("X-Version", "changed") |
| 679 | h["Content-Type"][0] = "text/evil" |
| 680 | h.Add("X-New", "x") |
| 681 | |
| 682 | check := func(name string, e *Entry) { |
| 683 | t.Helper() |
| 684 | got := e.Meta.Header |
| 685 | if got.Get("X-Version") != "1" || got.Get("Content-Type") != "text/css" || got.Get("X-New") != "" { |
| 686 | t.Fatalf("%s: Header = %v, want the header as committed", name, got) |
| 687 | } |
| 688 | } |
| 689 | check("hit", mustGet(t, s, k)) |
| 690 | check("after reopen", mustGet(t, reopen(t, dir), k)) |
| 691 | } |
| 692 | |
| 693 | // TestBodyCloseFailureAborts is not acceptance: it uses a fake FS. A body |
| 694 | // that fails to close before publication aborts the commit, and the old entry |
| 695 | // stays as it was. |
| 696 | func TestBodyCloseFailureAborts(t *testing.T) { |
| 697 | s, _, dir := entryStore(t, 4<<30) |
| 698 | k := testKey("shop", "/a.css") |
| 699 | commit(t, s, k, []byte("old bytes"), cssHeader("old"), false) |
| 700 | oldFiles := listDir(t, dir) |
| 701 | before := s.Stats() |
| 702 | |
| 703 | f := fill(t, s, k, cssText) |
| 704 | f.failAt = faultBodyClose |
| 705 | if err := f.Commit(Meta{Header: cssHeader("new"), Lifetime: time.Hour, Expires: testNow.Add(time.Hour)}, true); !errors.Is(err, errInjected) { |
| 706 | t.Fatalf("Commit = %v, want the injected close failure", err) |
| 707 | } |
| 708 | f.Abort() |
| 709 | sameNames(t, listDir(t, dir), oldFiles) |
| 710 | if st := s.Stats(); st != before { |
| 711 | t.Fatalf("Stats = %+v, want %+v", st, before) |
| 712 | } |
| 713 | if box, apps := slots(s); box != 0 || apps != 0 { |
| 714 | t.Fatalf("slots: box %d, apps %d", box, apps) |
| 715 | } |
| 716 | e := mustGet(t, s, k) |
| 717 | if e.Meta.Header.Get("X-Version") != "old" || string(readAll(t, e, false)) != "old bytes" { |
| 718 | t.Fatalf("entry after a failed commit = %+v", e.Meta) |
| 719 | } |
| 720 | } |
| 721 | |
| 722 | // TestUnsyncedMetaRenameKeepsTheOldBodies is not acceptance: it uses a fake |
| 723 | // FS. When the directory sync after the meta rename fails, Commit reports it, |
| 724 | // the new entry (whose files are on disk) stays indexed, and the old bodies |
| 725 | // are kept, still counted, for the next Open: a crash could yet bring back |
| 726 | // the old meta, which must find its bodies. |
| 727 | func TestUnsyncedMetaRenameKeepsTheOldBodies(t *testing.T) { |
| 728 | s, _, dir := entryStore(t, 4<<30) |
| 729 | k := testKey("shop", "/a.css") |
| 730 | old := cssText[:1000] |
| 731 | commit(t, s, k, old, cssHeader("old"), true) |
| 732 | oldMeta := mustGet(t, s, k).Meta |
| 733 | oldBytes := oldMeta.Size + oldMeta.GzipSize |
| 734 | |
| 735 | f := fill(t, s, k, []byte("new body")) |
| 736 | f.failAt = faultMetaDirSync |
| 737 | if err := f.Commit(Meta{Header: cssHeader("new"), Lifetime: time.Hour, Expires: testNow.Add(time.Hour)}, false); !errors.Is(err, errInjected) { |
| 738 | t.Fatalf("Commit = %v, want the injected sync failure", err) |
| 739 | } |
| 740 | f.Abort() // a no-op: the entry is published |
| 741 | |
| 742 | e := mustGet(t, s, k) |
| 743 | if e.Meta.Header.Get("X-Version") != "new" || string(readAll(t, e, false)) != "new body" { |
| 744 | t.Fatalf("entry = %+v, want the new one", e.Meta) |
| 745 | } |
| 746 | sameNames(t, listDir(t, dir), []string{ |
| 747 | k.ID() + ".meta", bodyName(k.ID(), e.Meta.Version), |
| 748 | bodyName(k.ID(), oldMeta.Version), gzipName(k.ID(), oldMeta.Version), |
| 749 | }) |
| 750 | metaSize := fileSize(t, filepath.Join(dir, k.ID()+".meta")) |
| 751 | want := Stats{Entries: 1, Bytes: int64(len("new body")) + metaSize + oldBytes, Retired: oldBytes, Apps: 1} |
| 752 | if st := s.Stats(); st != want { |
| 753 | t.Fatalf("Stats = %+v, want %+v", st, want) |
| 754 | } |
| 755 | if box, apps := slots(s); box != 0 || apps != 0 { |
| 756 | t.Fatalf("slots: box %d, apps %d", box, apps) |
| 757 | } |
| 758 | // Later commits and drops do not delete the kept bodies either. |
| 759 | commit(t, s, testKey("blog", "/b.css"), []byte("b"), cssHeader("b"), false) |
| 760 | if _, err := os.Stat(filepath.Join(dir, bodyName(k.ID(), oldMeta.Version))); err != nil { |
| 761 | t.Fatalf("old body after a later commit: %v", err) |
| 762 | } |
| 763 | |
| 764 | s2 := reopen(t, dir) |
| 765 | e2 := mustGet(t, s2, k) |
| 766 | if e2.Meta.Version != e.Meta.Version { |
| 767 | t.Fatalf("after reopen: version %q, want %q", e2.Meta.Version, e.Meta.Version) |
| 768 | } |
| 769 | if _, err := os.Stat(filepath.Join(dir, bodyName(k.ID(), oldMeta.Version))); !errors.Is(err, os.ErrNotExist) { |
| 770 | t.Fatalf("old body after reopen: %v, want removed", err) |
| 771 | } |
| 772 | } |
| 773 | |
amadan