| 1 | // Cache eviction for the on-box version store (discussion #111). |
| 2 | // |
| 3 | // StageArtifact writes every version this box is ever asked to run under |
| 4 | // <opt>/versions/<ver>/... and nothing removes one. Retention |
| 5 | // (internal/host/retention.go, issue #136) prunes the BUCKET; the box's copy |
| 6 | // is a cache of it, and an uncapped cache on a 20 GB root filled the flagship |
| 7 | // on 2026-09-06 — 369 staged versions, 9.3 GB, at 40 to 60 new versions a day. |
| 8 | // |
| 9 | // Eviction, not retention: ensureStaged re-downloads a version that is |
| 10 | // missing before it adopts it, so removing one costs a download, never a |
| 11 | // deploy. What it must NOT do is remove a version something is USING, which is |
| 12 | // the whole content of the pin set below. |
| 13 | package release |
| 14 | |
| 15 | import ( |
| 16 | "fmt" |
| 17 | "io/fs" |
| 18 | "os" |
| 19 | "path/filepath" |
| 20 | "time" |
| 21 | ) |
| 22 | |
| 23 | // DefaultSweepGrace is how long an unpinned version survives after its last |
| 24 | // write. Time-driven rather than size-driven on purpose: a size trigger only |
| 25 | // fires once the disk is ALREADY tight, which is the state in which every |
| 26 | // other thing on the box (the bake clock, the registry write, the next stage) |
| 27 | // has begun to fail — see RunAdopt's placement comment. Six hours is long |
| 28 | // enough to cover the window between staging a version and pointing a live |
| 29 | // link at it (promotions, canaries and rollbacks all happen inside minutes), |
| 30 | // and short enough that a day's shipping cannot accumulate. |
| 31 | const DefaultSweepGrace = 6 * time.Hour |
| 32 | |
| 33 | // SweepResult is one sweep's accounting, for the log line. Errs holds the |
| 34 | // per-version failures: a version that cannot be scanned or removed is |
| 35 | // skipped, never fatal, because one unreadable tree must not stop the sweep |
| 36 | // freeing the rest of the disk. |
| 37 | type SweepResult struct { |
| 38 | Scanned int // entries under <opt>/versions |
| 39 | Skipped int // not a directory, or not a version name |
| 40 | Pinned int // named by a live link or a registry row |
| 41 | Young int // unpinned but written to inside the grace window |
| 42 | Removed []string // version directories deleted |
| 43 | Bytes int64 // bytes those directories held |
| 44 | Errs []error |
| 45 | } |
| 46 | |
| 47 | // SweepStaged removes every <optDir>/versions/<ver> that `pinned` does not |
| 48 | // name and that nothing has written to for `grace`. |
| 49 | // |
| 50 | // The granularity is the VERSION directory, not the per-app subtree inside |
| 51 | // it: a version dir is shared by every account and app staged at that version |
| 52 | // (VersionPath), so one pin on any of them keeps the whole directory. Coarse |
| 53 | // on purpose — the cost is a few megabytes held slightly longer, and the |
| 54 | // alternative (deleting one app's binary out from under a shared version) |
| 55 | // buys nothing a later sweep would not collect anyway. |
| 56 | // |
| 57 | // Entries that are not directories, and directories whose name ValidVersion |
| 58 | // rejects, are left alone and counted in Skipped. Nothing this package wrote |
| 59 | // has such a name, so such an entry is somebody else's — an operator's backup, |
| 60 | // a hand-placed tree — and the sweep is not entitled to it. |
| 61 | // |
| 62 | // The returned error is reserved for "could not read the versions directory |
| 63 | // at all", the one case in which the caller learned nothing and should not |
| 64 | // treat an empty Removed list as "there was nothing to remove". |
| 65 | func SweepStaged(optDir string, pinned map[string]bool, grace time.Duration, now time.Time) (SweepResult, error) { |
| 66 | var res SweepResult |
| 67 | vdir := versionsDir(optDir) |
| 68 | entries, err := os.ReadDir(vdir) |
| 69 | if err != nil { |
| 70 | if os.IsNotExist(err) { |
| 71 | // A box that has never staged anything. Not an error. |
| 72 | return res, nil |
| 73 | } |
| 74 | return res, fmt.Errorf("read %s: %w", vdir, err) |
| 75 | } |
| 76 | for _, e := range entries { |
| 77 | res.Scanned++ |
| 78 | name := e.Name() |
| 79 | // e.IsDir() is false for a symlink (ReadDir does not follow), which |
| 80 | // is the answer we want: a link into the store is not a staged tree. |
| 81 | if !e.IsDir() || !ValidVersion(name) { |
| 82 | res.Skipped++ |
| 83 | continue |
| 84 | } |
| 85 | if pinned[name] { |
| 86 | res.Pinned++ |
| 87 | continue |
| 88 | } |
| 89 | path := filepath.Join(vdir, name) |
| 90 | newest, size, err := treeStats(path) |
| 91 | if err != nil { |
| 92 | res.Errs = append(res.Errs, fmt.Errorf("scan %s: %w", path, err)) |
| 93 | continue |
| 94 | } |
| 95 | // A negative Sub (a file dated in the future, from a clock step or a |
| 96 | // tarball's preserved mtime) reads as "younger than the grace", which |
| 97 | // is the cautious direction. |
| 98 | if now.Sub(newest) < grace { |
| 99 | res.Young++ |
| 100 | continue |
| 101 | } |
| 102 | if err := os.RemoveAll(path); err != nil { |
| 103 | res.Errs = append(res.Errs, fmt.Errorf("remove %s: %w", path, err)) |
| 104 | continue |
| 105 | } |
| 106 | res.Removed = append(res.Removed, name) |
| 107 | res.Bytes += size |
| 108 | } |
| 109 | return res, nil |
| 110 | } |
| 111 | |
| 112 | // treeStats reports the newest FILE mtime under root, and the bytes the |
| 113 | // regular files hold. |
| 114 | // |
| 115 | // Files, not directories, and that is the subtle part: removing entries from a |
| 116 | // directory bumps ITS mtime, so a tree the previous sweep half-deleted (a |
| 117 | // RemoveAll that hit an error partway) would look freshly written and buy |
| 118 | // itself another full grace period, every period, forever. The files that |
| 119 | // survive carry the real age. |
| 120 | // |
| 121 | // A tree with no files left at all falls back to the newest directory mtime, |
| 122 | // so an empty husk is still collectable rather than permanently ageless. |
| 123 | func treeStats(root string) (newest time.Time, bytes int64, err error) { |
| 124 | var dirNewest time.Time |
| 125 | walkErr := filepath.WalkDir(root, func(_ string, d fs.DirEntry, err error) error { |
| 126 | if err != nil { |
| 127 | // A concurrent removal (an operator, an interrupted sweep) is not |
| 128 | // this walk's problem: skip what vanished under it. |
| 129 | if os.IsNotExist(err) { |
| 130 | return nil |
| 131 | } |
| 132 | return err |
| 133 | } |
| 134 | info, err := d.Info() |
| 135 | if err != nil { |
| 136 | if os.IsNotExist(err) { |
| 137 | return nil |
| 138 | } |
| 139 | return err |
| 140 | } |
| 141 | if d.IsDir() { |
| 142 | if info.ModTime().After(dirNewest) { |
| 143 | dirNewest = info.ModTime() |
| 144 | } |
| 145 | return nil |
| 146 | } |
| 147 | if info.ModTime().After(newest) { |
| 148 | newest = info.ModTime() |
| 149 | } |
| 150 | if info.Mode().IsRegular() { |
| 151 | bytes += info.Size() |
| 152 | } |
| 153 | return nil |
| 154 | }) |
| 155 | if walkErr != nil { |
| 156 | return time.Time{}, 0, walkErr |
| 157 | } |
| 158 | if newest.IsZero() { |
| 159 | newest = dirNewest |
| 160 | } |
| 161 | return newest, bytes, nil |
| 162 | } |
| 163 | |
| 164 | // LivePins is the set of versions the box's live symlinks name — the hard |
| 165 | // pins, because each one is a path some unit's ExecStart resolves through. |
| 166 | // |
| 167 | // Read from the LINKS and not from the registry's per-route Version column, |
| 168 | // deliberately. The two agree on a healthy box, and the case where they do |
| 169 | // not is exactly the incident's: PointLive succeeded, reg.SetVersions then |
| 170 | // failed on a full /data, and the link is the only record of what is running. |
| 171 | // Believing the registry there would delete a live binary. |
| 172 | // |
| 173 | // Every entry is read, including PointLive's `<host>.tmp` leftovers from a |
| 174 | // crashed swap: a leftover names a real staged tree and costs one version dir |
| 175 | // of disk, which is a better trade than interpreting which links are current. |
| 176 | // |
| 177 | // A link this package cannot read at all is an error, not a skipped entry. |
| 178 | // The sweep's caller declines to delete anything when pins cannot be |
| 179 | // established — losing a pin is how a running app gets its binary removed, |
| 180 | // so an unreadable live dir must stop the sweep and be loud, not quietly |
| 181 | // narrow the pin set. |
| 182 | func LivePins(optDir string) (map[string]bool, error) { |
| 183 | pins := map[string]bool{} |
| 184 | entries, err := os.ReadDir(liveDir(optDir)) |
| 185 | if err != nil { |
| 186 | if os.IsNotExist(err) { |
| 187 | return pins, nil |
| 188 | } |
| 189 | return nil, fmt.Errorf("read %s: %w", liveDir(optDir), err) |
| 190 | } |
| 191 | for _, e := range entries { |
| 192 | ver, err := LiveVersion(optDir, e.Name()) |
| 193 | if err != nil { |
| 194 | return nil, fmt.Errorf("read live link %s: %w", LiveLink(optDir, e.Name()), err) |
| 195 | } |
| 196 | if ver != "" { |
| 197 | pins[ver] = true |
| 198 | } |
| 199 | } |
| 200 | entries, err = os.ReadDir(filepath.Join(optDir, "previous")) |
| 201 | if err != nil && !os.IsNotExist(err) { |
| 202 | return nil, err |
| 203 | } |
| 204 | for _, e := range entries { |
| 205 | target, err := os.Readlink(filepath.Join(optDir, "previous", e.Name())) |
| 206 | if err != nil { |
| 207 | return nil, err |
| 208 | } |
| 209 | if ver := versionOfTarget(optDir, target); ver != "" { |
| 210 | pins[ver] = true |
| 211 | } |
| 212 | } |
| 213 | return pins, nil |
| 214 | } |
| 215 | |
amadan