| 1 | // The static serving class (routes with Kind=="static"): channel-tracked |
| 2 | // file trees served at the edge. A static route names an app + release |
| 3 | // channel; the release for that channel carries ONE artifact, site.tar.gz |
| 4 | // (a gzipped tar of the site tree, produced by PackDir). The handler |
| 5 | // resolves the signed channel pointer (cached, re-checked every 60s), |
| 6 | // extracts the sha256-verified tar into <cacheDir>/<app>/<version>/, and |
| 7 | // serves it with http.FileServer semantics — index.html for directories, |
| 8 | // clean 404s. A pointer advance atomically swaps the served version (and the |
| 9 | // `current` symlink) without dropping a request. |
| 10 | package edge |
| 11 | |
| 12 | import ( |
| 13 | "archive/tar" |
| 14 | "bytes" |
| 15 | "compress/gzip" |
| 16 | "context" |
| 17 | "crypto/ed25519" |
| 18 | "crypto/sha256" |
| 19 | "encoding/hex" |
| 20 | "errors" |
| 21 | "fmt" |
| 22 | "io" |
| 23 | "io/fs" |
| 24 | "log" |
| 25 | "mime" |
| 26 | "net/http" |
| 27 | "os" |
| 28 | "path" |
| 29 | "path/filepath" |
| 30 | "sort" |
| 31 | "strconv" |
| 32 | "strings" |
| 33 | "sync" |
| 34 | "sync/atomic" |
| 35 | "time" |
| 36 | |
| 37 | "github.com/carlosframework/platform/internal/apperr" |
| 38 | "github.com/carlosframework/platform/internal/registry" |
| 39 | "github.com/carlosframework/platform/internal/release" |
| 40 | ) |
| 41 | |
| 42 | // siteArtifactFile is the one artifact a static release carries: the gzipped |
| 43 | // tar of the site tree (`carlos ship -kind static` packs it via PackDir). |
| 44 | const siteArtifactFile = "site.tar.gz" |
| 45 | |
| 46 | // staticRecheck is how long a resolved channel pointer is trusted before the |
| 47 | // handler re-reads it from the store. Tests drop it to 0 (re-check every |
| 48 | // request) via the staticSite field or Edge.SetStaticRecheck. |
| 49 | const staticRecheck = 60 * time.Second |
| 50 | |
| 51 | // PackDir packs a site tree into the gzipped tar the static handler expects. |
| 52 | // The output is deterministic — sorted paths, zeroed times/uid/gid, fixed |
| 53 | // 0644 modes — so re-shipping an unchanged tree produces byte-identical |
| 54 | // artifacts (and Ship's immutable-put treats it as an idempotent no-op). |
| 55 | // Only regular files are packed: directories are implied by file paths, and |
| 56 | // anything else (symlinks, devices) in a site tree is a mistake, not content. |
| 57 | func PackDir(dir string) ([]byte, error) { |
| 58 | var names []string |
| 59 | err := filepath.WalkDir(dir, func(p string, de fs.DirEntry, err error) error { |
| 60 | if err != nil { |
| 61 | return err |
| 62 | } |
| 63 | if de.IsDir() { |
| 64 | return nil |
| 65 | } |
| 66 | if !de.Type().IsRegular() { |
| 67 | return fmt.Errorf("pack %s: %s is not a regular file", dir, p) |
| 68 | } |
| 69 | rel, err := filepath.Rel(dir, p) |
| 70 | if err != nil { |
| 71 | return err |
| 72 | } |
| 73 | names = append(names, filepath.ToSlash(rel)) |
| 74 | return nil |
| 75 | }) |
| 76 | if err != nil { |
| 77 | return nil, err |
| 78 | } |
| 79 | sort.Strings(names) |
| 80 | var buf bytes.Buffer |
| 81 | gz := gzip.NewWriter(&buf) |
| 82 | tw := tar.NewWriter(gz) |
| 83 | for _, name := range names { |
| 84 | b, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(name))) |
| 85 | if err != nil { |
| 86 | return nil, err |
| 87 | } |
| 88 | if err := tw.WriteHeader(&tar.Header{ |
| 89 | Typeflag: tar.TypeReg, |
| 90 | Name: name, |
| 91 | Mode: 0o644, |
| 92 | Size: int64(len(b)), |
| 93 | ModTime: time.Unix(0, 0), // zeroed: determinism over provenance |
| 94 | }); err != nil { |
| 95 | return nil, err |
| 96 | } |
| 97 | if _, err := tw.Write(b); err != nil { |
| 98 | return nil, err |
| 99 | } |
| 100 | if name == redirectsFile { |
| 101 | // Refused here rather than skipped at the edge, so a rule the |
| 102 | // edge cannot use is an error the shipper sees, not a redirect |
| 103 | // that silently never happens. |
| 104 | if _, err := parseRedirects(b); err != nil { |
| 105 | return nil, fmt.Errorf("pack %s: %s: %w", dir, name, err) |
| 106 | } |
| 107 | } |
| 108 | } |
| 109 | if err := tw.Close(); err != nil { |
| 110 | return nil, err |
| 111 | } |
| 112 | if err := gz.Close(); err != nil { |
| 113 | return nil, err |
| 114 | } |
| 115 | return buf.Bytes(), nil |
| 116 | } |
| 117 | |
| 118 | // staticSite serves one static route. All mutable state (resolved version, |
| 119 | // last pointer check) sits behind mu; requests serve from an immutable |
| 120 | // per-version directory, so a swap never races a response in flight. |
| 121 | type staticSite struct { |
| 122 | // acct is the account sqid whose bucket keys this app's objects live |
| 123 | // under — part of every key, and part of the cache path: app names are |
| 124 | // unique per ACCOUNT, so two accounts can both have a "docs" static site |
| 125 | // and their extracted trees must never share a directory. |
| 126 | acct string |
| 127 | app, channel string |
| 128 | store release.Store |
| 129 | pub ed25519.PublicKey |
| 130 | cacheDir string |
| 131 | // recheck is how long a resolved pointer is trusted; 0 = every request. |
| 132 | recheck time.Duration |
| 133 | |
| 134 | mu sync.Mutex |
| 135 | version string // currently served version; "" until first resolve |
| 136 | checkedAt time.Time // when the pointer was last read |
| 137 | digest string // the served tree's manifest digest |
| 138 | name string // the served tree's directory: siteDirName(version, digest) |
| 139 | redirects *redirectRules // the served tree's _redirects; nil when it has none |
| 140 | |
| 141 | // ratchet is #253's per-box ratchet for this process's pointer reads |
| 142 | // (Edge.staticRatchet); nil admits every verified pointer. |
| 143 | ratchet *release.DigestRatchet |
| 144 | } |
| 145 | |
| 146 | // newStaticHandler builds the serving handler for one static route. The |
| 147 | // route's empty Channel defaults to "stable", mirroring `carlos add`. |
| 148 | func newStaticHandler(rt registry.Route, store release.Store, pub ed25519.PublicKey, cacheDir string) http.Handler { |
| 149 | ch := rt.Channel |
| 150 | if ch == "" { |
| 151 | ch = "stable" |
| 152 | } |
| 153 | return &staticSite{ |
| 154 | acct: release.RouteAccount(rt), |
| 155 | app: rt.App, |
| 156 | channel: ch, |
| 157 | store: store, |
| 158 | pub: pub, |
| 159 | cacheDir: cacheDir, |
| 160 | recheck: staticRecheck, |
| 161 | } |
| 162 | } |
| 163 | |
| 164 | func (s *staticSite) ServeHTTP(w http.ResponseWriter, r *http.Request) { |
| 165 | tree, err := s.currentDir(r.Context()) |
| 166 | if err != nil { |
| 167 | // Nothing has ever resolved here, so there is no version to name: |
| 168 | // this 502 stays unstamped rather than claiming a build it isn't |
| 169 | // serving. |
| 170 | log.Printf("edge: static %s/%s: %v", s.app, s.channel, err) |
| 171 | noteCause(r, apperr.CauseStatic) |
| 172 | http.Error(w, "static site unavailable", http.StatusBadGateway) |
| 173 | return |
| 174 | } |
| 175 | // X-Carlos-Version for the static class (issue #112). Binary routes get |
| 176 | // this from the route table (edge.go, versionHeader), which the adopt |
| 177 | // pass stamps; a static route is never adopted, so the truth lives here |
| 178 | // instead — and it is PER-REQUEST truth, the version whose tree answers |
| 179 | // THIS request, which is strictly better than a table stamp that can lag |
| 180 | // the process it describes. That includes the degraded path: currentDir |
| 181 | // returns the CACHED version when the pointer is unreadable, and this |
| 182 | // stamps what is actually served, never the pointer's unreachable head. |
| 183 | // Set before any body is written, so it lands on the tree's own 404s too |
| 184 | // (the handler serving that tree IS a version). |
| 185 | w.Header().Set("X-Carlos-Version", tree.version) |
| 186 | // A page out of the tree is revalidated before it is reused, and so is |
| 187 | // the tree's own 404. With Last-Modified alone a browser GUESSES a |
| 188 | // freshness lifetime — typically a tenth of the file's age — and reuses |
| 189 | // its copy without asking, so a site last deployed a month ago kept |
| 190 | // showing the old page for days after the next deploy (discussion #195). |
| 191 | // no-cache is "store it, but ask first"; the ETag serveTree sets is what |
| 192 | // makes asking cost a 304. It also keeps these responses out of the |
| 193 | // edge's own shared cache, which admits only a positive lifetime — as |
| 194 | // they always were, having named none. (net/http strips the header again |
| 195 | // from the errors it writes itself, a 416 or a 400: neither is a status a |
| 196 | // browser caches on a guess.) |
| 197 | w.Header().Set("Cache-Control", "no-cache") |
| 198 | s.serveTree(tree, w, r) |
| 199 | } |
| 200 | |
| 201 | // servedTree is what one request is answered from: an extracted tree, the |
| 202 | // version it holds, and the rules its _redirects declared. They travel |
| 203 | // together so a request never pairs one version's files with another's rules. |
| 204 | type servedTree struct { |
| 205 | dir, version string |
| 206 | redirects *redirectRules |
| 207 | } |
| 208 | |
| 209 | // dir is where the tree named name lives once extracted. |
| 210 | func (s *staticSite) dir(name string) string { |
| 211 | return filepath.Join(s.appDir(), name) |
| 212 | } |
| 213 | |
| 214 | // siteDirName keys the extracted-site cache by the verified manifest digest |
| 215 | // as well as the version (#253, spec section 4.6), so a tree extracted from |
| 216 | // other bytes under a reused name is never reused. A bare version name, the |
| 217 | // key before this change, never matches. |
| 218 | func siteDirName(version, digest string) string { return version + "@" + digest } |
| 219 | |
| 220 | // appDir is this app's cache root, under its account: app names are unique |
| 221 | // per ACCOUNT, so two accounts' "docs" sites must never share a tree. |
| 222 | func (s *staticSite) appDir() string { |
| 223 | return filepath.Join(s.cacheDir, s.acct, s.app) |
| 224 | } |
| 225 | |
| 226 | // currentDir returns the directory to serve this request from AND the version |
| 227 | // that directory holds, re-resolving the channel pointer when the cache window |
| 228 | // has lapsed. A resolution or staging failure NEVER takes down a site that was |
| 229 | // already serving: the cached version keeps answering and the pointer is |
| 230 | // retried next window. Only a handler that has never resolved (nothing safe to |
| 231 | // serve) errors out. |
| 232 | // |
| 233 | // The version comes back with the dir rather than being read off s.version |
| 234 | // afterwards so ServeHTTP can stamp X-Carlos-Version without re-taking mu (and |
| 235 | // without the window in between, where a concurrent resolve could swap |
| 236 | // s.version and make the header describe a different tree than the one served). |
| 237 | // The tree's redirect rules come back with it for the same reason. |
| 238 | func (s *staticSite) currentDir(ctx context.Context) (servedTree, error) { |
| 239 | s.mu.Lock() |
| 240 | defer s.mu.Unlock() |
| 241 | if s.version != "" && s.recheck > 0 && time.Since(s.checkedAt) < s.recheck { |
| 242 | // Inside the trust window: this request is answered from the cached |
| 243 | // version, and stamping that version is honest by construction — the |
| 244 | // header names what the bytes are, not what the pointer may have |
| 245 | // advanced to since the last check. |
| 246 | return s.served(), nil |
| 247 | } |
| 248 | cp, err := release.Channel(ctx, s.store, s.pub, s.acct, s.app, s.channel) |
| 249 | if err != nil { |
| 250 | return s.stale(fmt.Errorf("resolve channel: %w", err)) |
| 251 | } |
| 252 | if err := s.ratchet.Admit(s.acct, cp); err != nil { |
| 253 | return s.stale(err) |
| 254 | } |
| 255 | if want := cp.Digest; cp.Version != s.version || (want != "" && want != s.digest) { |
| 256 | name, digest, err := s.ensureExtracted(ctx, cp.Version, want) |
| 257 | if err != nil { |
| 258 | return s.stale(fmt.Errorf("stage %s: %w", cp.Version, err)) |
| 259 | } |
| 260 | if err := s.pointCurrent(name); err != nil { |
| 261 | return s.stale(fmt.Errorf("swap current -> %s: %w", name, err)) |
| 262 | } |
| 263 | s.version, s.digest, s.name = cp.Version, digest, name |
| 264 | s.redirects = s.loadRedirects(s.dir(name), cp.Version) |
| 265 | } |
| 266 | s.checkedAt = time.Now() |
| 267 | return s.served(), nil |
| 268 | } |
| 269 | |
| 270 | // served is the tree currently being served. The caller holds mu. |
| 271 | func (s *staticSite) served() servedTree { |
| 272 | return servedTree{dir: s.dir(s.name), version: s.version, redirects: s.redirects} |
| 273 | } |
| 274 | |
| 275 | // loadRedirects parses the _redirects at the root of an extracted tree, once |
| 276 | // per version swap. A tree without one has no rules. A line that does not |
| 277 | // parse is logged and skipped: ship refuses such a tree (PackDir), so only a |
| 278 | // tree packed some other way gets here, and its remaining rules still apply. |
| 279 | func (s *staticSite) loadRedirects(dir, version string) *redirectRules { |
| 280 | data, err := os.ReadFile(filepath.Join(dir, redirectsFile)) |
| 281 | if errors.Is(err, fs.ErrNotExist) { |
| 282 | return nil |
| 283 | } |
| 284 | if err != nil { |
| 285 | log.Printf("edge: static %s/%s: %s %s: %v — serving without its rules", s.app, s.channel, version, redirectsFile, err) |
| 286 | return nil |
| 287 | } |
| 288 | rules, err := parseRedirects(data) |
| 289 | if err != nil { |
| 290 | log.Printf("edge: static %s/%s: %s %s: %s", s.app, s.channel, version, redirectsFile, strings.ReplaceAll(err.Error(), "\n", "; ")) |
| 291 | } |
| 292 | return rules |
| 293 | } |
| 294 | |
| 295 | // stale is the degraded path: keep serving the version we already verified |
| 296 | // and extracted — and name it, so the response header describes the CACHED |
| 297 | // tree rather than the head this handler could not reach — or fail if there |
| 298 | // has never been one. |
| 299 | func (s *staticSite) stale(err error) (servedTree, error) { |
| 300 | if s.version == "" { |
| 301 | return servedTree{}, err |
| 302 | } |
| 303 | log.Printf("edge: static %s/%s: %v — still serving cached %s", s.app, s.channel, err, s.version) |
| 304 | s.checkedAt = time.Now() // back off: retry next recheck window, not next request |
| 305 | return s.served(), nil |
| 306 | } |
| 307 | |
| 308 | // ensureExtracted makes the version's verified tree exist and returns its |
| 309 | // directory name and manifest digest. want is the pointer entry's signed |
| 310 | // digest: when set, the manifest must hash to it before anything is |
| 311 | // downloaded (#253, spec section 4.6), and either way the tree is keyed by the |
| 312 | // digest of the manifest actually used. The tar's sha256 is checked against |
| 313 | // that manifest before extraction, and extraction goes to a temp dir renamed |
| 314 | // into place, so a directory's existence means "complete and verified". app |
| 315 | // and version are path-safe by construction: both came out of a |
| 316 | // signature-verified pointer whose fields passed ValidApp/ValidVersion. |
| 317 | func (s *staticSite) ensureExtracted(ctx context.Context, version, want string) (string, string, error) { |
| 318 | m, raw, err := release.CheckManifestDigest(ctx, s.store, s.acct, s.app, version, want) |
| 319 | if err != nil { |
| 320 | return "", "", err |
| 321 | } |
| 322 | digest := release.ManifestDigest(raw) |
| 323 | name := siteDirName(version, digest) |
| 324 | dst := s.dir(name) |
| 325 | if _, err := os.Stat(dst); err == nil { |
| 326 | return name, digest, nil // extracted by an earlier check or a sibling handler |
| 327 | } |
| 328 | if m.Kind != "static" { |
| 329 | return "", "", fmt.Errorf("release %s/%s is kind %q, not static", s.app, version, m.Kind) |
| 330 | } |
| 331 | var art *release.Artifact |
| 332 | for _, a := range m.Artifacts { |
| 333 | if a.File == siteArtifactFile { |
| 334 | art = &a |
| 335 | break |
| 336 | } |
| 337 | } |
| 338 | if art == nil { |
| 339 | return "", "", fmt.Errorf("release %s/%s has no %s artifact", s.app, version, siteArtifactFile) |
| 340 | } |
| 341 | data, _, err := release.GetQualified(ctx, s.store, release.ArtifactKey(s.acct, s.app, version, art.File)) |
| 342 | if err != nil { |
| 343 | return "", "", err |
| 344 | } |
| 345 | sum := sha256.Sum256(data) |
| 346 | if hex.EncodeToString(sum[:]) != art.SHA256 { |
| 347 | return "", "", fmt.Errorf("%s sha256 mismatch (manifest %s, stored %s): refusing to extract", |
| 348 | siteArtifactFile, art.SHA256, hex.EncodeToString(sum[:])) |
| 349 | } |
| 350 | if int64(len(data)) != art.Size { |
| 351 | return "", "", fmt.Errorf("%s size mismatch (manifest %d, stored %d)", siteArtifactFile, art.Size, len(data)) |
| 352 | } |
| 353 | appDir := s.appDir() |
| 354 | if err := os.MkdirAll(appDir, 0o755); err != nil { |
| 355 | return "", "", err |
| 356 | } |
| 357 | tmp, err := os.MkdirTemp(appDir, ".extract-") |
| 358 | if err != nil { |
| 359 | return "", "", err |
| 360 | } |
| 361 | if err := extractSiteTar(data, tmp); err != nil { |
| 362 | os.RemoveAll(tmp) |
| 363 | return "", "", err |
| 364 | } |
| 365 | if err := os.Rename(tmp, dst); err != nil { |
| 366 | os.RemoveAll(tmp) |
| 367 | if _, serr := os.Stat(dst); serr == nil { |
| 368 | return name, digest, nil // lost the staging race; the winner's tree is identical |
| 369 | } |
| 370 | return "", "", err |
| 371 | } |
| 372 | return name, digest, nil |
| 373 | } |
| 374 | |
| 375 | // pointCurrent atomically swaps the app's `current` symlink to the tree named name |
| 376 | // (temp symlink + rename — the same swap versions.go's PointLive uses), so |
| 377 | // anything looking at the cache dir sees the version flip as one atomic step. |
| 378 | func (s *staticSite) pointCurrent(name string) error { |
| 379 | link := filepath.Join(s.appDir(), "current") |
| 380 | tmp := link + ".tmp" |
| 381 | _ = os.Remove(tmp) // a leftover from a crashed swap |
| 382 | if err := os.Symlink(name, tmp); err != nil { |
| 383 | return err |
| 384 | } |
| 385 | if err := os.Rename(tmp, link); err != nil { |
| 386 | os.Remove(tmp) |
| 387 | return err |
| 388 | } |
| 389 | return nil |
| 390 | } |
| 391 | |
| 392 | // extractSiteTar unpacks a verified site tar into dst, rejecting anything |
| 393 | // that would write outside it: absolute names, ".." traversal, and non-file |
| 394 | // entry types (a symlink in a site tar could alias the extraction outside |
| 395 | // the cache — PackDir never emits one, so none is ever legitimate). |
| 396 | func extractSiteTar(data []byte, dst string) error { |
| 397 | gz, err := gzip.NewReader(bytes.NewReader(data)) |
| 398 | if err != nil { |
| 399 | return fmt.Errorf("site tar: %v", err) |
| 400 | } |
| 401 | defer gz.Close() |
| 402 | tr := tar.NewReader(gz) |
| 403 | for { |
| 404 | hdr, err := tr.Next() |
| 405 | if err == io.EOF { |
| 406 | return nil |
| 407 | } |
| 408 | if err != nil { |
| 409 | return fmt.Errorf("site tar: %v", err) |
| 410 | } |
| 411 | name := filepath.FromSlash(path.Clean(hdr.Name)) |
| 412 | if !filepath.IsLocal(name) { |
| 413 | return fmt.Errorf("site tar: entry %q escapes the extraction dir", hdr.Name) |
| 414 | } |
| 415 | switch hdr.Typeflag { |
| 416 | case tar.TypeDir: |
| 417 | if err := os.MkdirAll(filepath.Join(dst, name), 0o755); err != nil { |
| 418 | return err |
| 419 | } |
| 420 | case tar.TypeReg: |
| 421 | p := filepath.Join(dst, name) |
| 422 | if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { |
| 423 | return err |
| 424 | } |
| 425 | f, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) |
| 426 | if err != nil { |
| 427 | return err |
| 428 | } |
| 429 | if _, err := io.Copy(f, tr); err != nil { |
| 430 | f.Close() |
| 431 | return err |
| 432 | } |
| 433 | if err := f.Close(); err != nil { |
| 434 | return err |
| 435 | } |
| 436 | case tar.TypeXGlobalHeader: |
| 437 | // pax global header: metadata, nothing to write |
| 438 | default: |
| 439 | return fmt.Errorf("site tar: entry %q has unsupported type %d", hdr.Name, hdr.Typeflag) |
| 440 | } |
| 441 | } |
| 442 | } |
| 443 | |
| 444 | // serveTree answers one request from an extracted version dir: |
| 445 | // http.FileServer semantics with index.html for directories and a clean 404 |
| 446 | // (never a directory listing) when neither the file nor an index exists — |
| 447 | // after the tree's _redirects (redirects.go) has had its say. A rule applies |
| 448 | // only where no file answers the path, unless it is forced. |
| 449 | // |
| 450 | // It serves the file itself rather than through http.ServeFile, because |
| 451 | // ServeFile offers the file's mtime as Last-Modified and answers |
| 452 | // If-Modified-Since from it — and an extracted file's mtime is when THIS box |
| 453 | // unpacked the tree, which is not a fact about the content. It differs |
| 454 | // between edges serving the same release, and it runs backwards on a |
| 455 | // rollback: the tree rolled back to was unpacked before the one the browser |
| 456 | // holds, so "modified since?" answers no and the browser keeps the build that |
| 457 | // was withdrawn. No date is offered, so there is none to compare. |
| 458 | // |
| 459 | // The validator is the file's content (tagCache). What ServeFile did besides |
| 460 | // is kept by hand: the refusal of a ".." element, and the redirect off a |
| 461 | // trailing /index.html onto its directory. |
| 462 | func (s *staticSite) serveTree(tree servedTree, w http.ResponseWriter, r *http.Request) { |
| 463 | if containsDotDot(r.URL.Path) { |
| 464 | // The Clean below would resolve it harmlessly inside the tree; this |
| 465 | // keeps the answer ServeFile always gave such a path. |
| 466 | http.Error(w, "invalid URL path", http.StatusBadRequest) |
| 467 | return |
| 468 | } |
| 469 | // Clean rooted at "/" so no request path can climb out of the tree. |
| 470 | p := path.Clean("/" + r.URL.Path) |
| 471 | status, rewritten := http.StatusOK, false |
| 472 | for { |
| 473 | fp := filepath.Join(tree.dir, filepath.FromSlash(p)) |
| 474 | fi, err := os.Stat(fp) |
| 475 | if err == nil && fi.IsDir() { |
| 476 | fp = filepath.Join(fp, "index.html") |
| 477 | fi, err = os.Stat(fp) |
| 478 | } |
| 479 | // The _redirects file is configuration and answers nothing. |
| 480 | found := err == nil && !fi.IsDir() && p != "/"+redirectsFile |
| 481 | if !rewritten { |
| 482 | if act, ok := tree.redirects.resolve(requestHost(r.Host), p, r.URL.Path, r.URL.RawQuery, found); ok { |
| 483 | if act.location != "" { |
| 484 | http.Redirect(w, r, act.location, act.status) |
| 485 | return |
| 486 | } |
| 487 | // A rewrite serves another file of the tree at this URL. It is |
| 488 | // looked up once: rules are not consulted a second time, and a |
| 489 | // destination with no file is a plain 404. |
| 490 | p, status, rewritten = act.file, act.status, true |
| 491 | continue |
| 492 | } |
| 493 | } |
| 494 | if !found { |
| 495 | http.NotFound(w, r) |
| 496 | return |
| 497 | } |
| 498 | if !rewritten && strings.HasSuffix(r.URL.Path, "/index.html") { |
| 499 | // Relative, as ServeFile wrote it, so it survives any prefix the |
| 500 | // request arrived under. |
| 501 | loc := "./" |
| 502 | if q := r.URL.RawQuery; q != "" { |
| 503 | loc += "?" + q |
| 504 | } |
| 505 | w.Header().Set("Location", loc) |
| 506 | w.WriteHeader(http.StatusMovedPermanently) |
| 507 | return |
| 508 | } |
| 509 | s.serveFile(fp, p, fi, status, w, r) |
| 510 | return |
| 511 | } |
| 512 | } |
| 513 | |
| 514 | // serveFile sends the tree file at fp, whose tree path is p (for logs). A 200 |
| 515 | // gets a content tag and net/http's conditional and range handling. Any other |
| 516 | // status — a _redirects 404 page — is the file's bytes under that status, |
| 517 | // with no validator: ServeContent knows only how to succeed. |
| 518 | func (s *staticSite) serveFile(fp, p string, fi os.FileInfo, status int, w http.ResponseWriter, r *http.Request) { |
| 519 | f, err := os.Open(fp) |
| 520 | if err != nil { |
| 521 | // ServeFile's mapping, kept: a file that vanished since the Stat is |
| 522 | // a 404, but one that is there and will not open is not a missing |
| 523 | // page, and must not read as one in an access log. |
| 524 | switch { |
| 525 | case errors.Is(err, fs.ErrNotExist): |
| 526 | http.NotFound(w, r) |
| 527 | case errors.Is(err, fs.ErrPermission): |
| 528 | http.Error(w, "403 Forbidden", http.StatusForbidden) |
| 529 | default: |
| 530 | log.Printf("edge: static %s/%s: open %s: %v", s.app, s.channel, p, err) |
| 531 | http.Error(w, "500 Internal Server Error", http.StatusInternalServerError) |
| 532 | } |
| 533 | return |
| 534 | } |
| 535 | defer f.Close() |
| 536 | if status != http.StatusOK { |
| 537 | ctype := mime.TypeByExtension(filepath.Ext(fp)) |
| 538 | if ctype == "" { |
| 539 | var head [512]byte |
| 540 | n, _ := io.ReadFull(f, head[:]) |
| 541 | ctype = http.DetectContentType(head[:n]) |
| 542 | if _, err := f.Seek(0, io.SeekStart); err != nil { |
| 543 | log.Printf("edge: static %s/%s: rewind %s: %v", s.app, s.channel, p, err) |
| 544 | http.Error(w, "500 Internal Server Error", http.StatusInternalServerError) |
| 545 | return |
| 546 | } |
| 547 | } |
| 548 | w.Header().Set("Content-Type", ctype) |
| 549 | w.Header().Set("Content-Length", strconv.FormatInt(fi.Size(), 10)) |
| 550 | w.WriteHeader(status) |
| 551 | if r.Method != http.MethodHead { |
| 552 | io.Copy(w, f) |
| 553 | } |
| 554 | return |
| 555 | } |
| 556 | // Stat the OPEN file: its identity is what a remembered tag is checked |
| 557 | // against, and it is the file whose bytes are about to be sent. |
| 558 | if ofi, err := f.Stat(); err == nil && !ofi.IsDir() { |
| 559 | fi = ofi |
| 560 | if tag, ctype, err := staticTags.tag(r.Context(), fp, f, fi); err == nil { |
| 561 | // The type the tag was computed over is the type sent: set here, |
| 562 | // ServeContent leaves it alone. |
| 563 | w.Header().Set("ETag", tag) |
| 564 | w.Header().Set("Content-Type", ctype) |
| 565 | } else { |
| 566 | if !errors.Is(err, errTagBusy) && r.Context().Err() == nil { |
| 567 | // No tag is the honest answer: the response then carries no |
| 568 | // validator and no-cache makes the browser refetch it whole. |
| 569 | log.Printf("edge: static %s/%s: tag %s: %v", s.app, s.channel, p, err) |
| 570 | } |
| 571 | // A failed hash rewinds, but if even that failed the file's |
| 572 | // position is unknown and ServeContent would sniff its type |
| 573 | // from the middle of it. |
| 574 | if _, serr := f.Seek(0, io.SeekStart); serr != nil { |
| 575 | log.Printf("edge: static %s/%s: rewind %s: %v", s.app, s.channel, p, serr) |
| 576 | http.Error(w, "500 Internal Server Error", http.StatusInternalServerError) |
| 577 | return |
| 578 | } |
| 579 | } |
| 580 | } |
| 581 | // ServeContent brings Content-Type, ranges and the conditional-request |
| 582 | // handling; the zero time is "no Last-Modified, and no date to test". |
| 583 | http.ServeContent(w, r, fi.Name(), time.Time{}, f) |
| 584 | } |
| 585 | |
| 586 | // tagCache computes and remembers the entity tag of each extracted file: a |
| 587 | // hash of its bytes and of the one thing besides them that decides what the |
| 588 | // client receives, its Content-Type. |
| 589 | // |
| 590 | // The content, and nothing that merely names it. A version name is reused — |
| 591 | // a pruned version re-shipped under its old name is a new release |
| 592 | // (release.putImmutable), and a hostname can move between two apps that both |
| 593 | // have a "v1" — so a tag made of the name would hand a browser a 304 over |
| 594 | // bytes it does not hold. Hashing also means identical bytes get one tag on |
| 595 | // every edge, and a file a deploy did not touch keeps its tag across it: the |
| 596 | // release that changed one page does not make every browser refetch every |
| 597 | // asset. |
| 598 | // |
| 599 | // The type is hashed in because the same bytes can be two representations. |
| 600 | // /about served from an extensionless file is sniffed as text/plain; the same |
| 601 | // bytes at about/index.html are text/html; and two boxes whose MIME tables |
| 602 | // differ type one .md file differently. A 304 carries no Content-Type, so |
| 603 | // bytes alone would leave the browser rendering the old one. The type is |
| 604 | // resolved here exactly as net/http resolves it — the extension's mapping, |
| 605 | // else a sniff of the first 512 bytes — and handed back with the tag, so the |
| 606 | // type the tag covers is the type the response carries. |
| 607 | // |
| 608 | // WEAK, although such a hash is as strong as a validator gets. The edge gzips |
| 609 | // compressible responses above this handler and weakens a strong tag when it |
| 610 | // does (compress.go, weakenETag) — but only on the 200. A 304 and a HEAD pass |
| 611 | // through as they are, so a strong tag would come back weak on the page and |
| 612 | // strong on its own revalidation, and a cache may not update the one from the |
| 613 | // other. Weak from the start, it reads the same on all three. The cost is |
| 614 | // that If-Range falls back to the whole file. |
| 615 | // |
| 616 | // One cache for the process rather than one per handler: the edge rebuilds |
| 617 | // every static handler on each route-table reload, and a per-handler memo |
| 618 | // would re-hash every site each time a route anywhere on the box changed. |
| 619 | // What makes a shared, long-lived memo safe: |
| 620 | // |
| 621 | // - An entry is checked against the file it was computed FROM, not the path |
| 622 | // it was computed at (os.SameFile, size, mtime). A tree replaced on disk |
| 623 | // under a live handler — cache maintenance, then a re-shipped version of |
| 624 | // the same name extracted again — is new files, and hashes again. |
| 625 | // - A cold file is hashed once however many requests want it: the rest |
| 626 | // wait for the one in flight. |
| 627 | // - A file over tagInline hashes under a semaphore, tagHashers at a time, |
| 628 | // so a sweep of a site's large assets cannot take every core and the disk |
| 629 | // at once. Small files never queue behind them: their work is bounded by |
| 630 | // their size and they hash inline. |
| 631 | // - Nobody waits long. A tag is an optimization, and a request that has |
| 632 | // waited `wait` for a hasher or for someone else's hash is served without |
| 633 | // one (errTagBusy): correct, uncached, and tried again on the next |
| 634 | // request. Only the request doing a hash sees it through, because every |
| 635 | // later request needs the result. |
| 636 | // - It is bounded: past tagCacheMax entries it is emptied and refills on |
| 637 | // demand, which costs hashing and never correctness. |
| 638 | type tagCache struct { |
| 639 | mu sync.Mutex |
| 640 | entries map[string]tagEntry |
| 641 | flights map[string]chan struct{} // closed when the hash for that path is done |
| 642 | sem chan struct{} |
| 643 | wait time.Duration |
| 644 | |
| 645 | // Evidence and a seam for the tests: files hashed, callers currently |
| 646 | // waiting on another's hash, and a hook run inside a hash (nil in |
| 647 | // production) so a test can hold one open. |
| 648 | hashed atomic.Int64 |
| 649 | waiting atomic.Int64 |
| 650 | hold func() |
| 651 | } |
| 652 | |
| 653 | type tagEntry struct { |
| 654 | fi os.FileInfo |
| 655 | tag, ctype string |
| 656 | } |
| 657 | |
| 658 | const ( |
| 659 | tagHashers = 4 |
| 660 | tagInline = 1 << 20 // files up to this size hash without a hasher slot |
| 661 | tagWait = 250 * time.Millisecond |
| 662 | tagCacheMax = 1 << 16 |
| 663 | ) |
| 664 | |
| 665 | // errTagBusy: the tag could not be had within the wait budget. The caller |
| 666 | // serves the file untagged. |
| 667 | var errTagBusy = errors.New("static tag: hashers busy") |
| 668 | |
| 669 | func newTagCache() *tagCache { |
| 670 | return &tagCache{ |
| 671 | entries: map[string]tagEntry{}, |
| 672 | flights: map[string]chan struct{}{}, |
| 673 | sem: make(chan struct{}, tagHashers), |
| 674 | wait: tagWait, |
| 675 | } |
| 676 | } |
| 677 | |
| 678 | var staticTags = newTagCache() |
| 679 | |
| 680 | // current reports whether e still describes the file fi is. |
| 681 | func (e tagEntry) current(fi os.FileInfo) bool { |
| 682 | return e.fi != nil && os.SameFile(e.fi, fi) && e.fi.Size() == fi.Size() && e.fi.ModTime().Equal(fi.ModTime()) |
| 683 | } |
| 684 | |
| 685 | // tag returns the entity tag and Content-Type for the open file f at fp, |
| 686 | // hashing it if no current entry exists. f is left positioned at its start. |
| 687 | // ctx and the wait budget bound the WAITING — on another request's hash, or |
| 688 | // on a free hasher — never a hash in progress. |
| 689 | func (c *tagCache) tag(ctx context.Context, fp string, f io.ReadSeeker, fi os.FileInfo) (tag, ctype string, err error) { |
| 690 | var budget <-chan time.Time |
| 691 | for { |
| 692 | c.mu.Lock() |
| 693 | if e, ok := c.entries[fp]; ok && e.current(fi) { |
| 694 | c.mu.Unlock() |
| 695 | return e.tag, e.ctype, nil |
| 696 | } |
| 697 | if budget == nil { |
| 698 | t := time.NewTimer(c.wait) |
| 699 | defer t.Stop() |
| 700 | budget = t.C |
| 701 | } |
| 702 | if flight, ok := c.flights[fp]; ok { |
| 703 | c.mu.Unlock() |
| 704 | c.waiting.Add(1) |
| 705 | select { |
| 706 | case <-flight: |
| 707 | c.waiting.Add(-1) |
| 708 | continue // look again: usually a hit, a miss if the file differs |
| 709 | case <-ctx.Done(): |
| 710 | c.waiting.Add(-1) |
| 711 | return "", "", ctx.Err() |
| 712 | case <-budget: |
| 713 | c.waiting.Add(-1) |
| 714 | return "", "", errTagBusy |
| 715 | } |
| 716 | } |
| 717 | done := make(chan struct{}) |
| 718 | c.flights[fp] = done |
| 719 | c.mu.Unlock() |
| 720 | |
| 721 | tag, ctype, err = c.hash(ctx, budget, f, fi) |
| 722 | c.mu.Lock() |
| 723 | if err == nil { |
| 724 | if len(c.entries) >= tagCacheMax { |
| 725 | c.entries = map[string]tagEntry{} |
| 726 | } |
| 727 | c.entries[fp] = tagEntry{fi: fi, tag: tag, ctype: ctype} |
| 728 | } |
| 729 | delete(c.flights, fp) |
| 730 | c.mu.Unlock() |
| 731 | close(done) |
| 732 | return tag, ctype, err |
| 733 | } |
| 734 | } |
| 735 | |
| 736 | func (c *tagCache) hash(ctx context.Context, budget <-chan time.Time, f io.ReadSeeker, fi os.FileInfo) (tag, ctype string, err error) { |
| 737 | if fi.Size() > tagInline { |
| 738 | select { |
| 739 | case c.sem <- struct{}{}: |
| 740 | defer func() { <-c.sem }() |
| 741 | case <-ctx.Done(): |
| 742 | return "", "", ctx.Err() |
| 743 | case <-budget: |
| 744 | return "", "", errTagBusy |
| 745 | } |
| 746 | } |
| 747 | c.hashed.Add(1) |
| 748 | if c.hold != nil { |
| 749 | c.hold() |
| 750 | } |
| 751 | // From here the file is being read, and it goes back to its start on |
| 752 | // EVERY path out — a failed hash included. The caller serves the file |
| 753 | // either way, and net/http sniffs a Content-Type from wherever the file |
| 754 | // is positioned. |
| 755 | defer func() { |
| 756 | if _, serr := f.Seek(0, io.SeekStart); serr != nil && err == nil { |
| 757 | tag, ctype, err = "", "", serr |
| 758 | } |
| 759 | }() |
| 760 | // net/http's own rule (serveContent): the extension's type, else a sniff |
| 761 | // of the first 512 bytes. |
| 762 | var head [512]byte |
| 763 | n, err := io.ReadFull(f, head[:]) |
| 764 | if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { |
| 765 | return "", "", err |
| 766 | } |
| 767 | ctype = mime.TypeByExtension(filepath.Ext(fi.Name())) |
| 768 | if ctype == "" { |
| 769 | ctype = http.DetectContentType(head[:n]) |
| 770 | } |
| 771 | h := sha256.New() |
| 772 | io.WriteString(h, ctype) |
| 773 | h.Write([]byte{0}) |
| 774 | h.Write(head[:n]) |
| 775 | if _, err := io.Copy(h, f); err != nil { |
| 776 | return "", "", err |
| 777 | } |
| 778 | return `W/"` + hex.EncodeToString(h.Sum(nil)[:16]) + `"`, ctype, nil |
| 779 | } |
| 780 | |
| 781 | // containsDotDot reports whether a URL path has a ".." element — net/http's |
| 782 | // own test, which http.ServeFile applied before this handler served files |
| 783 | // itself. |
| 784 | func containsDotDot(p string) bool { |
| 785 | if !strings.Contains(p, "..") { |
| 786 | return false |
| 787 | } |
| 788 | for _, el := range strings.FieldsFunc(p, func(r rune) bool { return r == '/' || r == '\\' }) { |
| 789 | if el == ".." { |
| 790 | return true |
| 791 | } |
| 792 | } |
| 793 | return false |
| 794 | } |
| 795 | |
amadan