| 1 | // The on-box version store — ported from Eleven versions.go. |
| 2 | // |
| 3 | // Versioned, side-by-side deploys: the box runs one binary per host, but |
| 4 | // they need not be the SAME binary. Each host's unit execs |
| 5 | // <opt>/live/<host>, a symlink into a content store at |
| 6 | // <opt>/versions/<ver>/<app>. Swapping one link (and restarting one unit) |
| 7 | // moves one host to a new build while every other host stays put — so a new |
| 8 | // build can be canaried on one host before it's promoted fleet-wide, and |
| 9 | // rolled back per host. |
| 10 | // |
| 11 | // The opt dir defaults from CARLOS_OPT_DIR (default /opt/carlos), but every |
| 12 | // function takes the base dir as a parameter so tests never touch /opt. |
| 13 | package release |
| 14 | |
| 15 | import ( |
| 16 | "context" |
| 17 | "crypto/sha256" |
| 18 | "encoding/hex" |
| 19 | "encoding/json" |
| 20 | "errors" |
| 21 | "fmt" |
| 22 | "io" |
| 23 | "net" |
| 24 | "net/http" |
| 25 | "os" |
| 26 | "path/filepath" |
| 27 | "regexp" |
| 28 | "strings" |
| 29 | "time" |
| 30 | |
| 31 | "github.com/carlosframework/platform/internal/safefile" |
| 32 | "github.com/carlosframework/platform/internal/unixdial" |
| 33 | ) |
| 34 | |
| 35 | // OptDir is the on-box install root: CARLOS_OPT_DIR, default /opt/carlos. |
| 36 | // Resolved at call time (not init) so a test's Setenv is honoured. |
| 37 | func OptDir() string { |
| 38 | if d := os.Getenv("CARLOS_OPT_DIR"); d != "" { |
| 39 | return d |
| 40 | } |
| 41 | return "/opt/carlos" |
| 42 | } |
| 43 | |
| 44 | func versionsDir(optDir string) string { return filepath.Join(optDir, "versions") } |
| 45 | func liveDir(optDir string) string { return filepath.Join(optDir, "live") } |
| 46 | |
| 47 | // VersionPath is where a staged build of app lives. App names are unique per |
| 48 | // ACCOUNT, so two accounts can both have a "blog" — and, since versions are |
| 49 | // caller-chosen strings, both can have a "v1" whose bytes differ. A single |
| 50 | // <opt>/versions/<ver>/<app> would let one account's binary be staged over |
| 51 | // another's and then executed under its host. |
| 52 | // |
| 53 | // The ops account deliberately keeps the pre-accounts path byte for byte |
| 54 | // (<opt>/versions/<ver>/<app>): every existing box's live symlinks point |
| 55 | // there, and moving them would restage and restart the whole fleet on |
| 56 | // rollout for no benefit. Everyone else gets a level in between. |
| 57 | func VersionPath(optDir, acct, ver, app string) string { |
| 58 | if acct == "" || acct == OpsAccount { |
| 59 | return filepath.Join(versionsDir(optDir), ver, app) |
| 60 | } |
| 61 | return filepath.Join(versionsDir(optDir), ver, acct, app) |
| 62 | } |
| 63 | |
| 64 | // LiveLink is the per-host pointer the unit execs (<opt>/live/<host>) — a |
| 65 | // symlink to a VersionPath. Swapping it atomically is how one host moves |
| 66 | // between versions independently of the others. |
| 67 | func LiveLink(optDir, host string) string { return filepath.Join(liveDir(optDir), host) } |
| 68 | |
| 69 | // versionRe keeps a version usable as a directory name (no slashes, no |
| 70 | // traversal). |
| 71 | var versionRe = regexp.MustCompile(`^[a-zA-Z0-9._-]+$`) |
| 72 | |
| 73 | func ValidVersion(v string) bool { |
| 74 | return v != "" && len(v) <= 64 && v != "." && v != ".." && versionRe.MatchString(v) |
| 75 | } |
| 76 | |
| 77 | // CopyFileAtomic copies src to dst via a temp file + rename, so a reader |
| 78 | // never sees a half-written binary (and a crashed copy leaves the old one |
| 79 | // intact). |
| 80 | func CopyFileAtomic(src, dst string, mode os.FileMode) error { |
| 81 | in, err := safefile.OpenFile(src, os.O_RDONLY, 0) |
| 82 | if err != nil { |
| 83 | return err |
| 84 | } |
| 85 | defer in.Close() |
| 86 | return safefile.Write(dst, mode, func(out io.Writer) error { _, err := io.Copy(out, in); return err }) |
| 87 | } |
| 88 | |
| 89 | // WriteFileAtomic is CopyFileAtomic for in-memory bytes (a downloaded, |
| 90 | // sha256-verified artifact). |
| 91 | func WriteFileAtomic(dst string, data []byte, mode os.FileMode) error { |
| 92 | return safefile.WriteFile(dst, data, mode) |
| 93 | } |
| 94 | |
| 95 | // StageArtifact puts verified artifact bytes into the store under ver, so |
| 96 | // hosts can be pointed at it. Idempotent: re-staging overwrites. |
| 97 | func StageArtifact(optDir, acct, ver, app string, data []byte) error { |
| 98 | dst, err := stagePath(optDir, acct, ver, app) |
| 99 | if err != nil { |
| 100 | return err |
| 101 | } |
| 102 | return WriteFileAtomic(dst, data, 0o755) |
| 103 | } |
| 104 | |
| 105 | // errArtifactMismatch is stageArtifactFrom refusing bytes that do not hash |
| 106 | // to the manifest's sha256. Nothing is left behind when it is returned. |
| 107 | var errArtifactMismatch = errors.New("sha256 mismatch") |
| 108 | |
| 109 | // stageArtifactFrom is StageArtifact for a build read from the store as a |
| 110 | // stream (#269). The bytes go straight into the temp file beside the |
| 111 | // destination and are hashed on the way, so memory stays flat whatever the |
| 112 | // build's size: the root stage pass runs under MemoryMax=256M, and reading a |
| 113 | // ~99 MB build whole peaked past it. Only bytes that hash to wantSHA are |
| 114 | // moved into place. A mismatch or a failed read removes the temp file and |
| 115 | // leaves whatever was staged under the name untouched, exactly as verifying |
| 116 | // in memory first did. |
| 117 | // |
| 118 | // open is called only once the version dir is writable, so a sandboxed |
| 119 | // agent fails on the mkdir (stageBlocked relays that to root) without |
| 120 | // opening the build at all. |
| 121 | func stageArtifactFrom(optDir, acct, ver, app, wantSHA string, open func() (io.ReadCloser, error)) error { |
| 122 | dst, err := stagePath(optDir, acct, ver, app) |
| 123 | if err != nil { |
| 124 | return err |
| 125 | } |
| 126 | in, err := open() |
| 127 | if err != nil { |
| 128 | return fmt.Errorf("download %s: %w", ver, err) |
| 129 | } |
| 130 | defer in.Close() |
| 131 | return safefile.Write(dst, 0o755, func(out io.Writer) error { |
| 132 | h := sha256.New() |
| 133 | if _, err := io.Copy(io.MultiWriter(out, h), in); err != nil { |
| 134 | return fmt.Errorf("download %s: %w", ver, err) |
| 135 | } |
| 136 | if hex.EncodeToString(h.Sum(nil)) != wantSHA { |
| 137 | return fmt.Errorf("download %s: %w (refusing to stage)", ver, errArtifactMismatch) |
| 138 | } |
| 139 | return nil |
| 140 | }) |
| 141 | } |
| 142 | |
| 143 | // stagePath validates the names, makes the version dir traversable, and |
| 144 | // returns where the build goes. |
| 145 | func stagePath(optDir, acct, ver, app string) (string, error) { |
| 146 | if !ValidVersion(ver) { |
| 147 | return "", fmt.Errorf("invalid version %q (allowed: letters, digits, . _ -)", ver) |
| 148 | } |
| 149 | if !ValidApp(app) { |
| 150 | return "", fmt.Errorf("invalid app %q", app) |
| 151 | } |
| 152 | if acct != "" && !ValidAccount(acct) { |
| 153 | return "", fmt.Errorf("invalid account %q", acct) |
| 154 | } |
| 155 | dst := VersionPath(optDir, acct, ver, app) |
| 156 | dir := filepath.Dir(dst) |
| 157 | if err := safefile.MkdirAll(dir, 0o755); err != nil { |
| 158 | return "", err |
| 159 | } |
| 160 | // MkdirAll's mode is masked by the caller's umask, so a tight operator |
| 161 | // umask (e.g. 077 from a hardened shell) leaves the version dir 0700 — |
| 162 | // the sandboxed service user then can't traverse in to exec its binary |
| 163 | // (203/EXEC "Permission denied", found staging a canary on Eleven). |
| 164 | // Force it, like the write forces the binary's +x. |
| 165 | if err := os.Chmod(dir, 0o755); err != nil { |
| 166 | return "", err |
| 167 | } |
| 168 | return dst, nil |
| 169 | } |
| 170 | |
| 171 | // StagedManifestPath is the staged binary's SIBLING file: |
| 172 | // |
| 173 | // <opt>/versions/<ver>/[<acct>/]<app>.manifest.json |
| 174 | // |
| 175 | // A sibling, not a new directory, so an old box's tree stays valid and no |
| 176 | // directory shape changes. This is what lets the converge pass read a |
| 177 | // RUNNING version's capabilities without a bucket round-trip on every tick. |
| 178 | func StagedManifestPath(optDir, acct, ver, app string) string { |
| 179 | return VersionPath(optDir, acct, ver, app) + ".manifest.json" |
| 180 | } |
| 181 | |
| 182 | // StageManifest writes the manifest bytes atomically beside the binary, |
| 183 | // 0644 — root-writable, world-readable, never executable (it's JSON, not a |
| 184 | // unit the sandboxed service execs). The write is temp-file-then-rename |
| 185 | // (WriteFileAtomic), so a crashed pass never leaves a truncated sibling — |
| 186 | // that atomicity is relied on but not itself exercised by this package's |
| 187 | // suite (no test kills a fault injected between the write and the rename); |
| 188 | // it is the same primitive StageArtifact already trusts for the binary. |
| 189 | // |
| 190 | // raw is the canonical manifest's bytes as the store returned them |
| 191 | // (GetManifestRaw), so the sibling is lossless (zero-downtime deploys spec |
| 192 | // §4.2). This is the ONE write path that skips StageArtifact, so it repeats |
| 193 | // StageArtifact's three charset gates itself rather than relying on a caller |
| 194 | // that staged the binary first: the back-fill branch calls this with no |
| 195 | // StageArtifact call in the same pass, and the reserved-word/traversal |
| 196 | // protection those gates give the binary's path must cover this sibling path |
| 197 | // too. |
| 198 | func StageManifest(optDir, acct, ver, app string, raw []byte) error { |
| 199 | if !ValidVersion(ver) { |
| 200 | return fmt.Errorf("invalid version %q (allowed: letters, digits, . _ -)", ver) |
| 201 | } |
| 202 | if !ValidApp(app) { |
| 203 | return fmt.Errorf("invalid app %q", app) |
| 204 | } |
| 205 | if acct != "" && !ValidAccount(acct) { |
| 206 | return fmt.Errorf("invalid account %q", acct) |
| 207 | } |
| 208 | // A version staged by a pre-capability binary has no <acct>/ subdirectory |
| 209 | // at all (issue #58): WriteFileAtomic's tmp-file create fails ENOENT the |
| 210 | // first time a sibling is back-filled for it under a real account. Create |
| 211 | // the parent dir first, exactly as StageArtifact does for the binary — |
| 212 | // but only Chmod it when WE are the one creating it. The umask-masking |
| 213 | // problem StageArtifact's Chmod exists for (a tight operator umask |
| 214 | // leaving the dir non-traversable) only applies to a dir this call |
| 215 | // creates; Chmod-ing a dir that already existed is a new failure mode of |
| 216 | // its own — a non-owning uid that can still write into the dir (and so |
| 217 | // can legitimately reach this branch) gets EPERM trying to chmod it, |
| 218 | // and the sibling write below never runs. |
| 219 | dir := filepath.Dir(StagedManifestPath(optDir, acct, ver, app)) |
| 220 | _, statErr := os.Stat(dir) |
| 221 | dirExisted := statErr == nil |
| 222 | if err := safefile.MkdirAll(dir, 0o755); err != nil { |
| 223 | return err |
| 224 | } |
| 225 | if !dirExisted { |
| 226 | if err := os.Chmod(dir, 0o755); err != nil { |
| 227 | return err |
| 228 | } |
| 229 | } |
| 230 | return WriteFileAtomic(StagedManifestPath(optDir, acct, ver, app), raw, 0o644) |
| 231 | } |
| 232 | |
| 233 | // ReadStagedManifest parses the sibling. It returns (nil, nil) — NOT an |
| 234 | // error — when the file is absent: a box staged before this feature existed |
| 235 | // has a binary and no sibling, and the converge pass must read that as |
| 236 | // "unknown", not as "no capabilities". The distinction is load-bearing: |
| 237 | // unknown LEAVES THE DROP-INS ALONE (spec §4d), none REMOVES them. |
| 238 | // |
| 239 | // A present-but-unreadable or unparsable sibling is a real error (fail |
| 240 | // toward leave-alone, the same three-state rule): the caller cannot tell |
| 241 | // "unknown" from "corrupt" if both return nil, and the converge pass must |
| 242 | // not treat a half-written file as "nothing declared". |
| 243 | // |
| 244 | // The parsed manifest runs through the same validateManifest gate every |
| 245 | // other manifest read does (GetManifest's), so a stale sibling written by a |
| 246 | // NEWER binary — one that understood a capability this binary does not — |
| 247 | // refuses cleanly here instead of being silently trusted. It also runs |
| 248 | // GetManifest's identity cross-check: a sibling whose body names a |
| 249 | // different app or version than the path it was read from is refused, |
| 250 | // rather than trusted against the PATH's name — without this, a sibling at |
| 251 | // keymail's path claiming app "console" would pass every charset and |
| 252 | // reserved-word gate under keymail's own identity, and its capabilities |
| 253 | // (host_role's User=, runtime_dir, ports) would land on keymail's unit. |
| 254 | // |
| 255 | // A sibling staged by this binary or later is the canonical manifest's exact |
| 256 | // bytes. One staged by an older binary is a LOSSY re-marshal: it drops any |
| 257 | // top-level key that binary's Manifest struct did not know, `swap` included. |
| 258 | // So the bucket object stays the authority for anything a box acts on |
| 259 | // (CanonicalDemand); this sibling exists so the converge pass can read a |
| 260 | // running version's capabilities without a bucket round trip on every tick. |
| 261 | func ReadStagedManifest(optDir, acct, ver, app string) (*Manifest, error) { |
| 262 | raw, err := safefile.ReadFile(StagedManifestPath(optDir, acct, ver, app)) |
| 263 | if err != nil { |
| 264 | if os.IsNotExist(err) { |
| 265 | return nil, nil |
| 266 | } |
| 267 | return nil, err |
| 268 | } |
| 269 | var m Manifest |
| 270 | if err := json.Unmarshal(raw, &m); err != nil { |
| 271 | return nil, fmt.Errorf("staged manifest %s/%s: %v", app, ver, err) |
| 272 | } |
| 273 | if err := validateManifest(&m); err != nil { |
| 274 | return nil, fmt.Errorf("staged manifest %s/%s: %w", app, ver, err) |
| 275 | } |
| 276 | if m.App != app || m.Version != ver { |
| 277 | return nil, fmt.Errorf("staged manifest at %s names %s/%s", StagedManifestPath(optDir, acct, ver, app), m.App, m.Version) |
| 278 | } |
| 279 | return &m, nil |
| 280 | } |
| 281 | |
| 282 | // PointLive atomically repoints a host's live symlink at ver's build of app. |
| 283 | // It verifies ver is staged first, so a typo can't leave a dangling link |
| 284 | // that would fail the unit's ExecStart on the next restart. |
| 285 | func PointLive(optDir, host, acct, ver, app string, runningVersion ...string) error { |
| 286 | target := VersionPath(optDir, acct, ver, app) |
| 287 | if _, err := os.Stat(target); err != nil { |
| 288 | return fmt.Errorf("version %q of %s is not staged (%s)", ver, app, target) |
| 289 | } |
| 290 | if err := safefile.MkdirAll(liveDir(optDir), 0o755); err != nil { |
| 291 | return err |
| 292 | } |
| 293 | link := LiveLink(optDir, host) |
| 294 | if old, err := os.Readlink(link); err == nil && old != target { |
| 295 | dir := filepath.Join(optDir, "previous") |
| 296 | // Root may have crashed after the last repoint but before restart. |
| 297 | // Keep the verified running build's pointer instead of replacing its |
| 298 | // last authority with an intermediate build that never ran. |
| 299 | if len(runningVersion) == 1 && runningVersion[0] != "" { |
| 300 | if !ValidVersion(runningVersion[0]) { |
| 301 | return os.ErrPermission |
| 302 | } |
| 303 | running := VersionPath(optDir, acct, runningVersion[0], app) |
| 304 | current, err := filepath.EvalSymlinks(link) |
| 305 | if err != nil { |
| 306 | return err |
| 307 | } |
| 308 | if running != current { |
| 309 | prior, err := filepath.EvalSymlinks(filepath.Join(dir, host)) |
| 310 | if err != nil || prior != running { |
| 311 | return os.ErrPermission |
| 312 | } |
| 313 | } |
| 314 | old = running |
| 315 | } |
| 316 | if err := safefile.MkdirAll(dir, 0755); err != nil { |
| 317 | return err |
| 318 | } |
| 319 | previous := filepath.Join(dir, host) |
| 320 | pending := previous + ".tmp" |
| 321 | _ = os.Remove(pending) |
| 322 | if err := os.Symlink(old, pending); err != nil { |
| 323 | return err |
| 324 | } |
| 325 | if err := os.Rename(pending, previous); err != nil { |
| 326 | os.Remove(pending) |
| 327 | return err |
| 328 | } |
| 329 | f, err := os.Open(dir) |
| 330 | if err != nil { |
| 331 | return err |
| 332 | } |
| 333 | err = f.Sync() |
| 334 | f.Close() |
| 335 | if err != nil { |
| 336 | return err |
| 337 | } |
| 338 | } else if err != nil && !os.IsNotExist(err) { |
| 339 | return err |
| 340 | } |
| 341 | tmp := link + ".tmp" |
| 342 | _ = os.Remove(tmp) // a leftover temp link from a crashed swap |
| 343 | if err := os.Symlink(target, tmp); err != nil { |
| 344 | return err |
| 345 | } |
| 346 | if err := os.Rename(tmp, link); err != nil { |
| 347 | os.Remove(tmp) |
| 348 | return err |
| 349 | } |
| 350 | return nil |
| 351 | } |
| 352 | |
| 353 | // LiveVersion reports the version a host's live symlink currently points at, |
| 354 | // or "" (no error) if the host hasn't been put under version management yet. |
| 355 | func LiveVersion(optDir, host string) (string, error) { |
| 356 | target, err := os.Readlink(LiveLink(optDir, host)) |
| 357 | if err != nil { |
| 358 | if os.IsNotExist(err) { |
| 359 | return "", nil |
| 360 | } |
| 361 | return "", err |
| 362 | } |
| 363 | return versionOfTarget(optDir, target), nil |
| 364 | } |
| 365 | |
| 366 | // versionOfTarget is the version a live link's RAW target names, or "" when |
| 367 | // the target is not under the version store. target is |
| 368 | // <opt>/versions/<ver>/<app> (ops) or <opt>/versions/<ver>/<acct>/<app> |
| 369 | // (everyone else), so the version is the FIRST segment under the versions |
| 370 | // dir, not simply the parent directory, which would read an account sqid as |
| 371 | // a version. A target outside the store, or a relative one, is not ours to |
| 372 | // interpret: "" is the same answer an absent link gets. |
| 373 | func versionOfTarget(optDir, target string) string { |
| 374 | return versionUnder(versionsDir(optDir), target) |
| 375 | } |
| 376 | |
| 377 | // versionUnder is versionOfTarget against a given versions dir: the first |
| 378 | // segment of target under vdir, or "" when target is not below it. |
| 379 | func versionUnder(vdir, target string) string { |
| 380 | rel, err := filepath.Rel(vdir, target) |
| 381 | if err != nil || strings.HasPrefix(rel, "..") { |
| 382 | return "" |
| 383 | } |
| 384 | ver, _, ok := strings.Cut(filepath.ToSlash(rel), "/") |
| 385 | if !ok { |
| 386 | return "" |
| 387 | } |
| 388 | return ver |
| 389 | } |
| 390 | |
| 391 | // WaitHealthyVersion waits until the host behind socket not only answers but |
| 392 | // reports the expected build at /api/version — so a deploy fails loudly if |
| 393 | // the process is still the old binary (a stale socket, or a restart that |
| 394 | // didn't take), rather than silently "succeeding". |
| 395 | func WaitHealthyVersion(socket, wantBuild string, timeout time.Duration) error { |
| 396 | deadline := time.Now().Add(timeout) |
| 397 | client := unixHTTPClient(socket) |
| 398 | var last error |
| 399 | for time.Now().Before(deadline) { |
| 400 | resp, err := client.Get("http://carlos/api/version") |
| 401 | if err == nil { |
| 402 | var body struct { |
| 403 | Build string `json:"build"` |
| 404 | } |
| 405 | derr := json.NewDecoder(io.LimitReader(resp.Body, 1<<16)).Decode(&body) |
| 406 | resp.Body.Close() |
| 407 | if derr == nil && body.Build == wantBuild { |
| 408 | return nil |
| 409 | } |
| 410 | if derr != nil { |
| 411 | last = fmt.Errorf("read /api/version: %v", derr) |
| 412 | } else { |
| 413 | last = fmt.Errorf("live build %q != expected %q", body.Build, wantBuild) |
| 414 | } |
| 415 | } else { |
| 416 | last = err |
| 417 | } |
| 418 | time.Sleep(150 * time.Millisecond) |
| 419 | } |
| 420 | return fmt.Errorf("host didn't report build %q within %s: %v", wantBuild, timeout, last) |
| 421 | } |
| 422 | |
| 423 | // ProbeVersion reads the build the process behind a socket is ACTUALLY |
| 424 | // serving (its /api/version), not the symlink target — how a status probe |
| 425 | // catches an awake hibernating host that repointed its symlink but still |
| 426 | // runs the old binary. "" + ok=false when the socket doesn't answer (asleep, |
| 427 | // or no such host). |
| 428 | func ProbeVersion(socket string) (build string, ok bool) { |
| 429 | resp, err := unixHTTPClient(socket).Get("http://carlos/api/version") |
| 430 | if err != nil { |
| 431 | return "", false |
| 432 | } |
| 433 | defer resp.Body.Close() |
| 434 | var body struct { |
| 435 | Build string `json:"build"` |
| 436 | } |
| 437 | if json.NewDecoder(io.LimitReader(resp.Body, 1<<16)).Decode(&body) != nil { |
| 438 | return "", false |
| 439 | } |
| 440 | return body.Build, body.Build != "" |
| 441 | } |
| 442 | |
| 443 | // unixHTTPClient dials a unix socket over HTTP — the edge<->host transport, |
| 444 | // reused by the health checks. |
| 445 | func unixHTTPClient(socket string) *http.Client { |
| 446 | return &http.Client{ |
| 447 | Timeout: 3 * time.Second, |
| 448 | Transport: &http.Transport{ |
| 449 | DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { |
| 450 | return unixdial.Checked(ctx, &net.Dialer{}, socket) |
| 451 | }, |
| 452 | }, |
| 453 | } |
| 454 | } |
| 455 | |
amadan