eleven / messenger Public

server: mirror the signed release feed to S3 — updater falls back on failure or staleness (#662)

GitHub's asset CDN is the fleet's update SPOF and has served
freshly-clobbered manifests stale (and new binaries 404) for minutes after
a publish. The feed is Ed25519-signed with sha256-pinned binaries, so a
mirror is trust-preserving by construction — it only has to be available.

- publish side: LCHAT_RELEASE_MIRROR_BUCKET makes release-server.sh copy
  manifest.json (on publish AND promote) + server-<ver>/ binaries to S3 via
  the aws CLI; unset, the script is unchanged. Best-effort with a loud
  warning — a lagging mirror is harmless because updaters take whichever
  verified feed is fresher.
- fetch side: LCHAT_RELEASE_MIRROR_URL is a plain HTTPS base the updater
  falls back to when the primary manifest fetch fails, or verifies but is
  stale (generated_at older than the anti-replay stamp — exactly the
  post-publish CDN window); binary downloads fall back the same way.
  Signature + sha256 checks identical on both paths; fallbacks log loudly;
  a feed equal to the stamp never consults the mirror, so steady-state
  passes stay single-request.
- infra/prod/releases.tf: versioned eleven-releases-<account> bucket,
  GET-only public-read policy (the one deliberately public bucket — public,
  signed, non-secret content), IAM-only writes via a no-delete
  release-publisher user. Planned (7 add / 0 change / 0 destroy), NOT
  applied — needs a named tofu apply.
- tests: manifest fallback on failure, wrong-key mirror rejected, fresher
  mirror wins over a stale primary, equally-stale mirror never masks the
  replay refusal, mirror binary fallback stages + applies, tampered mirror
  binary refused.
- also fixes a pre-existing go vet failure (unchecked http.Get) + gofmt
  drift in handlers_test.go so the suite runs clean.

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Paul Campbell 3 months ago 3492e3f2c117987a5a06fad907294116c6f7b7ce parent ae85660

7 files changed, +589 −18

deploy/release-server.sh +36 −0
docs/server-releases.md +29 −0
handlers_test.go +1 −3
infra/prod/releases.tf +126 −0
release.go +33 −0
updater.go +88 −15
updater_test.go +276 −0

diff --git a/deploy/release-server.sh b/deploy/release-server.sh

index 6129d44..8a86eab 100755

--- a/deploy/release-server.sh

+++ b/deploy/release-server.sh

@@ -22,10 +22,20 @@

# Needs: gh (authed, write access to $LCHAT_RELEASES_REPO), jq, go, and the

# signing key at $LCHAT_RELEASE_KEY_FILE (1Password: "Eleven Messenger server

# release signing key").

+# Optional S3 mirror: set LCHAT_RELEASE_MIRROR_BUCKET (needs the aws CLI) and

+# every feed artifact is also copied to s3://<bucket>/releases/… — GitHub's

+# asset CDN serves freshly-clobbered manifests stale (and new binaries 404)

+# for minutes after a publish, and it's the fleet's only update path, so the

+# mirror is a second boring home for the same signed bytes. Updaters fall back

+# to it via LCHAT_RELEASE_MIRROR_URL (docs/server-releases.md § The S3 mirror);

+# the feed's Ed25519 signature + sha256s make the mirror availability-only,

+# never trusted. Unset, this script is unchanged.

set -eu

REPO="${LCHAT_RELEASES_REPO:-elevenmessenger/releases}"

KEY="${LCHAT_RELEASE_KEY_FILE:-}"

+MIRROR_BUCKET="${LCHAT_RELEASE_MIRROR_BUCKET:-}"

BIN=lchat-linux-amd64

BIN_ARM=lchat-linux-arm64

@@ -37,6 +47,8 @@ case "${1:-}" in publish|promote)

echo "release-server: set LCHAT_RELEASE_KEY_FILE to the signing key file" >&2; exit 1; }

command -v jq >/dev/null || { echo "release-server: jq is required" >&2; exit 1; }

command -v gh >/dev/null || { echo "release-server: gh is required" >&2; exit 1; }

+ [ -z "$MIRROR_BUCKET" ] || command -v aws >/dev/null || {

+ echo "release-server: LCHAT_RELEASE_MIRROR_BUCKET is set but the aws CLI is missing" >&2; exit 1; }

;;

esac

@@ -58,6 +70,20 @@ fetch_payload() {

}

+# mirror_upload <file> <key>: copy an artifact into the S3 mirror (no-op when

+# LCHAT_RELEASE_MIRROR_BUCKET is unset). Best-effort by design: by the time

+# this runs the GitHub half has already landed, so a failed mirror write warns

+# instead of aborting a half-done publish — a lagging mirror is harmless

+# (updaters use whichever verified feed is fresher).

+mirror_upload() {

+ [ -n "$MIRROR_BUCKET" ] || return 0

+ if aws s3 cp "$1" "s3://$MIRROR_BUCKET/releases/$2" >/dev/null; then

+ echo "mirrored $2 -> s3://$MIRROR_BUCKET/releases/$2"

+ else

+ echo "release-server: WARNING: mirror upload of $2 failed — the S3 mirror lags until the next publish/promote" >&2

+ fi

# sign_and_upload: sign $1 (payload json) and clobber the rolling feed asset.

sign_and_upload() {

"$TOOL" release sign --key "$KEY" --in "$1" --out "$TMP/manifest.json"

@@ -66,6 +92,9 @@ sign_and_upload() {

--notes "Rolling release: manifest.json is the channel feed instances update from. See the messenger repo's docs/server-releases.md."

gh release upload server "$TMP/manifest.json" -R "$REPO" --clobber

"$TOOL" release verify --in "$TMP/manifest.json"

+ # Both publish and promote come through here, so the mirror's feed tracks

+ # every channel move, not just new builds.

+ mirror_upload "$TMP/manifest.json" manifest.json

}

cmd="${1:-}"; shift || true

@@ -90,6 +119,13 @@ publish)

gh release create "server-$VER" -R "$REPO" --title "Server $VER" --notes "$NOTES" \

"$TMP/$BIN" "$TMP/$BIN.sha256" "$TMP/$BIN_ARM" "$TMP/$BIN_ARM.sha256"

+ # Binaries mirror before the manifest that points at them, so the mirror

+ # never serves a feed whose mirror-side binaries don't exist yet. Keys copy

+ # the GitHub layout (server-<ver>/<asset>) — that's what the updater's

+ # mirror fallback derives from a primary binary URL.

+ for A in "$BIN" "$BIN.sha256" "$BIN_ARM" "$BIN_ARM.sha256"; do

+ mirror_upload "$TMP/$A" "server-$VER/$A"

+ done

fetch_payload | jq \

--arg ver "$VER" --arg sha "$SHA" --argjson size "$SIZE" --arg notes "$NOTES" \

diff --git a/docs/server-releases.md b/docs/server-releases.md

index a54b9e1..b521b53 100644

--- a/docs/server-releases.md

+++ b/docs/server-releases.md

@@ -103,6 +103,35 @@ The updater downloads and *executes* native code, so the chain matters:

validly signed) manifest can't downgrade a fleet. An intentional pull-back is

a *re-signed* manifest with a newer timestamp, which still applies.

+## The S3 mirror

+GitHub's asset CDN is the feed's single point of failure — and it has served

+freshly-clobbered manifests stale (and brand-new binaries 404) for minutes

+after a publish (the gotcha above). The feed therefore also mirrors to S3

+(bucket: `infra/prod/releases.tf` — public-read GETs so boxes fetch

+anonymously, IAM-only writes, versioned for manifest rollback):

+- **Publish side:** set `LCHAT_RELEASE_MIRROR_BUCKET` and

+ `deploy/release-server.sh` copies everything it uploads to GitHub into

+ `s3://<bucket>/releases/…` too — `manifest.json` at the top (refreshed on

+ publish *and* promote), binaries under `server-<ver>/`. Unset, the script

+ is unchanged. Mirror writes are best-effort: a failure warns rather than

+ aborting a publish whose GitHub half already landed — a lagging mirror is

+ harmless because updaters use whichever verified feed is fresher.

+- **Fetch side:** set `LCHAT_RELEASE_MIRROR_URL` (a plain HTTPS base, e.g.

+ `https://<bucket>.s3.<region>.amazonaws.com/releases`) and the updater

+ falls back to the mirror when the primary fetch fails — or when it verifies

+ but is *stale*: generated_at older than the newest feed the box ever

+ accepted (the anti-replay stamp), which is exactly what the post-publish

+ CDN window looks like; a fresher mirror then wins. Binary downloads fall

+ back the same way. Every fallback logs loudly; a feed merely *equal* to the

+ stamp is indistinguishable from "no new release", so steady-state passes

+ never touch the mirror.

+The mirror adds availability, never trust: both paths run the identical

+Ed25519 manifest verification and sha256 binary check, so a compromised

+mirror can only cause a refusal, not an install.

## Publishing

```sh

diff --git a/handlers_test.go b/handlers_test.go

index 1fe9f9e..ba184a2 100644

--- a/handlers_test.go

+++ b/handlers_test.go

@@ -6,8 +6,8 @@ import (

"encoding/json"

"io"

"net/http"

- "net/url"

"net/http/httptest"

+ "net/url"

"os"

"strconv"

"strings"

@@ -113,7 +113,6 @@ func TestMessagesThreadScoping(t *testing.T) {

}

func TestKeyInviteFlow(t *testing.T) {

ts, _ := newTestServer(t)

@@ -2457,7 +2456,6 @@ func mustUID(t *testing.T, srv *Server, token string) string {

return uid

}

// TestTierKeyLifecycle pins the one-way mint and the honest dead ends: a

// minted level whose holders are all gone is LOST (410, never re-minted), a

// level still gating groups can't be deleted, and removing a user cascades

diff --git a/infra/prod/releases.tf b/infra/prod/releases.tf

new file mode 100644

index 0000000..356e646

--- /dev/null

+++ b/infra/prod/releases.tf

@@ -0,0 +1,126 @@

+# 🤖 The server-release-feed mirror (issue #550 Part 12): a second, boring

+# home for the signed update feed — manifest.json plus the per-version

+# binaries. GitHub's asset CDN has served freshly-clobbered manifests stale

+# (and brand-new binaries 404) for minutes after a publish — documented

+# operator pain — and that feed is the entire fleet's update path.

+# deploy/release-server.sh copies every publish/promote here

+# (LCHAT_RELEASE_MIRROR_BUCKET); updaters fall back to it

+# (LCHAT_RELEASE_MIRROR_URL). Versioning doubles as the rollback story for a

+# clobbered manifest.json.

+# NOT YET APPLIED — needs a deliberate, named

+# `AWS_PROFILE=… tofu apply` in infra/prod; plan only until then.

+resource "aws_s3_bucket" "releases" {

+ bucket = "eleven-releases-${local.account_id}"

+resource "aws_s3_bucket_versioning" "releases" {

+ bucket = aws_s3_bucket.releases.id

+ versioning_configuration {

+ status = "Enabled"

+ }

+# Manifest clobbers are tiny and binaries land under fresh server-<ver>/ keys,

+# so noncurrent versions exist only as the manifest's rollback window — 90

+# days is generous without growing forever.

+resource "aws_s3_bucket_lifecycle_configuration" "releases" {

+ bucket = aws_s3_bucket.releases.id

+ rule {

+ id = "expire-noncurrent-90d"

+ status = "Enabled"

+ filter {}

+ noncurrent_version_expiration {

+ noncurrent_days = 90

+ }

+# This bucket is deliberately PUBLIC-READ — the one bucket in the account

+# where that is the point, against the everything-else-blocked posture

+# (backup.tf, hibernate.tf). Instances and self-hosted boxes must fetch

+# updates anonymously, and the content is public, non-secret, and

+# trust-anchored elsewhere: the manifest is Ed25519-signed against a key baked

+# into every binary, and binaries are sha256-pinned by that signed manifest,

+# so the mirror provides availability only — a tampered object can only cause

+# a refusal, never an install (docs/server-releases.md § Trust model). ACLs

+# stay fully blocked; only the bucket *policy* door is opened, and the

+# GET-only policy below is the entire public surface (no anonymous List).

+# Writes stay IAM-only via the release-publisher user.

+resource "aws_s3_bucket_public_access_block" "releases" {

+ bucket = aws_s3_bucket.releases.id

+ block_public_acls = true

+ ignore_public_acls = true

+ block_public_policy = false

+ restrict_public_buckets = false

+data "aws_iam_policy_document" "releases_public_read" {

+ statement {

+ sid = "PublicReadReleaseObjects"

+ actions = ["s3:GetObject"]

+ resources = ["${aws_s3_bucket.releases.arn}/*"]

+ principals {

+ type = "*"

+ identifiers = ["*"]

+ }

+resource "aws_s3_bucket_policy" "releases" {

+ bucket = aws_s3_bucket.releases.id

+ policy = data.aws_iam_policy_document.releases_public_read.json

+ # The policy grants public access, so the access block must relax

+ # block_public_policy before it can be attached.

+ depends_on = [aws_s3_bucket_public_access_block.releases]

+# The publisher's credentials: object put + list on this bucket, nothing else

+# in the account — and no DeleteObject, so a leaked publisher key can

+# overwrite the current manifest (noncurrent versions survive, versioning is

+# the rollback) but never destroy history. Mint the access key by hand

+# (aws iam create-access-key --user-name release-publisher) — keys never land

+# in tofu state.

+resource "aws_iam_user" "release_publisher" {

+ name = "release-publisher"

+data "aws_iam_policy_document" "release_publisher" {

+ statement {

+ sid = "ListReleasesBucket"

+ actions = ["s3:ListBucket"]

+ resources = [aws_s3_bucket.releases.arn]

+ }

+ statement {

+ sid = "WriteReleaseObjects"

+ actions = [

+ "s3:GetObject",

+ "s3:PutObject",

+ ]

+ resources = ["${aws_s3_bucket.releases.arn}/*"]

+ }

+resource "aws_iam_user_policy" "release_publisher" {

+ name = "release-publisher-bucket"

+ user = aws_iam_user.release_publisher.name

+ policy = data.aws_iam_policy_document.release_publisher.json

+output "releases_bucket" {

+ value = aws_s3_bucket.releases.bucket

+# What instances set as LCHAT_RELEASE_MIRROR_URL.

+output "releases_mirror_url" {

+ value = "https://${aws_s3_bucket.releases.bucket_regional_domain_name}/releases"

diff --git a/release.go b/release.go

index 581be52..cc42ab0 100644

--- a/release.go

+++ b/release.go

@@ -61,6 +61,39 @@ var releasePubKeyB64 = "6pL2QKDCfXqcYGR6Vr5uUZZKrpV6YVwg9rV0Okb8LY8="

var releaseManifestURL = envOr("LCHAT_UPDATE_MANIFEST_URL",

"https://github.com/elevenmessenger/releases/releases/download/server/manifest.json")

+// releaseMirrorURL is the optional second home for the feed — a plain HTTPS

+// base (e.g. https://<bucket>.s3.<region>.amazonaws.com/releases) that

+// deploy/release-server.sh keeps in sync when LCHAT_RELEASE_MIRROR_BUCKET is

+// set at publish time. GitHub's asset CDN has served freshly-clobbered

+// manifests stale (and brand-new binaries 404) for minutes after a publish,

+// and that feed is the whole fleet's update path — the mirror exists purely

+// for availability. It needs no integrity trust of its own: everything

+// fetched from it passes the exact same Ed25519 signature and sha256 checks

+// as the primary, so a compromised mirror can only cause a refusal, never an

+// install. Empty (the default) disables all mirror behaviour.

+var releaseMirrorURL = envOr("LCHAT_RELEASE_MIRROR_URL", "")

+// mirrorManifestURL is where a mirror base keeps the feed itself.

+func mirrorManifestURL(base string) string {

+ return strings.TrimRight(base, "/") + "/manifest.json"

+// mirrorBinaryURL maps a binary's primary URL onto the mirror, which stores

+// each artifact under its last two path segments

+// (server-<ver>/lchat-linux-<arch> — the GitHub layout, copied verbatim by

+// release-server.sh). "" when the primary URL doesn't have that shape.

+func mirrorBinaryURL(base, primary string) string {

+ u, err := url.Parse(primary)

+ if err != nil {

+ return ""

+ }

+ segs := strings.Split(strings.Trim(u.Path, "/"), "/")

+ if len(segs) < 2 || segs[len(segs)-2] == "" || segs[len(segs)-1] == "" {

+ return ""

+ }

+ return strings.TrimRight(base, "/") + "/" + segs[len(segs)-2] + "/" + segs[len(segs)-1]

type releaseEntry struct {

Version string `json:"version"`

// The flat fields are the amd64 binary — kept flat (not per-arch keyed) so

diff --git a/updater.go b/updater.go

index 7fb3802..49b5210 100644

--- a/updater.go

+++ b/updater.go

@@ -8,6 +8,7 @@ import (

"flag"

"fmt"

"io"

+ "log"

"net/http"

"os"

"path/filepath"

@@ -52,9 +53,9 @@ func writeManifestStamp(t time.Time) error {

return os.WriteFile(manifestStampPath(), []byte(t.Format(time.RFC3339)+"\n"), 0o644)

}

-// fetchReleaseManifest downloads and signature-verifies the channel feed.

-// Shared with the instance-side availability checker (updatecheck.go).

-func fetchReleaseManifest(url string) (*releaseManifest, error) {

+// fetchReleaseManifestFrom downloads and signature-verifies the channel feed

+// from one URL.

+func fetchReleaseManifestFrom(url string) (*releaseManifest, error) {

pub, err := releasePubKey()

if err != nil {

return nil, err

@@ -75,6 +76,52 @@ func fetchReleaseManifest(url string) (*releaseManifest, error) {

return verifyReleaseManifest(pub, raw)

}

+// fetchReleaseManifest reads the channel feed, falling back to the S3 mirror

+// (LCHAT_RELEASE_MIRROR_URL) when the primary fetch fails — or when it

+// verifies but is STALE: GitHub's CDN has served a pre-publish manifest for

+// minutes after a clobber, which errors nowhere. Stale here means a

+// generated_at strictly older than the newest feed this box ever accepted

+// (the anti-replay stamp); a feed equal to the stamp is indistinguishable

+// from "no new release", so it never consults the mirror — the hourly pass

+// stays a single request in the steady state. Signature verification is

+// identical on both paths (the mirror adds availability, never trust), and

+// every fallback logs loudly. Shared with the instance-side availability

+// checker (updatecheck.go), where no stamp file exists and so only outright

+// primary failure falls back.

+func fetchReleaseManifest(url string) (*releaseManifest, error) {

+ m, err := fetchReleaseManifestFrom(url)

+ base := releaseMirrorURL

+ if base == "" {

+ return m, err

+ }

+ stamp := readManifestStamp()

+ stale := err == nil && m.GeneratedAt.Before(stamp)

+ if err == nil && !stale {

+ return m, nil

+ }

+ murl := mirrorManifestURL(base)

+ mm, merr := fetchReleaseManifestFrom(murl)

+ switch {

+ case err != nil && merr != nil:

+ log.Printf("update: primary feed failed (%v); mirror %s failed too (%v)", err, murl, merr)

+ return nil, err

+ case err != nil:

+ log.Printf("update: primary feed failed (%v) — using mirror %s (signature verified)", err, murl)

+ return mm, nil

+ case merr != nil:

+ log.Printf("update: primary feed is stale (generated_at %s < last accepted %s) and mirror %s failed (%v) — keeping the primary result",

+ m.GeneratedAt.Format(time.RFC3339), stamp.Format(time.RFC3339), murl, merr)

+ return m, nil

+ case mm.GeneratedAt.After(m.GeneratedAt):

+ log.Printf("update: primary feed is stale (generated_at %s < last accepted %s) — using fresher mirror %s (generated_at %s)",

+ m.GeneratedAt.Format(time.RFC3339), stamp.Format(time.RFC3339), murl, mm.GeneratedAt.Format(time.RFC3339))

+ return mm, nil

+ }

+ // The mirror is no fresher: the stale primary stands, and updatePass's

+ // replay check refuses it — the mirror never masks a replayed feed.

+ return m, nil

// updatePrefs is what an instance's admin chose (GET /api/update-prefs).

type updatePrefs struct {

AutoUpdate bool `json:"auto_update"`

@@ -119,16 +166,9 @@ func fetchUpdatePrefs(socket string) (updatePrefs, error) {

// the arm64 resolution without faking runtime.GOARCH.

var updaterArch = runtime.GOARCH

-// downloadReleaseBinary fetches this box's arch of a release into the incoming

-// dir and refuses anything whose sha256 doesn't match the signed manifest

-// entry. Returns the temp path; the caller stages it and removes the temp.

-func downloadReleaseBinary(e releaseEntry) (string, error) {

- url, sha, ok := e.forArch(updaterArch)

- if !ok {

- // An arm64 box facing a pre-arm64 release: skip, never install the

- // wrong arch. The next release publishes both and the box catches up.

- return "", fmt.Errorf("release %s has no %s build — skipping", e.Version, updaterArch)

- }

+// downloadBinaryToTemp streams one URL into a temp file in the incoming dir

+// and refuses bytes whose sha256 doesn't match the signed manifest entry.

+func downloadBinaryToTemp(url, sha, ver string) (string, error) {

if err := os.MkdirAll(incomingDir(), 0o755); err != nil {

return "", err

}

@@ -148,18 +188,51 @@ func downloadReleaseBinary(e releaseEntry) (string, error) {

}

defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {

- return fail(fmt.Errorf("download %s: %s", e.Version, resp.Status))

+ return fail(fmt.Errorf("download %s: %s", ver, resp.Status))

}

h := sha256.New()

if _, err := io.Copy(io.MultiWriter(f, h), resp.Body); err != nil {

return fail(err)

}

if got := hex.EncodeToString(h.Sum(nil)); got != sha {

- return fail(fmt.Errorf("download %s: sha256 mismatch (refusing to stage)", e.Version))

+ return fail(fmt.Errorf("download %s: sha256 mismatch (refusing to stage)", ver))

}

return f.Name(), nil

}

+// downloadReleaseBinary fetches this box's arch of a release into the incoming

+// dir and refuses anything whose sha256 doesn't match the signed manifest

+// entry. A failed primary download (GitHub 404s brand-new assets for minutes

+// after a publish) falls back to the S3 mirror, loudly; the sha256 gate is

+// identical on both paths, so a mirror serving the wrong bytes only ever

+// causes a refusal. Returns the temp path; the caller stages it and removes

+// the temp.

+func downloadReleaseBinary(e releaseEntry) (string, error) {

+ url, sha, ok := e.forArch(updaterArch)

+ if !ok {

+ // An arm64 box facing a pre-arm64 release: skip, never install the

+ // wrong arch. The next release publishes both and the box catches up.

+ return "", fmt.Errorf("release %s has no %s build — skipping", e.Version, updaterArch)

+ }

+ path, err := downloadBinaryToTemp(url, sha, e.Version)

+ if err == nil {

+ return path, nil

+ }

+ murl := ""

+ if releaseMirrorURL != "" {

+ murl = mirrorBinaryURL(releaseMirrorURL, url)

+ }

+ if murl == "" {

+ return "", err

+ }

+ log.Printf("update: primary download of %s failed (%v) — trying mirror %s", e.Version, err, murl)

+ path, merr := downloadBinaryToTemp(murl, sha, e.Version)

+ if merr != nil {

+ return "", fmt.Errorf("download %s: primary failed (%v); mirror failed too (%v)", e.Version, err, merr)

+ }

+ return path, nil

// ensureVersionStaged makes a manifest entry's binary available in the version

// store, downloading it if needed. Idempotent across hosts in one pass.

func ensureVersionStaged(e releaseEntry) error {

diff --git a/updater_test.go b/updater_test.go

index 4c82e61..2e521ba 100644

--- a/updater_test.go

+++ b/updater_test.go

@@ -127,6 +127,35 @@ func startFeed(t *testing.T, binaries map[string]string) (*httptest.Server, func

return ts, setChannels

}

+// signManifestAt builds a signed wrapper with a caller-controlled

+// generated_at — signReleaseManifest always stamps now, but the mirror tests

+// need feeds whose relative ages are exact.

+func signManifestAt(t *testing.T, priv ed25519.PrivateKey, m *releaseManifest, at time.Time) []byte {

+ t.Helper()

+ m.Schema = 1

+ m.GeneratedAt = at.UTC().Truncate(time.Second)

+ payload, err := json.Marshal(m)

+ if err != nil {

+ t.Fatal(err)

+ }

+ b64 := base64.StdEncoding.EncodeToString(payload)

+ sig := ed25519.Sign(priv, []byte(b64))

+ w, err := json.Marshal(signedManifest{Payload: b64, Sig: base64.StdEncoding.EncodeToString(sig)})

+ if err != nil {

+ t.Fatal(err)

+ }

+ return w

+// swapFeedGlobals points the trust root, primary URL, and mirror base at test

+// servers, restoring them after.

+func swapFeedGlobals(t *testing.T, pubB64, primaryURL, mirrorBase string) {

+ t.Helper()

+ oldKey, oldURL, oldMirror := releasePubKeyB64, releaseManifestURL, releaseMirrorURL

+ releasePubKeyB64, releaseManifestURL, releaseMirrorURL = pubB64, primaryURL, mirrorBase

+ t.Cleanup(func() { releasePubKeyB64, releaseManifestURL, releaseMirrorURL = oldKey, oldURL, oldMirror })

// stubSystemctl records unit restarts and leaves the "restart" to the fake

// instance's symlink-following /api/version.

func stubSystemctl(t *testing.T) *[]string {

@@ -539,6 +568,253 @@ func TestPinHostWritesViaAdminAPI(t *testing.T) {

}

+// entryFor is a syntactically valid manifest entry (validateManifest passes)

+// whose binary URL nothing in the manifest tests ever downloads.

+func entryFor(ver string) releaseEntry {

+ return releaseEntry{

+ Version: ver,

+ URL: "https://example.com/server-" + ver + "/lchat-linux-amd64",

+ SHA256: strings.Repeat("ab", 32),

+ }

+func TestFetchManifestFallsBackToMirrorOnFailure(t *testing.T) {

+ withTempOpt(t) // the staleness check reads the stamp from optDir

+ pub, priv, err := ed25519.GenerateKey(rand.Reader)

+ if err != nil {

+ t.Fatal(err)

+ }

+ good := signManifestAt(t, priv,

+ &releaseManifest{Channels: map[string]releaseEntry{"stable": entryFor("new1234")}}, time.Now())

+ primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

+ http.Error(w, "gone", http.StatusNotFound) // GitHub's brand-new-asset 404 window

+ }))

+ defer primary.Close()

+ mirror := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

+ // The mirror keeps the feed at <base>/manifest.json.

+ if r.URL.Path != "/releases/manifest.json" {

+ http.NotFound(w, r)

+ return

+ }

+ w.Write(good)

+ }))

+ defer mirror.Close()

+ swapFeedGlobals(t, base64.StdEncoding.EncodeToString(pub),

+ primary.URL+"/manifest.json", mirror.URL+"/releases")

+ m, err := fetchReleaseManifest(releaseManifestURL)

+ if err != nil {

+ t.Fatalf("mirror fallback failed: %v", err)

+ }

+ if m.Channels["stable"].Version != "new1234" {

+ t.Fatalf("mirror manifest channels = %+v", m.Channels)

+ }

+func TestFetchManifestRejectsBadMirrorSignature(t *testing.T) {

+ withTempOpt(t)

+ pub, _, err := ed25519.GenerateKey(rand.Reader) // the trusted root...

+ if err != nil {

+ t.Fatal(err)

+ }

+ _, evilPriv, err := ed25519.GenerateKey(rand.Reader) // ...and an untrusted signer

+ if err != nil {

+ t.Fatal(err)

+ }

+ forged := signManifestAt(t, evilPriv,

+ &releaseManifest{Channels: map[string]releaseEntry{"stable": entryFor("evil999")}}, time.Now())

+ primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

+ http.Error(w, "down", http.StatusServiceUnavailable)

+ }))

+ defer primary.Close()

+ mirror := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

+ w.Write(forged)

+ }))

+ defer mirror.Close()

+ swapFeedGlobals(t, base64.StdEncoding.EncodeToString(pub),

+ primary.URL+"/manifest.json", mirror.URL+"/releases")

+ if _, err := fetchReleaseManifest(releaseManifestURL); err == nil {

+ t.Fatal("a mirror manifest signed by the wrong key was accepted")

+ }

+func TestFetchManifestPrefersFresherMirrorWhenPrimaryStale(t *testing.T) {

+ withTempOpt(t)

+ pub, priv, err := ed25519.GenerateKey(rand.Reader)

+ if err != nil {

+ t.Fatal(err)

+ }

+ now := time.Now()

+ staleFeed := signManifestAt(t, priv,

+ &releaseManifest{Channels: map[string]releaseEntry{"stable": entryFor("old1111")}}, now.Add(-2*time.Hour))

+ freshFeed := signManifestAt(t, priv,

+ &releaseManifest{Channels: map[string]releaseEntry{"stable": entryFor("new1234")}}, now)

+ // The box last accepted a feed newer than what the primary now serves —

+ // exactly what GitHub's stale CDN looks like right after a publish.

+ if err := writeManifestStamp(now.Add(-time.Hour)); err != nil {

+ t.Fatal(err)

+ }

+ primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

+ w.Write(staleFeed)

+ }))

+ defer primary.Close()

+ mirror := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

+ w.Write(freshFeed)

+ }))

+ defer mirror.Close()

+ swapFeedGlobals(t, base64.StdEncoding.EncodeToString(pub),

+ primary.URL+"/manifest.json", mirror.URL+"/releases")

+ m, err := fetchReleaseManifest(releaseManifestURL)

+ if err != nil {

+ t.Fatal(err)

+ }

+ if got := m.Channels["stable"].Version; got != "new1234" {

+ t.Fatalf("stale primary won over a fresher mirror: stable = %q", got)

+ }

+func TestFetchManifestKeepsStalePrimaryWhenMirrorNoFresher(t *testing.T) {

+ withTempOpt(t)

+ pub, priv, err := ed25519.GenerateKey(rand.Reader)

+ if err != nil {

+ t.Fatal(err)

+ }

+ now := time.Now()

+ staleFeed := signManifestAt(t, priv,

+ &releaseManifest{Channels: map[string]releaseEntry{"stable": entryFor("old1111")}}, now.Add(-2*time.Hour))

+ if err := writeManifestStamp(now); err != nil {

+ t.Fatal(err)

+ }

+ serveStale := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write(staleFeed) })

+ primary := httptest.NewServer(serveStale)

+ defer primary.Close()

+ mirror := httptest.NewServer(serveStale)

+ defer mirror.Close()

+ swapFeedGlobals(t, base64.StdEncoding.EncodeToString(pub),

+ primary.URL+"/manifest.json", mirror.URL+"/releases")

+ // Both stale: the primary result stands, and updatePass's replay check is

+ // what refuses it — the mirror must never mask a genuinely replayed feed.

+ m, err := fetchReleaseManifest(releaseManifestURL)

+ if err != nil {

+ t.Fatal(err)

+ }

+ if !m.GeneratedAt.Before(readManifestStamp()) {

+ t.Fatal("expected the stale primary manifest back (for the replay check to refuse)")

+ }

+ reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db"))

+ if err != nil {

+ t.Fatal(err)

+ }

+ if err := updatePass(reg, "lchat@", false); err == nil || !strings.Contains(err.Error(), "replayed") {

+ t.Fatalf("want replay refusal, got %v", err)

+ }

+// mirrorBinaryFeed wires a primary feed whose manifest is fine but whose

+// binary URL 404s, plus a mirror holding mirrorContent under the derived

+// server-<ver>/<name> key — the GitHub new-asset-404 shape.

+func mirrorBinaryFeed(t *testing.T, binaryContent, mirrorContent string) {

+ t.Helper()

+ pub, priv, err := ed25519.GenerateKey(rand.Reader)

+ if err != nil {

+ t.Fatal(err)

+ }

+ sum := sha256.Sum256([]byte(binaryContent))

+ primaryMux := http.NewServeMux() // serves only the manifest; the binary path 404s

+ primary := httptest.NewServer(primaryMux)

+ t.Cleanup(primary.Close)

+ entry := releaseEntry{

+ Version: "new1234",

+ URL: primary.URL + "/server-new1234/lchat-linux-amd64",

+ SHA256: hex.EncodeToString(sum[:]),

+ }

+ wrapper := signManifestAt(t, priv,

+ &releaseManifest{Channels: map[string]releaseEntry{"stable": entry}}, time.Now())

+ primaryMux.HandleFunc("/manifest.json", func(w http.ResponseWriter, r *http.Request) { w.Write(wrapper) })

+ mirror := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

+ // The mirror stores binaries under the primary URL's last two segments.

+ if r.URL.Path != "/releases/server-new1234/lchat-linux-amd64" {

+ http.NotFound(w, r)

+ return

+ }

+ w.Write([]byte(mirrorContent))

+ }))

+ t.Cleanup(mirror.Close)

+ swapFeedGlobals(t, base64.StdEncoding.EncodeToString(pub),

+ primary.URL+"/manifest.json", mirror.URL+"/releases")

+ prevArch := updaterArch

+ updaterArch = "amd64"

+ t.Cleanup(func() { updaterArch = prevArch })

+func TestUpdatePassFallsBackToMirrorBinary(t *testing.T) {

+ withTempOpt(t)

+ stubSystemctl(t)

+ shrink := deployHealthTimeout

+ deployHealthTimeout = 2 * time.Second

+ t.Cleanup(func() { deployHealthTimeout = shrink })

+ content := "mirrored-binary-content"

+ mirrorBinaryFeed(t, content, content)

+ reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db"))

+ if err != nil {

+ t.Fatal(err)

+ }

+ s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "")

+ seedInstance(t, reg, "h1.example.com", s1, "old1111")

+ if err := updatePass(reg, "lchat@", false); err != nil {

+ t.Fatal(err)

+ }

+ if v, _ := liveVersion("h1.example.com"); v != "new1234" {

+ t.Fatalf("mirror binary fallback didn't land: live = %q", v)

+ }

+ staged, err := os.ReadFile(versionBinPath("new1234"))

+ if err != nil || string(staged) != content {

+ t.Fatalf("staged binary = %q, %v — must be the mirror's bytes", staged, err)

+ }

+func TestUpdatePassRejectsBadMirrorBinary(t *testing.T) {

+ withTempOpt(t)

+ calls := stubSystemctl(t)

+ // The mirror serves bytes that don't hash to the signed sha256 — a

+ // tampered (or stale) mirror must cause a refusal, never an install.

+ mirrorBinaryFeed(t, "the-real-binary", "tampered-bytes")

+ reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db"))

+ if err != nil {

+ t.Fatal(err)

+ }

+ s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "")

+ seedInstance(t, reg, "h1.example.com", s1, "old1111")

+ if err := updatePass(reg, "lchat@", false); err != nil {

+ t.Fatal(err)

+ }

+ if v, _ := liveVersion("h1.example.com"); v != "old1111" {

+ t.Fatalf("tampered mirror binary was installed: live = %q", v)

+ }

+ if _, err := os.Stat(versionBinPath("new1234")); !os.IsNotExist(err) {

+ t.Fatal("tampered mirror binary was staged")

+ }

+ if len(*calls) != 0 {

+ t.Fatalf("restarts despite a refused download: %v", *calls)

+ }

func TestAcquireDeployLockExcludes(t *testing.T) {

withTempOpt(t)

unlock, err := acquireDeployLock()