amadan
Supply-chain bake window for beta/stable auto-updates (#815)
* server: supply-chain bake window for beta/stable auto-updates beta and stable channel heads are held 24h/72h (+ per-box jitter) after the box FIRST SEES them at the channel head before installing — a compromised release caught and pulled back inside the window never reaches subscribers. The clock is the box's own observation (/opt/lchat/channel-first-seen.json), never a feed timestamp, so a stolen signing key can't backdate past it; a pull-back to a previously-seen version applies immediately. No publisher-side bypass by design — the override is local root (update run --ignore-bake or a manual deploy). Held heads surface in update status and the admin consoles. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * deploy: ship treats bake-held rows like pinned ones — reported, not waited on A held beta/stable head is measured in days; ship's deadline is 10 minutes. Without this, promote-then-ship on a stable-riding fleet would spin to a false failure. ---------
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
11 files changed,
+629
−17
bake.go+151 −0bake_test.go+327 −0changelog/2026-07-13-update-safety-hold.md+7 −0deploy/release-server.sh+13 −2docs/instance-router.md+7 −0docs/server-releases.md+44 −0updatecheck.go+7 −0updater.go+36 −3updater_test.go+29 −12web/static/admin.js+4 −0web/static/apps/admin.js+4 −0
diff --git a/bake.go b/bake.go| new file mode 100644 |
| index 0000000..890d4bf |
| --- /dev/null |
| +++ b/bake.go |
| @@ -0,0 +1,151 @@ |
| +package main |
| + |
| +import ( |
| + "encoding/json" |
| + "fmt" |
| + "math/rand" |
| + "os" |
| + "path/filepath" |
| + "time" |
| +) |
| + |
| +// The supply-chain bake window: beta and stable subscribers don't install a |
| +// release the moment it appears — the box waits until the release has sat at |
| +// the channel head for a while first, so a compromised release that gets |
| +// caught and pulled back (docs/server-releases.md) never reaches them. The |
| +// clock is THIS BOX's own observation ("when did I first see this version at |
| +// this channel head"), never a manifest timestamp: an attacker holding the |
| +// signing key controls every timestamp in the feed and would simply backdate |
| +// one, but cannot backdate what the box already saw. |
| +// |
| +// edge has no window — it IS the operator's own deploy path (`publish && |
| +// ship`). beta's 24h absorbs the caught-within-a-day mass of real |
| +// supply-chain incidents; stable's 72h additionally spans a weekend, so a |
| +// Friday-night malicious publish is seen by a human on Monday with margin to |
| +// pull it back. |
| +// |
| +// There is deliberately NO publisher-side bypass ("critical fix, skip the |
| +// hold") — that flag is the first thing an attacker sets. The bypass is |
| +// local and explicit: `lchat update run --ignore-bake` or a plain |
| +// `lchat deploy <host> --version <ver>`, both of which need root on the box. |
| +var bakeDelays = map[string]time.Duration{ |
| + "edge": 0, |
| + "beta": 24 * time.Hour, |
| + "stable": 72 * time.Hour, |
| +} |
| + |
| +// bakeHours is the display form consoles show ("held up to N hours"). |
| +func bakeHours(channel string) int { return int(bakeDelays[channel].Hours()) } |
| + |
| +// firstSeenPath persists the bake clock, next to the anti-replay stamp. |
| +func firstSeenPath() string { return filepath.Join(optDir, "channel-first-seen.json") } |
| + |
| +// firstSeenKeep bounds each channel's history. Past heads are kept (not just |
| +// the current one) so a pull-back to a previously-seen version is immediately |
| +// eligible — evacuating a bad release must never itself sit in a bake window. |
| +const firstSeenKeep = 12 |
| + |
| +// firstSeenState is the bake clock: for each channel, when this box first |
| +// observed each version at its head. |
| +type firstSeenState struct { |
| + // Jitter desynchronises a fleet of boxes: each waits delay*(1+Jitter), so |
| + // self-hosted boxes don't all install in the same hour and the earliest |
| + // become canaries for the rest. Minted once per box, in [0, 0.25). |
| + Jitter float64 `json:"jitter"` |
| + Channels map[string]map[string]time.Time `json:"channels"` |
| +} |
| + |
| +// loadFirstSeen reads the bake clock. A missing or corrupt file is an empty |
| +// clock — every head then bakes from "now", which fails closed (a wiped clock |
| +// delays updates; it never lets one through early). |
| +func loadFirstSeen() *firstSeenState { |
| + st := &firstSeenState{} |
| + if raw, err := os.ReadFile(firstSeenPath()); err == nil { |
| + _ = json.Unmarshal(raw, st) |
| + } |
| + if st.Channels == nil { |
| + st.Channels = map[string]map[string]time.Time{} |
| + } |
| + if st.Jitter < 0 || st.Jitter >= 0.25 { |
| + st.Jitter = 0 |
| + } |
| + return st |
| +} |
| + |
| +// observe records every channel head the manifest shows that this box hasn't |
| +// seen at that head before. Returns whether anything changed (the caller |
| +// persists). Never called on a manifest that failed verification — the clock |
| +// only ever ticks for signed feeds. |
| +func (st *firstSeenState) observe(m *releaseManifest, now time.Time) bool { |
| + changed := false |
| + if st.Jitter == 0 && len(st.Channels) == 0 { |
| + // A fresh box mints its jitter once, alongside its first observation. |
| + st.Jitter = rand.Float64() * 0.25 |
| + changed = true |
| + } |
| + for ch, e := range m.Channels { |
| + seen := st.Channels[ch] |
| + if seen == nil { |
| + seen = map[string]time.Time{} |
| + st.Channels[ch] = seen |
| + } |
| + if _, ok := seen[e.Version]; !ok { |
| + seen[e.Version] = now.UTC().Truncate(time.Second) |
| + changed = true |
| + } |
| + // Prune oldest entries beyond the cap — the just-recorded head is the |
| + // newest, so it always survives. |
| + for len(seen) > firstSeenKeep { |
| + oldest, at := "", time.Time{} |
| + for v, t := range seen { |
| + if oldest == "" || t.Before(at) { |
| + oldest, at = v, t |
| + } |
| + } |
| + delete(seen, oldest) |
| + } |
| + } |
| + return changed |
| +} |
| + |
| +func (st *firstSeenState) save() error { |
| + raw, err := json.MarshalIndent(st, "", " ") |
| + if err != nil { |
| + return err |
| + } |
| + return os.WriteFile(firstSeenPath(), append(raw, '\n'), 0o644) |
| +} |
| + |
| +// bakeEligibleAt is the moment a version first seen at a channel head may be |
| +// installed: firstSeen + delay*(1+jitter). ok=false when the version has no |
| +// record at this channel's head — brand new, so not eligible (updatePass |
| +// calls observe() first, which makes that case unreachable there; status |
| +// reads the clock without ticking it and can legitimately see it). |
| +func (st *firstSeenState) bakeEligibleAt(channel, version string) (time.Time, bool) { |
| + seen, ok := st.Channels[channel][version] |
| + if !ok { |
| + return time.Time{}, false |
| + } |
| + d := bakeDelays[channel] |
| + if d == 0 { |
| + return seen, true |
| + } |
| + return seen.Add(time.Duration(float64(d) * (1 + st.Jitter))), true |
| +} |
| + |
| +// bakeHold returns a non-empty human reason while a channel head must not yet |
| +// be installed on this box, "" once it may be. |
| +func bakeHold(st *firstSeenState, channel, version string, ignore bool, now time.Time) string { |
| + if ignore { |
| + return "" |
| + } |
| + at, ok := st.bakeEligibleAt(channel, version) |
| + if !ok { |
| + // Fail closed: no record means no clock has run, so nothing installs. |
| + return "in its supply-chain bake window (clock starts this pass)" |
| + } |
| + if now.Before(at) { |
| + return fmt.Sprintf("in its supply-chain bake window until %s", at.UTC().Format("2006-01-02 15:04 MST")) |
| + } |
| + return "" |
| +} |
diff --git a/bake_test.go b/bake_test.go| new file mode 100644 |
| index 0000000..7d828d8 |
| --- /dev/null |
| +++ b/bake_test.go |
| @@ -0,0 +1,327 @@ |
| +package main |
| + |
| +import ( |
| + "net/http" |
| + "os" |
| + "path/filepath" |
| + "testing" |
| + "time" |
| +) |
| + |
| +// The supply-chain bake window (bake.go). These tests run with the real |
| +// delays (stable 72h / beta 24h) — everything else in updater_test.go runs |
| +// zeroBake()d so the two concerns stay separate. |
| + |
| +func TestUpdatePassBakeHoldsNewStableHead(t *testing.T) { |
| + withTempOpt(t) |
| + calls := stubSystemctl(t) |
| + shrink := deployHealthTimeout |
| + deployHealthTimeout = 2 * time.Second |
| + t.Cleanup(func() { deployHealthTimeout = shrink }) |
| + |
| + _, setChannels := startFeed(t, map[string]string{"new1234": "held-binary"}) |
| + setChannels(map[string]string{"stable": "new1234"}) |
| + |
| + reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "") |
| + seedInstance(t, reg, "h1.example.com", s1, "old1111") |
| + |
| + // First sighting: the clock starts and NOTHING moves — no download, no |
| + // restart. The window is measured from this box's own observation. |
| + if err := updatePass(reg, "lchat@", false, false); err != nil { |
| + t.Fatalf("updatePass: %v", err) |
| + } |
| + if v, _ := liveVersion("h1.example.com"); v != "old1111" { |
| + t.Fatalf("brand-new stable head installed immediately: live = %q", v) |
| + } |
| + if _, err := os.Stat(versionBinPath("new1234")); !os.IsNotExist(err) { |
| + t.Fatal("binary downloaded during the bake window") |
| + } |
| + if len(*calls) != 0 { |
| + t.Fatalf("restarts during the bake window: %v", *calls) |
| + } |
| + st := loadFirstSeen() |
| + seen, ok := st.Channels["stable"]["new1234"] |
| + if !ok { |
| + t.Fatal("first sighting not recorded") |
| + } |
| + if st.Jitter < 0 || st.Jitter >= 0.25 { |
| + t.Fatalf("minted jitter %v outside [0, 0.25)", st.Jitter) |
| + } |
| + |
| + // A second pass inside the window holds too, and doesn't move the clock. |
| + if err := updatePass(reg, "lchat@", false, false); err != nil { |
| + t.Fatal(err) |
| + } |
| + if v, _ := liveVersion("h1.example.com"); v != "old1111" { |
| + t.Fatalf("head installed inside the window: live = %q", v) |
| + } |
| + if again := loadFirstSeen().Channels["stable"]["new1234"]; !again.Equal(seen) { |
| + t.Fatalf("re-observation moved the clock: %v -> %v", seen, again) |
| + } |
| + |
| + // Once the window (72h plus any jitter, ≤ 90h) has passed, it installs. |
| + st.Channels["stable"]["new1234"] = time.Now().Add(-91 * time.Hour) |
| + if err := st.save(); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := updatePass(reg, "lchat@", false, false); err != nil { |
| + t.Fatal(err) |
| + } |
| + if v, _ := liveVersion("h1.example.com"); v != "new1234" { |
| + t.Fatalf("head not installed after the window: live = %q", v) |
| + } |
| +} |
| + |
| +// Pulling a channel back to a previously-seen version applies immediately — |
| +// evacuating a bad release must never itself sit in a bake window. |
| +func TestUpdatePassBakePullBackIsImmediate(t *testing.T) { |
| + withTempOpt(t) |
| + stubSystemctl(t) |
| + shrink := deployHealthTimeout |
| + deployHealthTimeout = 2 * time.Second |
| + t.Cleanup(func() { deployHealthTimeout = shrink }) |
| + |
| + _, setChannels := startFeed(t, map[string]string{"goodaaa": "good-binary", "badbbbb": "bad-binary"}) |
| + setChannels(map[string]string{"stable": "badbbbb"}) |
| + |
| + // goodaaa sat at the stable head long ago (this box saw it); badbbbb is |
| + // the fresh, about-to-be-pulled release. |
| + st := loadFirstSeen() |
| + st.Jitter = 0.1 |
| + st.Channels = map[string]map[string]time.Time{ |
| + "stable": {"goodaaa": time.Now().Add(-200 * time.Hour)}, |
| + } |
| + if err := st.save(); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "") |
| + seedInstance(t, reg, "h1.example.com", s1, "old1111") |
| + |
| + // The fresh head is held... |
| + if err := updatePass(reg, "lchat@", false, false); err != nil { |
| + t.Fatal(err) |
| + } |
| + if v, _ := liveVersion("h1.example.com"); v != "old1111" { |
| + t.Fatalf("fresh head installed: live = %q", v) |
| + } |
| + |
| + // ...the operator pulls stable back to the known-good version, and the |
| + // next pass applies it with no further wait. |
| + setChannels(map[string]string{"stable": "goodaaa"}) |
| + if err := updatePass(reg, "lchat@", false, false); err != nil { |
| + t.Fatal(err) |
| + } |
| + if v, _ := liveVersion("h1.example.com"); v != "goodaaa" { |
| + t.Fatalf("pull-back target was held: live = %q", v) |
| + } |
| +} |
| + |
| +// --ignore-bake is the deliberate local override: root on the box applying a |
| +// held head NOW (there is intentionally no publisher-side equivalent). |
| +func TestUpdatePassIgnoreBakeOverrides(t *testing.T) { |
| + withTempOpt(t) |
| + stubSystemctl(t) |
| + shrink := deployHealthTimeout |
| + deployHealthTimeout = 2 * time.Second |
| + t.Cleanup(func() { deployHealthTimeout = shrink }) |
| + |
| + _, setChannels := startFeed(t, map[string]string{"new1234": "content"}) |
| + setChannels(map[string]string{"stable": "new1234"}) |
| + |
| + reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "") |
| + seedInstance(t, reg, "h1.example.com", s1, "old1111") |
| + |
| + if err := updatePass(reg, "lchat@", false, true); err != nil { |
| + t.Fatal(err) |
| + } |
| + if v, _ := liveVersion("h1.example.com"); v != "new1234" { |
| + t.Fatalf("--ignore-bake didn't install: live = %q", v) |
| + } |
| +} |
| + |
| +// edge has no window: it's the operator's own deploy path (`publish && ship` |
| +// must keep converging the fleet in one pass). |
| +func TestUpdatePassEdgeHasNoBake(t *testing.T) { |
| + withTempOpt(t) |
| + stubSystemctl(t) |
| + shrink := deployHealthTimeout |
| + deployHealthTimeout = 2 * time.Second |
| + t.Cleanup(func() { deployHealthTimeout = shrink }) |
| + |
| + _, setChannels := startFeed(t, map[string]string{"edge777": "edge-binary"}) |
| + setChannels(map[string]string{"edge": "edge777"}) |
| + |
| + reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + s1 := startFakeInstance(t, "h1.example.com", http.StatusOK, |
| + updatePrefs{AutoUpdate: true, AutoDownload: true, Channel: "edge"}, "") |
| + seedInstance(t, reg, "h1.example.com", s1, "old1111") |
| + |
| + if err := updatePass(reg, "lchat@", false, false); err != nil { |
| + t.Fatal(err) |
| + } |
| + if v, _ := liveVersion("h1.example.com"); v != "edge777" { |
| + t.Fatalf("edge head was held: live = %q", v) |
| + } |
| +} |
| + |
| +// Hibernating hosts honour the window too — their repoints have no health |
| +// check, so they must never be the first to run a fresh head anyway. |
| +func TestUpdatePassBakeHoldsHibernating(t *testing.T) { |
| + withTempOpt(t) |
| + calls := stubSystemctl(t) |
| + |
| + oldData := instanceDataDir |
| + instanceDataDir = t.TempDir() |
| + t.Cleanup(func() { instanceDataDir = oldData }) |
| + |
| + _, setChannels := startFeed(t, map[string]string{"new1234": "content"}) |
| + setChannels(map[string]string{"stable": "new1234"}) |
| + |
| + reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + src := filepath.Join(t.TempDir(), "old") |
| + writeFakeBinary(t, src, "old-binary") |
| + if err := stageVersion(src, "old1111"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := pointLive("h1.hib", "old1111"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := reg.upsert(Route{Host: "h1.hib", Kind: "instance", Status: "ready", |
| + Socket: filepath.Join(t.TempDir(), "no.sock"), Hibernate: true, Version: "old1111"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + db, err := openDB(filepath.Join(instanceDataDir, "h1.hib.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + db.sql.Close() |
| + |
| + if err := updatePass(reg, "lchat@", false, false); err != nil { |
| + t.Fatal(err) |
| + } |
| + if v, _ := liveVersion("h1.hib"); v != "old1111" { |
| + t.Fatalf("hibernating host repointed during the bake window: live = %q", v) |
| + } |
| + if len(*calls) != 0 { |
| + t.Fatalf("systemctl touched: %v", *calls) |
| + } |
| + |
| + // After the window it repoints as usual. |
| + st := loadFirstSeen() |
| + st.Channels["stable"]["new1234"] = time.Now().Add(-91 * time.Hour) |
| + if err := st.save(); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := updatePass(reg, "lchat@", false, false); err != nil { |
| + t.Fatal(err) |
| + } |
| + if v, _ := liveVersion("h1.hib"); v != "new1234" { |
| + t.Fatalf("hibernating host not repointed after the window: live = %q", v) |
| + } |
| +} |
| + |
| +// Dry runs compute bake eligibility but never persist the clock — a dry run |
| +// must not start windows ticking. |
| +func TestUpdatePassBakeDryRunPersistsNothing(t *testing.T) { |
| + withTempOpt(t) |
| + stubSystemctl(t) |
| + |
| + _, setChannels := startFeed(t, map[string]string{"new1234": "content"}) |
| + setChannels(map[string]string{"stable": "new1234"}) |
| + |
| + reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "") |
| + seedInstance(t, reg, "h1.example.com", s1, "old1111") |
| + |
| + if err := updatePass(reg, "lchat@", true, false); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := os.Stat(firstSeenPath()); !os.IsNotExist(err) { |
| + t.Fatal("dry run persisted the bake clock") |
| + } |
| +} |
| + |
| +func TestFirstSeenObservePruneAndEligibility(t *testing.T) { |
| + withTempOpt(t) |
| + |
| + // A fresh box mints its jitter alongside the first observation. |
| + st := loadFirstSeen() |
| + now := time.Now() |
| + if !st.observe(&releaseManifest{Channels: map[string]releaseEntry{"stable": {Version: "v0"}}}, now) { |
| + t.Fatal("first observation reported no change") |
| + } |
| + if st.Jitter < 0 || st.Jitter >= 0.25 { |
| + t.Fatalf("jitter %v outside [0, 0.25)", st.Jitter) |
| + } |
| + // Re-observing the same head is a no-op. |
| + if st.observe(&releaseManifest{Channels: map[string]releaseEntry{"stable": {Version: "v0"}}}, now.Add(time.Hour)) { |
| + t.Fatal("re-observation reported a change") |
| + } |
| + |
| + // Eligibility: firstSeen + delay*(1+jitter), exactly. |
| + st.Jitter = 0.2 // 0.25 is exclusive — loadFirstSeen clamps it as corrupt |
| + seen := st.Channels["stable"]["v0"] |
| + at, ok := st.bakeEligibleAt("stable", "v0") |
| + if want := seen.Add(time.Duration(float64(72*time.Hour) * 1.2)); !ok || !at.Equal(want) { |
| + t.Fatalf("eligibleAt = %v ok=%v, want %v", at, ok, want) |
| + } |
| + if _, ok := st.bakeEligibleAt("stable", "never-seen"); ok { |
| + t.Fatal("unseen version reported eligible time") |
| + } |
| + // An unseen version is held, an old one isn't, ignore always passes. |
| + if bakeHold(st, "stable", "never-seen", false, now) == "" { |
| + t.Fatal("unseen version not held") |
| + } |
| + if bakeHold(st, "stable", "never-seen", true, now) != "" { |
| + t.Fatal("ignore-bake didn't override") |
| + } |
| + st.Channels["stable"]["ancient"] = now.Add(-100 * time.Hour) |
| + if hold := bakeHold(st, "stable", "ancient", false, now); hold != "" { |
| + t.Fatalf("aged-out version still held: %s", hold) |
| + } |
| + |
| + // History is pruned to firstSeenKeep entries, oldest first — the current |
| + // head always survives. |
| + for i := 0; i < firstSeenKeep+5; i++ { |
| + st.observe(&releaseManifest{Channels: map[string]releaseEntry{ |
| + "stable": {Version: string(rune('a' + i))}, |
| + }}, now.Add(time.Duration(i)*time.Minute)) |
| + } |
| + if n := len(st.Channels["stable"]); n != firstSeenKeep { |
| + t.Fatalf("history size = %d, want %d", n, firstSeenKeep) |
| + } |
| + if _, ok := st.Channels["stable"][string(rune('a'+firstSeenKeep+4))]; !ok { |
| + t.Fatal("newest head was pruned") |
| + } |
| + |
| + // The clock round-trips through disk. |
| + if err := st.save(); err != nil { |
| + t.Fatal(err) |
| + } |
| + got := loadFirstSeen() |
| + if got.Jitter != st.Jitter || len(got.Channels["stable"]) != firstSeenKeep { |
| + t.Fatalf("reload = jitter %v, %d entries", got.Jitter, len(got.Channels["stable"])) |
| + } |
| +} |
diff --git a/changelog/2026-07-13-update-safety-hold.md b/changelog/2026-07-13-update-safety-hold.md| new file mode 100644 |
| index 0000000..afb865f |
| --- /dev/null |
| +++ b/changelog/2026-07-13-update-safety-hold.md |
| @@ -0,0 +1,7 @@ |
| +🤖 Server updates now pass through a safety hold. Spaces on the beta and |
| +stable update channels wait 24 and 72 hours after a release appears before |
| +installing it automatically — so if a bad or tampered-with release is caught |
| +and withdrawn in that window, your space never runs it. Withdrawing a release |
| +takes effect immediately, the hold only ever delays installs. Admins can see |
| +the hold in the admin console's update section and can still apply an update |
| +straight away by hand; the edge channel is unchanged. |
diff --git a/deploy/release-server.sh b/deploy/release-server.sh| index 8a86eab..33831cf 100755 |
| --- a/deploy/release-server.sh |
| +++ b/deploy/release-server.sh |
| @@ -216,11 +216,14 @@ ship) |
| STATUS="$(ssh "$BOX" /opt/lchat/lchat update status || true)" |
| # Done when no un-pinned instance with a known channel head is behind, and |
| # at least one row proves a feed landed at all. Pinned rows are someone's |
| - # canary: reported, never waited on. |
| - BEHIND="$(printf '%s\n' "$STATUS" | awk 'NR>1 && $3!="pinned" && $5!="-" && $0 !~ /\(current\)/ {print $1}')" |
| + # canary: reported, never waited on. Held rows (a beta/stable head still |
| + # inside its supply-chain bake window) are likewise reported, not waited |
| + # on — the hold is measured in days and ship's deadline in minutes. |
| + BEHIND="$(printf '%s\n' "$STATUS" | awk 'NR>1 && $3!="pinned" && $5!="-" && $0 !~ /\(current\)/ && $0 !~ /\(held/ {print $1}')" |
| if [ -z "$BEHIND" ] && printf '%s\n' "$STATUS" | grep -q "(current)"; then |
| printf '%s\n' "$STATUS" |
| printf '%s\n' "$STATUS" | awk 'NR>1 && $3=="pinned" {print "note: " $1 " is pinned (a canary) - left alone"}' |
| + printf '%s\n' "$STATUS" | awk 'NR>1 && /\(held/ {print "note: " $1 " is holding its channel head (safety hold) - not waited on"}' |
| if printf '%s\n' "$STATUS" | grep -q "$VER (current)"; then |
| echo "shipped $VER - every un-pinned instance is current" |
| else |
| @@ -228,6 +231,14 @@ ship) |
| fi |
| exit 0 |
| fi |
| + # Nothing behind and nothing current can still be a clean stop: every |
| + # remaining row is inside its safety hold (e.g. the whole fleet rides |
| + # stable right after a promote). They install when the window elapses. |
| + if [ -z "$BEHIND" ] && printf '%s\n' "$STATUS" | grep -q "(held"; then |
| + printf '%s\n' "$STATUS" |
| + echo "shipped the feed - every remaining instance is holding its channel head inside the safety window (installs when it elapses; per-host override: lchat deploy <host> --version <ver>)" |
| + exit 0 |
| + fi |
| if [ "$(date +%s)" -ge "$DEADLINE" ]; then |
| printf '%s\n' "$STATUS" |
| echo "ship: gave up after 10m - still behind: $BEHIND (GitHub CDN can serve a fresh manifest stale; retry in a few minutes)" >&2 |
diff --git a/docs/instance-router.md b/docs/instance-router.md| index fc86ddd..1627b4b 100644 |
| --- a/docs/instance-router.md |
| +++ b/docs/instance-router.md |
| @@ -512,6 +512,13 @@ described in `docs/server-releases.md`: |
| `/opt/lchat/manifest-stamp`; older (validly signed but stale) manifests are |
| refused, so the feed can be pulled backward only by *re-signing* — a |
| deliberate act with the key. |
| +- **The bake window (supply-chain hold)**: beta/stable heads aren't installed |
| + until they've sat at the channel head for 24 h/72 h (+ per-box jitter) by |
| + the box's *own* clock (`/opt/lchat/channel-first-seen.json`) — a compromised |
| + release caught and pulled back inside the window never reaches subscribers, |
| + and a pull-back to a previously-seen version applies immediately. Local |
| + override: `lchat update run --ignore-bake` (root); design + numbers in |
| + `docs/server-releases.md § The bake window`. |
| - **GC**: after a pass, version-store dirs nothing references (no route's |
| current/previous, no channel head, no live symlink, not the control binary's |
| own build) are pruned, and crashed `dl-*` downloads in `incoming/` are |
diff --git a/docs/server-releases.md b/docs/server-releases.md| index 1f37462..bcb3a08 100644 |
| --- a/docs/server-releases.md |
| +++ b/docs/server-releases.md |
| @@ -103,6 +103,50 @@ The updater downloads and *executes* native code, so the chain matters: |
| validly signed) manifest can't downgrade a fleet. An intentional pull-back is |
| a *re-signed* manifest with a newer timestamp, which still applies. |
| +### The bake window (supply-chain hold) |
| + |
| +Everything above assumes the signing key is honest. The bake window is the |
| +defence for the day it isn't — a stolen seed, or a compromised operator |
| +machine signing a malicious build. beta and stable subscribers don't install |
| +a release the moment it appears; the box waits until the release has **sat at |
| +the channel head for a while**: |
| + |
| +| channel | hold | |
| +|---------|------| |
| +| `edge` | none — it *is* the operator's own deploy path | |
| +| `beta` | 24 h | |
| +| `stable` | 72 h | |
| + |
| +Real supply-chain incidents are bimodal: noisy ones (malicious npm/PyPI |
| +releases, stolen-token publishes) are overwhelmingly detected and yanked |
| +within hours to ~3 days; stealthy targeted ones take weeks and no tolerable |
| +hold defends against them (that's what key hygiene is for). 24 h absorbs the |
| +caught-within-a-day mass; 72 h additionally **spans a weekend** — a |
| +Friday-night malicious publish is seen by a human on Monday with margin to |
| +pull the channel back, and a pull-back to a previously-seen version applies |
| +*immediately* (evacuation never bakes). This is also why stable-channel lag |
| +saved every distro from xz-utils. |
| + |
| +The mechanics (`bake.go`): |
| + |
| +- **The clock is the box's own first sighting** — `lchat update run` records |
| + "first saw `<ver>` at the head of `<chan>`" in |
| + `/opt/lchat/channel-first-seen.json` and installs only once that record is |
| + old enough. Never a manifest timestamp: an attacker holding the key |
| + controls every timestamp in the feed and would simply backdate one, but |
| + cannot backdate what the box already saw. A missing/corrupt clock file |
| + fails **closed** (everything bakes from now). |
| +- **Jitter**: each box waits `hold × (1 + j)`, `j ∈ [0, 0.25)` minted once |
| + per box — a fleet of self-hosted boxes doesn't install in the same hour, |
| + so the earliest are canaries for the rest. |
| +- **There is deliberately NO publisher-side bypass** ("critical fix, skip the |
| + hold") — that flag is the first thing an attacker would set. The bypass is |
| + local and explicit, requiring root on the box: |
| + `lchat update run --ignore-bake`, or a plain |
| + `lchat deploy <host> --version <ver>`. A held release shows in |
| + `lchat update status` as `(held until <time>)` and in the admin consoles as |
| + "auto-installs after a N-hour safety hold". |
| + |
| ## The S3 mirror |
| GitHub's asset CDN is the feed's single point of failure — and it has served |
diff --git a/updatecheck.go b/updatecheck.go| index e6a1e6c..06055a9 100644 |
| --- a/updatecheck.go |
| +++ b/updatecheck.go |
| @@ -78,6 +78,13 @@ func (s *Server) updateInfo() map[string]any { |
| if e, ok := m.Channels[s.db.updateChannel()]; ok { |
| info["available_version"] = e.Version |
| info["available_notes"] = e.Notes |
| + // The channel's supply-chain bake window (bake.go), so the console can |
| + // explain why a fresh release hasn't auto-installed yet. The instance |
| + // can't see the box's bake clock — only the policy — so this is an |
| + // "up to N hours" statement, never a countdown. |
| + if h := bakeHours(s.db.updateChannel()); h > 0 { |
| + info["bake_hours"] = h |
| + } |
| } |
| return info |
| } |
diff --git a/updater.go b/updater.go| index 74bb0cc..324a39f 100644 |
| --- a/updater.go |
| +++ b/updater.go |
| @@ -340,7 +340,9 @@ func ensureVersionStaged(e releaseEntry) error { |
| // updatePass is one full check-and-apply sweep, split from runUpdateRun so |
| // tests drive it directly (die()-free: per-host problems log and continue — |
| // an unreachable instance must not strand the rest of the fleet unchecked). |
| -func updatePass(reg *Registry, unitTmpl string, dryRun bool) error { |
| +// ignoreBake is the local, root-only override of the supply-chain bake window |
| +// (bake.go) — deliberately never a publisher-side flag. |
| +func updatePass(reg *Registry, unitTmpl string, dryRun, ignoreBake bool) error { |
| m, err := fetchReleaseManifest(releaseManifestURL) |
| if err != nil { |
| // No feed is a normal state (the releases repo may not exist yet, or |
| @@ -357,6 +359,16 @@ func updatePass(reg *Registry, unitTmpl string, dryRun bool) error { |
| return fmt.Errorf("record manifest stamp: %v", err) |
| } |
| } |
| + // Tick the bake clock: record every channel head this box hasn't seen at |
| + // that head before, so the supply-chain window is measured from the box's |
| + // own first sighting, never a feed timestamp (bake.go). Dry runs compute |
| + // eligibility but persist nothing. |
| + bake := loadFirstSeen() |
| + if bake.observe(m, time.Now()) && !dryRun { |
| + if err := bake.save(); err != nil { |
| + return fmt.Errorf("record bake clock: %v", err) |
| + } |
| + } |
| routes, err := reg.list() |
| if err != nil { |
| @@ -407,6 +419,11 @@ func updatePass(reg *Registry, unitTmpl string, dryRun bool) error { |
| if cur == e.Version { |
| continue // up to date — stay quiet on the hourly tick |
| } |
| + if hold := bakeHold(bake, prefs.Channel, e.Version, ignoreBake, time.Now()); hold != "" { |
| + fmt.Printf(" %s: %s (%s) is %s — apply now: lchat deploy %s --version %s\n", |
| + rt.Host, e.Version, prefs.Channel, hold, rt.Host, e.Version) |
| + continue |
| + } |
| if !prefs.AutoDownload && !prefs.AutoUpdate { |
| fmt.Printf(" %s: %s available on %s (downloads off — admin's call)\n", rt.Host, e.Version, prefs.Channel) |
| continue |
| @@ -473,6 +490,11 @@ func updatePass(reg *Registry, unitTmpl string, dryRun bool) error { |
| if cur == e.Version { |
| continue // up to date — stay quiet on the hourly tick |
| } |
| + if hold := bakeHold(bake, prefs.Channel, e.Version, ignoreBake, time.Now()); hold != "" { |
| + fmt.Printf(" %s: %s (%s) is %s — apply now: lchat deploy %s --version %s\n", |
| + rt.Host, e.Version, prefs.Channel, hold, rt.Host, e.Version) |
| + continue |
| + } |
| if !prefs.AutoDownload && !prefs.AutoUpdate { |
| fmt.Printf(" %s: %s available on %s (downloads off — admin's call)\n", rt.Host, e.Version, prefs.Channel) |
| continue |
| @@ -669,6 +691,7 @@ func runUpdateRun(args []string) { |
| registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") |
| unitTmpl := fs.String("unit", envOr("LCHAT_UNIT", "lchat@"), "systemd instance unit template prefix") |
| dryRun := fs.Bool("dry-run", false, "report what would change without staging or restarting anything") |
| + ignoreBake := fs.Bool("ignore-bake", false, "install channel heads still inside their supply-chain bake window (the deliberate local override)") |
| _ = fs.Parse(reorderFlagsFirst(fs, args)) |
| unlock, err := acquireDeployLock() |
| @@ -681,7 +704,7 @@ func runUpdateRun(args []string) { |
| if err != nil { |
| die("update: open registry: %v", err) |
| } |
| - if err := updatePass(reg, *unitTmpl, *dryRun); err != nil { |
| + if err := updatePass(reg, *unitTmpl, *dryRun, *ignoreBake); err != nil { |
| die("update: %v", err) |
| } |
| } |
| @@ -714,6 +737,7 @@ func runUpdateStatus(args []string) { |
| die("update: %v", err) |
| } |
| m, merr := fetchReleaseManifest(releaseManifestURL) |
| + bake := loadFirstSeen() |
| fmt.Printf("%-34s %-8s %-8s %-12s %s\n", "HOST", "CHANNEL", "AUTO", "CURRENT", "AVAILABLE") |
| for _, rt := range routes { |
| @@ -744,8 +768,17 @@ func runUpdateStatus(args []string) { |
| if m != nil { |
| if e, ok := m.Channels[channel]; ok { |
| avail = e.Version |
| - if e.Version == cur { |
| + switch { |
| + case e.Version == cur: |
| avail += " (current)" |
| + case bakeDelays[channel] > 0: |
| + // Status reads the bake clock without ticking it — only an |
| + // update pass observes. An unticked head shows as held too. |
| + if at, ok := bake.bakeEligibleAt(channel, e.Version); !ok { |
| + avail += " (held — bake starts on the next update pass)" |
| + } else if time.Now().Before(at) { |
| + avail += " (held until " + at.UTC().Format("2006-01-02 15:04 MST") + ")" |
| + } |
| } |
| } |
| } |
diff --git a/updater_test.go b/updater_test.go| index 0b6fde1..584f657 100644 |
| --- a/updater_test.go |
| +++ b/updater_test.go |
| @@ -185,8 +185,18 @@ func seedInstance(t *testing.T, reg *Registry, host, socket, ver string) { |
| } |
| } |
| +// zeroBake disables the supply-chain bake window (bake.go) for tests that |
| +// exercise the rest of the pipeline; bake behaviour has its own tests below. |
| +func zeroBake(t *testing.T) { |
| + t.Helper() |
| + prev := bakeDelays |
| + bakeDelays = map[string]time.Duration{"edge": 0, "beta": 0, "stable": 0} |
| + t.Cleanup(func() { bakeDelays = prev }) |
| +} |
| + |
| func TestUpdatePassAppliesPerPrefs(t *testing.T) { |
| withTempOpt(t) |
| + zeroBake(t) |
| calls := stubSystemctl(t) |
| shrink := deployHealthTimeout |
| deployHealthTimeout = 2 * time.Second |
| @@ -216,7 +226,7 @@ func TestUpdatePassAppliesPerPrefs(t *testing.T) { |
| seedInstance(t, reg, "h3.example.com", s3, "old1111") |
| seedInstance(t, reg, "h4.example.com", s4, "old1111") |
| - if err := updatePass(reg, "lchat@", false); err != nil { |
| + if err := updatePass(reg, "lchat@", false, false); err != nil { |
| t.Fatalf("updatePass: %v", err) |
| } |
| @@ -274,6 +284,7 @@ func TestUpdatePassAppliesPerPrefs(t *testing.T) { |
| func TestUpdatePassRollsBackAndPoisons(t *testing.T) { |
| withTempOpt(t) |
| + zeroBake(t) |
| calls := stubSystemctl(t) |
| shrink := deployHealthTimeout |
| deployHealthTimeout = 500 * time.Millisecond |
| @@ -292,7 +303,7 @@ func TestUpdatePassRollsBackAndPoisons(t *testing.T) { |
| seedInstance(t, reg, "h1.example.com", s1, "old1111") |
| seedInstance(t, reg, "h2.example.com", s2, "old1111") |
| - if err := updatePass(reg, "lchat@", false); err != nil { |
| + if err := updatePass(reg, "lchat@", false, false); err != nil { |
| t.Fatalf("updatePass: %v", err) |
| } |
| @@ -327,13 +338,14 @@ func TestUpdatePassRefusesReplayedManifest(t *testing.T) { |
| if err != nil { |
| t.Fatal(err) |
| } |
| - if err := updatePass(reg, "lchat@", false); err == nil || !strings.Contains(err.Error(), "replayed") { |
| + if err := updatePass(reg, "lchat@", false, false); err == nil || !strings.Contains(err.Error(), "replayed") { |
| t.Fatalf("want replay refusal, got %v", err) |
| } |
| } |
| func TestUpdatePassDryRunTouchesNothing(t *testing.T) { |
| withTempOpt(t) |
| + zeroBake(t) |
| calls := stubSystemctl(t) |
| _, setChannels := startFeed(t, map[string]string{"new1234": "content"}) |
| @@ -346,7 +358,7 @@ func TestUpdatePassDryRunTouchesNothing(t *testing.T) { |
| s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "") |
| seedInstance(t, reg, "h1.example.com", s1, "old1111") |
| - if err := updatePass(reg, "lchat@", true); err != nil { |
| + if err := updatePass(reg, "lchat@", true, false); err != nil { |
| t.Fatal(err) |
| } |
| if v, _ := liveVersion("h1.example.com"); v != "old1111" { |
| @@ -365,6 +377,7 @@ func TestUpdatePassDryRunTouchesNothing(t *testing.T) { |
| func TestUpdatePassStagesTheRightArch(t *testing.T) { |
| withTempOpt(t) |
| + zeroBake(t) |
| stubSystemctl(t) |
| _, setChannels := startFeed(t, map[string]string{"new1234": "binary-content"}) |
| @@ -379,7 +392,7 @@ func TestUpdatePassStagesTheRightArch(t *testing.T) { |
| s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "") |
| seedInstance(t, reg, "h1.example.com", s1, "old1111") |
| - if err := updatePass(reg, "lchat@", false); err != nil { |
| + if err := updatePass(reg, "lchat@", false, false); err != nil { |
| t.Fatal(err) |
| } |
| staged, err := os.ReadFile(versionBinPath("new1234")) |
| @@ -393,6 +406,7 @@ func TestUpdatePassStagesTheRightArch(t *testing.T) { |
| func TestUpdatePassSkipsArchlessRelease(t *testing.T) { |
| withTempOpt(t) |
| + zeroBake(t) |
| calls := stubSystemctl(t) |
| prevArch := updaterArch |
| updaterArch = "arm64" |
| @@ -420,7 +434,7 @@ func TestUpdatePassSkipsArchlessRelease(t *testing.T) { |
| s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "") |
| seedInstance(t, reg, "h1.example.com", s1, "old1111") |
| - if err := updatePass(reg, "lchat@", false); err != nil { |
| + if err := updatePass(reg, "lchat@", false, false); err != nil { |
| t.Fatal(err) |
| } |
| if v, _ := liveVersion("h1.example.com"); v != "old1111" { |
| @@ -456,7 +470,7 @@ func TestUpdatePassSkipsPinnedInstance(t *testing.T) { |
| t.Fatal(err) |
| } |
| - if err := updatePass(reg, "lchat@", false); err != nil { |
| + if err := updatePass(reg, "lchat@", false, false); err != nil { |
| t.Fatal(err) |
| } |
| @@ -712,7 +726,7 @@ func TestFetchManifestKeepsStalePrimaryWhenMirrorNoFresher(t *testing.T) { |
| if err != nil { |
| t.Fatal(err) |
| } |
| - if err := updatePass(reg, "lchat@", false); err == nil || !strings.Contains(err.Error(), "replayed") { |
| + if err := updatePass(reg, "lchat@", false, false); err == nil || !strings.Contains(err.Error(), "replayed") { |
| t.Fatalf("want replay refusal, got %v", err) |
| } |
| } |
| @@ -759,6 +773,7 @@ func mirrorBinaryFeed(t *testing.T, binaryContent, mirrorContent string) { |
| func TestUpdatePassFallsBackToMirrorBinary(t *testing.T) { |
| withTempOpt(t) |
| + zeroBake(t) |
| stubSystemctl(t) |
| shrink := deployHealthTimeout |
| deployHealthTimeout = 2 * time.Second |
| @@ -774,7 +789,7 @@ func TestUpdatePassFallsBackToMirrorBinary(t *testing.T) { |
| s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "") |
| seedInstance(t, reg, "h1.example.com", s1, "old1111") |
| - if err := updatePass(reg, "lchat@", false); err != nil { |
| + if err := updatePass(reg, "lchat@", false, false); err != nil { |
| t.Fatal(err) |
| } |
| if v, _ := liveVersion("h1.example.com"); v != "new1234" { |
| @@ -788,6 +803,7 @@ func TestUpdatePassFallsBackToMirrorBinary(t *testing.T) { |
| func TestUpdatePassRejectsBadMirrorBinary(t *testing.T) { |
| withTempOpt(t) |
| + zeroBake(t) |
| calls := stubSystemctl(t) |
| // The mirror serves bytes that don't hash to the signed sha256 — a |
| @@ -801,7 +817,7 @@ func TestUpdatePassRejectsBadMirrorBinary(t *testing.T) { |
| s1 := startFakeInstance(t, "h1.example.com", http.StatusNotFound, updatePrefs{}, "") |
| seedInstance(t, reg, "h1.example.com", s1, "old1111") |
| - if err := updatePass(reg, "lchat@", false); err != nil { |
| + if err := updatePass(reg, "lchat@", false, false); err != nil { |
| t.Fatal(err) |
| } |
| if v, _ := liveVersion("h1.example.com"); v != "old1111" { |
| @@ -837,6 +853,7 @@ func TestAcquireDeployLockExcludes(t *testing.T) { |
| // Prefs come from the parked DB; no DB reads as the defaults; pins hold. |
| func TestUpdatePassRepointsHibernating(t *testing.T) { |
| withTempOpt(t) |
| + zeroBake(t) |
| calls := stubSystemctl(t) |
| oldData := instanceDataDir |
| @@ -895,7 +912,7 @@ func TestUpdatePassRepointsHibernating(t *testing.T) { |
| seedDB("h3.hib", map[string]string{"update_pin": "old1111"}) |
| seedDB("h4.hib", map[string]string{"update_auto": "0", "update_download": "0"}) |
| - if err := updatePass(reg, "lchat@", false); err != nil { |
| + if err := updatePass(reg, "lchat@", false, false); err != nil { |
| t.Fatalf("updatePass: %v", err) |
| } |
| @@ -923,7 +940,7 @@ func TestUpdatePassRepointsHibernating(t *testing.T) { |
| } |
| // A second pass is quiet + idempotent. |
| - if err := updatePass(reg, "lchat@", false); err != nil { |
| + if err := updatePass(reg, "lchat@", false, false); err != nil { |
| t.Fatalf("second pass: %v", err) |
| } |
| assertLive("h1.hib", "new1234") |
diff --git a/web/static/admin.js b/web/static/admin.js| index dfcab64..d31b0e7 100644 |
| --- a/web/static/admin.js |
| +++ b/web/static/admin.js |
| @@ -127,6 +127,10 @@ async function wireUpdates() { |
| line += ` — pinned to ${s.update_pin}; automatic updates are paused (pick a channel to resume)`; |
| } else if (u.available_version && u.available_version !== u.build) { |
| line += ` — ${u.available_version} is available on ${s.update_channel}`; |
| + if (u.bake_hours) { |
| + // The supply-chain safety hold: new releases wait before auto-installing. |
| + line += ` (auto-installs after a ${u.bake_hours}-hour safety hold)`; |
| + } |
| } else if (u.checked_at) { |
| line += ` — up to date on ${s.update_channel}`; |
| } else { |
diff --git a/web/static/apps/admin.js b/web/static/apps/admin.js| index 5d493ad..c66fc91 100644 |
| --- a/web/static/apps/admin.js |
| +++ b/web/static/apps/admin.js |
| @@ -1213,6 +1213,10 @@ export const app = defineApp({ |
| line += ` — pinned to ${pin}; automatic updates are paused (pick a channel to resume)`; |
| } else if (u.available_version && u.available_version !== u.build) { |
| line += ` — ${u.available_version} is available on ${channel}`; |
| + if (u.bake_hours) { |
| + // The supply-chain safety hold: new releases wait before auto-installing. |
| + line += ` (auto-installs after a ${u.bake_hours}-hour safety hold)`; |
| + } |
| } else if (u.checked_at) { |
| line += ` — up to date on ${channel}`; |
| } else { |