amadan

carlos / platform Public

release: roll back a unit's whole process group and refuse the build when it does not come up

A named unit whose restart returns *UnitNotUpError now puts every route sharing its process back on the previous version, writes a refusal for each, and restarts the unit with the dedicated rollback restart. Plain restart errors keep today's behaviour.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 5893dffc5c9f5cfc8872258b03d015ea4c34daec parent 828fe89
3 files changed, +247 −2
  • internal/release/adopt.go +29 −2
  • internal/release/rollback.go +99 −0
  • internal/release/rollback_test.go +119 −0
diff --git a/internal/release/adopt.go b/internal/release/adopt.go
index 755926370..0362a84ca 100644
--- a/internal/release/adopt.go
+++ b/internal/release/adopt.go
@@ -65,6 +65,12 @@ type passConfig struct {
// keeps today's behaviour byte-identically: the rollback repoints and
// stops.
startUnit func(host string) error
+
+ // rollbackRestart replaces startUnit on D1's path only — a named unit
+ // whose restart returned *UnitNotUpError. It must REPLACE whatever is
+ // running (reset-failed + restart), because a deadline failure can leave
+ // the rejected build running unready and `start` is a no-op on it.
+ rollbackRestart func(host string) error
}
// NagExpiredCanaries opts a Pass/PassFilter call into spec §5's nag: one
@@ -247,6 +253,11 @@ func WithStartUnit(start func(host string) error) PassOption {
return func(c *passConfig) { c.startUnit = start }
}
+// WithRollbackRestart sets D1's rollback restart (see passConfig).
+func WithRollbackRestart(fn func(host string) error) PassOption {
+ return func(c *passConfig) { c.rollbackRestart = fn }
+}
+
// stageBlocked reports whether err is the version store refusing a write to
// THIS uid, rather than any of the many other ways staging fails. errors.Is
// unwraps the *os.PathError that mkdir/openat return.
@@ -1127,6 +1138,8 @@ func adoptHost(ctx context.Context, reg *registry.Registry, s Store, optDir stri
// whether the rollback actually HAPPENED, not on whether one was
// attempted.
poisoned[ver] = true
+ var notUp *UnitNotUpError
+ isNotUp := errors.As(rerr, &notUp)
rolledBack := false
if cur != "" {
if perr := PointLive(optDir, rt.Host, RouteAccount(rt), cur, rt.App); perr != nil {
@@ -1203,8 +1216,22 @@ func adoptHost(ctx context.Context, reg *registry.Registry, s Store, optDir stri
// anything (a permission-denied systemctl) fails its start the
// same way while the old process still serves — the two errors on
// the line tell them apart.
- if cfg.startUnit != nil && rolledBack && !daemonRestart {
- if serr := cfg.startUnit(rt.Host); serr != nil {
+ start := cfg.startUnit
+ if isNotUp && cfg.rollbackRestart != nil {
+ start = cfg.rollbackRestart
+ }
+ if isNotUp {
+ sha := ""
+ if m != nil {
+ sha = m.Artifacts[adoptPlatform].SHA256
+ }
+ rollbackGroup(reg, optDir, cfg, rt, ptr, ver, cur, sha, adoptPlatform, notUp, now)
+ if notUp.Journal != "" {
+ fmt.Printf(" %s: %s's last lines:\n%s\n", rt.Host, notUp.Unit, indent(notUp.Journal, " "))
+ }
+ }
+ if start != nil && rolledBack && !daemonRestart {
+ if serr := start(rt.Host); serr != nil {
fmt.Printf(" %s: rolled back to %s but could not start it: %v — this host may be serving nothing\n",
rt.Host, cur, serr)
}
diff --git a/internal/release/rollback.go b/internal/release/rollback.go
new file mode 100644
index 000000000..98b53309d
--- /dev/null
+++ b/internal/release/rollback.go
@@ -0,0 +1,99 @@
+// The D1 rollback (spec 2026-10-06-console-release-defences-design.md §2.3):
+// what adoptHost does when a named unit's restart reports that the new build
+// did not come up, as opposed to systemctl itself failing.
+package release
+
+import (
+ "fmt"
+ "strings"
+ "time"
+
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+// UnitNotUpError is a named unit that was restarted onto a new build and did
+// not settle: it failed, restarted again, or was not ready by the deadline.
+// The host package's settle check returns it (internal/host/unitsettle.go).
+type UnitNotUpError struct {
+ Unit string
+ Reason string
+ Journal string // the unit's last journal lines, for the rollback line and the alert
+}
+
+func (e *UnitNotUpError) Error() string {
+ return fmt.Sprintf("%s did not come up: %s", e.Unit, e.Reason)
+}
+
+// upstream is a route's effective upstream: Socket wins over Addr, as at the
+// edge (registry.Route.Socket's doc).
+func upstream(rt registry.Route) string {
+ if rt.Socket != "" {
+ return rt.Socket
+ }
+ return rt.Addr
+}
+
+// processGroup is every OTHER route served by bound's process: unstamped,
+// same account and app, same non-empty upstream. A bound route with no
+// upstream is a group of one.
+func processGroup(reg *registry.Registry, bound registry.Route) []registry.Route {
+ up := upstream(bound)
+ if up == "" {
+ return nil
+ }
+ all, err := reg.List()
+ if err != nil {
+ fmt.Printf(" %s: listing routes for the rollback group: %v (rolling back this host only)\n", bound.Host, err)
+ return nil
+ }
+ var out []registry.Route
+ for _, r := range all {
+ if r.Host == bound.Host || r.Unit != "" || r.App != bound.App ||
+ RouteAccount(r) != RouteAccount(bound) || upstream(r) != up {
+ continue
+ }
+ out = append(out, r)
+ }
+ return out
+}
+
+// rollbackGroup puts every group route whose link names the rejected version
+// onto restored (what bound now runs), stamps it to match, and writes a
+// refusal for bound and every group host.
+func rollbackGroup(reg *registry.Registry, optDir string, cfg passConfig, bound registry.Route,
+ ptr *ChannelPointer, rejected, restored, sha, platform string, nu *UnitNotUpError, now time.Time) {
+ hosts := []string{bound.Host}
+ for _, g := range processGroup(reg, bound) {
+ hosts = append(hosts, g.Host)
+ v, err := LiveVersion(optDir, g.Host)
+ if err != nil || v != rejected || restored == "" {
+ continue
+ }
+ if err := PointLive(optDir, g.Host, RouteAccount(g), restored, g.App); err != nil {
+ fmt.Printf(" %s: could not roll the symlink back to %s with its process: %v\n", g.Host, restored, err)
+ continue
+ }
+ if err := reg.SetVersions(g.Host, restored, g.PrevVersion); err != nil {
+ fmt.Printf(" %s: rolled back to %s but recording the version failed: %v\n", g.Host, restored, err)
+ }
+ }
+ if cfg.bakeDir == "" {
+ return
+ }
+ reason := nu.Reason
+ if j := strings.TrimSpace(nu.Journal); j != "" {
+ lines := strings.Split(j, "\n")
+ reason += " — " + lines[len(lines)-1]
+ }
+ for _, h := range hosts {
+ r := Refusal{Host: h, App: bound.App, Version: rejected, SHA256: sha, Platform: platform,
+ PointerPromotedAt: ptr.PromotedAt, RefusedAt: now.UTC(), Reason: reason}
+ if err := WriteRefusal(cfg.bakeDir, r); err != nil {
+ fmt.Printf(" %s: writing refusal record: %v\n", h, err)
+ }
+ }
+}
+
+func indent(s, pre string) string {
+ return pre + strings.ReplaceAll(strings.TrimRight(s, "\n"), "\n", "\n"+pre)
+}
diff --git a/internal/release/rollback_test.go b/internal/release/rollback_test.go
new file mode 100644
index 000000000..b7604c80c
--- /dev/null
+++ b/internal/release/rollback_test.go
@@ -0,0 +1,119 @@
+package release
+
+import (
+ "bytes"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+func groupFixtureRoute(t *testing.T, reg *registry.Registry, host, socket, addr, unit string) {
+ t.Helper()
+ if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", App: "console",
+ Channel: "canary/x", Socket: socket, Addr: addr, Unit: unit}); err != nil {
+ t.Fatalf("upsert %s: %v", host, err)
+ }
+}
+
+func TestProcessGroupKeysOnTheEffectiveUpstream(t *testing.T) {
+ reg := testRegistry(t)
+ groupFixtureRoute(t, reg, "platform.example.com", "", "127.0.0.1:8090", "console.service")
+ groupFixtureRoute(t, reg, "console.example.com", "", "127.0.0.1:8090", "")
+ groupFixtureRoute(t, reg, "other.example.com", "", "127.0.0.1:9000", "")
+ groupFixtureRoute(t, reg, "sock.example.com", "/run/x.sock", "127.0.0.1:8090", "")
+ bound, _ := reg.Get("platform.example.com")
+ got := processGroup(reg, *bound)
+ if len(got) != 1 || got[0].Host != "console.example.com" {
+ t.Fatalf("group = %+v, want only console.example.com (socket wins over addr; 9000 is another process)", got)
+ }
+}
+
+func TestProcessGroupNeverMatchesEmptyUpstreams(t *testing.T) {
+ reg := testRegistry(t)
+ groupFixtureRoute(t, reg, "a.example.com", "", "", "a.service")
+ groupFixtureRoute(t, reg, "b.example.com", "", "", "")
+ bound, _ := reg.Get("a.example.com")
+ if got := processGroup(reg, *bound); len(got) != 0 {
+ t.Fatalf("group = %+v, want none: two empty upstreams are not one process", got)
+ }
+}
+
+// The flagship shape: three hosts, one process, the unit runs the LAST
+// host's link. Hosts before it were repointed and stamped; the bound host's
+// restart reports the unit did not come up.
+func TestNotUpRollsTheWholeGroupBackAndRefusesEveryHost(t *testing.T) {
+ ctx, reg, s, pub, opt := startUnitFixture(t, "console.example.com")
+ if err := reg.SetUnit("console.example.com", ""); err != nil {
+ t.Fatal(err)
+ }
+ groupFixtureRoute(t, reg, "platform.example.com", "", "127.0.0.1:8090", "console.service")
+ if err := reg.SetUpstreams("console.example.com", "127.0.0.1:8090", "", ""); err != nil {
+ t.Fatal(err)
+ }
+ if err := PointLive(opt, "platform.example.com", "bab", "old", "console"); err != nil {
+ t.Fatal(err)
+ }
+ bake := t.TempDir()
+ restart := func(h string) error {
+ if h == "platform.example.com" {
+ return &UnitNotUpError{Unit: "console.service", Reason: "exited: status=2"}
+ }
+ return nil
+ }
+ var rolledBackHost string
+ rb := func(h string) error { rolledBackHost = h; return nil }
+ var out bytes.Buffer
+ withCapturedStdout(t, &out, func() {
+ if err := Pass(ctx, reg, s, pub, opt, restart, BakeDir(bake), WithRollbackRestart(rb)); err != nil {
+ t.Fatalf("Pass: %v", err)
+ }
+ })
+ for _, h := range []string{"console.example.com", "platform.example.com"} {
+ if v, _ := LiveVersion(opt, h); v != "old" {
+ t.Fatalf("%s live = %q, want old", h, v)
+ }
+ rt, _ := reg.Get(h)
+ if rt.Version == "new" {
+ t.Fatalf("%s still stamped new: %+v", h, rt)
+ }
+ if r, _ := ReadRefusal(bake, h); r == nil || r.Version != "new" {
+ t.Fatalf("%s refusal = %+v, want one naming new", h, r)
+ }
+ }
+ if rolledBackHost != "platform.example.com" {
+ t.Fatalf("rollback restart ran for %q, want the bound host", rolledBackHost)
+ }
+}
+
+func TestRollbackGroupLeavesOtherUpstreamAlone(t *testing.T) {
+ ctx, reg, s, pub, opt := startUnitFixture(t, "console.example.com")
+ if err := reg.SetUnit("console.example.com", "console.service"); err != nil {
+ t.Fatal(err)
+ }
+ if err := reg.SetUpstreams("console.example.com", "127.0.0.1:8090", "", ""); err != nil {
+ t.Fatal(err)
+ }
+ groupFixtureRoute(t, reg, "zz-other.example.com", "", "127.0.0.1:9000", "")
+ if err := StageArtifact(opt, "bab", "new", "console", []byte("bad build")); err != nil {
+ t.Fatal(err)
+ }
+ if err := PointLive(opt, "zz-other.example.com", "bab", "new", "console"); err != nil {
+ t.Fatal(err)
+ }
+ restart := func(h string) error {
+ if h == "console.example.com" {
+ return &UnitNotUpError{Unit: "console.service", Reason: "not ready after 30s"}
+ }
+ return nil
+ }
+ bake := t.TempDir()
+ if err := Pass(ctx, reg, s, pub, opt, restart, BakeDir(bake), WithRollbackRestart(func(string) error { return nil })); err != nil {
+ t.Fatalf("Pass: %v", err)
+ }
+ if v, _ := LiveVersion(opt, "zz-other.example.com"); v != "new" {
+ t.Fatalf("another process's route was rolled back to %q", v)
+ }
+ if r, _ := ReadRefusal(bake, "zz-other.example.com"); r != nil {
+ t.Fatalf("another process's route was refused: %+v", r)
+ }
+}