amadan

carlos / platform Public

host: settle check for a named unit after try-restart

Implements unitsettle.go with parseShow, settleUnit, unitSettleTimeout and supporting types. Uses systemctl show, journalctl, and os.Readlink to watch unit health after try-restart and detect deployment failures (restart loops, failed state, old binary). Returns release.UnitNotUpError with unit, reason, and journal tail for use in console rollback.
Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
claude 85472d8321ed91678c0f366b972ead4113a5b8c3 parent 5893dff
2 files changed, +320 −0
  • internal/host/unitsettle.go +170 −0
  • internal/host/unitsettle_test.go +150 −0
diff --git a/internal/host/unitsettle.go b/internal/host/unitsettle.go
new file mode 100644
index 000000000..69bf568b8
--- /dev/null
+++ b/internal/host/unitsettle.go
@@ -0,0 +1,170 @@
+// The settle check — D1 of the console release defences (spec
+// 2026-10-06-console-release-defences-design.md §2.2-§2.4).
+//
+// `systemctl try-restart` of a Type=simple unit returns 0 as soon as the start
+// job is queued, so it says nothing about whether the new build runs. On
+// 2026-10-06 the console's new process printed the CLI's usage and exited 2
+// after 55 ms while try-restart reported success, and adoptHost's rollback
+// never fired. settleUnit watches the unit after the restart and returns
+// *release.UnitNotUpError when the new build does not come up, which drives
+// that rollback.
+package host
+
+import (
+ "fmt"
+ "os"
+ "os/exec"
+ "regexp"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/carlosframework/platform/internal/release"
+)
+
+type unitProps struct {
+ ActiveState string
+ InvocationID string
+ MainPID int
+ NRestarts int
+ ExecPath string // ExecStart's executable path
+}
+
+type unitSystem interface {
+ Show(unit string) (unitProps, error)
+ TryRestart(unit string) error
+ Restart(unit string) error
+ ResetFailed(unit string) error
+ JournalTail(unit string, n int) string
+ Exe(pid int) (string, error)
+}
+
+type systemdUnits struct{}
+
+func (systemdUnits) Show(unit string) (unitProps, error) {
+ out, err := exec.Command("systemctl", "show", unit,
+ "-p", "ActiveState", "-p", "InvocationID", "-p", "MainPID", "-p", "NRestarts", "-p", "ExecStart").Output()
+ if err != nil {
+ return unitProps{}, fmt.Errorf("systemctl show %s: %w", unit, err)
+ }
+ return parseShow(string(out)), nil
+}
+
+func (systemdUnits) TryRestart(unit string) error { return systemctlTryRestartUnit(unit) }
+func (systemdUnits) Restart(unit string) error { return systemctlVerb("restart", unit) }
+func (systemdUnits) ResetFailed(unit string) error { return systemctlVerb("reset-failed", unit) }
+
+func (systemdUnits) JournalTail(unit string, n int) string {
+ out, _ := exec.Command("journalctl", "-u", unit, "-o", "cat", "-n", strconv.Itoa(n), "--no-pager").Output()
+ return strings.TrimSpace(string(out))
+}
+
+func (systemdUnits) Exe(pid int) (string, error) { return os.Readlink(fmt.Sprintf("/proc/%d/exe", pid)) }
+
+func systemctlVerb(verb, unit string) error {
+ out, err := exec.Command("systemctl", verb, unit).CombinedOutput()
+ if err != nil {
+ return fmt.Errorf("systemctl %s %s: %w: %s", verb, unit, err, strings.TrimSpace(string(out)))
+ }
+ return nil
+}
+
+var execPathRE = regexp.MustCompile(`path=([^ ;]+)`)
+
+func parseShow(out string) unitProps {
+ var p unitProps
+ for _, line := range strings.Split(out, "\n") {
+ k, v, ok := strings.Cut(strings.TrimSpace(line), "=")
+ if !ok {
+ continue
+ }
+ switch k {
+ case "ActiveState":
+ p.ActiveState = v
+ case "InvocationID":
+ p.InvocationID = v
+ case "MainPID":
+ p.MainPID, _ = strconv.Atoi(v)
+ case "NRestarts":
+ p.NRestarts, _ = strconv.Atoi(v)
+ case "ExecStart":
+ if m := execPathRE.FindStringSubmatch(v); m != nil {
+ p.ExecPath = m[1]
+ }
+ }
+ }
+ return p
+}
+
+type settleOpts struct {
+ Timeout time.Duration // the whole window (CARLOS_UNIT_SETTLE)
+ Poll time.Duration // 500ms
+ Hold time.Duration // no-probe readiness: active this long in one invocation
+ WantDir string // the new version's directory; MainPID's exe must be under it
+ Ready func() error // nil = no upstream to probe; Hold applies
+ Now func() time.Time
+ Sleep func(time.Duration)
+}
+
+// settleUnit watches unit after a try-restart. beforeInvocation is the
+// InvocationID before the restart; "" means the unit was not running, and an
+// inactive unit is then "nothing started", not a failure.
+func settleUnit(sys unitSystem, unit, beforeInvocation string, o settleOpts) error {
+ deadline := o.Now().Add(o.Timeout)
+ var inv string // the new invocation, once seen active
+ var activeSince time.Time
+ var lastReady error
+ notUp := func(reason string) error {
+ return &release.UnitNotUpError{Unit: unit, Reason: reason, Journal: sys.JournalTail(unit, 20)}
+ }
+ for {
+ p, err := sys.Show(unit)
+ if err != nil {
+ return notUp(err.Error())
+ }
+ if beforeInvocation == "" && p.ActiveState == "inactive" {
+ return nil
+ }
+ switch {
+ case p.ActiveState == "failed":
+ return notUp("unit entered failed state")
+ case inv != "" && p.InvocationID != inv:
+ return notUp("systemd restarted it (new invocation)")
+ case inv != "" && p.NRestarts > 0:
+ return notUp(fmt.Sprintf("systemd restarted it %d time(s)", p.NRestarts))
+ case inv == "" && p.ActiveState == "active" && p.InvocationID != beforeInvocation:
+ inv, activeSince = p.InvocationID, o.Now()
+ }
+ if inv != "" && p.ActiveState == "active" {
+ exe, err := sys.Exe(p.MainPID)
+ if err == nil && o.WantDir != "" && !strings.HasPrefix(exe, strings.TrimSuffix(o.WantDir, "/")+"/") {
+ return notUp(fmt.Sprintf("running %s, not the new build in %s", exe, o.WantDir))
+ }
+ if o.Ready != nil {
+ if lastReady = o.Ready(); lastReady == nil {
+ return nil
+ }
+ } else if o.Now().Sub(activeSince) >= o.Hold {
+ return nil
+ }
+ }
+ if !o.Now().Before(deadline) {
+ if lastReady != nil {
+ return notUp(fmt.Sprintf("not ready after %s: %v", o.Timeout, lastReady))
+ }
+ return notUp(fmt.Sprintf("not ready after %s", o.Timeout))
+ }
+ o.Sleep(o.Poll)
+ }
+}
+
+func unitSettleTimeout(getenv func(string) string) time.Duration {
+ d, err := time.ParseDuration(strings.TrimSpace(getenv("CARLOS_UNIT_SETTLE")))
+ switch {
+ case err != nil || d <= 0:
+ return 30 * time.Second
+ case d < 5*time.Second:
+ return 5 * time.Second
+ }
+ return d
+}
diff --git a/internal/host/unitsettle_test.go b/internal/host/unitsettle_test.go
new file mode 100644
index 000000000..fb92cb4ae
--- /dev/null
+++ b/internal/host/unitsettle_test.go
@@ -0,0 +1,150 @@
+package host
+
+import (
+ "errors"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/release"
+)
+
+type fakeUnits struct {
+ shows []unitProps // returned in order; the last repeats
+ n int
+ exe map[int]string
+ journal string
+ restarts []string
+}
+
+func (f *fakeUnits) Show(string) (unitProps, error) {
+ p := f.shows[min(f.n, len(f.shows)-1)]
+ f.n++
+ return p, nil
+}
+func (f *fakeUnits) TryRestart(u string) error { f.restarts = append(f.restarts, "try-restart "+u); return nil }
+func (f *fakeUnits) Restart(u string) error { f.restarts = append(f.restarts, "restart "+u); return nil }
+func (f *fakeUnits) ResetFailed(u string) error { f.restarts = append(f.restarts, "reset-failed "+u); return nil }
+func (f *fakeUnits) JournalTail(string, int) string { return f.journal }
+func (f *fakeUnits) Exe(pid int) (string, error) {
+ if e, ok := f.exe[pid]; ok {
+ return e, nil
+ }
+ return "", errors.New("no such pid")
+}
+
+// settleClock, not fakeClock: stage_relay_test.go already declares a
+// fakeClock type in this package.
+func settleClock() (func() time.Time, func(time.Duration)) {
+ t := time.Unix(0, 0)
+ return func() time.Time { return t }, func(d time.Duration) { t = t.Add(d) }
+}
+
+func opts(ready func() error) settleOpts {
+ now, sleep := settleClock()
+ return settleOpts{Timeout: 30 * time.Second, Poll: 500 * time.Millisecond, Hold: 5 * time.Second,
+ WantDir: "/opt/carlos/versions/new", Ready: ready, Now: now, Sleep: sleep}
+}
+
+func TestParseShow(t *testing.T) {
+ p := parseShow("ActiveState=active\nInvocationID=abc\nMainPID=42\nNRestarts=0\n" +
+ "ExecStart={ path=/opt/carlos/live/platform.example.com ; argv[]=/opt/carlos/live/platform.example.com serve ; ignore_errors=no }\n")
+ if p.ActiveState != "active" || p.InvocationID != "abc" || p.MainPID != 42 ||
+ p.ExecPath != "/opt/carlos/live/platform.example.com" {
+ t.Fatalf("parseShow = %+v", p)
+ }
+}
+
+func TestSettleHealthyNewInvocation(t *testing.T) {
+ f := &fakeUnits{shows: []unitProps{
+ {ActiveState: "activating", InvocationID: "new"},
+ {ActiveState: "active", InvocationID: "new", MainPID: 7},
+ }, exe: map[int]string{7: "/opt/carlos/versions/new/console"}}
+ if err := settleUnit(f, "console.service", "old", opts(func() error { return nil })); err != nil {
+ t.Fatalf("settle = %v, want healthy", err)
+ }
+}
+
+// systemd resets NRestarts on a manual start: a healthy unit that had
+// auto-restarted before must still pass (spec §2.3, codex 4).
+func TestSettleIgnoresThePreviousInvocationsCounter(t *testing.T) {
+ f := &fakeUnits{shows: []unitProps{{ActiveState: "active", InvocationID: "new", MainPID: 7, NRestarts: 0}},
+ exe: map[int]string{7: "/opt/carlos/versions/new/console"}}
+ if err := settleUnit(f, "console.service", "old", opts(func() error { return nil })); err != nil {
+ t.Fatalf("settle = %v", err)
+ }
+}
+
+// Today's outage: the process exits in 55 ms and systemd cycles it.
+func TestSettleCatchesARestartLoop(t *testing.T) {
+ f := &fakeUnits{shows: []unitProps{
+ {ActiveState: "active", InvocationID: "new1", MainPID: 7},
+ {ActiveState: "activating", InvocationID: "new2", NRestarts: 1},
+ }, exe: map[int]string{7: "/opt/carlos/versions/new/console"}, journal: "usage: carlos <command> [flags]"}
+ err := settleUnit(f, "console.service", "old", opts(func() error { return errors.New("connection refused") }))
+ var nu *release.UnitNotUpError
+ if !errors.As(err, &nu) || !strings.Contains(nu.Journal, "usage: carlos") {
+ t.Fatalf("settle = %v, want UnitNotUpError carrying the journal", err)
+ }
+}
+
+func TestSettleCatchesFailed(t *testing.T) {
+ f := &fakeUnits{shows: []unitProps{{ActiveState: "failed", InvocationID: "new"}}}
+ var nu *release.UnitNotUpError
+ if err := settleUnit(f, "console.service", "old", opts(nil)); !errors.As(err, &nu) {
+ t.Fatalf("settle = %v, want UnitNotUpError", err)
+ }
+}
+
+func TestSettleDeadlineSaysNotReady(t *testing.T) {
+ f := &fakeUnits{shows: []unitProps{{ActiveState: "active", InvocationID: "new", MainPID: 7}},
+ exe: map[int]string{7: "/opt/carlos/versions/new/console"}}
+ err := settleUnit(f, "console.service", "old", opts(func() error { return errors.New("dial: refused") }))
+ var nu *release.UnitNotUpError
+ if !errors.As(err, &nu) || !strings.Contains(nu.Reason, "not ready after 30s") {
+ t.Fatalf("settle = %v, want a not-ready deadline", err)
+ }
+}
+
+func TestSettleRejectsTheOldBinary(t *testing.T) {
+ f := &fakeUnits{shows: []unitProps{{ActiveState: "active", InvocationID: "new", MainPID: 7}},
+ exe: map[int]string{7: "/opt/carlos/versions/old/console"}}
+ var nu *release.UnitNotUpError
+ if err := settleUnit(f, "console.service", "old", opts(func() error { return nil })); !errors.As(err, &nu) {
+ t.Fatalf("settle = %v, want not-up: the process runs the old build", err)
+ }
+}
+
+func TestSettleWithoutReadyHoldsFiveSeconds(t *testing.T) {
+ f := &fakeUnits{shows: []unitProps{{ActiveState: "active", InvocationID: "new", MainPID: 7}},
+ exe: map[int]string{7: "/opt/carlos/versions/new/console"}}
+ o := opts(nil)
+ start := o.Now()
+ if err := settleUnit(f, "console.service", "old", o); err != nil {
+ t.Fatalf("settle = %v", err)
+ }
+ if got := o.Now().Sub(start); got < 5*time.Second {
+ t.Fatalf("settled after %v, want at least the 5s hold", got)
+ }
+}
+
+func TestSettleSkipsAUnitThatWasNotRunning(t *testing.T) {
+ // beforeInvocation "" means the caller saw the unit inactive; settleUnit
+ // is not called at all in that case (Task 6). This pins the caller's
+ // contract from this side: an inactive unit that stays inactive after
+ // try-restart is "nothing started", not a failure.
+ f := &fakeUnits{shows: []unitProps{{ActiveState: "inactive"}}}
+ if err := settleUnit(f, "console.service", "", opts(nil)); err != nil {
+ t.Fatalf("settle on a stopped unit = %v, want nil", err)
+ }
+}
+
+func TestUnitSettleTimeoutFloor(t *testing.T) {
+ env := func(v string) func(string) string { return func(string) string { return v } }
+ for in, want := range map[string]time.Duration{"": 30 * time.Second, "1s": 5 * time.Second,
+ "45s": 45 * time.Second, "junk": 30 * time.Second} {
+ if got := unitSettleTimeout(env(in)); got != want {
+ t.Fatalf("CARLOS_UNIT_SETTLE=%q -> %v, want %v", in, got, want)
+ }
+ }
+}