| 1 | // The settle check — D1 of the console release defences (spec |
| 2 | // 2026-10-06-console-release-defences-design.md §2.2-§2.4). |
| 3 | // |
| 4 | // `systemctl try-restart` of a Type=simple unit returns 0 as soon as the start |
| 5 | // job is queued, so it says nothing about whether the new build runs. On |
| 6 | // 2026-10-06 the console's new process printed the CLI's usage and exited 2 |
| 7 | // after 55 ms while try-restart reported success, and adoptHost's rollback |
| 8 | // never fired. settleUnit watches the unit after the restart and returns |
| 9 | // *release.UnitNotUpError when the new build does not come up, which drives |
| 10 | // that rollback. |
| 11 | package host |
| 12 | |
| 13 | import ( |
| 14 | "context" |
| 15 | "encoding/json" |
| 16 | "errors" |
| 17 | "fmt" |
| 18 | "io" |
| 19 | "log" |
| 20 | "net" |
| 21 | "net/http" |
| 22 | "os" |
| 23 | "os/exec" |
| 24 | "regexp" |
| 25 | "strconv" |
| 26 | "strings" |
| 27 | "time" |
| 28 | |
| 29 | "github.com/carlosframework/platform/internal/provision" |
| 30 | "github.com/carlosframework/platform/internal/registry" |
| 31 | "github.com/carlosframework/platform/internal/release" |
| 32 | ) |
| 33 | |
| 34 | type unitProps struct { |
| 35 | Name string |
| 36 | FragmentPath string |
| 37 | DropInPaths []string |
| 38 | ActiveState string |
| 39 | SubState string // "dead" with ActiveState inactive is a stop, not a crash |
| 40 | InvocationID string |
| 41 | MainPID int |
| 42 | NRestarts int |
| 43 | ExecPath string // ExecStart's executable path |
| 44 | |
| 45 | ActiveEnterTimestamp time.Time // when the unit last entered active; zero when unknown |
| 46 | } |
| 47 | |
| 48 | type unitSystem interface { |
| 49 | Show(unit string) (unitProps, error) |
| 50 | TryRestart(unit string) error |
| 51 | Restart(unit string) error |
| 52 | ResetFailed(unit string) error |
| 53 | JournalTail(unit string, n int) string |
| 54 | Exe(pid int) (string, error) |
| 55 | StartTime(pid int) (string, error) |
| 56 | } |
| 57 | |
| 58 | type systemdUnits struct{} |
| 59 | |
| 60 | func (systemdUnits) Show(unit string) (unitProps, error) { |
| 61 | out, err := exec.Command("systemctl", "show", unit, |
| 62 | "-p", "Id", "-p", "FragmentPath", "-p", "DropInPaths", "-p", "ActiveState", "-p", "SubState", "-p", "InvocationID", "-p", "MainPID", "-p", "NRestarts", "-p", "ExecStart", "-p", "ActiveEnterTimestamp").Output() |
| 63 | if err != nil { |
| 64 | return unitProps{}, fmt.Errorf("systemctl show %s: %w", unit, err) |
| 65 | } |
| 66 | return parseShow(string(out)), nil |
| 67 | } |
| 68 | |
| 69 | func (systemdUnits) TryRestart(unit string) error { return systemctlTryRestartUnit(unit) } |
| 70 | func (systemdUnits) Restart(unit string) error { return systemctlVerb("restart", unit) } |
| 71 | func (systemdUnits) ResetFailed(unit string) error { return systemctlVerb("reset-failed", unit) } |
| 72 | |
| 73 | func (systemdUnits) JournalTail(unit string, n int) string { |
| 74 | out, _ := exec.Command("journalctl", "-u", unit, "-o", "cat", "-n", strconv.Itoa(n), "--no-pager").Output() |
| 75 | return strings.TrimSpace(string(out)) |
| 76 | } |
| 77 | |
| 78 | func (systemdUnits) Exe(pid int) (string, error) { |
| 79 | return os.Readlink(fmt.Sprintf("/proc/%d/exe", pid)) |
| 80 | } |
| 81 | |
| 82 | func (systemdUnits) StartTime(pid int) (string, error) { |
| 83 | if pid <= 0 { |
| 84 | return "", os.ErrPermission |
| 85 | } |
| 86 | body, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid)) |
| 87 | if err != nil { |
| 88 | return "", err |
| 89 | } |
| 90 | i := strings.LastIndexByte(string(body), ')') |
| 91 | if i < 0 { |
| 92 | return "", os.ErrPermission |
| 93 | } |
| 94 | fields := strings.Fields(string(body[i+1:])) |
| 95 | if len(fields) <= 19 { |
| 96 | return "", os.ErrPermission |
| 97 | } |
| 98 | if n, err := strconv.ParseUint(fields[19], 10, 64); err != nil || n == 0 { |
| 99 | return "", os.ErrPermission |
| 100 | } |
| 101 | return fields[19], nil |
| 102 | } |
| 103 | |
| 104 | func systemctlVerb(verb, unit string) error { |
| 105 | out, err := exec.Command("systemctl", verb, unit).CombinedOutput() |
| 106 | if err != nil { |
| 107 | return fmt.Errorf("systemctl %s %s: %w: %s", verb, unit, err, strings.TrimSpace(string(out))) |
| 108 | } |
| 109 | return nil |
| 110 | } |
| 111 | |
| 112 | var execPathRE = regexp.MustCompile(`path=([^ ;]+)`) |
| 113 | |
| 114 | func parseShow(out string) unitProps { |
| 115 | var p unitProps |
| 116 | for _, line := range strings.Split(out, "\n") { |
| 117 | k, v, ok := strings.Cut(strings.TrimSpace(line), "=") |
| 118 | if !ok { |
| 119 | continue |
| 120 | } |
| 121 | switch k { |
| 122 | case "Id": |
| 123 | p.Name = v |
| 124 | case "FragmentPath": |
| 125 | p.FragmentPath = v |
| 126 | case "DropInPaths": |
| 127 | p.DropInPaths = strings.Fields(v) |
| 128 | case "ActiveState": |
| 129 | p.ActiveState = v |
| 130 | case "SubState": |
| 131 | p.SubState = v |
| 132 | case "InvocationID": |
| 133 | p.InvocationID = v |
| 134 | case "MainPID": |
| 135 | p.MainPID, _ = strconv.Atoi(v) |
| 136 | case "NRestarts": |
| 137 | p.NRestarts, _ = strconv.Atoi(v) |
| 138 | case "ExecStart": |
| 139 | if m := execPathRE.FindStringSubmatch(v); m != nil { |
| 140 | p.ExecPath = m[1] |
| 141 | } |
| 142 | case "ActiveEnterTimestamp": |
| 143 | p.ActiveEnterTimestamp = parseSystemdTime(v) |
| 144 | } |
| 145 | } |
| 146 | return p |
| 147 | } |
| 148 | |
| 149 | // parseSystemdTime reads systemctl show's default timestamp form, |
| 150 | // "Mon 2026-10-06 09:30:30 UTC". Boxes run in UTC. "" and "n/a" are zero, |
| 151 | // which every caller reads as "unknown", never as "long ago". |
| 152 | func parseSystemdTime(v string) time.Time { |
| 153 | t, err := time.Parse("Mon 2006-01-02 15:04:05 MST", strings.TrimSpace(v)) |
| 154 | if err != nil { |
| 155 | return time.Time{} |
| 156 | } |
| 157 | return t |
| 158 | } |
| 159 | |
| 160 | type settleOpts struct { |
| 161 | RequireExe bool |
| 162 | Timeout time.Duration // the whole window (CARLOS_UNIT_SETTLE) |
| 163 | Poll time.Duration // 500ms |
| 164 | Hold time.Duration // no-probe readiness: active this long in one invocation |
| 165 | WantDir string // the new version's directory; MainPID's exe must be under it |
| 166 | Ready func() error // nil = no upstream to probe; Hold applies |
| 167 | Now func() time.Time |
| 168 | Sleep func(time.Duration) |
| 169 | } |
| 170 | |
| 171 | // errUnitStopped is settleUnit's answer for a unit that was stopped while it |
| 172 | // was being watched (inactive, SubState dead): an operator's stop, not a |
| 173 | // crash. It is neither a healthy build nor a failed one, so the caller judges |
| 174 | // nothing and restarts nothing. |
| 175 | var errUnitStopped = errors.New("unit was stopped during settle") |
| 176 | |
| 177 | // settleUnit watches unit after a try-restart. beforeInvocation is the |
| 178 | // InvocationID before the restart; "" means the unit was not running, and an |
| 179 | // inactive unit is then "nothing started", not a failure. Otherwise a unit |
| 180 | // that goes inactive/dead returns errUnitStopped. |
| 181 | func settleUnit(sys unitSystem, unit, beforeInvocation string, o settleOpts) error { |
| 182 | deadline := o.Now().Add(o.Timeout) |
| 183 | var inv string // the new invocation, once seen active |
| 184 | var activeSince time.Time |
| 185 | var lastReady error |
| 186 | var lastExeErr error |
| 187 | // wrongExe is the executable MainPID ran on this poll when it was not |
| 188 | // under WantDir. Not a failure on its own: for Type=simple systemd reports |
| 189 | // the new invocation active as soon as it has forked, BEFORE it execs the |
| 190 | // ExecStart binary, so /proc/<pid>/exe can briefly name systemd's own |
| 191 | // executor (systemd-executor, or systemd itself on older releases). It is |
| 192 | // treated like not-ready, and only a deadline that still sees it fails. |
| 193 | var wrongExe string |
| 194 | var binCheckSkipped bool // permission error on Exe; skip binary check for rest of settle |
| 195 | notUp := func(reason string) error { |
| 196 | return &release.UnitNotUpError{Unit: unit, Reason: reason, Journal: sys.JournalTail(unit, 20)} |
| 197 | } |
| 198 | for { |
| 199 | p, err := sys.Show(unit) |
| 200 | if err != nil { |
| 201 | // One failed `systemctl show` says nothing about the build (a |
| 202 | // busy bus, a timed-out D-Bus call). Retried until the deadline, |
| 203 | // and only a deadline that still cannot observe the unit fails. |
| 204 | if !o.Now().Before(deadline) { |
| 205 | return notUp(fmt.Sprintf("cannot observe %s: %v", unit, err)) |
| 206 | } |
| 207 | o.Sleep(o.Poll) |
| 208 | continue |
| 209 | } |
| 210 | // Inactive with SubState dead and nothing pending is a STOP: a crash |
| 211 | // reads failed, and a crash systemd will restart reads activating |
| 212 | // (auto-restart). try-restart blocks until its start job is done, so |
| 213 | // this is not the gap between the stop and start halves of the |
| 214 | // restart either. Somebody stopped the unit on purpose, before or |
| 215 | // during this check; the caller must neither judge nor restart it. |
| 216 | if beforeInvocation != "" && p.ActiveState == "inactive" && p.SubState == "dead" { |
| 217 | return errUnitStopped |
| 218 | } |
| 219 | if beforeInvocation == "" && p.ActiveState == "inactive" { |
| 220 | return nil |
| 221 | } |
| 222 | // NRestarts check on fresh invocation, before the switch |
| 223 | if p.InvocationID != beforeInvocation && p.InvocationID != "" && p.NRestarts > 0 { |
| 224 | return notUp(fmt.Sprintf("systemd restarted it %d time(s)", p.NRestarts)) |
| 225 | } |
| 226 | switch { |
| 227 | case p.ActiveState == "failed": |
| 228 | return notUp("unit entered failed state") |
| 229 | case inv != "" && p.InvocationID != inv: |
| 230 | return notUp("systemd restarted it (new invocation)") |
| 231 | case inv != "" && p.NRestarts > 0: |
| 232 | return notUp(fmt.Sprintf("systemd restarted it %d time(s)", p.NRestarts)) |
| 233 | case inv == "" && p.ActiveState == "active" && p.InvocationID != beforeInvocation: |
| 234 | inv, activeSince = p.InvocationID, o.Now() |
| 235 | } |
| 236 | wrongExe = "" |
| 237 | if inv != "" && p.ActiveState == "active" { |
| 238 | // Check the binary only if we haven't hit a permission error |
| 239 | if !binCheckSkipped { |
| 240 | exe, err := sys.Exe(p.MainPID) |
| 241 | if err != nil { |
| 242 | if errors.Is(err, os.ErrPermission) && !o.RequireExe { |
| 243 | // Permission error; log once and skip binary check for rest |
| 244 | log.Printf("adopt: %s: cannot read /proc/%d/exe (%v); binary check skipped", unit, p.MainPID, err) |
| 245 | binCheckSkipped = true |
| 246 | } else { |
| 247 | // Other error; remember it and skip Ready/Hold on this poll |
| 248 | lastExeErr = err |
| 249 | // Continue to next poll without calling Ready/Hold |
| 250 | } |
| 251 | } else if o.WantDir != "" && !strings.HasPrefix(exe, strings.TrimSuffix(o.WantDir, "/")+"/") { |
| 252 | wrongExe = exe // not yet the new build: not ready, poll again |
| 253 | } |
| 254 | } |
| 255 | // Only call Ready/Hold once this poll saw the new build running |
| 256 | if lastExeErr == nil && wrongExe == "" { |
| 257 | if o.Ready != nil { |
| 258 | if lastReady = o.Ready(); lastReady == nil { |
| 259 | return nil |
| 260 | } |
| 261 | } else if o.Now().Sub(activeSince) >= o.Hold { |
| 262 | return nil |
| 263 | } |
| 264 | } |
| 265 | } |
| 266 | if !o.Now().Before(deadline) { |
| 267 | if wrongExe != "" { |
| 268 | return notUp(fmt.Sprintf("running %s, not the new build in %s", wrongExe, o.WantDir)) |
| 269 | } |
| 270 | if lastExeErr != nil { |
| 271 | return notUp(fmt.Sprintf("exe unreadable: %v", lastExeErr)) |
| 272 | } |
| 273 | if lastReady != nil { |
| 274 | return notUp(fmt.Sprintf("not ready after %s: %v", o.Timeout, lastReady)) |
| 275 | } |
| 276 | return notUp(fmt.Sprintf("not ready after %s", o.Timeout)) |
| 277 | } |
| 278 | lastExeErr = nil // reset exe error for next poll |
| 279 | o.Sleep(o.Poll) |
| 280 | } |
| 281 | } |
| 282 | |
| 283 | func unitSettleTimeout(getenv func(string) string) time.Duration { |
| 284 | d, err := time.ParseDuration(strings.TrimSpace(getenv("CARLOS_UNIT_SETTLE"))) |
| 285 | switch { |
| 286 | case err != nil || d <= 0: |
| 287 | return 30 * time.Second |
| 288 | case d < 5*time.Second: |
| 289 | return 5 * time.Second |
| 290 | } |
| 291 | return d |
| 292 | } |
| 293 | |
| 294 | // routeReady is the readiness check for rt's effective upstream (spec §2.4): |
| 295 | // Socket wins over Addr. With a configured probe it is an HTTP GET; without |
| 296 | // one, a connect. nil when the route has no upstream: settle then requires |
| 297 | // the unit to stay active for its Hold. |
| 298 | func routeReady(rt registry.Route, timeout time.Duration) func() error { |
| 299 | network, address := "unix", rt.Socket |
| 300 | if address == "" { |
| 301 | network, address = "tcp", rt.Addr |
| 302 | } |
| 303 | if address == "" { |
| 304 | return nil |
| 305 | } |
| 306 | if rt.Health == "" { |
| 307 | return func() error { |
| 308 | c, err := net.DialTimeout(network, address, timeout) |
| 309 | if err != nil { |
| 310 | return fmt.Errorf("connect %s: %w", address, err) |
| 311 | } |
| 312 | return c.Close() |
| 313 | } |
| 314 | } |
| 315 | var h provision.Health |
| 316 | if err := json.Unmarshal([]byte(rt.Health), &h); err != nil { |
| 317 | return func() error { return fmt.Errorf("route health probe does not parse: %w", err) } |
| 318 | } |
| 319 | h = h.WithDefaults() |
| 320 | hostHdr := h.HostHeader |
| 321 | if hostHdr == "" { |
| 322 | hostHdr = rt.Host |
| 323 | } |
| 324 | client := &http.Client{Timeout: timeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, Transport: &http.Transport{ |
| 325 | DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { |
| 326 | var d net.Dialer |
| 327 | return d.DialContext(ctx, network, address) |
| 328 | }, |
| 329 | }} |
| 330 | return func() error { |
| 331 | req, err := http.NewRequest(http.MethodGet, "http://upstream"+h.Path, nil) |
| 332 | if err != nil { |
| 333 | return err |
| 334 | } |
| 335 | req.Host = hostHdr |
| 336 | resp, err := client.Do(req) |
| 337 | if err != nil { |
| 338 | return fmt.Errorf("dial: %v", err) |
| 339 | } |
| 340 | io.Copy(io.Discard, io.LimitReader(resp.Body, 4<<10)) |
| 341 | resp.Body.Close() |
| 342 | if resp.StatusCode != h.Status { |
| 343 | return fmt.Errorf("status %d (want %d)", resp.StatusCode, h.Status) |
| 344 | } |
| 345 | return nil |
| 346 | } |
| 347 | } |
| 348 | |
amadan