carlos / platform Public

host: named-unit restarts settle and roll back by restart (console defences D1)

checkedUnitRestart verifies the unit's binding then settles; rollbackUnitRestart resets, restarts and settles the restored build. Wired into the adopt loop, fast loop and RunAdopt.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

claude 2 weeks ago fb6d12fbd071cc53bbbdd8bb588395964bd9f796 parent b429670

4 files changed, +201 −32

internal/host/agent.go +89 −9
internal/host/agent_test.go +23 −18
internal/host/converge_adopted_test.go +5 −5
internal/host/unitsettle_test.go +84 −0

diff --git a/internal/host/agent.go b/internal/host/agent.go

index 328c83bef..b124c4d13 100644

--- a/internal/host/agent.go

+++ b/internal/host/agent.go

@@ -817,7 +817,7 @@ func adoptLoop(reg *registry.Registry, svc *activator.Service, sd *scheduleDeps)

return

}

optDir := release.OptDir()

- restart := makeRestartRoute(reg, systemctlTryRestart, systemctlTryRestartUnit, adoptBounceFn(svc))

+ restart := makeRestartRoute(reg, systemctlTryRestart, checkedUnitRestart(systemdUnits{}, optDir, os.Getenv), adoptBounceFn(svc))

log.Printf("agent: adopt loop armed (hourly + jitter)")

// First pass shortly after boot (Eleven's OnBootSec=5min, compressed —

// the edge is already up by the time this fires), then hourly + jitter.

@@ -868,7 +868,8 @@ func adoptLoop(reg *registry.Registry, svc *activator.Service, sd *scheduleDeps)

if err := release.Pass(ctx, reg, store, pub, optDir, restart,

release.BakeDir(dataDir()), release.OnStageBlocked(stageRequester(dataDir())),

release.RestartHibernating(restart),

- release.WithStartUnit(makeStartRoute(reg, systemctlStartUnit))); err != nil {

+ release.WithStartUnit(makeStartRoute(reg, systemctlStartUnit)),

+ release.WithRollbackRestart(rollbackUnitRestart(reg, systemdUnits{}, optDir, os.Getenv))); err != nil {

log.Printf("agent: adopt pass: %v", err)

}

// The schedule converge's hourly floor (schedule spec §6.2), AFTER the

@@ -1928,10 +1929,13 @@ func fastConvergeLoop(reg *registry.Registry, svc *activator.Service, sd *schedu

// sweep instead of the sweep re-cycling the service every tick. The

// remaining fields are filled in right below.

d := &fastDeps{reg: reg, store: store, pub: pub, optDir: optDir}

- restart := d.observeRestart(makeRestartRoute(reg, systemctlTryRestart, systemctlTryRestartUnit, adoptBounceFn(svc)))

+ restart := d.observeRestart(makeRestartRoute(reg, systemctlTryRestart, checkedUnitRestart(systemdUnits{}, optDir, os.Getenv), adoptBounceFn(svc)))

// Built once, used by both pass shapes below — see fastPassOpts for the

// DO-NOT list and why the stage requester is per-loop, not per-tick.

opts := fastPassOpts(dd, stageRequester(dd), restart, makeStartRoute(reg, systemctlStartUnit))

+ // The rollback's restart (D1): checked like the forward one, and a replace

+ // rather than a start, so a rejected build left up is cycled out.

+ opts = append(opts, release.WithRollbackRestart(rollbackUnitRestart(reg, systemdUnits{}, optDir, os.Getenv)))

// The restart check's memory (issue #113, narrowed requests added by

// discussion #16), seeded ONCE here — two reads per app at arming (the

// app-wide generation and the narrowed request log) and never again except

@@ -3098,14 +3102,14 @@ func statusTick(ctx context.Context, s release.Store, reg *registry.Registry, p

// merely unattended. Returned, adoptHost rolls the symlink back to the

// version that IS running, skips SetVersions, prints the failure, and the

// next pass retries; the header stays honest on the old version throughout.

-func makeRestartRoute(reg *registry.Registry, tryRestart func(host string), tryRestartUnit func(unit string) error, bounce func(host string) bool) func(host string) error {

+func makeRestartRoute(reg *registry.Registry, tryRestart func(host string), tryRestartUnit func(rt registry.Route) error, bounce func(host string) bool) func(host string) error {

return func(host string) error {

rt, err := reg.Get(host)

if err != nil {

log.Printf("adopt: registry lookup %s: %v (restarting as a unit)", host, err)

}

if err == nil && rt != nil && rt.Unit != "" {

- return tryRestartUnit(rt.Unit)

+ return tryRestartUnit(*rt)

}

if err == nil && rt != nil && rt.Backing != registry.BackingUnit {

if bounce != nil && bounce(host) {

@@ -3201,6 +3205,81 @@ func systemctlTryRestartUnit(unit string) error {

return nil

}

+// unitOnly adapts a unit-name restart to makeRestartRoute's route-aware arm.

+func unitOnly(f func(unit string) error) func(rt registry.Route) error {

+ return func(rt registry.Route) error { return f(rt.Unit) }

+// checkedUnitRestart is the named-unit arm with D1's settle check (spec

+// §2.2-§2.3). The binding is verified first: the check judges a restart only

+// when the unit's ExecStart runs THIS route's live link. Anything it cannot

+// establish (show denied, wrong binding, unit stopped) falls back to today's

+// plain try-restart and returns that command's own error.

+func checkedUnitRestart(sys unitSystem, optDir string, getenv func(string) string) func(rt registry.Route) error {

+ return func(rt registry.Route) error {

+ before, err := sys.Show(rt.Unit)

+ if err != nil {

+ log.Printf("adopt: %s: cannot read %s (%v); restart not checked", rt.Host, rt.Unit, err)

+ return sys.TryRestart(rt.Unit)

+ }

+ link := filepath.Join(optDir, "live", rt.Host)

+ if before.ExecPath != link {

+ log.Printf("adopt: binding: %s runs %s, but the unit stamp is on %s; restart not checked", rt.Unit, before.ExecPath, rt.Host)

+ return sys.TryRestart(rt.Unit)

+ }

+ if before.ActiveState != "active" {

+ return sys.TryRestart(rt.Unit) // stopped by an operator: stays a no-op

+ }

+ want, err := filepath.EvalSymlinks(link)

+ if err != nil {

+ return sys.TryRestart(rt.Unit)

+ }

+ if err := sys.TryRestart(rt.Unit); err != nil {

+ return err

+ }

+ timeout := unitSettleTimeout(getenv)

+ err = settleUnit(sys, rt.Unit, before.InvocationID, settleOpts{

+ Timeout: timeout, Poll: 500 * time.Millisecond, Hold: 5 * time.Second,

+ WantDir: filepath.Dir(want), Ready: routeReady(rt, 2*time.Second),

+ Now: time.Now, Sleep: time.Sleep})

+ if err == nil {

+ log.Printf("adopt: %s: %s settled on %s", rt.Host, rt.Unit, filepath.Dir(want))

+ }

+ return err

+ }

+// rollbackUnitRestart is D1's rollback restart (release.WithRollbackRestart):

+// the live link already names the restored build; replace whatever is

+// running (a deadline failure can leave the rejected build up) and check

+// the restored build comes up too.

+func rollbackUnitRestart(reg *registry.Registry, sys unitSystem, optDir string, getenv func(string) string) func(host string) error {

+ return func(host string) error {

+ rt, err := reg.Get(host)

+ if err != nil || rt == nil || rt.Unit == "" {

+ return nil

+ }

+ before, _ := sys.Show(rt.Unit)

+ if err := sys.ResetFailed(rt.Unit); err != nil {

+ log.Printf("adopt: %s: reset-failed %s: %v (continuing)", host, rt.Unit, err)

+ }

+ if err := sys.Restart(rt.Unit); err != nil {

+ return err

+ }

+ want, err := filepath.EvalSymlinks(filepath.Join(optDir, "live", host))

+ if err != nil {

+ return err

+ }

+ if err := settleUnit(sys, rt.Unit, before.InvocationID, settleOpts{

+ Timeout: unitSettleTimeout(getenv), Poll: 500 * time.Millisecond, Hold: 5 * time.Second,

+ WantDir: filepath.Dir(want), Ready: routeReady(*rt, 2*time.Second),

+ Now: time.Now, Sleep: time.Sleep}); err != nil {

+ return fmt.Errorf("rollback did not come up either: %w", err)

+ }

+ return nil

+ }

// systemctlStartUnit is makeStartRoute's production path: bring the named

// unit up after a rolled-back restart left it down. START, not try-restart —

// try-restart is a no-op on a stopped unit, which on this path would leave

@@ -3239,8 +3318,8 @@ func systemctlStartUnit(unit string) error {

// staging, the repoint, the restart and the version stamp. A restart arm

// wired only into the agent would therefore be a restart arm the consoles

// never reach.

-func rootRestartRoute(reg *registry.Registry) func(host string) error {

- return makeRestartRoute(reg, systemctlTryRestart, systemctlTryRestartUnit, nil)

+func rootRestartRoute(reg *registry.Registry, optDir string) func(host string) error {

+ return makeRestartRoute(reg, systemctlTryRestart, checkedUnitRestart(systemdUnits{}, optDir, os.Getenv), nil)

}

// bounceFn adapts an activator to makeRestartRoute's bounce parameter,

@@ -3533,7 +3612,7 @@ func RunAdopt(args []string) {

// -unit` refuses to stamp — so config sync sees no unit-named route by

// construction. It is handed the same callback anyway because there should

// be ONE restart dispatch on this pass, not two that could drift.

- restart := rootRestartRoute(reg)

+ restart := rootRestartRoute(reg, release.OptDir())

// The restart-request half, and the ONE place it dispatches from —

// including for exec-backed routes, whose children belong to the agent:

// rootRestartRoute's try-restart arm reaches a carlos-app@ unit, and an

@@ -3658,7 +3737,8 @@ func RunAdopt(args []string) {

// run). -stage-only keeps it for the same reason it keeps the converge:

// a relayed stage is exactly the request that performs such a restart.

passOpts := []release.PassOption{release.WithCapabilities(caps.Ops{}), release.BakeDir(dataDir()),

- release.WithStartUnit(makeStartRoute(reg, systemctlStartUnit))}

+ release.WithStartUnit(makeStartRoute(reg, systemctlStartUnit)),

+ release.WithRollbackRestart(rollbackUnitRestart(reg, systemdUnits{}, release.OptDir(), os.Getenv))}

if !*stageOnly {

passOpts = append(passOpts, release.NagExpiredCanaries(os.Stdout))

}

diff --git a/internal/host/agent_test.go b/internal/host/agent_test.go

index ca22797b1..fcdd5ee2a 100644

--- a/internal/host/agent_test.go

+++ b/internal/host/agent_test.go

@@ -551,10 +551,10 @@ func TestMakeRestartRouteDispatchesOnBacking(t *testing.T) {

var restarted, bounced []string

restart := makeRestartRoute(reg,

func(h string) { restarted = append(restarted, h) },

- func(u string) error {

+ unitOnly(func(u string) error {

t.Fatalf("named-unit arm fired for %q on a route that names no unit", u)

return nil

- },

+ }),

func(h string) bool { bounced = append(bounced, h); return true })

if err := restart("unit.test"); err != nil {

t.Fatalf("restart unit.test: %v", err)

@@ -577,10 +577,10 @@ func TestMakeRestartRouteUnknownHostFallsBackToUnit(t *testing.T) {

reg := restartTestRegistry(t)

var restarted []string

restart := makeRestartRoute(reg, func(h string) { restarted = append(restarted, h) },

- func(u string) error {

+ unitOnly(func(u string) error {

t.Fatalf("named-unit arm fired for %q on a host the registry cannot answer for", u)

return nil

- }, nil)

+ }), nil)

if err := restart("ghost.test"); err != nil {

t.Fatalf("restart ghost.test: %v", err)

}

@@ -616,10 +616,10 @@ func TestMakeRestartRouteExecFallsThroughWhenNothingBounced(t *testing.T) {

}

var restarted []string

restart := makeRestartRoute(reg, func(h string) { restarted = append(restarted, h) },

- func(u string) error {

+ unitOnly(func(u string) error {

t.Fatalf("named-unit arm fired for %q on a route that names no unit", u)

return nil

- }, tc.bounce)

+ }), tc.bounce)

if err := restart("exec.test"); err != nil {

t.Fatalf("restart exec.test: %v", err)

}

@@ -647,7 +647,7 @@ func TestMakeRestartRouteRestartsTheUnitARouteNames(t *testing.T) {

var units, restarted, bounced []string

restart := makeRestartRoute(reg,

func(h string) { restarted = append(restarted, h) },

- func(u string) error { units = append(units, u); return nil },

+ unitOnly(func(u string) error { units = append(units, u); return nil }),

func(h string) bool { bounced = append(bounced, h); return true })

if err := restart("console.carloku.com"); err != nil {

t.Fatalf("restart console.carloku.com: %v", err)

@@ -679,7 +679,7 @@ func TestMakeRestartRouteUnitNameBeatsBacking(t *testing.T) {

var units, restarted []string

restart := makeRestartRoute(reg,

func(h string) { restarted = append(restarted, h) },

- func(u string) error { units = append(units, u); return nil },

+ unitOnly(func(u string) error { units = append(units, u); return nil }),

nil)

if err := restart("odd.test"); err != nil {

t.Fatalf("restart odd.test: %v", err)

@@ -715,14 +715,17 @@ func TestRootRestartRouteCarriesTheNamedUnitArm(t *testing.T) {

}

t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))

- if err := rootRestartRoute(reg)("console.carloku.com"); err != nil {

+ if err := rootRestartRoute(reg, t.TempDir())("console.carloku.com"); err != nil {

t.Fatalf("root restart: %v", err)

}

got, err := os.ReadFile(callLog)

if err != nil {

t.Fatalf("the root pass ran no systemctl at all: %v", err)

}

- if want := "try-restart console.service\n"; string(got) != want {

+ // The checked arm reads the unit first (show), finds no binding it can

+ // verify from the fake's empty answer, and falls back to the plain

+ // try-restart: the argv this test pins is the last call.

+ if want := "show console.service -p ActiveState -p InvocationID -p MainPID -p NRestarts -p ExecStart\ntry-restart console.service\n"; string(got) != want {

t.Fatalf("systemctl calls = %q, want %q", got, want)

}

@@ -762,7 +765,7 @@ func TestAdoptPassRestartsANamedUnitOnAdoption(t *testing.T) {

var units []string

restart := makeRestartRoute(reg,

func(h string) { t.Fatalf("carlos-app@ arm fired for %q — the console has no such unit", h) },

- func(u string) error { units = append(units, u); return nil },

+ unitOnly(func(u string) error { units = append(units, u); return nil }),

func(h string) bool {

t.Fatalf("bounce arm fired for %q — the activator owns no console child", h)

return false

@@ -833,7 +836,7 @@ func TestAdoptPassRefusesToStampAVersionAWrongUnitDidNotStart(t *testing.T) {

log.SetOutput(&journal)

t.Cleanup(func() { log.SetOutput(os.Stderr) })

- restart := rootRestartRoute(reg)

+ restart := rootRestartRoute(reg, optDir)

if err := release.Pass(ctx, reg, s, pub, optDir, restart); err != nil {

t.Fatalf("Pass: %v", err)

}

@@ -960,7 +963,7 @@ func TestAdoptPassStartsTheNamedUnitAgainAfterAFailedRestart(t *testing.T) {

log.SetOutput(&journal)

t.Cleanup(func() { log.SetOutput(os.Stderr) })

- if err := release.Pass(ctx, reg, s, pub, optDir, rootRestartRoute(reg),

+ if err := release.Pass(ctx, reg, s, pub, optDir, rootRestartRoute(reg, optDir),

release.WithStartUnit(makeStartRoute(reg, systemctlStartUnit))); err != nil {

t.Fatalf("Pass: %v", err)

}

@@ -969,7 +972,9 @@ func TestAdoptPassStartsTheNamedUnitAgainAfterAFailedRestart(t *testing.T) {

if err != nil {

t.Fatalf("no systemctl calls at all: %v", err)

}

- if want := "try-restart console.service\nstart console.service\n"; string(got) != want {

+ // The checked arm's show comes first (its empty fake answer verifies no

+ // binding, so the restart falls back to the plain try-restart).

+ if want := "show console.service -p ActiveState -p InvocationID -p MainPID -p NRestarts -p ExecStart\ntry-restart console.service\nstart console.service\n"; string(got) != want {

t.Fatalf("systemctl calls = %q, want %q — the rolled-back unit must be STARTED (try-restart would no-op on it)", got, want)

}

if v, err := release.LiveVersion(optDir, "console.test"); err != nil || v != "old" {

@@ -1032,10 +1037,10 @@ func TestAdoptPassBouncesAWarmHibernatingInstance(t *testing.T) {

// it never starts anything.

restart := makeRestartRoute(reg,

func(h string) { tried = append(tried, h) },

- func(u string) error {

+ unitOnly(func(u string) error {

t.Fatalf("named-unit arm fired for %q on a route that names no unit", u)

return nil

- },

+ }),

func(h string) bool {

bounced = append(bounced, h)

return h == "warm.test"

@@ -1378,10 +1383,10 @@ func TestFastSweepBouncesAWarmHibernatingCanary(t *testing.T) {

var bounced []string

restart := makeRestartRoute(env.reg,

func(string) {},

- func(u string) error {

+ unitOnly(func(u string) error {

t.Fatalf("named-unit arm fired for %q on a route that names no unit", u)

return nil

- },

+ }),

func(h string) bool { bounced = append(bounced, h); return true })

d := env.deps(nil)

d.restart = restart

diff --git a/internal/host/converge_adopted_test.go b/internal/host/converge_adopted_test.go

index ea527cd27..70c0346f6 100644

--- a/internal/host/converge_adopted_test.go

+++ b/internal/host/converge_adopted_test.go

@@ -119,7 +119,7 @@ func TestRootAdoptionOfAWarmExecInstanceIsFinishedByTheAgent(t *testing.T) {

// rootRestartRoute with systemctl stubbed out: same dispatch, bounce nil.

rootPass := func() {

t.Helper()

- restart := makeRestartRoute(env.reg, func(string) {}, func(string) error { return nil }, nil)

+ restart := makeRestartRoute(env.reg, func(string) {}, unitOnly(func(string) error { return nil }), nil)

if err := release.Pass(ctx, env.reg, env.store, env.pub, env.optDir, restart, release.BakeDir(env.dataDir)); err != nil {

t.Fatalf("root pass: %v", err)

}

@@ -147,7 +147,7 @@ func TestRootAdoptionOfAWarmExecInstanceIsFinishedByTheAgent(t *testing.T) {

// The agent's tick. Its pass has a bounce to offer and never gets to use

// it: the host is already up to date.

var bounced, cycled []string

- restart := makeRestartRoute(env.reg, func(string) {}, func(string) error { return nil },

+ restart := makeRestartRoute(env.reg, func(string) {}, unitOnly(func(string) error { return nil }),

func(h string) bool { bounced = append(bounced, h); return true })

d := env.deps(nil)

d.restart = restart

@@ -301,7 +301,7 @@ func TestConvergeLegCyclesARootAdoptedInstanceOnARealActivator(t *testing.T) {

t.Fatalf("Promote %s: %v", ver, err)

}

// The root pass: rootRestartRoute's dispatch (bounce nil), systemctl stubbed.

- restart := makeRestartRoute(env.reg, func(string) {}, func(string) error { return nil }, nil)

+ restart := makeRestartRoute(env.reg, func(string) {}, unitOnly(func(string) error { return nil }), nil)

if err := release.Pass(ctx, env.reg, env.store, env.pub, env.optDir, restart, release.BakeDir(env.dataDir)); err != nil {

t.Fatalf("root pass for %s: %v", ver, err)

}

@@ -400,10 +400,10 @@ func TestAdoptRestartDispatchSparesAConvergedChild(t *testing.T) {

t.Fatalf("wake request = %d", rec.Code)

}

- restart := makeRestartRoute(env.reg, func(string) {}, func(u string) error {

+ restart := makeRestartRoute(env.reg, func(string) {}, unitOnly(func(u string) error {

t.Fatalf("named-unit arm fired for %q on a route that names no unit", u)

return nil

- }, adoptBounceFn(svc))

+ }), adoptBounceFn(svc))

// Asked of the spawn count, after a request — see the note in

// TestConvergeLegCyclesARootAdoptedInstanceOnARealActivator.

diff --git a/internal/host/unitsettle_test.go b/internal/host/unitsettle_test.go

index 2ec404380..e2ff0a97d 100644

--- a/internal/host/unitsettle_test.go

+++ b/internal/host/unitsettle_test.go

@@ -286,3 +286,87 @@ func TestRouteReadyDoesNotFollowRedirects(t *testing.T) {

t.Fatalf("ready expecting 302 = %v, want nil", err)

}

+func boundRoute() registry.Route {

+ return registry.Route{Host: "platform.example.com", App: "console", Unit: "console.service", Addr: "127.0.0.1:1"}

+func liveTo(t *testing.T, opt, host, ver string) {

+ t.Helper()

+ dir := filepath.Join(opt, "versions", ver)

+ if err := os.MkdirAll(dir, 0o755); err != nil {

+ t.Fatal(err)

+ }

+ if err := os.WriteFile(filepath.Join(dir, "console"), []byte("x"), 0o755); err != nil {

+ t.Fatal(err)

+ }

+ if err := os.MkdirAll(filepath.Join(opt, "live"), 0o755); err != nil {

+ t.Fatal(err)

+ }

+ os.Remove(filepath.Join(opt, "live", host))

+ if err := os.Symlink(filepath.Join(dir, "console"), filepath.Join(opt, "live", host)); err != nil {

+ t.Fatal(err)

+ }

+func TestNamedRestartBindingMismatchKeepsTheError(t *testing.T) {

+ opt := t.TempDir()

+ liveTo(t, opt, "platform.example.com", "new")

+ f := &fakeUnits{shows: []unitProps{{ActiveState: "active", InvocationID: "old",

+ ExecPath: filepath.Join(opt, "live", "console.example.com")}}}

+ failing := &failingTry{fakeUnits: f}

+ err := checkedUnitRestart(failing, opt, func(string) string { return "" })(boundRoute())

+ if err == nil || !strings.Contains(err.Error(), "try-restart failed") {

+ t.Fatalf("err = %v, want the try-restart error propagated (codex 12)", err)

+ }

+type failingTry struct{ *fakeUnits }

+func (failingTry) TryRestart(string) error { return errors.New("try-restart failed") }

+func TestNamedRestartShowFailsFallsBack(t *testing.T) {

+ opt := t.TempDir()

+ sys := &showFails{fakeUnits: &fakeUnits{}}

+ if err := checkedUnitRestart(sys, opt, func(string) string { return "" })(boundRoute()); err != nil {

+ t.Fatalf("err = %v, want today's behaviour (plain try-restart, its nil)", err)

+ }

+ if len(sys.restarts) != 1 || sys.restarts[0] != "try-restart console.service" {

+ t.Fatalf("restarts = %v", sys.restarts)

+ }

+type showFails struct{ *fakeUnits }

+func (showFails) Show(string) (unitProps, error) { return unitProps{}, errors.New("Access denied") }

+func TestNamedRestartStoppedUnitIsNotChecked(t *testing.T) {

+ opt := t.TempDir()

+ liveTo(t, opt, "platform.example.com", "new")

+ f := &fakeUnits{shows: []unitProps{{ActiveState: "inactive", ExecPath: filepath.Join(opt, "live", "platform.example.com")}}}

+ if err := checkedUnitRestart(f, opt, func(string) string { return "" })(boundRoute()); err != nil {

+ t.Fatalf("err = %v, want nil for an operator-stopped unit", err)

+ }

+func TestRollbackRestartResetsThenRestarts(t *testing.T) {

+ opt := t.TempDir()

+ liveTo(t, opt, "platform.example.com", "old")

+ reg := restartTestRegistry(t)

+ r := boundRoute()

+ r.Addr = "" // no upstream: readiness is nil and settle holds instead of dialling

+ if err := reg.Upsert(r); err != nil {

+ t.Fatal(err)

+ }

+ f := &fakeUnits{shows: []unitProps{

+ {ActiveState: "active", InvocationID: "bad"},

+ {ActiveState: "active", InvocationID: "restored", MainPID: 9},

+ }, exe: map[int]string{9: filepath.Join(opt, "versions", "old", "console")}}

+ rb := rollbackUnitRestart(reg, f, opt, func(string) string { return "" })

+ if err := rb("platform.example.com"); err != nil {

+ t.Fatalf("rollback = %v", err)

+ }

+ if strings.Join(f.restarts, ",") != "reset-failed console.service,restart console.service" {

+ t.Fatalf("restarts = %v, want reset-failed then restart (never start)", f.restarts)

+ }