amadan
host: arming verifies a real shim placement; decline cgroup-placement-failed (#272)
swapArming, once claims and delegation pass, now asks placementCheck: cgroup.Manager.VerifyPlacement places a probe through the shim into a scratch child (carlos-placement-check) with the default tier, checks the probe saw itself there and memory.max reads back, and removes the child once the probe is reaped. A failure declines the flag with the new code cgroup-placement-failed, which reads as the approved declined phrase and counts as a decline in rolloutwords. honourDeclinedFailures keys on the activator built without generations, so it covers the new code as is.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
8 files changed,
+469
−20
internal/cgroup/verify_linux.go+108 −0internal/cgroup/verify_linux_test.go+166 −0internal/cgroup/verify_other.go+17 −0internal/host/rollout.go+27 −2internal/host/rollout_test.go+134 −11internal/rolloutwords/words.go+2 −1internal/rolloutwords/words_test.go+11 −3internal/status/snapshot.go+4 −3
diff --git a/internal/cgroup/verify_linux.go b/internal/cgroup/verify_linux.go| new file mode 100644 |
| index 000000000..0b1de37b7 |
| --- /dev/null |
| +++ b/internal/cgroup/verify_linux.go |
| @@ -0,0 +1,108 @@ |
| +//go:build linux |
| + |
| +package cgroup |
| + |
| +import ( |
| + "bytes" |
| + "context" |
| + "errors" |
| + "fmt" |
| + "os" |
| + "os/exec" |
| + "path/filepath" |
| + "strconv" |
| + "strings" |
| + |
| + "golang.org/x/sys/unix" |
| +) |
| + |
| +// placementCheckName is the scratch child of the delegated root that |
| +// VerifyPlacement places its probe in. It is no carlos-exec@ name, so no |
| +// host's cleanup or the stale sweep ever takes it for a host's cgroup. |
| +const placementCheckName = "carlos-placement-check" |
| + |
| +// placementProbeArgs is the probe tenant's argv after shimExe. Only tests |
| +// change it, to stand in a probe that misreports or never exits. |
| +var placementProbeArgs = []string{ProbeVerb} |
| + |
| +// VerifyPlacement does one real placement, through the shim, into a scratch |
| +// child cgroup with the default tier's limits, and answers nil only if the |
| +// placed process saw itself in that cgroup and the limit reads back. It is |
| +// the arm-time question for CARLOS_SWAP=overlap (#272): the second dev-box |
| +// check found a box that armed and then failed every generation start, |
| +// because no placement was possible under carlos-edge.service's hardening. |
| +// |
| +// It calls startViaShim itself, never the placeStart seam, so a test's |
| +// FakePlacementForTest cannot make it pass. The scratch cgroup is removed |
| +// once the probe is reaped; a removal failure is logged, not returned. |
| +func (m *Manager) VerifyPlacement(ctx context.Context) error { |
| + if m == nil { |
| + return ErrNoDelegation |
| + } |
| + if !shimAvailable() { |
| + return errors.New("cgroup: placement check: the placement shim is not available in this process") |
| + } |
| + dir := filepath.Join(m.root, placementCheckName) |
| + // A leftover from an earlier run is empty: the unit's restart killed |
| + // its members. One that will not go fails the Mkdir below. |
| + _ = os.Remove(dir) |
| + // Nothing is created under a root that is no cgroup: the limit files |
| + // would be plain files there, and the directory could not be removed. |
| + var st unix.Statfs_t |
| + if err := unix.Statfs(m.root, &st); err != nil { |
| + return fmt.Errorf("cgroup: placement check: statfs %s: %w", m.root, err) |
| + } |
| + if st.Type != unix.CGROUP2_SUPER_MAGIC { |
| + return fmt.Errorf("cgroup: placement check: %s is not a cgroup2 directory", m.root) |
| + } |
| + if err := os.Mkdir(dir, 0o755); err != nil { |
| + return fmt.Errorf("cgroup: placement check: %w", err) |
| + } |
| + // Every path below reaps any probe it started before it returns, so |
| + // this runs on an empty cgroup. |
| + defer func() { |
| + if err := os.Remove(dir); err != nil { |
| + m.logf("cgroup: placement check: %s not removed: %v", dir, err) |
| + } |
| + }() |
| + if err := m.applyLimit("placement check", dir, TierDefault, true); err != nil { |
| + return fmt.Errorf("cgroup: placement check: %w", err) |
| + } |
| + |
| + var out bytes.Buffer |
| + cmd := exec.Command(shimExe, placementProbeArgs...) |
| + cmd.Env = []string{} |
| + cmd.Stdout = &out |
| + if _, err := m.startViaShim(dir, cmd, nil); err != nil { |
| + // A placement error or not, the check failed; startViaShim has |
| + // reaped anything it started. |
| + return fmt.Errorf("cgroup: placement check: %w", err) |
| + } |
| + waited := make(chan error, 1) |
| + go func() { waited <- cmd.Wait() }() |
| + var werr error |
| + select { |
| + case werr = <-waited: |
| + case <-ctx.Done(): |
| + _ = cmd.Process.Kill() |
| + <-waited // bounded: the probe is dead and holds no other process |
| + return fmt.Errorf("cgroup: placement check: the probe did not exit: %w", ctx.Err()) |
| + } |
| + if werr != nil { |
| + return fmt.Errorf("cgroup: placement check: the probe failed: %w", werr) |
| + } |
| + // What the probe saw in /proc/self/cgroup: dir as startViaShim names it |
| + // to the shim, below /sys/fs/cgroup. |
| + expected := strings.TrimPrefix(dir, "/sys/fs/cgroup") |
| + if got := out.String(); got != expected+"\n" { |
| + return fmt.Errorf("cgroup: placement check: the probe reported cgroup %q, want %q", got, expected+"\n") |
| + } |
| + raw, err := os.ReadFile(filepath.Join(dir, "memory.max")) |
| + if err != nil { |
| + return fmt.Errorf("cgroup: placement check: %w", err) |
| + } |
| + if got, want := strings.TrimSpace(string(raw)), strconv.FormatInt(TierDefault.MemoryMax, 10); got != want { |
| + return fmt.Errorf("cgroup: placement check: memory.max reads %q, want %q", got, want) |
| + } |
| + return nil |
| +} |
diff --git a/internal/cgroup/verify_linux_test.go b/internal/cgroup/verify_linux_test.go| new file mode 100644 |
| index 000000000..dfdbedd98 |
| --- /dev/null |
| +++ b/internal/cgroup/verify_linux_test.go |
| @@ -0,0 +1,166 @@ |
| +//go:build linux |
| + |
| +package cgroup |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "os" |
| + "path/filepath" |
| + "strings" |
| + "testing" |
| + "time" |
| +) |
| + |
| +// The arm-time placement check (zdd readiness 3, Task 3). The ungated tests |
| +// run anywhere; the kernel ones need CARLOS_SHIM_KERNEL=1 inside a unit with |
| +// Delegate=yes, as shim_kernel_linux_test.go's do: |
| +// |
| +// systemd-run --user --wait --pipe --collect -p Delegate=yes \ |
| +// -E TMPDIR=/tmp/k/zddr3 -E CARLOS_SHIM_KERNEL=1 -E CARLOS_GEN_KERNEL=1 \ |
| +// cgroup.test -test.run 'VerifyPlacement' -test.v -test.count=1 |
| + |
| +func assertNoPlacementCheckDir(t *testing.T, root string) { |
| + t.Helper() |
| + if _, err := os.Lstat(filepath.Join(root, placementCheckName)); !errors.Is(err, os.ErrNotExist) { |
| + t.Fatalf("%s left behind under %s (%v)", placementCheckName, root, err) |
| + } |
| +} |
| + |
| +func TestVerifyPlacementNilManager(t *testing.T) { |
| + var m *Manager |
| + if err := m.VerifyPlacement(context.Background()); !errors.Is(err, ErrNoDelegation) { |
| + t.Fatalf("nil manager: %v, want ErrNoDelegation", err) |
| + } |
| +} |
| + |
| +func TestVerifyPlacementWithoutTheShim(t *testing.T) { |
| + prev := shimHooked.Load() |
| + shimHooked.Store(false) |
| + t.Cleanup(func() { shimHooked.Store(prev) }) |
| + root := t.TempDir() |
| + m := &Manager{root: root, logf: t.Logf} |
| + err := m.VerifyPlacement(context.Background()) |
| + if err == nil || !strings.Contains(err.Error(), "shim") { |
| + t.Fatalf("no shim: %v, want an error naming the shim", err) |
| + } |
| + assertNoPlacementCheckDir(t, root) |
| +} |
| + |
| +// A manager over a directory that is no cgroup answers an error and leaves |
| +// nothing behind, a leftover from an earlier run included. |
| +func TestVerifyPlacementOverATempRootFails(t *testing.T) { |
| + root := t.TempDir() |
| + if err := os.Mkdir(filepath.Join(root, placementCheckName), 0o755); err != nil { |
| + t.Fatal(err) |
| + } |
| + m := &Manager{root: root, logf: t.Logf} |
| + err := m.VerifyPlacement(context.Background()) |
| + if err == nil { |
| + t.Fatal("a temp directory passed the placement check") |
| + } |
| + t.Logf("temp root: %v", err) |
| + assertNoPlacementCheckDir(t, root) |
| + if m.placementsUnder(root) != 0 { |
| + t.Fatal("a placement is still registered") |
| + } |
| +} |
| + |
| +// TestVerifyPlacementKernelChild is the probe that never exits: it sleeps |
| +// when it finds itself in the placement check's cgroup, and otherwise (a |
| +// normal test run) returns at once. |
| +func TestVerifyPlacementKernelChild(t *testing.T) { |
| + raw, err := os.ReadFile("/proc/self/cgroup") |
| + if err != nil { |
| + return |
| + } |
| + rel, ok := ParseSelfCgroup(string(raw)) |
| + if !ok || filepath.Base(rel) != placementCheckName { |
| + return |
| + } |
| + time.Sleep(time.Minute) |
| +} |
| + |
| +func TestVerifyPlacementKernel(t *testing.T) { |
| + m := shimKernelManager(t) |
| + if err := m.VerifyPlacement(context.Background()); err != nil { |
| + t.Fatalf("VerifyPlacement: %v", err) |
| + } |
| + assertNoPlacementCheckDir(t, m.root) |
| + if m.placementsUnder(m.root) != 0 { |
| + t.Fatal("a placement is still registered") |
| + } |
| +} |
| + |
| +// A leftover from an earlier run (empty: the unit's restart killed its |
| +// members) is replaced, and the check still passes. |
| +func TestVerifyPlacementKernelReplacesALeftover(t *testing.T) { |
| + m := shimKernelManager(t) |
| + if err := os.Mkdir(filepath.Join(m.root, placementCheckName), 0o755); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := m.VerifyPlacement(context.Background()); err != nil { |
| + t.Fatalf("VerifyPlacement over a leftover: %v", err) |
| + } |
| + assertNoPlacementCheckDir(t, m.root) |
| +} |
| + |
| +// A shim that cannot place (here, one that is not the carlos binary at all) |
| +// fails the check, and the scratch cgroup is removed. |
| +func TestVerifyPlacementKernelBrokenShim(t *testing.T) { |
| + m := shimKernelManager(t) |
| + prev := shimExe |
| + shimExe = "/bin/true" |
| + t.Cleanup(func() { shimExe = prev }) |
| + err := m.VerifyPlacement(context.Background()) |
| + if err == nil { |
| + t.Fatal("a shim that cannot place passed the check") |
| + } |
| + t.Logf("broken shim: %v", err) |
| + assertNoPlacementCheckDir(t, m.root) |
| +} |
| + |
| +// A probe whose output is not its cgroup, or that exits non-zero, fails |
| +// the check: placement is proven only by what the placed process saw. |
| +func TestVerifyPlacementKernelChecksTheProbe(t *testing.T) { |
| + m := shimKernelManager(t) |
| + for name, args := range map[string][]string{ |
| + // The test binary with no tests: exits 0, prints "PASS" and a warning. |
| + "wrong output": {"-test.run=^$"}, |
| + // An unknown flag: exits 2. |
| + "non-zero exit": {"-test.run=^$", "-test.no-such-flag"}, |
| + } { |
| + t.Run(name, func(t *testing.T) { |
| + prev := placementProbeArgs |
| + placementProbeArgs = args |
| + t.Cleanup(func() { placementProbeArgs = prev }) |
| + err := m.VerifyPlacement(context.Background()) |
| + if err == nil { |
| + t.Fatal("passed with a probe that did not report its cgroup") |
| + } |
| + t.Logf("%s: %v", name, err) |
| + assertNoPlacementCheckDir(t, m.root) |
| + }) |
| + } |
| +} |
| + |
| +// A probe that does not exit is killed when ctx ends, and reaped before the |
| +// scratch cgroup is removed: the removal succeeds. |
| +func TestVerifyPlacementKernelKillsAProbeThatHangs(t *testing.T) { |
| + m := shimKernelManager(t) |
| + prev := placementProbeArgs |
| + placementProbeArgs = []string{"-test.run=^TestVerifyPlacementKernelChild$", "-test.count=1"} |
| + t.Cleanup(func() { placementProbeArgs = prev }) |
| + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) |
| + defer cancel() |
| + start := time.Now() |
| + err := m.VerifyPlacement(ctx) |
| + if !errors.Is(err, context.DeadlineExceeded) { |
| + t.Fatalf("hung probe: %v, want the context's deadline", err) |
| + } |
| + if d := time.Since(start); d > 10*time.Second { |
| + t.Fatalf("took %v to give up on a hung probe", d) |
| + } |
| + t.Logf("hung probe: %v", err) |
| + assertNoPlacementCheckDir(t, m.root) |
| +} |
diff --git a/internal/cgroup/verify_other.go b/internal/cgroup/verify_other.go| new file mode 100644 |
| index 000000000..d0c6f398f |
| --- /dev/null |
| +++ b/internal/cgroup/verify_other.go |
| @@ -0,0 +1,17 @@ |
| +//go:build !linux |
| + |
| +package cgroup |
| + |
| +import ( |
| + "context" |
| + "errors" |
| +) |
| + |
| +// VerifyPlacement needs cgroup v2 and the placement shim, so it is linux |
| +// only: elsewhere every check fails. |
| +func (m *Manager) VerifyPlacement(ctx context.Context) error { |
| + if m == nil { |
| + return ErrNoDelegation |
| + } |
| + return errors.New("cgroup: placement check: placement before exec needs linux") |
| +} |
diff --git a/internal/host/rollout.go b/internal/host/rollout.go| index d57f6024b..d09feb71a 100644 |
| --- a/internal/host/rollout.go |
| +++ b/internal/host/rollout.go |
| @@ -13,6 +13,7 @@ import ( |
| "github.com/carlosframework/platform/internal/activator" |
| "github.com/carlosframework/platform/internal/appconfig" |
| + "github.com/carlosframework/platform/internal/cgroup" |
| "github.com/carlosframework/platform/internal/registry" |
| "github.com/carlosframework/platform/internal/release" |
| "github.com/carlosframework/platform/internal/rollout" |
| @@ -33,6 +34,11 @@ const ( |
| // generations file or the good build (#272, "claims-on swaps"), so the |
| // box declines and stays on today's paths. |
| rolloutHostClaims = "host-claims-on" |
| + // The flag is on and the box has delegation, but a test placement into |
| + // a scratch cgroup through the shim failed (placementCheck): generation |
| + // starts could not be placed inside their limits (#272, the second |
| + // dev-box check), so the box declines and stays on today's paths. |
| + rolloutPlacementFailed = "cgroup-placement-failed" |
| // The flag is on and generations are on, but the scheduler's required |
| // state is unavailable, so every managed start is refused |
| // (refuseManaged). |
| @@ -87,7 +93,8 @@ func nudgeStatus() { |
| var rolloutDown atomic.Pointer[string] |
| // rolloutDeclined is why CARLOS_SWAP=overlap is set but the box stayed on |
| -// today's paths: it cannot contain generations (swapArming). Unlike |
| +// today's paths: it cannot contain generations, has host claims on, or |
| +// failed its arm-time test placement (swapArming). Unlike |
| // rolloutDown nothing is refused and no leg changes, except where failure |
| // records an earlier armed run left hold a build for an instance |
| // (honourDeclinedFailures, final review F6). The reason is published in the |
| @@ -138,6 +145,16 @@ func swapDelegation(eb *activator.ExecBackend) string { |
| return "" |
| } |
| +// placementCheck is swapArming's arm-time placement check: one real shim |
| +// placement into a scratch child of the delegated subtree |
| +// (cgroup.Manager.VerifyPlacement), bounded at 15 s. A seam host tests |
| +// replace, since a temp-dir root can never pass it. |
| +var placementCheck = func(m *cgroup.Manager) error { |
| + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) |
| + defer cancel() |
| + return m.VerifyPlacement(ctx) |
| +} |
| + |
| // swapArming decides, before the activator exists, whether CARLOS_SWAP=overlap |
| // turns generations on (spec §15; plan 3's review: missing delegation must |
| // fail closed, CODEX-FINDINGS-ZDD-IMPL-3.md:138). With the flag off it is |
| @@ -146,7 +163,10 @@ func swapDelegation(eb *activator.ExecBackend) string { |
| // ruling, #272): generations, generation containment and the scheduler all stay |
| // off, every leg keeps today's paths but for the failure records armRollout |
| // honours (honourDeclinedFailures), and the refusal is logged loudly and |
| -// published as a reason code in the box's status (rolloutStatusProbe). |
| +// published as a reason code in the box's status (rolloutStatusProbe). With |
| +// containment present it asks for one real placement last (placementCheck), |
| +// and declines the same way when it fails: an armed box whose generation |
| +// starts can never be placed fails every one of them (#272). |
| func swapArming(getenv func(string) string, eb *activator.ExecBackend) bool { |
| if !rollout.Enabled(getenv) { |
| return false |
| @@ -164,6 +184,11 @@ func swapArming(getenv func(string) string, eb *activator.ExecBackend) bool { |
| declineRollout(why) |
| return false |
| } |
| + if err := placementCheck(eb.Cgroups); err != nil { |
| + log.Printf("agent: REFUSING %s=overlap (%s): a test placement into a scratch cgroup failed (%v), so generations could not be started inside their limits; this box stays on today's stop-then-wake paths", rollout.SwapEnv, rolloutPlacementFailed, err) |
| + declineRollout(rolloutPlacementFailed) |
| + return false |
| + } |
| return true |
| } |
diff --git a/internal/host/rollout_test.go b/internal/host/rollout_test.go| index c80d80c08..56698fb39 100644 |
| --- a/internal/host/rollout_test.go |
| +++ b/internal/host/rollout_test.go |
| @@ -74,6 +74,49 @@ func overlapEnv(pub string) func(string) string { |
| func noEnv(string) string { return "" } |
| +// stubPlacementCheck replaces swapArming's arm-time placement check with |
| +// one that answers err, and counts its calls. A temp-dir cgroup root can |
| +// never pass the real check (cgroup.Manager.VerifyPlacement), so every test |
| +// that expects swapArming to arm uses this with a nil err. |
| +func stubPlacementCheck(t *testing.T, err error) *int { |
| + t.Helper() |
| + orig := placementCheck |
| + calls := new(int) |
| + placementCheck = func(*cgroup.Manager) error { *calls++; return err } |
| + t.Cleanup(func() { placementCheck = orig }) |
| + return calls |
| +} |
| + |
| +// forbidPlacementCheck fails the test if swapArming runs the placement |
| +// check: with the flag off, or after an earlier check declined, it must not. |
| +func forbidPlacementCheck(t *testing.T) { |
| + t.Helper() |
| + orig := placementCheck |
| + placementCheck = func(*cgroup.Manager) error { |
| + t.Error("swapArming ran the placement check") |
| + return nil |
| + } |
| + t.Cleanup(func() { placementCheck = orig }) |
| +} |
| + |
| +// armableBackend is an exec backend that passes every check swapArming makes |
| +// before the placement check: claims off, a delegated subtree (over a temp |
| +// root), and a control channel that speaks the generation verbs. |
| +func armableBackend(t *testing.T) *activator.ExecBackend { |
| + t.Helper() |
| + root := t.TempDir() |
| + for _, f := range []string{"cgroup.procs", "cgroup.subtree_control"} { |
| + if err := os.WriteFile(filepath.Join(root, f), nil, 0o644); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + cgm, err := cgroup.SetupAt(root, t.Logf) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + return &activator.ExecBackend{Cgroups: cgm, Control: instancectl.NewClient(filepath.Join(t.TempDir(), "ctl.sock"))} |
| +} |
| + |
| func unusedStore(context.Context) (release.Store, error) { return nil, errors.New("unused") } |
| func funcPtr(f any) uintptr { return reflect.ValueOf(f).Pointer() } |
| @@ -121,6 +164,7 @@ func TestTheFlagOffArmsNothing(t *testing.T) { |
| if adoptBounceFn(svc) == nil { |
| t.Fatal("the adopt bounce must stay BounceAdopted") |
| } |
| + forbidPlacementCheck(t) |
| if swapArming(noEnv, nil) || activator.GenerationsEnabled() { |
| t.Fatal("CARLOS_SWAP unset turned generations on") |
| } |
| @@ -472,6 +516,7 @@ func TestNoDelegationDeclinesTheFlag(t *testing.T) { |
| t.Fatal(err) |
| } |
| resetRollout(t) |
| + forbidPlacementCheck(t) |
| eb := &activator.ExecBackend{Control: instancectl.NewClient(filepath.Join(t.TempDir(), "ctl.sock"))} |
| if swapArming(overlapEnv(pub), eb) { |
| t.Fatal("armed generations with no delegated cgroup subtree") |
| @@ -512,27 +557,103 @@ func TestDelegationArmsTheFlag(t *testing.T) { |
| t.Fatal(err) |
| } |
| resetRollout(t) |
| - root := t.TempDir() |
| - for _, f := range []string{"cgroup.procs", "cgroup.subtree_control"} { |
| - if err := os.WriteFile(filepath.Join(root, f), nil, 0o644); err != nil { |
| - t.Fatal(err) |
| - } |
| - } |
| - cgm, err := cgroup.SetupAt(root, t.Logf) |
| - if err != nil { |
| - t.Fatal(err) |
| - } |
| - eb := &activator.ExecBackend{Cgroups: cgm, Control: instancectl.NewClient(filepath.Join(t.TempDir(), "ctl.sock"))} |
| + calls := stubPlacementCheck(t, nil) |
| + eb := armableBackend(t) |
| if !swapArming(overlapEnv(pub), eb) || rolloutUnarmed() != "" { |
| t.Fatalf("delegation present: declined with %q", rolloutUnarmed()) |
| } |
| + if *calls != 1 { |
| + t.Fatalf("the armed flag ran the placement check %d times, want once", *calls) |
| + } |
| if swapArming(noEnv, eb) { |
| t.Fatal("the flag off armed generations") |
| } |
| + if *calls != 1 { |
| + t.Fatal("the flag off ran the placement check") |
| + } |
| eb.Control = nil |
| if swapArming(overlapEnv(pub), eb) || rolloutUnarmed() != rolloutNoGenControl { |
| t.Fatalf("no generation control: armed, or declined with %q", rolloutUnarmed()) |
| } |
| + if *calls != 1 { |
| + t.Fatal("a box that declined on its generation control still ran the placement check") |
| + } |
| +} |
| + |
| +// The second dev-box check (#272): a box armed CARLOS_SWAP=overlap and then |
| +// failed every generation start, because no process could be placed in its |
| +// cgroup under carlos-edge.service's hardening. Arming now does one real |
| +// placement into a scratch cgroup and declines when it fails, with a code |
| +// of its own, a loud operator line, and the code in the box's status. |
| +func TestSwapArmingDeclinesWhenPlacementFails(t *testing.T) { |
| + _, pub, err := release.GenerateKey() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + resetRollout(t) |
| + var buf bytes.Buffer |
| + log.SetOutput(&buf) |
| + t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| + eb := armableBackend(t) |
| + var got *cgroup.Manager |
| + orig := placementCheck |
| + placementCheck = func(m *cgroup.Manager) error { got = m; return errors.New("probe exited before exec") } |
| + t.Cleanup(func() { placementCheck = orig }) |
| + if swapArming(overlapEnv(pub), eb) { |
| + t.Fatal("armed with a failed placement check") |
| + } |
| + if got != eb.Cgroups { |
| + t.Fatal("the placement check was not run against the backend's cgroup manager") |
| + } |
| + if rolloutUnarmed() != rolloutPlacementFailed { |
| + t.Fatalf("declined as %q", rolloutUnarmed()) |
| + } |
| + if rolloutUnavailable() != "" { |
| + t.Fatal("a placement decline refuses managed starts") |
| + } |
| + if s := buf.String(); !strings.Contains(s, "REFUSING CARLOS_SWAP=overlap (cgroup-placement-failed)") || |
| + !strings.Contains(s, "probe exited before exec") { |
| + t.Fatalf("no operator line naming the placement refusal and its error: %q", s) |
| + } |
| + if r := rolloutStatusProbe(); r == nil || r.Unavailable != rolloutPlacementFailed { |
| + t.Fatalf("status %+v: want Unavailable %q", r, rolloutPlacementFailed) |
| + } |
| +} |
| + |
| +// A box that declined for a failed placement honours the failure records an |
| +// earlier armed run left, exactly as the other declined codes do |
| +// (honourDeclinedFailures keys on the activator built without generations, |
| +// not on the code), and publishes its own code beside the honoured book. |
| +func TestPlacementDeclineHonoursFailureRecords(t *testing.T) { |
| + t.Setenv("CARLOS_OPT_DIR", t.TempDir()) |
| + svc, reg, dd := rolloutFixture(t, false) |
| + acct := release.RouteAccount(managedRoute("x", "")) |
| + writeFailureBook(t, dd, []rollout.FailureRecord{{ |
| + Account: acct, App: "blog", Build: "v2", Host: "sib.example.com", Cause: "adopt", Mode: "hold", Reason: "health", |
| + }}, nil) |
| + _, pub, err := release.GenerateKey() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + stubPlacementCheck(t, errors.New("the shim exited before placement")) |
| + if swapArming(overlapEnv(pub), armableBackend(t)) { |
| + t.Fatal("armed with a failed placement check") |
| + } |
| + m := armRollout(context.Background(), overlapEnv(pub), svc, reg, dd, nil, unusedStore) |
| + if m != nil || armedRollout() != nil { |
| + t.Fatal("a declined box armed the scheduler") |
| + } |
| + if heldHolds() == nil || svc.Planner == nil { |
| + t.Fatal("a placement decline did not honour the failure records") |
| + } |
| + blocked := managedRoute("a.example.com", "v2") |
| + blocked.GoodVersion, blocked.GoodDigest = "v1", "sha256:v1" |
| + if p := svc.Planner(context.Background(), blocked); p.Refuse != refuseNoSwaps { |
| + t.Fatalf("a wake onto a held build planned %+v, want refused", p) |
| + } |
| + if r := rolloutStatusProbe(); r == nil || r.Unavailable != rolloutPlacementFailed || r.FailureBook != rolloutBookHonoured { |
| + t.Fatalf("status = %+v, want the decline code %q and the book %q", r, rolloutPlacementFailed, rolloutBookHonoured) |
| + } |
| } |
| // A box with host claims on declines the swap flag (#272): claims-backed |
| @@ -541,6 +662,7 @@ func TestDelegationArmsTheFlag(t *testing.T) { |
| // so a box that has neither says claims. |
| func TestSwapArmingDeclinesWhenHostClaimsAreOn(t *testing.T) { |
| resetRollout(t) |
| + forbidPlacementCheck(t) |
| var buf bytes.Buffer |
| log.SetOutput(&buf) |
| t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| @@ -564,6 +686,7 @@ func TestSwapArmingDeclinesWhenHostClaimsAreOn(t *testing.T) { |
| func TestSwapArmingIgnoresClaimsWithTheFlagOff(t *testing.T) { |
| resetRollout(t) |
| + forbidPlacementCheck(t) |
| eb := &activator.ExecBackend{Cfg: activator.Config{HostWrites: hostwrite.Config{Enabled: true}}} |
| if swapArming(noEnv, eb) || rolloutUnarmed() != "" { |
| t.Fatal("flag off: a claims box must say nothing") |
diff --git a/internal/rolloutwords/words.go b/internal/rolloutwords/words.go| index 6652d0901..0e0edb74a 100644 |
| --- a/internal/rolloutwords/words.go |
| +++ b/internal/rolloutwords/words.go |
| @@ -85,6 +85,7 @@ var unavailablePhrases = map[string]string{ |
| "no-generation-control": unavailableDeclined, |
| "generations-off": unavailableDeclined, |
| "host-claims-on": unavailableDeclined, |
| + "cgroup-placement-failed": unavailableDeclined, |
| "no-deploy-store": "its box cannot start instances: the deploy store is unavailable", |
| "no-release-key": "its box cannot start instances: the release key is unavailable", |
| "failure-book-unreadable": "its box cannot start instances: its record of failed builds cannot be read", |
| @@ -108,7 +109,7 @@ func Unavailable(code string) string { |
| // are safe to wake. |
| func UnavailableDeclines(code string) bool { |
| switch code { |
| - case "no-cgroup-delegation", "no-generation-control", "generations-off", "host-claims-on": |
| + case "no-cgroup-delegation", "no-generation-control", "generations-off", "host-claims-on", "cgroup-placement-failed": |
| return true |
| default: |
| return false |
diff --git a/internal/rolloutwords/words_test.go b/internal/rolloutwords/words_test.go| index 06e5aaf74..d0e62ba62 100644 |
| --- a/internal/rolloutwords/words_test.go |
| +++ b/internal/rolloutwords/words_test.go |
| @@ -83,7 +83,7 @@ func TestEveryCauseAndReasonHasAPhrase(t *testing.T) { |
| func TestEveryUnavailableCodeHasAPhrase(t *testing.T) { |
| // Declined codes |
| - for _, code := range []string{"no-cgroup-delegation", "no-generation-control", "generations-off", "host-claims-on"} { |
| + for _, code := range []string{"no-cgroup-delegation", "no-generation-control", "generations-off", "host-claims-on", "cgroup-placement-failed"} { |
| phrase := Unavailable(code) |
| if phrase == "" { |
| t.Errorf("Unavailable(%q) = empty string", code) |
| @@ -117,12 +117,12 @@ func TestEveryUnavailableCodeHasAPhrase(t *testing.T) { |
| } |
| } |
| -// UnavailableDeclines is true for exactly the four codes under which a box |
| +// UnavailableDeclines is true for exactly the five codes under which a box |
| // runs today's stop-then-start paths. A refusing code is not one, and nor is |
| // a code this binary does not know: a reader cannot assume a newer box's |
| // problem leaves its instances safe to wake. |
| func TestUnavailableDeclinesOnlyTheKnownDeclinedCodes(t *testing.T) { |
| - for _, code := range []string{"no-cgroup-delegation", "no-generation-control", "generations-off", "host-claims-on"} { |
| + for _, code := range []string{"no-cgroup-delegation", "no-generation-control", "generations-off", "host-claims-on", "cgroup-placement-failed"} { |
| if !UnavailableDeclines(code) { |
| t.Errorf("UnavailableDeclines(%q) = false, want true", code) |
| } |
| @@ -139,3 +139,11 @@ func TestHostClaimsDeclineReadsAsADecline(t *testing.T) { |
| t.Fatal("host-claims-on must read and act as a decline") |
| } |
| } |
| + |
| +// A box whose arm-time test placement failed (#272) declines like the |
| +// others, in the same approved words: it adds no member-facing text. |
| +func TestPlacementFailedDeclineReadsAsADecline(t *testing.T) { |
| + if Unavailable("cgroup-placement-failed") != unavailableDeclined || !UnavailableDeclines("cgroup-placement-failed") || UnavailableRefuses("cgroup-placement-failed") { |
| + t.Fatal("cgroup-placement-failed must read and act as a decline") |
| + } |
| +} |
diff --git a/internal/status/snapshot.go b/internal/status/snapshot.go| index b031f5bf0..4dd9ef44a 100644 |
| --- a/internal/status/snapshot.go |
| +++ b/internal/status/snapshot.go |
| @@ -299,14 +299,15 @@ type RolloutReport struct { |
| // Unavailable, when set, is a reason code (not member-facing text) for |
| // why CARLOS_SWAP=overlap is on but the scheduler could not arm. Either |
| // the box cannot contain generations ("no-cgroup-delegation", |
| - // "no-generation-control", "generations-off") or has host claims on |
| - // ("host-claims-on"), and stays on today's |
| + // "no-generation-control", "generations-off"), has host claims on |
| + // ("host-claims-on"), or failed its arm-time test placement |
| + // ("cgroup-placement-failed"), and stays on today's |
| // paths, or its required state is unavailable ("no-deploy-store", |
| // "no-release-key", "failure-book-unreadable"), and every managed start |
| // is refused meanwhile (Task 14, internal/host's swapArming and |
| // refuseManaged). The rest of the report is then empty. Plan 6 words it. |
| Unavailable string `json:"unavailable,omitempty"` |
| - // FailureBook is set only beside a decline code (the first four): what a |
| + // FailureBook is set only beside a decline code (the first five): what a |
| // box that cannot contain generations does with the failure records an |
| // earlier armed run left (final review F6). "honoured": a managed wake, |
| // or a bounce, onto a build they hold for that instance is refused. |