| 1 | package activator |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "slices" |
| 7 | "sync" |
| 8 | "testing" |
| 9 | "time" |
| 10 | |
| 11 | "github.com/carlosframework/platform/internal/registry" |
| 12 | "github.com/carlosframework/platform/internal/release" |
| 13 | ) |
| 14 | |
| 15 | // stagedHostKill is stagedFake plus KillHost, failing while bad: a staged |
| 16 | // worker backend whose host-cgroup proof the test controls. |
| 17 | type stagedHostKill struct { |
| 18 | *stagedFake |
| 19 | hmu sync.Mutex |
| 20 | bad bool |
| 21 | kills int |
| 22 | } |
| 23 | |
| 24 | func (h *stagedHostKill) KillHost(context.Context, registry.Route) error { |
| 25 | h.hmu.Lock() |
| 26 | defer h.hmu.Unlock() |
| 27 | h.kills++ |
| 28 | if h.bad { |
| 29 | return errors.New("the host cgroup still holds a process") |
| 30 | } |
| 31 | return nil |
| 32 | } |
| 33 | |
| 34 | func (h *stagedHostKill) setBad(bad bool) { |
| 35 | h.hmu.Lock() |
| 36 | h.bad = bad |
| 37 | h.hmu.Unlock() |
| 38 | } |
| 39 | |
| 40 | func (h *stagedHostKill) killCount() int { |
| 41 | h.hmu.Lock() |
| 42 | defer h.hmu.Unlock() |
| 43 | return h.kills |
| 44 | } |
| 45 | |
| 46 | // generationsHost is the versions the generations file lists for host, |
| 47 | // read after a publication of a's state as it is now (persistGenerations is |
| 48 | // serialized, so nothing older can land after it). The caller makes sure no |
| 49 | // other writer is under way. |
| 50 | func generationsHost(t *testing.T, a *Service, path, host string) []string { |
| 51 | t.Helper() |
| 52 | if err := a.persistGenerations(); err != nil { |
| 53 | t.Fatalf("persistGenerations: %v", err) |
| 54 | } |
| 55 | g, err := release.ReadGenerations(path) |
| 56 | if err != nil { |
| 57 | t.Fatalf("ReadGenerations: %v", err) |
| 58 | } |
| 59 | if g == nil { |
| 60 | return nil |
| 61 | } |
| 62 | return g.Hosts[host] |
| 63 | } |
| 64 | |
| 65 | // Final review F12: a failed worker wake that is retained, because its |
| 66 | // termination is not proven, keeps its build in the generations file |
| 67 | // (root's pin set) until the proof holds. The process may still run those |
| 68 | // bytes, so root must not restage or sweep them. |
| 69 | func TestARetainedFailedWorkerWakeKeepsItsBuildPinned(t *testing.T) { |
| 70 | old := sidecarSettle |
| 71 | sidecarSettle = 20 * time.Millisecond |
| 72 | t.Cleanup(func() { sidecarSettle = old }) |
| 73 | a, b, _, _, _ := stagedActivator(t) |
| 74 | a.gens = true |
| 75 | hk := &stagedHostKill{stagedFake: b, bad: true} |
| 76 | a.backend = hk |
| 77 | path := release.GenerationsPath(t.TempDir()) |
| 78 | a.GenerationsFile = path |
| 79 | host := "bot8.example.com" |
| 80 | b.mu.Lock() |
| 81 | b.workerDeadOnStart = true // the wake fails its settle |
| 82 | b.mu.Unlock() |
| 83 | if err := a.WakeWorker(testWorkerRoute(host)); err == nil { |
| 84 | t.Fatal("a worker dead on start woke") |
| 85 | } |
| 86 | a.mu.Lock() |
| 87 | _, retained := a.unprovenWakes[host] |
| 88 | a.mu.Unlock() |
| 89 | if !retained { |
| 90 | t.Fatal("the failed wake was not retained over its unproven host cgroup") |
| 91 | } |
| 92 | if got := generationsHost(t, a, path, host); !slices.Equal(got, []string{"v1"}) { |
| 93 | t.Fatalf("generations file lists %v for %s while its failed wake is retained; want [v1]", got, host) |
| 94 | } |
| 95 | hk.setBad(false) |
| 96 | a.sweepOnce() |
| 97 | waitFor(t, "the retained wake's proof and its unpin", func() bool { |
| 98 | g, err := release.ReadGenerations(path) |
| 99 | return err == nil && g != nil && len(g.Hosts[host]) == 0 |
| 100 | }) |
| 101 | } |
| 102 | |
| 103 | // Final review F12, T11: a respawn whose child is shed (here: its lease |
| 104 | // is lost after the exec) and cannot be proven gone keeps that child's |
| 105 | // build pinned while the member is retained, and drops it once the reaper |
| 106 | // proves the child gone. The serving build stays pinned throughout. |
| 107 | func TestARetainedShedRespawnKeepsItsBuildPinned(t *testing.T) { |
| 108 | t.Parallel() |
| 109 | a, b, _, _, ver := stagedActivator(t) |
| 110 | a.gens = true |
| 111 | hk := &stagedHostKill{stagedFake: b} |
| 112 | a.backend = hk |
| 113 | path := release.GenerationsPath(t.TempDir()) |
| 114 | a.GenerationsFile = path |
| 115 | rt := alwaysOnWorkerRoute("bot9.example.com") |
| 116 | if err := a.WakeWorker(rt); err != nil { |
| 117 | t.Fatal(err) |
| 118 | } |
| 119 | rep := b.lastRep() |
| 120 | ver.Store("v2") // the respawn (from the live link) execs v2 |
| 121 | b.mu.Lock() |
| 122 | b.workerStartHook = func(registry.Route) { |
| 123 | // After the exec: the lease fails, and the host cgroup will not |
| 124 | // prove the shed child gone. |
| 125 | rep.setErr(errors.New("renewal failed")) |
| 126 | hk.setBad(true) |
| 127 | } |
| 128 | b.mu.Unlock() |
| 129 | a.mu.Lock() |
| 130 | e := a.entries[rt.Host] |
| 131 | a.respawnWorkerLocked(e, nil, "test: shed after the exec", nil) |
| 132 | done := e.respawnDone |
| 133 | a.mu.Unlock() |
| 134 | <-done // the respawn goroutine has finished, its shed and publication included |
| 135 | if _, members := a.genStateForTest(rt.Host); len(members) != 1 { |
| 136 | t.Fatalf("members = %v: want the shed child retained", members) |
| 137 | } |
| 138 | if got := generationsHost(t, a, path, rt.Host); !slices.Equal(got, []string{"v1", "v2"}) { |
| 139 | t.Fatalf("generations file lists %v while the shed v2 child is retained; want [v1 v2]", got) |
| 140 | } |
| 141 | hk.setBad(false) |
| 142 | a.sweepOnce() |
| 143 | waitFor(t, "the shed child's proof and its unpin", func() bool { |
| 144 | g, err := release.ReadGenerations(path) |
| 145 | return err == nil && g != nil && !slices.Contains(g.Hosts[rt.Host], "v2") |
| 146 | }) |
| 147 | } |
| 148 | |
amadan