| 1 | package activator |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "fmt" |
| 7 | "os" |
| 8 | "path/filepath" |
| 9 | "strings" |
| 10 | "sync" |
| 11 | "testing" |
| 12 | "time" |
| 13 | |
| 14 | "github.com/carlosframework/platform/internal/cgroup" |
| 15 | "github.com/carlosframework/platform/internal/instancectl" |
| 16 | "github.com/carlosframework/platform/internal/registry" |
| 17 | ) |
| 18 | |
| 19 | // exeControl is a root control channel with the generation verbs. Its plain |
| 20 | // StartGen refuses, so a test sees which start was used. |
| 21 | type exeControl struct { |
| 22 | mu sync.Mutex |
| 23 | got []instancectl.StagedExe |
| 24 | sidecarExes []instancectl.StagedExe |
| 25 | plainStarts []instancectl.Role // Start calls: the live-link exe layout |
| 26 | tiers []int64 // StartGenExe's memoryMax, per call |
| 27 | sidecarTiers []int64 // StartSidecarExe's memoryMax, per call |
| 28 | exeErr error // what StartGenExe answers, when set |
| 29 | // exeAnswer, when set, is what StartGenExe answers for each staged exe |
| 30 | // (in place of exeErr): an empty invocation with no error is a unit |
| 31 | // that exited before root observed it. |
| 32 | exeAnswer func(instancectl.StagedExe) (string, error) |
| 33 | stopGens int // StopGen calls |
| 34 | stopGenErr error // what StopGen answers |
| 35 | cmdExes []instancectl.StagedExe // Cmd (a zero StagedExe: the live link) and CmdExe calls |
| 36 | } |
| 37 | |
| 38 | func (c *exeControl) Start(_ context.Context, _ string, role instancectl.Role, _ int64) (string, error) { |
| 39 | c.mu.Lock() |
| 40 | defer c.mu.Unlock() |
| 41 | c.plainStarts = append(c.plainStarts, role) |
| 42 | return "inv", nil |
| 43 | } |
| 44 | func (c *exeControl) StartSidecarExe(_ context.Context, _ string, memoryMax int64, x instancectl.StagedExe) (string, error) { |
| 45 | c.mu.Lock() |
| 46 | defer c.mu.Unlock() |
| 47 | c.sidecarExes = append(c.sidecarExes, x) |
| 48 | c.sidecarTiers = append(c.sidecarTiers, memoryMax) |
| 49 | return "inv-sc", nil |
| 50 | } |
| 51 | func (c *exeControl) Seed(context.Context, string, int64) (instancectl.SeedResponse, error) { |
| 52 | return instancectl.SeedResponse{}, nil |
| 53 | } |
| 54 | func (c *exeControl) Stop(context.Context, string, instancectl.Role) error { return nil } |
| 55 | func (c *exeControl) Reset(context.Context, string, instancectl.Role) error { return nil } |
| 56 | func (c *exeControl) Sweep(context.Context) (instancectl.SweepResponse, error) { |
| 57 | return instancectl.SweepResponse{}, nil |
| 58 | } |
| 59 | func (c *exeControl) Ready(context.Context) error { return nil } |
| 60 | func (c *exeControl) Cmd(context.Context, string, int64) (string, error) { |
| 61 | c.mu.Lock() |
| 62 | defer c.mu.Unlock() |
| 63 | c.cmdExes = append(c.cmdExes, instancectl.StagedExe{}) |
| 64 | return "inv-cmd", nil |
| 65 | } |
| 66 | func (c *exeControl) CmdExe(_ context.Context, _ string, _ int64, x instancectl.StagedExe) (string, error) { |
| 67 | c.mu.Lock() |
| 68 | defer c.mu.Unlock() |
| 69 | c.cmdExes = append(c.cmdExes, x) |
| 70 | return "inv-cmd", nil |
| 71 | } |
| 72 | func (c *exeControl) CmdStatus(context.Context, string, string) (instancectl.CmdStatus, error) { |
| 73 | return instancectl.CmdStatus{State: instancectl.CmdDone}, nil |
| 74 | } |
| 75 | func (c *exeControl) StartGen(context.Context, string, uint64, int64) (string, error) { |
| 76 | return "", errors.New("test: the live-link generation start was used") |
| 77 | } |
| 78 | func (c *exeControl) StopGen(context.Context, string, uint64, bool) error { |
| 79 | c.mu.Lock() |
| 80 | defer c.mu.Unlock() |
| 81 | c.stopGens++ |
| 82 | return c.stopGenErr |
| 83 | } |
| 84 | func (c *exeControl) ResetGen(context.Context, string, uint64) error { return nil } |
| 85 | func (c *exeControl) Parent(context.Context, string, int64) error { return nil } |
| 86 | func (c *exeControl) StartGenExe(_ context.Context, _ string, _ uint64, memoryMax int64, x instancectl.StagedExe) (string, error) { |
| 87 | c.mu.Lock() |
| 88 | defer c.mu.Unlock() |
| 89 | c.got = append(c.got, x) |
| 90 | c.tiers = append(c.tiers, memoryMax) |
| 91 | if c.exeAnswer != nil { |
| 92 | return c.exeAnswer(x) |
| 93 | } |
| 94 | if c.exeErr != nil { |
| 95 | return "", c.exeErr |
| 96 | } |
| 97 | return "inv-7", nil |
| 98 | } |
| 99 | |
| 100 | // A nested staged generation names its executable to root. Without one, |
| 101 | // the generation start is plan 2's, on the live link. |
| 102 | func TestStartRoleNamesTheStagedExe(t *testing.T) { |
| 103 | b := testExecBackend(t) |
| 104 | b.GenContainment = true |
| 105 | ctl := &exeControl{} |
| 106 | b.Control = ctl |
| 107 | rt := registry.Route{Host: "n.example.com"} |
| 108 | x := instancectl.StagedExe{Path: "/opt/carlos/versions/v2/blog", Version: "v2", Digest: strings.Repeat("a", 64)} |
| 109 | inv, err := b.startRole(context.Background(), rt, instancectl.RoleRun, 7, &x, cgroup.Limit{}) |
| 110 | if err != nil || inv != "inv-7" || len(ctl.got) != 1 || ctl.got[0] != x { |
| 111 | t.Fatalf("startRole = %q, %v; root was sent %+v", inv, err, ctl.got) |
| 112 | } |
| 113 | if _, err := b.startRole(context.Background(), rt, instancectl.RoleRun, 7, nil, cgroup.Limit{}); err == nil || !strings.Contains(err.Error(), "live-link") { |
| 114 | t.Fatalf("no staged exe: %v, want plan 2's start", err) |
| 115 | } |
| 116 | } |
| 117 | |
| 118 | // A nested legacy sidecar is started on the serving generation's captured |
| 119 | // staged executable, which root validates (spec §7.5). The live link names |
| 120 | // v1 while the generation serves v2, as after a fallback, and the sidecar |
| 121 | // still starts, on v2. Task 7's interim nested arm refused it |
| 122 | // (errNestedSidecarGen), and today's StartSidecar would bind the live v1. |
| 123 | func TestStartSidecarGenNestedNamesTheStagedExe(t *testing.T) { |
| 124 | b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{}) |
| 125 | b.GenContainment = true |
| 126 | ctl := &exeControl{} |
| 127 | b.Control = ctl |
| 128 | stageLive(t, b, rt.Host, "v1") |
| 129 | if err := os.MkdirAll(filepath.Dir(b.pathsFor(rt).sidecarEnv), 0o750); err != nil { |
| 130 | t.Fatal(err) |
| 131 | } |
| 132 | if err := os.WriteFile(b.pathsFor(rt).sidecarEnv, []byte("X=1\n"), 0o600); err != nil { |
| 133 | t.Fatal(err) |
| 134 | } |
| 135 | // The generation's own tier: 1 GiB, while the live link's (b.tier) is |
| 136 | // the default 512 MiB. After a fallback the two differ. |
| 137 | g := Gen{N: 4, Build: "v2", StagedPath: "/opt/carlos/versions/v2/blog", Digest: strings.Repeat("b", 64), Tier: cgroup.TierChildren} |
| 138 | p, err := b.StartSidecarGen(context.Background(), rt, g) |
| 139 | if err != nil { |
| 140 | t.Fatalf("StartSidecarGen: %v", err) |
| 141 | } |
| 142 | defer p.Stop(context.Background()) //nolint:errcheck |
| 143 | want := instancectl.StagedExe{Path: g.StagedPath, Version: "v2", Digest: g.Digest} |
| 144 | ctl.mu.Lock() |
| 145 | defer ctl.mu.Unlock() |
| 146 | if len(ctl.sidecarExes) != 1 || ctl.sidecarExes[0] != want { |
| 147 | t.Fatalf("root was sent %+v for the sidecar, want %+v", ctl.sidecarExes, want) |
| 148 | } |
| 149 | if len(ctl.plainStarts) != 0 { |
| 150 | t.Fatalf("a live-link start ran too: %v", ctl.plainStarts) |
| 151 | } |
| 152 | if ctl.sidecarTiers[0] != cgroup.TierChildren.MemoryMax { |
| 153 | t.Fatalf("root was sent memory_max %d, want the generation's own %d", ctl.sidecarTiers[0], cgroup.TierChildren.MemoryMax) |
| 154 | } |
| 155 | } |
| 156 | |
| 157 | // Root refusing the staged path abandons the swap (the one classifier, |
| 158 | // verdictOf, reads instancectl.ErrStagedExe as NoOwnership). It is not a |
| 159 | // failure of the build, and the incumbent keeps serving. |
| 160 | func TestSwapAbandonsWhenRootRefusesTheStagedExe(t *testing.T) { |
| 161 | f := newSwapFixture(t) |
| 162 | f.b.genErr = fmt.Errorf("%w: not blog's staged v2", instancectl.ErrStagedExe) |
| 163 | res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) |
| 164 | if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, "staged executable refused") { |
| 165 | t.Fatalf("Swap = %+v, want abandoned as a root refusal", res) |
| 166 | } |
| 167 | if got := f.appSocket(t); got != f.incumbent.Socket { |
| 168 | t.Fatalf("AppSocket = %q, want the incumbent", got) |
| 169 | } |
| 170 | } |
| 171 | |
| 172 | // A nested staged generation is returned as soon as root has started its |
| 173 | // unit, with no health wait inside (spec §8.2): the socket here is never |
| 174 | // bound, and the real 30 s spawnHealthTimeout is left alone. root is sent |
| 175 | // the generation's own tier. |
| 176 | func TestStartStagedNestedReturnsTheUnitAtOnce(t *testing.T) { |
| 177 | b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{}) |
| 178 | b.GenContainment = true |
| 179 | ctl := &exeControl{} |
| 180 | b.Control = ctl |
| 181 | g := Gen{N: 7, Build: "v2", StagedPath: "/opt/carlos/versions/v2/blog", Digest: strings.Repeat("c", 64), Tier: cgroup.TierChildren} |
| 182 | done := make(chan error, 1) |
| 183 | var proc Proc |
| 184 | go func() { |
| 185 | var err error |
| 186 | _, proc, err = b.StartStaged(context.Background(), rt, g) |
| 187 | done <- err |
| 188 | }() |
| 189 | select { |
| 190 | case err := <-done: |
| 191 | if err != nil || proc == nil { |
| 192 | t.Fatalf("StartStaged = %v, %v: want the started unit", proc, err) |
| 193 | } |
| 194 | case <-time.After(5 * time.Second): |
| 195 | t.Fatal("StartStaged waited for health: it must return the started unit at once") |
| 196 | } |
| 197 | defer proc.Stop(context.Background()) //nolint:errcheck |
| 198 | ctl.mu.Lock() |
| 199 | defer ctl.mu.Unlock() |
| 200 | if len(ctl.got) != 1 || ctl.tiers[0] != cgroup.TierChildren.MemoryMax { |
| 201 | t.Fatalf("root was sent %+v at %v, want one start at the generation's 1 GiB", ctl.got, ctl.tiers) |
| 202 | } |
| 203 | } |
| 204 | |
| 205 | // Root refusing the staged build started nothing, so the agent kills |
| 206 | // nothing and holds nothing unproven. The refusal reaches the swap as |
| 207 | // ErrStagedExe alone (verdictOf: abandonment), with the generation number |
| 208 | // the caller keeps. |
| 209 | func TestStartStagedNestedRootRefusalIsNotUnproven(t *testing.T) { |
| 210 | b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{}) |
| 211 | b.GenContainment = true |
| 212 | ctl := &exeControl{exeErr: fmt.Errorf("%w: not blog's staged v2", instancectl.ErrStagedExe)} |
| 213 | b.Control = ctl |
| 214 | g := Gen{N: 7, Build: "v2", StagedPath: "/opt/carlos/versions/v2/blog", Digest: strings.Repeat("c", 64), Tier: cgroup.TierChildren} |
| 215 | got, proc, err := b.StartStaged(context.Background(), rt, g) |
| 216 | if !errors.Is(err, instancectl.ErrStagedExe) || errors.Is(err, errGenUnproven) || proc != nil || got.N != 7 { |
| 217 | t.Fatalf("StartStaged = %+v, %v, %v", got, proc, err) |
| 218 | } |
| 219 | if ctl.stopGens != 0 { |
| 220 | t.Fatalf("the agent killed %d generation(s) root never started", ctl.stopGens) |
| 221 | } |
| 222 | if verdictOf(err) != gateNoOwnership { |
| 223 | t.Fatalf("verdictOf(%v) = %d, want NoOwnership", err, verdictOf(err)) |
| 224 | } |
| 225 | // Final review T9: the launch file this start wrote (it carries the |
| 226 | // instance's environment) is removed after root's definite refusal. |
| 227 | launch := filepath.Join(b.pathsFor(rt).dataDir, instancectl.GenLaunchFileName(7)) |
| 228 | if _, err := os.Stat(launch); !os.IsNotExist(err) { |
| 229 | t.Fatalf("%s after root refused the start: %v, want it removed", launch, err) |
| 230 | } |
| 231 | } |
| 232 | |
| 233 | // A nested successor whose start failed and whose kill is unproven says so |
| 234 | // (errGenUnproven) and returns its generation number, so the member's stop |
| 235 | // can prove it (stopMember) and the operation set is not released. |
| 236 | func TestStartStagedNestedUnprovenKeepsTheGenerationNumber(t *testing.T) { |
| 237 | b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{}) |
| 238 | b.GenContainment = true |
| 239 | ctl := &exeControl{exeErr: errors.New("502 from root"), stopGenErr: errors.New("processes remain")} |
| 240 | b.Control = ctl |
| 241 | g := Gen{N: 7, Build: "v2", StagedPath: "/opt/carlos/versions/v2/blog", Digest: strings.Repeat("c", 64), Tier: cgroup.TierChildren} |
| 242 | got, proc, err := b.StartStaged(context.Background(), rt, g) |
| 243 | if !errors.Is(err, errGenUnproven) || proc != nil || got.N != 7 { |
| 244 | t.Fatalf("StartStaged = %+v, %v, %v: want errGenUnproven, no handle, generation 7", got, proc, err) |
| 245 | } |
| 246 | if ctl.stopGens != 1 { |
| 247 | t.Fatalf("StopGen calls = %d, want the one kill", ctl.stopGens) |
| 248 | } |
| 249 | if verdictOf(err) != gateProceed { |
| 250 | t.Fatalf("an unproven start must stay a failure, verdictOf = %d", verdictOf(err)) |
| 251 | } |
| 252 | } |
| 253 | |
| 254 | // A nested start with no staged path (a cold wake) is on the live link, as |
| 255 | // the flat arm is, and names no executable to root. |
| 256 | func TestStartStagedNestedWithoutAStagedPathUsesTheLiveLink(t *testing.T) { |
| 257 | b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{}) |
| 258 | b.GenContainment = true |
| 259 | ctl := &exeControl{} |
| 260 | b.Control = ctl |
| 261 | _, _, err := b.StartStaged(context.Background(), rt, Gen{N: 7}) |
| 262 | if err == nil || !strings.Contains(err.Error(), "live-link") || len(ctl.got) != 0 { |
| 263 | t.Fatalf("StartStaged = %v; root was sent %+v: want plan 2's live-link start", err, ctl.got) |
| 264 | } |
| 265 | } |
| 266 | |
amadan