| 1 | package activator |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "fmt" |
| 7 | "os" |
| 8 | "path/filepath" |
| 9 | "slices" |
| 10 | "strings" |
| 11 | "sync/atomic" |
| 12 | "testing" |
| 13 | "time" |
| 14 | |
| 15 | "github.com/carlosframework/platform/internal/cgroup" |
| 16 | "github.com/carlosframework/platform/internal/instancectl" |
| 17 | "github.com/carlosframework/platform/internal/registry" |
| 18 | "github.com/carlosframework/platform/internal/release" |
| 19 | ) |
| 20 | |
| 21 | // Settings spec §4.2, Task 5c: every unclaimed wake settles ONE executable |
| 22 | // identity after the restore, records it before the seed and the spawn, |
| 23 | // and runs only that file; legacy sidecars run their generation's recorded |
| 24 | // file by path whatever CARLOS_SWAP says. |
| 25 | |
| 26 | // identityFake is fakeBackend's identity capability. A test opts in |
| 27 | // (setIdentity); until then identityFor answers nil, so every test written |
| 28 | // before Task 5c keeps today's Start and Seed and their events. |
| 29 | type identityFake struct { |
| 30 | on bool |
| 31 | // live is the build live/<host> names, "" when the link is missing; |
| 32 | // LiveIdentity then materializes materializeTo first. |
| 33 | live string |
| 34 | materializeTo string |
| 35 | materializes int |
| 36 | captures int |
| 37 | // unversioned makes the live link name a file outside the version |
| 38 | // store (Version ""). |
| 39 | unversioned bool |
| 40 | // onLiveIdentity, when set, runs inside LiveIdentity before it resolves |
| 41 | // the link (under the fake's lock): a repoint that lands between a held |
| 42 | // wake's wait and its capture. |
| 43 | onLiveIdentity func(*identityFake) |
| 44 | exe []exeCall |
| 45 | last release.StagedIdentity |
| 46 | // buildIDs is every build BuildIdentity was asked to name, and |
| 47 | // buildIDErr its answer when set (a staged copy that cannot be read). |
| 48 | buildIDs []string |
| 49 | buildIDErr error |
| 50 | } |
| 51 | |
| 52 | // BuildIdentity is the fake's buildIdentifier: build's staged file as |
| 53 | // fakeIdentity names it, or buildIDErr. |
| 54 | func (b *fakeBackend) BuildIdentity(_ registry.Route, build string) (release.StagedIdentity, error) { |
| 55 | b.mu.Lock() |
| 56 | defer b.mu.Unlock() |
| 57 | b.ident.buildIDs = append(b.ident.buildIDs, build) |
| 58 | if b.ident.buildIDErr != nil { |
| 59 | return release.StagedIdentity{}, b.ident.buildIDErr |
| 60 | } |
| 61 | return fakeIdentity(build), nil |
| 62 | } |
| 63 | |
| 64 | func (b *fakeBackend) buildIdentities() []string { |
| 65 | b.mu.Lock() |
| 66 | defer b.mu.Unlock() |
| 67 | return slices.Clone(b.ident.buildIDs) |
| 68 | } |
| 69 | |
| 70 | type exeCall struct { |
| 71 | n uint64 |
| 72 | perGen bool |
| 73 | id release.StagedIdentity |
| 74 | } |
| 75 | |
| 76 | const unversionedPath = "/srv/hand-placed/app" |
| 77 | |
| 78 | func (b *fakeBackend) setIdentity(live string) { |
| 79 | b.mu.Lock() |
| 80 | defer b.mu.Unlock() |
| 81 | b.ident.on, b.ident.live = true, live |
| 82 | } |
| 83 | |
| 84 | func (b *fakeBackend) setLive(v string) { |
| 85 | b.mu.Lock() |
| 86 | defer b.mu.Unlock() |
| 87 | b.ident.live = v |
| 88 | } |
| 89 | |
| 90 | func (b *fakeBackend) identityFor(registry.Route) identityBackend { |
| 91 | b.mu.Lock() |
| 92 | defer b.mu.Unlock() |
| 93 | if !b.ident.on { |
| 94 | return nil |
| 95 | } |
| 96 | return b |
| 97 | } |
| 98 | |
| 99 | // LiveVersion makes New wire a.liveVer to the fake's link. "" (nothing |
| 100 | // opted in) is exactly the stub New used before. |
| 101 | func (b *fakeBackend) LiveVersion(string) (string, error) { |
| 102 | b.mu.Lock() |
| 103 | defer b.mu.Unlock() |
| 104 | return b.ident.live, nil |
| 105 | } |
| 106 | |
| 107 | func (b *fakeBackend) LiveIdentity(_ context.Context, rt registry.Route) (release.StagedIdentity, error) { |
| 108 | b.mu.Lock() |
| 109 | defer b.mu.Unlock() |
| 110 | if f := b.ident.onLiveIdentity; f != nil { |
| 111 | f(&b.ident) |
| 112 | } |
| 113 | if b.ident.unversioned { |
| 114 | b.ident.captures++ |
| 115 | return release.StagedIdentity{StagedPath: unversionedPath, Digest: "sha-of-hand-placed"}, nil |
| 116 | } |
| 117 | if b.ident.live == "" { |
| 118 | if b.ident.materializeTo == "" { |
| 119 | return release.StagedIdentity{}, fmt.Errorf("no live binary for %s", rt.Host) |
| 120 | } |
| 121 | b.ident.live = b.ident.materializeTo |
| 122 | b.ident.materializes++ |
| 123 | } |
| 124 | b.ident.captures++ |
| 125 | return fakeIdentity(b.ident.live), nil |
| 126 | } |
| 127 | |
| 128 | func (b *fakeBackend) StartGenExe(ctx context.Context, rt registry.Route, n uint64, perGen bool, id release.StagedIdentity) (Proc, Gen, error) { |
| 129 | // The exec seam, as the real backend's: the exec gate and the wake's |
| 130 | // pending record (amadan #272), before anything starts. |
| 131 | if err := runExecGate(ctx); err != nil { |
| 132 | return nil, Gen{}, err |
| 133 | } |
| 134 | b.mu.Lock() |
| 135 | defer b.mu.Unlock() |
| 136 | b.starts++ |
| 137 | b.order = append(b.order, "start "+id.Version) |
| 138 | b.startedRoutes = append(b.startedRoutes, rt) |
| 139 | b.ident.exe = append(b.ident.exe, exeCall{n: n, perGen: perGen, id: id}) |
| 140 | b.ident.last = id |
| 141 | sock := rt.Socket |
| 142 | if perGen { |
| 143 | sock = fmt.Sprintf("%s.g%d", rt.Socket, n) |
| 144 | } |
| 145 | g := Gen{N: n, Socket: sock, Control: sock + ".control"} |
| 146 | if b.startErr != nil { |
| 147 | return nil, g, b.startErr |
| 148 | } |
| 149 | proc := &fakeProc{label: "instance", rec: b, stuck: b.procStuck} |
| 150 | b.procs = append(b.procs, proc) |
| 151 | g.Build, g.StagedPath, g.Digest, g.Demand = id.Version, id.StagedPath, id.Digest, id.Demand |
| 152 | return proc, g, nil |
| 153 | } |
| 154 | |
| 155 | func (b *fakeBackend) SeedExe(ctx context.Context, rt registry.Route, id release.StagedIdentity) error { |
| 156 | if _, err := runPendingRecord(ctx); err != nil { |
| 157 | return err |
| 158 | } |
| 159 | b.mu.Lock() |
| 160 | defer b.mu.Unlock() |
| 161 | b.seeds++ |
| 162 | b.order = append(b.order, "seed "+id.Version) |
| 163 | b.seedRoutes = append(b.seedRoutes, rt) |
| 164 | b.ident.last = id |
| 165 | return b.seedErr |
| 166 | } |
| 167 | |
| 168 | var _ identityBackend = (*fakeBackend)(nil) |
| 169 | |
| 170 | // liveStarts counts Start calls: the live-link spawn the identity path |
| 171 | // replaces. |
| 172 | func (b *fakeBackend) liveStarts() int { |
| 173 | n := 0 |
| 174 | for _, ev := range b.events() { |
| 175 | if ev == "start:instance" { |
| 176 | n++ |
| 177 | } |
| 178 | } |
| 179 | return n |
| 180 | } |
| 181 | |
| 182 | // lastExeVersion is the build of the last file StartGenExe or SeedExe ran. |
| 183 | func (b *fakeBackend) lastExeVersion() string { |
| 184 | b.mu.Lock() |
| 185 | defer b.mu.Unlock() |
| 186 | return b.ident.last.Version |
| 187 | } |
| 188 | |
| 189 | // lastExe is the last StartGenExe call. |
| 190 | func (b *fakeBackend) lastExe() (exeCall, bool) { |
| 191 | b.mu.Lock() |
| 192 | defer b.mu.Unlock() |
| 193 | if len(b.ident.exe) == 0 { |
| 194 | return exeCall{}, false |
| 195 | } |
| 196 | return b.ident.exe[len(b.ident.exe)-1], true |
| 197 | } |
| 198 | |
| 199 | func (b *fakeBackend) identityCounts() (captures, materializes int) { |
| 200 | b.mu.Lock() |
| 201 | defer b.mu.Unlock() |
| 202 | return b.ident.captures, b.ident.materializes |
| 203 | } |
| 204 | |
| 205 | // identityActivator is testActivator with the identity capability on and |
| 206 | // live/<host> naming live. |
| 207 | func identityActivator(t *testing.T, live string) (*Service, *fakeBackend) { |
| 208 | t.Helper() |
| 209 | a, b, _, _ := testActivator(t, time.Hour) |
| 210 | b.setIdentity(live) |
| 211 | return a, b |
| 212 | } |
| 213 | |
| 214 | // identityRoute is testRoute with its socket and database in the test's |
| 215 | // own directory: a flag-on wake links the canonical path for real. |
| 216 | func identityRoute(t *testing.T, host string) registry.Route { |
| 217 | t.Helper() |
| 218 | rt := testRoute(host) |
| 219 | dir := t.TempDir() |
| 220 | rt.Socket, rt.DBPath = filepath.Join(dir, "app.sock"), filepath.Join(dir, "app.db") |
| 221 | return rt |
| 222 | } |
| 223 | |
| 224 | func seeded(rt registry.Route) registry.Route { |
| 225 | rt.Seed = `["seed"]` |
| 226 | return rt |
| 227 | } |
| 228 | |
| 229 | // recordInto is a BeforeExec that writes "record <build>" into b's event |
| 230 | // log before l's own record, so a test reads one ordered log. |
| 231 | func recordInto(b *fakeBackend, l *execLog) func(context.Context, string, string) error { |
| 232 | return func(ctx context.Context, host, build string) error { |
| 233 | b.record("record " + build) |
| 234 | return l.hook(ctx, host, build) |
| 235 | } |
| 236 | } |
| 237 | |
| 238 | // identityEvents is the record, seed and instance start events in b's log, |
| 239 | // in order: "record <v>", "seed <v>" and "start <v>" for the identity |
| 240 | // path, "seed" and "start:instance" for today's. |
| 241 | func identityEvents(b *fakeBackend) []string { |
| 242 | var out []string |
| 243 | for _, ev := range b.events() { |
| 244 | if strings.HasPrefix(ev, "record ") || strings.HasPrefix(ev, "seed") || strings.HasPrefix(ev, "start ") || ev == "start:instance" { |
| 245 | out = append(out, ev) |
| 246 | } |
| 247 | } |
| 248 | return out |
| 249 | } |
| 250 | |
| 251 | func (a *Service) entryGen(host string) (Gen, bool) { |
| 252 | a.mu.Lock() |
| 253 | defer a.mu.Unlock() |
| 254 | e := a.entries[host] |
| 255 | if e == nil || e.state != actAwake { |
| 256 | return Gen{}, false |
| 257 | } |
| 258 | return e.gen, true |
| 259 | } |
| 260 | |
| 261 | // --- the wake --- |
| 262 | |
| 263 | func TestTheBuildStateIsWrittenBeforeTheWake(t *testing.T) { |
| 264 | t.Parallel() |
| 265 | a, b := identityActivator(t, "v1") |
| 266 | l := &execLog{spawns: b.startCount} |
| 267 | a.BeforeExec = l.hook |
| 268 | rt := identityRoute(t, "one.example.com") |
| 269 | done, err := a.beginRequest(context.Background(), rt) |
| 270 | if err != nil { |
| 271 | t.Fatal(err) |
| 272 | } |
| 273 | done() |
| 274 | calls, at, locs := l.seen() |
| 275 | if len(calls) != 1 || calls[0] != "one.example.com@v1" || at[0] != 0 { |
| 276 | t.Fatalf("records %v at spawn counts %v, want one.example.com@v1 before the spawn", calls, at) |
| 277 | } |
| 278 | if locs[0] != (BuildStateLocation{Bucket: "test-bucket", Prefix: HibernatePrefix("", rt.Host)}) { |
| 279 | t.Fatalf("location = %+v, want the wake's lease", locs[0]) |
| 280 | } |
| 281 | x, ok := b.lastExe() |
| 282 | if b.startCount() != 1 || b.liveStarts() != 0 || !ok || x.id != fakeIdentity("v1") || x.perGen || x.n == 0 { |
| 283 | t.Fatalf("starts %d, live-link starts %d, exe %+v: want one flag-off start of the recorded v1", b.startCount(), b.liveStarts(), x) |
| 284 | } |
| 285 | if g, _ := a.entryGen(rt.Host); g.StagedPath != fakeIdentity("v1").StagedPath || g.Build != "v1" || g.N != x.n { |
| 286 | t.Fatalf("serving generation %+v, want the recorded file's", g) |
| 287 | } |
| 288 | } |
| 289 | |
| 290 | func TestAFailedWriteBlocksTheSpawn(t *testing.T) { |
| 291 | t.Parallel() |
| 292 | a, b := identityActivator(t, "v1") |
| 293 | b.restoreFresh = true |
| 294 | a.BeforeExec = (&execLog{fail: failAll}).hook |
| 295 | _, err := a.beginRequest(context.Background(), seeded(identityRoute(t, "one.example.com"))) |
| 296 | if !errors.Is(err, errBuildStateUnwritten) { |
| 297 | t.Fatalf("err = %v, want errBuildStateUnwritten", err) |
| 298 | } |
| 299 | if b.startCount() != 0 || b.seedCount() != 0 { |
| 300 | t.Fatalf("spawned %d, seeded %d with no build state", b.startCount(), b.seedCount()) |
| 301 | } |
| 302 | waitFor(t, "the lease to go back", func() bool { return b.releaseCount() == 1 }) |
| 303 | } |
| 304 | |
| 305 | // Review Focus 1, Astra I1 #5: root repoints the live link while the build |
| 306 | // state is being written. The wake still execs exactly the file it |
| 307 | // recorded, never the link, and records once. |
| 308 | func TestTheWakeExecsTheIdentityItRecorded(t *testing.T) { |
| 309 | a, b := identityActivator(t, "v1") |
| 310 | b.restoreFresh = true |
| 311 | l := &execLog{} |
| 312 | a.BeforeExec = func(ctx context.Context, host, build string) error { |
| 313 | b.setLive("v2") // root's repoint lands mid-write |
| 314 | return l.hook(ctx, host, build) |
| 315 | } |
| 316 | done, err := a.beginRequest(context.Background(), seeded(identityRoute(t, "one.example.com"))) |
| 317 | if err != nil { |
| 318 | t.Fatal(err) |
| 319 | } |
| 320 | done() |
| 321 | if calls, _, _ := l.seen(); len(calls) != 1 || calls[0] != "one.example.com@v1" { |
| 322 | t.Fatalf("calls = %v; want one record of v1", calls) |
| 323 | } |
| 324 | if got := identityEvents(b); !slices.Equal(got, []string{"seed v1", "start v1"}) { |
| 325 | t.Fatalf("events %v, want the recorded v1 seeded and started by path", got) |
| 326 | } |
| 327 | if c, _ := b.identityCounts(); b.liveStarts() != 0 || c != 1 { |
| 328 | t.Fatalf("live-link starts %d, identity captures %d: want one capture and no live-link exec", b.liveStarts(), c) |
| 329 | } |
| 330 | } |
| 331 | |
| 332 | // Astra I1 #5: the seed runs the recorded identity, after the record. |
| 333 | func TestTheSeedRunsTheRecordedIdentity(t *testing.T) { |
| 334 | a, b := identityActivator(t, "v1") |
| 335 | b.restoreFresh = true |
| 336 | l := &execLog{} |
| 337 | a.BeforeExec = recordInto(b, l) |
| 338 | done, err := a.beginRequest(context.Background(), seeded(identityRoute(t, "one.example.com"))) |
| 339 | if err != nil { |
| 340 | t.Fatal(err) |
| 341 | } |
| 342 | done() |
| 343 | if got := identityEvents(b); !slices.Equal(got, []string{"record v1", "seed v1", "start v1"}) { |
| 344 | t.Fatalf("events = %v; want record, seed and start, all v1, in that order", got) |
| 345 | } |
| 346 | } |
| 347 | |
| 348 | // Astra I1 #5: a planned wake records its boot build before the seed, seeds |
| 349 | // that build (never the live link), and startPinnedAt does not write it |
| 350 | // again. |
| 351 | func TestAPlannedWakeRecordsBeforeTheSeed(t *testing.T) { |
| 352 | t.Parallel() |
| 353 | next := stagedID(t, t.TempDir(), "v3") |
| 354 | a, b, _ := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap}) |
| 355 | b.fakeBackend.setIdentity("v1") |
| 356 | b.restoreFresh = true |
| 357 | l := &execLog{} |
| 358 | a.BeforeExec = recordInto(b.fakeBackend, l) |
| 359 | done, err := a.beginRequest(context.Background(), seeded(identityRoute(t, "one.example.com"))) |
| 360 | if err != nil { |
| 361 | t.Fatal(err) |
| 362 | } |
| 363 | done() |
| 364 | if got := identityEvents(b.fakeBackend); !slices.Equal(got, []string{"record v3", "seed v3"}) { |
| 365 | t.Fatalf("events = %v; want the planned build recorded before its seed", got) |
| 366 | } |
| 367 | if calls, _, _ := l.seen(); len(calls) != 1 { |
| 368 | t.Fatalf("calls = %v; startPinnedAt re-recorded the build the wake already recorded", calls) |
| 369 | } |
| 370 | if gens := b.started(); len(gens) != 1 || gens[0].Build != "v3" { |
| 371 | t.Fatalf("spawned %+v, want the planned v3", gens) |
| 372 | } |
| 373 | if c, _ := b.identityCounts(); c != 0 { |
| 374 | t.Fatalf("%d live-link captures: a planned wake runs its plan", c) |
| 375 | } |
| 376 | } |
| 377 | |
| 378 | // Ruling R-REPLAN: step 2b asks a planned wake's selection gate before it |
| 379 | // chooses the identity, so a replan lands first and the seed and the boot |
| 380 | // run one build, recorded once. |
| 381 | func TestAReplanBeforeTheSeedSeedsAndBootsOneBuild(t *testing.T) { |
| 382 | t.Parallel() |
| 383 | dir := t.TempDir() |
| 384 | stale, next := stagedID(t, dir, "v2"), stagedID(t, dir, "v3") |
| 385 | a, b, _ := planActivator(t, WakePlan{}) |
| 386 | b.fakeBackend.setIdentity("v1") |
| 387 | b.restoreFresh = true |
| 388 | var plans atomic.Int32 |
| 389 | a.Planner = func(context.Context, registry.Route) WakePlan { |
| 390 | if plans.Add(1) == 1 { |
| 391 | return WakePlan{Boot: stale, Mode: ModeOverlap, Check: func(string) error { return errors.New("superseded") }} |
| 392 | } |
| 393 | return WakePlan{Boot: next, Mode: ModeOverlap} |
| 394 | } |
| 395 | l := &execLog{} |
| 396 | a.BeforeExec = recordInto(b.fakeBackend, l) |
| 397 | done, err := a.beginRequest(context.Background(), seeded(identityRoute(t, "one.example.com"))) |
| 398 | if err != nil { |
| 399 | t.Fatal(err) |
| 400 | } |
| 401 | done() |
| 402 | if got := identityEvents(b.fakeBackend); !slices.Equal(got, []string{"record v3", "seed v3"}) { |
| 403 | t.Fatalf("events %v, want the replanned v3 recorded and seeded", got) |
| 404 | } |
| 405 | if calls, _, _ := l.seen(); !slices.Equal(calls, []string{"one.example.com@v3"}) { |
| 406 | t.Fatalf("records %v, want v3 once", calls) |
| 407 | } |
| 408 | if gens := b.started(); len(gens) != 1 || gens[0].Build != "v3" { |
| 409 | t.Fatalf("spawned %+v, want the replanned v3", gens) |
| 410 | } |
| 411 | } |
| 412 | |
| 413 | // D6: a replan that only the exec's own selection gate makes (after step |
| 414 | // 2b settled the first plan's boot) is recorded by startPinnedAt before it |
| 415 | // execs. The first plan's boot never ran, so its pending record is never |
| 416 | // written (amadan #272). |
| 417 | func TestAReplanAfterTheRecordRecordsTheNewBuild(t *testing.T) { |
| 418 | t.Parallel() |
| 419 | dir := t.TempDir() |
| 420 | stale, next := stagedID(t, dir, "v2"), stagedID(t, dir, "v3") |
| 421 | a, b, _ := planActivator(t, WakePlan{}) |
| 422 | b.fakeBackend.setIdentity("v1") |
| 423 | a.Planner = laterReplan(stale, next) |
| 424 | l := &execLog{spawns: func() int { return len(b.started()) }} |
| 425 | a.BeforeExec = l.hook |
| 426 | done, err := a.beginRequest(context.Background(), identityRoute(t, "one.example.com")) |
| 427 | if err != nil { |
| 428 | t.Fatal(err) |
| 429 | } |
| 430 | done() |
| 431 | calls, at, _ := l.seen() |
| 432 | if !slices.Equal(calls, []string{"one.example.com@v3"}) || at[0] != 0 { |
| 433 | t.Fatalf("records %v at spawn counts %v, want only v3, before the replanned exec", calls, at) |
| 434 | } |
| 435 | if gens := b.started(); len(gens) != 1 || gens[0].Build != "v3" { |
| 436 | t.Fatalf("spawned %+v, want the replanned v3", gens) |
| 437 | } |
| 438 | } |
| 439 | |
| 440 | // laterReplan plans stale first, whose selection gate passes once (step |
| 441 | // 2b) and refuses after (the exec's), and next on the replan. |
| 442 | func laterReplan(stale, next release.StagedIdentity) func(context.Context, registry.Route) WakePlan { |
| 443 | var plans, checks atomic.Int32 |
| 444 | return func(context.Context, registry.Route) WakePlan { |
| 445 | if plans.Add(1) == 1 { |
| 446 | return WakePlan{Boot: stale, Mode: ModeOverlap, Check: func(string) error { |
| 447 | if checks.Add(1) > 1 { |
| 448 | return errors.New("superseded") |
| 449 | } |
| 450 | return nil |
| 451 | }} |
| 452 | } |
| 453 | return WakePlan{Boot: next, Mode: ModeOverlap} |
| 454 | } |
| 455 | } |
| 456 | |
| 457 | // No hook (a box with no replica bucket): the wake still runs the file it |
| 458 | // captured, by path, and records nothing. |
| 459 | func TestANilHookRunsTheCapturedFileAndRecordsNothing(t *testing.T) { |
| 460 | a, b := identityActivator(t, "v1") |
| 461 | done, err := a.beginRequest(context.Background(), identityRoute(t, "one.example.com")) |
| 462 | if err != nil { |
| 463 | t.Fatal(err) |
| 464 | } |
| 465 | done() |
| 466 | if b.startCount() != 1 || b.lastExeVersion() != "v1" || b.liveStarts() != 0 { |
| 467 | t.Fatalf("starts %d of %q (live-link %d), want the captured v1 by path", b.startCount(), b.lastExeVersion(), b.liveStarts()) |
| 468 | } |
| 469 | } |
| 470 | |
| 471 | // Rulings R-EXEC-NOVERSION and -2: a live link outside the version store |
| 472 | // names no build. With no hook the wake is today's, in every layout: the |
| 473 | // live-link seed, start and sidecar, nothing published or recorded. With a |
| 474 | // hook it is refused. |
| 475 | func TestAnUnversionedLinkWithNoHookWakesAsToday(t *testing.T) { |
| 476 | t.Parallel() |
| 477 | a, b := sidecarActivator(t, "") |
| 478 | b.fakeBackend.setIdentity("") |
| 479 | b.mu.Lock() |
| 480 | b.ident.unversioned = true |
| 481 | b.mu.Unlock() |
| 482 | b.restoreFresh = true |
| 483 | a.GenerationsFile = filepath.Join(t.TempDir(), "zdd", "generations.json") |
| 484 | done, err := a.beginRequest(context.Background(), seeded(identityRoute(t, "two.example.com"))) |
| 485 | if err != nil { |
| 486 | t.Fatalf("an unversioned link with no hook: %v", err) |
| 487 | } |
| 488 | done() |
| 489 | if got := identityEvents(b.fakeBackend); !slices.Equal(got, []string{"seed", "start:instance"}) { |
| 490 | t.Fatalf("events %v, want today's live-link seed and start", got) |
| 491 | } |
| 492 | if _, ok := b.lastExe(); ok || len(b.sidecarsOn()) != 0 || b.sidecarStartCount() != 1 { |
| 493 | t.Fatalf("exe starts %v, generation sidecars %d, live-link sidecars %d; want today's wake", ok, len(b.sidecarsOn()), b.sidecarStartCount()) |
| 494 | } |
| 495 | if g, err := release.ReadGenerations(a.GenerationsFile); err == nil && g != nil && len(g.Paths["two.example.com"]) != 0 { |
| 496 | t.Fatalf("published %v for a file no build names", g.Paths) |
| 497 | } |
| 498 | |
| 499 | a, b = sidecarActivator(t, "") |
| 500 | b.fakeBackend.setIdentity("") |
| 501 | b.mu.Lock() |
| 502 | b.ident.unversioned = true |
| 503 | b.mu.Unlock() |
| 504 | a.BeforeExec = (&execLog{}).hook |
| 505 | if _, err := a.beginRequest(context.Background(), identityRoute(t, "two.example.com")); !errors.Is(err, errBuildStateUnwritten) { |
| 506 | t.Fatalf("err = %v, want an unversioned build refused while a build state is kept", err) |
| 507 | } |
| 508 | if b.startCount() != 0 || b.sidecarStartCount() != 0 { |
| 509 | t.Fatalf("%d starts, %d sidecars of a build no build state names", b.startCount(), b.sidecarStartCount()) |
| 510 | } |
| 511 | } |
| 512 | |
| 513 | // rootLikeControl refuses a staged executable with no version, as root's |
| 514 | // validateStagedExe answers 403, so a wake that sends one fails here as it |
| 515 | // would on a box. |
| 516 | type rootLikeControl struct{ identityControl } |
| 517 | |
| 518 | var errRootRefusesUnversioned = errors.New("instance-control: 403: the staged executable names no version") |
| 519 | |
| 520 | func (c rootLikeControl) StartExe(ctx context.Context, host string, role instancectl.Role, max int64, x instancectl.StagedExe) (string, error) { |
| 521 | if x.Version == "" { |
| 522 | c.rec("refused startexe") |
| 523 | return "", errRootRefusesUnversioned |
| 524 | } |
| 525 | return c.identityControl.StartExe(ctx, host, role, max, x) |
| 526 | } |
| 527 | func (c rootLikeControl) SeedExe(ctx context.Context, host string, max int64, x instancectl.StagedExe) (instancectl.SeedResponse, error) { |
| 528 | if x.Version == "" { |
| 529 | c.rec("refused seedexe") |
| 530 | return instancectl.SeedResponse{}, errRootRefusesUnversioned |
| 531 | } |
| 532 | return c.identityControl.SeedExe(ctx, host, max, x) |
| 533 | } |
| 534 | func (c rootLikeControl) StartGenExe(ctx context.Context, host string, n uint64, max int64, x instancectl.StagedExe) (string, error) { |
| 535 | if x.Version == "" { |
| 536 | c.rec("refused startgenexe") |
| 537 | return "", errRootRefusesUnversioned |
| 538 | } |
| 539 | return c.identityControl.StartGenExe(ctx, host, n, max, x) |
| 540 | } |
| 541 | func (c rootLikeControl) StartSidecarExe(ctx context.Context, host string, max int64, x instancectl.StagedExe) (string, error) { |
| 542 | if x.Version == "" { |
| 543 | c.rec("refused startsidecarexe") |
| 544 | return "", errRootRefusesUnversioned |
| 545 | } |
| 546 | return c.identityControl.StartSidecarExe(ctx, host, max, x) |
| 547 | } |
| 548 | |
| 549 | // leaselessExec is a real ExecBackend with the bucket's lease and restore |
| 550 | // faked, so a whole doWake runs over its real nested start and seed. |
| 551 | type leaselessExec struct { |
| 552 | *ExecBackend |
| 553 | fb *fakeBackend |
| 554 | } |
| 555 | |
| 556 | func (b *leaselessExec) Acquire(ctx context.Context, rt registry.Route) (Lease, error) { |
| 557 | return b.fb.Acquire(ctx, rt) |
| 558 | } |
| 559 | func (b *leaselessExec) Restore(context.Context, registry.Route) (bool, error) { return true, nil } |
| 560 | |
| 561 | // Ruling R-EXEC-NOVERSION-2 on the real nested backend: an unversioned live |
| 562 | // link with no hook seeds and starts through root's plain verbs, as today, |
| 563 | // where the staged verbs would name no version and root would refuse them. |
| 564 | func TestANestedUnversionedLinkWithNoHookWakesAsToday(t *testing.T) { |
| 565 | ctl := &fakeControl{inv: "inv1", seed: instancectl.SeedResponse{InvocationID: "s1", Result: "success", Status: "0", Code: "exited"}} |
| 566 | unit := &fakeUnit{} |
| 567 | eb, rt := nestedBackend(t, ctl, unit) |
| 568 | eb.Control = rootLikeControl{identityControl{ctl}} |
| 569 | hand := filepath.Join(t.TempDir(), "app") |
| 570 | if err := os.WriteFile(hand, []byte("#!/bin/sh\n"), 0o755); err != nil { |
| 571 | t.Fatal(err) |
| 572 | } |
| 573 | repointLiveLink(t, eb, rt.Host, hand) |
| 574 | if id, err := eb.LiveIdentity(context.Background(), rt); err != nil || id.Version != "" { |
| 575 | t.Fatalf("setup: live identity %+v, %v; want an unversioned file", id, err) |
| 576 | } |
| 577 | rt.Seed = `["seed"]` |
| 578 | // The seed unit's cgroup is empty after it (fakeUnit's zero state); the |
| 579 | // run unit comes up inside root's start, where the app would bind. |
| 580 | var stopServing func() |
| 581 | ctl.onStart = func() { |
| 582 | unit.set("active", "inv1", true, true) |
| 583 | stopServing = serveOK(t, rt.Socket) |
| 584 | } |
| 585 | defer func() { |
| 586 | if stopServing != nil { |
| 587 | stopServing() |
| 588 | } |
| 589 | }() |
| 590 | reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 591 | if err != nil { |
| 592 | t.Fatal(err) |
| 593 | } |
| 594 | a := New(Config{Bucket: "test-bucket", Idle: time.Hour}, &leaselessExec{ExecBackend: eb, fb: &fakeBackend{}}, reg) |
| 595 | done, err := a.beginRequest(context.Background(), rt) |
| 596 | if err != nil { |
| 597 | t.Fatalf("a nested unversioned link with no hook did not wake: %v\nroot was asked:\n%s", err, ctl.joined()) |
| 598 | } |
| 599 | done() |
| 600 | got := ctl.joined() |
| 601 | if !strings.Contains(got, "seed "+rt.Host) || !strings.Contains(got, "start "+rt.Host+" run") || strings.Contains(got, "exe") { |
| 602 | t.Fatalf("root was asked:\n%s\nwant today's plain seed and run start", got) |
| 603 | } |
| 604 | unit.set("inactive", "inv1", false, false) |
| 605 | } |
| 606 | |
| 607 | // A refused record starts nothing on any wake kind: no seed, no instance, |
| 608 | // no worker, no planned build, and the member reads the fence's wording. |
| 609 | func TestARefusingHookStartsNothing(t *testing.T) { |
| 610 | t.Parallel() |
| 611 | a, b := identityActivator(t, "v1") |
| 612 | b.restoreFresh, b.sidecarStaged = true, true |
| 613 | a.BeforeExec = (&execLog{fail: failAll}).hook |
| 614 | _, err := a.beginRequest(context.Background(), seeded(identityRoute(t, "one.example.com"))) |
| 615 | if !errors.Is(err, errBuildStateUnwritten) { |
| 616 | t.Fatalf("err = %v, want errBuildStateUnwritten", err) |
| 617 | } |
| 618 | if got := memberSafeWakeReason(err); got != memberSafeWakeReason(ErrFenced) { |
| 619 | t.Fatalf("a member reads %q, want the fence's wording", got) |
| 620 | } |
| 621 | if b.startCount() != 0 || b.seedCount() != 0 || b.sidecarStartCount() != 0 { |
| 622 | t.Fatalf("started %d, seeded %d, sidecars %d after a refused record", b.startCount(), b.seedCount(), b.sidecarStartCount()) |
| 623 | } |
| 624 | |
| 625 | sa, sb, _, _, _ := stagedActivator(t) |
| 626 | sb.fakeBackend.setIdentity("v1") |
| 627 | sa.BeforeExec = (&execLog{fail: failAll}).hook |
| 628 | if err := sa.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); !errors.Is(err, errBuildStateUnwritten) { |
| 629 | t.Fatalf("WakeWorker = %v, want errBuildStateUnwritten", err) |
| 630 | } |
| 631 | // The record is written at the worker's exec (amadan #272), so the |
| 632 | // spawn is entered and refused there: no child. |
| 633 | if sb.workerStarts() != 0 { |
| 634 | t.Fatalf("%d children after a refused record", sb.workerStarts()) |
| 635 | } |
| 636 | |
| 637 | next := stagedID(t, t.TempDir(), "v3") |
| 638 | pa, pb, _ := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap}) |
| 639 | pb.fakeBackend.setIdentity("v1") |
| 640 | pb.restoreFresh = true |
| 641 | pa.BeforeExec = (&execLog{fail: failAll}).hook |
| 642 | if _, err := pa.beginRequest(context.Background(), seeded(identityRoute(t, "one.example.com"))); !errors.Is(err, errBuildStateUnwritten) { |
| 643 | t.Fatalf("planned wake err = %v, want errBuildStateUnwritten", err) |
| 644 | } |
| 645 | if len(pb.started()) != 0 || pb.seedCount() != 0 { |
| 646 | t.Fatalf("planned: spawned %+v, seeded %d after a refused record", pb.started(), pb.seedCount()) |
| 647 | } |
| 648 | } |
| 649 | |
| 650 | // Astra I3 #3: identity capability is per route. On a SplitBackend whose |
| 651 | // exec half pins executables, a unit-backed route still wakes by today's |
| 652 | // Start and Seed, records nothing, and never reaches StartGenExe; an exec |
| 653 | // row on the same service takes the identity path. |
| 654 | func TestAUnitRouteOnASplitBackendKeepsTodaysWake(t *testing.T) { |
| 655 | exec, unit := &fakeBackend{}, &fakeBackend{} |
| 656 | exec.setIdentity("v1") |
| 657 | unit.setIdentity("v1") |
| 658 | // struct{ Backend } hides the unit fake's identity methods, as |
| 659 | // UnitBackend has none. |
| 660 | split := &SplitBackend{Exec: exec, Unit: struct{ Backend }{unit}} |
| 661 | reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 662 | if err != nil { |
| 663 | t.Fatal(err) |
| 664 | } |
| 665 | a := New(Config{Bucket: "test-bucket", Idle: time.Hour}, split, reg) // as agent.go builds it |
| 666 | l := &execLog{} |
| 667 | a.BeforeExec = l.hook |
| 668 | unit.restoreFresh = true |
| 669 | ur := seeded(identityRoute(t, "u.example.com")) |
| 670 | ur.Backing = registry.BackingUnit |
| 671 | done, err := a.beginRequest(context.Background(), ur) |
| 672 | if err != nil { |
| 673 | t.Fatalf("a unit route's wake failed on the split backend: %v", err) |
| 674 | } |
| 675 | done() |
| 676 | calls, _, _ := l.seen() |
| 677 | if unit.liveStarts() != 1 || unit.seedCount() != 1 || exec.startCount() != 0 || len(calls) != 0 { |
| 678 | t.Fatalf("unit live starts %d, seeds %d, exec starts %d, records %v; want today's unit wake, nothing recorded", |
| 679 | unit.liveStarts(), unit.seedCount(), exec.startCount(), calls) |
| 680 | } |
| 681 | if got := identityEvents(unit); !slices.Equal(got, []string{"seed", "start:instance"}) { |
| 682 | t.Fatalf("unit events %v, want today's seed and start", got) |
| 683 | } |
| 684 | done, err = a.beginRequest(context.Background(), identityRoute(t, "e.example.com")) |
| 685 | if err != nil { |
| 686 | t.Fatal(err) |
| 687 | } |
| 688 | done() |
| 689 | if calls, _, _ := l.seen(); exec.lastExeVersion() != "v1" || len(calls) != 1 || calls[0] != "e.example.com@v1" { |
| 690 | t.Fatalf("the exec row did not take the identity path: %q, %v", exec.lastExeVersion(), calls) |
| 691 | } |
| 692 | } |
| 693 | |
| 694 | // Astra I2 #6: a cold default instance with no live link still wakes: the |
| 695 | // Materialize hook runs before the identity is captured, and the identity |
| 696 | // is the build it materialized. |
| 697 | func TestADefaultWakeMaterializesBeforeItCapturesTheIdentity(t *testing.T) { |
| 698 | a, b := identityActivator(t, "") |
| 699 | b.mu.Lock() |
| 700 | b.ident.materializeTo = "v1" |
| 701 | b.mu.Unlock() |
| 702 | l := &execLog{} |
| 703 | a.BeforeExec = l.hook |
| 704 | done, err := a.beginRequest(context.Background(), identityRoute(t, "one.example.com")) |
| 705 | if err != nil { |
| 706 | t.Fatalf("a cold wake failed before materializing: %v", err) |
| 707 | } |
| 708 | done() |
| 709 | calls, _, _ := l.seen() |
| 710 | if _, m := b.identityCounts(); m != 1 || len(calls) != 1 || calls[0] != "one.example.com@v1" || b.lastExeVersion() != "v1" { |
| 711 | t.Fatalf("materialized %d, recorded %v, ran %q", m, calls, b.lastExeVersion()) |
| 712 | } |
| 713 | } |
| 714 | |
| 715 | // CARLOS_SWAP on, no plan: the recorded file runs as generation n on its |
| 716 | // own socket, exactly where startGen would have put it. |
| 717 | func TestAFlagOnUnplannedWakeRunsTheRecordedFileAtGenN(t *testing.T) { |
| 718 | a, b := identityActivator(t, "v1") |
| 719 | a.gens = true |
| 720 | l := &execLog{} |
| 721 | a.BeforeExec = l.hook |
| 722 | rt := identityRoute(t, "one.example.com") |
| 723 | done, err := a.beginRequest(context.Background(), rt) |
| 724 | if err != nil { |
| 725 | t.Fatal(err) |
| 726 | } |
| 727 | done() |
| 728 | x, ok := b.lastExe() |
| 729 | if !ok || !x.perGen || x.n == 0 || x.id != fakeIdentity("v1") || b.liveStarts() != 0 { |
| 730 | t.Fatalf("exe %+v (live-link starts %d), want the recorded v1 at its own generation", x, b.liveStarts()) |
| 731 | } |
| 732 | if calls, _, _ := l.seen(); len(calls) != 1 || calls[0] != "one.example.com@v1" { |
| 733 | t.Fatalf("records %v", calls) |
| 734 | } |
| 735 | g, _ := a.entryGen(rt.Host) |
| 736 | if g.N != x.n || g.Socket != fmt.Sprintf("%s.g%d", rt.Socket, x.n) || g.StagedPath != fakeIdentity("v1").StagedPath { |
| 737 | t.Fatalf("serving generation %+v, want generation %d of the recorded file", g, x.n) |
| 738 | } |
| 739 | } |
| 740 | |
| 741 | // Ruling R-COLDPUBLISH: the generations file names the captured build |
| 742 | // before its build state does, so root cannot sweep the file between the |
| 743 | // capture and the exec. A refused wake's pending build is dropped again. |
| 744 | func TestTheColdWakePublishesBeforeItRecords(t *testing.T) { |
| 745 | t.Parallel() |
| 746 | a, b := identityActivator(t, "v1") |
| 747 | a.GenerationsFile = filepath.Join(t.TempDir(), "zdd", "generations.json") |
| 748 | path := fakeIdentity("v1").StagedPath |
| 749 | names := func() bool { |
| 750 | g, err := release.ReadGenerations(a.GenerationsFile) |
| 751 | return err == nil && g != nil && slices.Contains(g.Paths["one.example.com"], path) |
| 752 | } |
| 753 | var publishedFirst atomic.Bool |
| 754 | l := &execLog{} |
| 755 | a.BeforeExec = func(ctx context.Context, host, build string) error { |
| 756 | publishedFirst.Store(names()) |
| 757 | return l.hook(ctx, host, build) |
| 758 | } |
| 759 | done, err := a.beginRequest(context.Background(), identityRoute(t, "one.example.com")) |
| 760 | if err != nil { |
| 761 | t.Fatal(err) |
| 762 | } |
| 763 | done() |
| 764 | if !publishedFirst.Load() { |
| 765 | t.Fatal("the build state was written before the generations file named the build") |
| 766 | } |
| 767 | waitFor(t, "the serving generation to stay published", names) |
| 768 | if b.startCount() != 1 { |
| 769 | t.Fatalf("starts = %d", b.startCount()) |
| 770 | } |
| 771 | |
| 772 | a, _ = identityActivator(t, "v1") |
| 773 | a.GenerationsFile = filepath.Join(t.TempDir(), "zdd", "generations.json") |
| 774 | a.BeforeExec = (&execLog{fail: failAll}).hook |
| 775 | if _, err := a.beginRequest(context.Background(), identityRoute(t, "one.example.com")); !errors.Is(err, errBuildStateUnwritten) { |
| 776 | t.Fatalf("err = %v", err) |
| 777 | } |
| 778 | waitFor(t, "the refused wake's pending build to be dropped", func() bool { return !names() }) |
| 779 | } |
| 780 | |
| 781 | // A hold swap's pinned wake records the pin before the seed, seeds the pin |
| 782 | // (never the live link), and startPinnedAt does not record it again. |
| 783 | func TestAPinnedWakeSeedsThePin(t *testing.T) { |
| 784 | f := newSwapFixture(t) |
| 785 | f.b.fakeBackend.setIdentity("v1") |
| 786 | f.b.restoreFresh = true |
| 787 | f.a.mu.Lock() |
| 788 | f.a.entries[f.host].route.Seed = `["seed"]` |
| 789 | f.a.mu.Unlock() |
| 790 | l := &execLog{} |
| 791 | f.a.BeforeExec = recordInto(f.b.fakeBackend, l) |
| 792 | res := f.a.Swap(context.Background(), f.attempt(ModeHold), f.newest) |
| 793 | if res.Outcome != SwapDone { |
| 794 | t.Fatalf("Swap = %+v", res) |
| 795 | } |
| 796 | // start:instance is the fixture's incumbent, woken before the test |
| 797 | // opted into the identity capability. |
| 798 | if got := identityEvents(f.b.fakeBackend); !slices.Equal(got, []string{"start:instance", "record v2", "seed v2"}) { |
| 799 | t.Fatalf("events %v, want the pin recorded, then seeded", got) |
| 800 | } |
| 801 | if calls, _, _ := l.seen(); len(calls) != 1 { |
| 802 | t.Fatalf("records %v: the pin was recorded twice", calls) |
| 803 | } |
| 804 | if c, _ := f.b.identityCounts(); c != 0 { |
| 805 | t.Fatalf("%d live-link captures: a pinned wake runs its pin", c) |
| 806 | } |
| 807 | } |
| 808 | |
| 809 | // A worker wake execs the identity it recorded at step 2b (onto), so a |
| 810 | // repoint during the record cannot change the worker's build, and its |
| 811 | // exec-time hook does not write the same build again. |
| 812 | func TestAWorkerWakeExecsTheIdentityItRecorded(t *testing.T) { |
| 813 | t.Parallel() |
| 814 | a, b, _, _, _ := stagedActivator(t) |
| 815 | b.fakeBackend.setIdentity("v1") |
| 816 | l := &execLog{} |
| 817 | a.BeforeExec = func(ctx context.Context, host, build string) error { |
| 818 | b.fakeBackend.setLive("v2") |
| 819 | return l.hook(ctx, host, build) |
| 820 | } |
| 821 | host := "bot.x.internal" |
| 822 | if err := a.WakeWorker(alwaysOnWorkerRoute(host)); err != nil { |
| 823 | t.Fatalf("WakeWorker: %v", err) |
| 824 | } |
| 825 | if got := b.calls(); len(got) != 1 || got[0] == nil || *got[0] != fakeIdentity("v1") { |
| 826 | t.Fatalf("staged spawns %v, want one onto the recorded v1", got) |
| 827 | } |
| 828 | if calls, _, _ := l.seen(); len(calls) != 1 || calls[0] != host+"@v1" { |
| 829 | t.Fatalf("records %v, want one of v1", calls) |
| 830 | } |
| 831 | if got := a.entryServing(host); got != fakeIdentity("v1") { |
| 832 | t.Fatalf("serving %+v, want the recorded v1", got) |
| 833 | } |
| 834 | } |
| 835 | |
| 836 | // --- legacy sidecars (ruling R-EXEC-SIDECAR) --- |
| 837 | |
| 838 | // sidecarActivator is a flag-off service whose backend can start a |
| 839 | // sidecar from a generation (planGenBackend) and pin an identity, with a |
| 840 | // sidecar staged for every host. |
| 841 | func sidecarActivator(t *testing.T, live string) (*Service, *planGenBackend) { |
| 842 | t.Helper() |
| 843 | b := newPlanGenBackend() |
| 844 | reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 845 | if err != nil { |
| 846 | t.Fatal(err) |
| 847 | } |
| 848 | a := New(Config{Bucket: "test-bucket", Idle: time.Hour}, b, reg) |
| 849 | b.sidecarStaged = true |
| 850 | if live != "" { |
| 851 | b.fakeBackend.setIdentity(live) |
| 852 | } |
| 853 | return a, b |
| 854 | } |
| 855 | |
| 856 | // crashSidecar kills host's legacy sidecar and runs the supervisor. |
| 857 | func crashSidecar(t *testing.T, a *Service, host string) { |
| 858 | t.Helper() |
| 859 | a.mu.Lock() |
| 860 | e := a.entries[host] |
| 861 | var p *fakeProc |
| 862 | if e != nil { |
| 863 | p, _ = e.sidecar.(*fakeProc) |
| 864 | } |
| 865 | a.mu.Unlock() |
| 866 | if p == nil { |
| 867 | t.Fatalf("%s has no sidecar to crash", host) |
| 868 | } |
| 869 | p.kill() |
| 870 | a.sweepOnce() |
| 871 | } |
| 872 | |
| 873 | // Flag off: the co-spawned sidecar runs the recorded file by path, not |
| 874 | // whatever the live link names by then, and the wake's record covers it. |
| 875 | func TestAFlagOffSidecarRunsTheRecordedFile(t *testing.T) { |
| 876 | a, b := sidecarActivator(t, "v1") |
| 877 | l := &execLog{} |
| 878 | a.BeforeExec = func(ctx context.Context, host, build string) error { |
| 879 | b.fakeBackend.setLive("v2") |
| 880 | return l.hook(ctx, host, build) |
| 881 | } |
| 882 | done, err := a.beginRequest(context.Background(), identityRoute(t, "one.example.com")) |
| 883 | if err != nil { |
| 884 | t.Fatal(err) |
| 885 | } |
| 886 | done() |
| 887 | sc := b.sidecarsOn() |
| 888 | if len(sc) != 1 || sc[0].StagedPath != fakeIdentity("v1").StagedPath || sc[0].Build != "v1" || b.sidecarStartCount() != 0 { |
| 889 | t.Fatalf("sidecars on %+v, %d live-link sidecar starts; want the recorded v1 by path", sc, b.sidecarStartCount()) |
| 890 | } |
| 891 | if calls, _, _ := l.seen(); len(calls) != 1 { |
| 892 | t.Fatalf("records %v: the co-spawn wrote the wake's build again", calls) |
| 893 | } |
| 894 | } |
| 895 | |
| 896 | // A supervisor restart is a new exec against the instance's data: it |
| 897 | // records its generation's build first, and runs that file. |
| 898 | func TestASidecarRestartRecordsItsBuild(t *testing.T) { |
| 899 | t.Parallel() |
| 900 | a, b := sidecarActivator(t, "v1") |
| 901 | l := &execLog{spawns: func() int { return len(b.sidecarsOn()) }} |
| 902 | a.BeforeExec = l.hook |
| 903 | host := "one.example.com" |
| 904 | done, err := a.beginRequest(context.Background(), identityRoute(t, host)) |
| 905 | if err != nil { |
| 906 | t.Fatal(err) |
| 907 | } |
| 908 | done() |
| 909 | b.fakeBackend.setLive("v2") // a repoint since the wake |
| 910 | crashSidecar(t, a, host) |
| 911 | waitFor(t, "the sidecar restart", func() bool { return len(b.sidecarsOn()) == 2 && a.sidecarSettled(host) }) |
| 912 | calls, at, _ := l.seen() |
| 913 | if !slices.Equal(calls, []string{host + "@v1", host + "@v1"}) || at[1] != 1 { |
| 914 | t.Fatalf("records %v at sidecar counts %v, want the restart's v1 before its exec", calls, at) |
| 915 | } |
| 916 | if sc := b.sidecarsOn(); sc[1].StagedPath != fakeIdentity("v1").StagedPath { |
| 917 | t.Fatalf("restarted on %+v, want the generation's recorded file", sc[1]) |
| 918 | } |
| 919 | } |
| 920 | |
| 921 | // A refused sidecar record starts no sidecar and is logged as a sidecar |
| 922 | // start failure; the instance keeps serving, at the wake and at a restart. |
| 923 | func TestARefusedSidecarRecordStartsNoSidecar(t *testing.T) { |
| 924 | t.Parallel() |
| 925 | a, b := sidecarActivator(t, "v1") |
| 926 | l := &execLog{} |
| 927 | a.BeforeExec = l.hook |
| 928 | host := "one.example.com" |
| 929 | done, err := a.beginRequest(context.Background(), identityRoute(t, host)) |
| 930 | if err != nil { |
| 931 | t.Fatal(err) |
| 932 | } |
| 933 | done() |
| 934 | l.mu.Lock() |
| 935 | l.fail = failAll |
| 936 | l.mu.Unlock() |
| 937 | crashSidecar(t, a, host) |
| 938 | waitFor(t, "the refused restart", func() bool { |
| 939 | calls, _, _ := l.seen() |
| 940 | return len(calls) == 2 && a.sidecarSettled(host) |
| 941 | }) |
| 942 | if n := len(b.sidecarsOn()); n != 1 || b.sidecarStartCount() != 0 { |
| 943 | t.Fatalf("%d generation sidecars, %d live-link sidecars after a refused record", n, b.sidecarStartCount()) |
| 944 | } |
| 945 | if _, ok := a.entryGen(host); !ok { |
| 946 | t.Fatal("the instance stopped serving over its sidecar's record") |
| 947 | } |
| 948 | |
| 949 | // At the wake: a replanned build is recorded again by its sidecar (the |
| 950 | // wake's context names the first plan's, which never ran and so is never |
| 951 | // written), and that record is refused. |
| 952 | dir := t.TempDir() |
| 953 | stale, next := stagedID(t, dir, "v2"), stagedID(t, dir, "v3") |
| 954 | pa, pb, _ := planActivator(t, WakePlan{}) |
| 955 | pb.fakeBackend.setIdentity("v1") |
| 956 | pb.sidecarStaged = true |
| 957 | pa.Planner = laterReplan(stale, next) |
| 958 | var n atomic.Int32 |
| 959 | pl := &execLog{fail: func(string) bool { return n.Add(1) == 2 }} |
| 960 | pa.BeforeExec = pl.hook |
| 961 | done, err = pa.beginRequest(context.Background(), identityRoute(t, host)) |
| 962 | if err != nil { |
| 963 | t.Fatalf("the wake failed over its sidecar's record: %v", err) |
| 964 | } |
| 965 | done() |
| 966 | if calls, _, _ := pl.seen(); !slices.Equal(calls, []string{host + "@v3", host + "@v3"}) { |
| 967 | t.Fatalf("records %v, want the replanned spawn's and the sidecar's", calls) |
| 968 | } |
| 969 | if sc := pb.sidecarsOn(); len(sc) != 0 || pb.sidecarStartCount() != 0 { |
| 970 | t.Fatalf("sidecars %+v (%d live): a sidecar ran unrecorded", sc, pb.sidecarStartCount()) |
| 971 | } |
| 972 | if g, ok := pa.entryGen(host); !ok || g.Build != "v3" { |
| 973 | t.Fatalf("serving %+v (%v), want v3", g, ok) |
| 974 | } |
| 975 | } |
| 976 | |
| 977 | // An instance whose generation names no staged file (awake from before |
| 978 | // this binary): its sidecar resolves the live link once, records that |
| 979 | // build, and runs that file by path. |
| 980 | func TestASidecarOnAnUnstagedGenerationResolvesRecordsAndRunsByPath(t *testing.T) { |
| 981 | t.Parallel() |
| 982 | a, b := sidecarActivator(t, "") |
| 983 | host := "one.example.com" |
| 984 | done, err := a.beginRequest(context.Background(), identityRoute(t, host)) |
| 985 | if err != nil { |
| 986 | t.Fatal(err) |
| 987 | } |
| 988 | done() |
| 989 | if g, _ := a.entryGen(host); g.StagedPath != "" || b.sidecarStartCount() != 1 { |
| 990 | t.Fatalf("setup: generation %+v, %d live-link sidecars", g, b.sidecarStartCount()) |
| 991 | } |
| 992 | b.fakeBackend.setIdentity("v1") |
| 993 | l := &execLog{} |
| 994 | a.BeforeExec = l.hook |
| 995 | b.lastSidecar().kill() |
| 996 | a.sweepOnce() |
| 997 | waitFor(t, "the sidecar restart", func() bool { return len(b.sidecarsOn()) == 1 && a.sidecarSettled(host) }) |
| 998 | if sc := b.sidecarsOn(); sc[0].StagedPath != fakeIdentity("v1").StagedPath || sc[0].Build != "v1" || sc[0].Tier != cgroup.TierDefault { |
| 999 | t.Fatalf("restarted on %+v, want the resolved v1 by path at its own tier (GenTier)", sc[0]) |
| 1000 | } |
| 1001 | if calls, _, _ := l.seen(); !slices.Equal(calls, []string{host + "@v1"}) { |
| 1002 | t.Fatalf("records %v, want the resolved v1", calls) |
| 1003 | } |
| 1004 | if c, _ := b.identityCounts(); c != 1 || b.sidecarStartCount() != 1 { |
| 1005 | t.Fatalf("captures %d, live-link sidecars %d: want one capture and no new live-link start", c, b.sidecarStartCount()) |
| 1006 | } |
| 1007 | } |
| 1008 | |
| 1009 | // Flag on: a sidecar attached late to the incumbent and its replacement |
| 1010 | // after an overlap switch each record their generation's build before |
| 1011 | // their exec (the replacement runs off any wake's context, so it writes). |
| 1012 | func TestASidecarReplacementRecordsTheNewBuild(t *testing.T) { |
| 1013 | t.Parallel() |
| 1014 | f := newSwapFixture(t) |
| 1015 | f.b.sidecarStaged = true |
| 1016 | l := &execLog{spawns: func() int { return len(f.b.sidecarGenPaths()) }} |
| 1017 | f.a.BeforeExec = l.hook |
| 1018 | f.a.sweepOnce() |
| 1019 | waitFor(t, "late attachment", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 1 }) |
| 1020 | if res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest); res.Outcome != SwapDone { |
| 1021 | t.Fatalf("Swap = %+v", res) |
| 1022 | } |
| 1023 | waitFor(t, "the replacement", func() bool { return f.a.sidecarSettled(f.host) && len(f.b.sidecarGenPaths()) == 2 }) |
| 1024 | calls, at, _ := l.seen() |
| 1025 | want := []string{f.host + "@v1", f.host + "@v2", f.host + "@v2"} |
| 1026 | if !slices.Equal(calls, want) || at[0] != 0 || at[2] != 1 { |
| 1027 | t.Fatalf("records %v at sidecar counts %v, want %v: the attachment's v1, the successor's v2, then the replacement's v2 before its exec", calls, at, want) |
| 1028 | } |
| 1029 | if paths := f.b.sidecarGenPaths(); paths[1] != f.staged { |
| 1030 | t.Fatalf("replacement ran %q, want %q", paths[1], f.staged) |
| 1031 | } |
| 1032 | } |
| 1033 | |
| 1034 | // A sidecar restart asks ownership again after its record: a lease lost |
| 1035 | // while the build state was written execs nothing. |
| 1036 | func TestASidecarExecsOnlyIfStillOwnedAfterItsRecord(t *testing.T) { |
| 1037 | t.Parallel() |
| 1038 | a, b := sidecarActivator(t, "v1") |
| 1039 | host := "one.example.com" |
| 1040 | var lose atomic.Bool |
| 1041 | l := &execLog{} |
| 1042 | a.BeforeExec = func(ctx context.Context, h, build string) error { |
| 1043 | if lose.Load() { |
| 1044 | b.lastRep().setErr(errors.New("lease lost")) |
| 1045 | } |
| 1046 | return l.hook(ctx, h, build) |
| 1047 | } |
| 1048 | done, err := a.beginRequest(context.Background(), identityRoute(t, host)) |
| 1049 | if err != nil { |
| 1050 | t.Fatal(err) |
| 1051 | } |
| 1052 | done() |
| 1053 | lose.Store(true) |
| 1054 | crashSidecar(t, a, host) |
| 1055 | waitFor(t, "the restart's record", func() bool { |
| 1056 | calls, _, _ := l.seen() |
| 1057 | return len(calls) == 2 && a.sidecarSettled(host) |
| 1058 | }) |
| 1059 | if n := len(b.sidecarsOn()); n != 1 || b.sidecarStartCount() != 0 { |
| 1060 | t.Fatalf("%d generation sidecars, %d live-link: a sidecar exec'd after its lease was lost", n, b.sidecarStartCount()) |
| 1061 | } |
| 1062 | } |
| 1063 | |
| 1064 | // A worker wake whose token cannot be minted fails after step 2b published |
| 1065 | // its build: the wake's pending build is dropped with it, and the lease |
| 1066 | // goes back. |
| 1067 | func TestAWorkerTokenMintFailureDropsThePendingBuild(t *testing.T) { |
| 1068 | a, b, _, _, _ := stagedActivator(t) |
| 1069 | b.fakeBackend.setIdentity("v1") |
| 1070 | a.GenerationsFile = filepath.Join(t.TempDir(), "zdd", "generations.json") |
| 1071 | old := randRead |
| 1072 | randRead = func([]byte) (int, error) { return 0, errors.New("entropy pool exhausted") } |
| 1073 | defer func() { randRead = old }() |
| 1074 | host := "bot.x.internal" |
| 1075 | if err := a.WakeWorker(testWorkerRoute(host)); err == nil || !strings.Contains(err.Error(), "mint worker token") { |
| 1076 | t.Fatalf("WakeWorker = %v, want the mint failure", err) |
| 1077 | } |
| 1078 | waitFor(t, "the pending build to be dropped", func() bool { |
| 1079 | g, err := release.ReadGenerations(a.GenerationsFile) |
| 1080 | return err == nil && g != nil && !slices.Contains(g.Paths[host], fakeIdentity("v1").StagedPath) |
| 1081 | }) |
| 1082 | if n := b.releaseCount(); n != 1 || b.workerStarts() != 0 { |
| 1083 | t.Fatalf("releases %d, workers %d, want the lease back and nothing spawned", n, b.workerStarts()) |
| 1084 | } |
| 1085 | } |
| 1086 | |
amadan