| index 465e27096..c98a63181 100644 |
| --- a/internal/activator/wakeidentity_test.go |
| +++ b/internal/activator/wakeidentity_test.go |
| @@ -4,6 +4,7 @@ import ( |
| "context" |
| "errors" |
| "fmt" |
| + "os" |
| "path/filepath" |
| "slices" |
| "strings" |
| @@ -11,6 +12,8 @@ import ( |
| "testing" |
| "time" |
| |
| + "github.com/carlosframework/platform/internal/cgroup" |
| + "github.com/carlosframework/platform/internal/instancectl" |
| "github.com/carlosframework/platform/internal/registry" |
| "github.com/carlosframework/platform/internal/release" |
| ) |
| @@ -332,13 +335,15 @@ func TestAPlannedWakeRecordsBeforeTheSeed(t *testing.T) { |
| } |
| } |
| |
| -// D6: checkPlan may replan after step 2b has recorded the old plan's |
| -// boot. startPinnedAt then records the new build before it execs. |
| -func TestAReplanAfterTheRecordRecordsTheNewBuild(t *testing.T) { |
| +// Ruling R-REPLAN: step 2b asks a planned wake's selection gate before it |
| +// chooses the identity, so a replan lands first and the seed and the boot |
| +// run one build, recorded once. |
| +func TestAReplanBeforeTheSeedSeedsAndBootsOneBuild(t *testing.T) { |
| dir := t.TempDir() |
| stale, next := stagedID(t, dir, "v2"), stagedID(t, dir, "v3") |
| a, b, _ := planActivator(t, WakePlan{}) |
| b.fakeBackend.setIdentity("v1") |
| + b.restoreFresh = true |
| var plans atomic.Int32 |
| a.Planner = func(context.Context, registry.Route) WakePlan { |
| if plans.Add(1) == 1 { |
| @@ -346,6 +351,33 @@ func TestAReplanAfterTheRecordRecordsTheNewBuild(t *testing.T) { |
| } |
| return WakePlan{Boot: next, Mode: ModeOverlap} |
| } |
| + l := &execLog{} |
| + a.BeforeExec = recordInto(b.fakeBackend, l) |
| + done, err := a.beginRequest(context.Background(), seeded(identityRoute(t, "one.example.com"))) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + done() |
| + if got := identityEvents(b.fakeBackend); !slices.Equal(got, []string{"record v3", "seed v3"}) { |
| + t.Fatalf("events %v, want the replanned v3 recorded and seeded", got) |
| + } |
| + if calls, _, _ := l.seen(); !slices.Equal(calls, []string{"one.example.com@v3"}) { |
| + t.Fatalf("records %v, want v3 once", calls) |
| + } |
| + if gens := b.started(); len(gens) != 1 || gens[0].Build != "v3" { |
| + t.Fatalf("spawned %+v, want the replanned v3", gens) |
| + } |
| +} |
| + |
| +// D6: a replan that only the exec's own selection gate makes (after step |
| +// 2b recorded the first plan's boot) is recorded again by startPinnedAt |
| +// before it execs. |
| +func TestAReplanAfterTheRecordRecordsTheNewBuild(t *testing.T) { |
| + dir := t.TempDir() |
| + stale, next := stagedID(t, dir, "v2"), stagedID(t, dir, "v3") |
| + a, b, _ := planActivator(t, WakePlan{}) |
| + b.fakeBackend.setIdentity("v1") |
| + a.Planner = laterReplan(stale, next) |
| l := &execLog{spawns: func() int { return len(b.started()) }} |
| a.BeforeExec = l.hook |
| done, err := a.beginRequest(context.Background(), identityRoute(t, "one.example.com")) |
| @@ -362,32 +394,66 @@ func TestAReplanAfterTheRecordRecordsTheNewBuild(t *testing.T) { |
| } |
| } |
| |
| -func TestANilHookIsToday(t *testing.T) { |
| +// laterReplan plans stale first, whose selection gate passes once (step |
| +// 2b) and refuses after (the exec's), and next on the replan. |
| +func laterReplan(stale, next release.StagedIdentity) func(context.Context, registry.Route) WakePlan { |
| + var plans, checks atomic.Int32 |
| + return func(context.Context, registry.Route) WakePlan { |
| + if plans.Add(1) == 1 { |
| + return WakePlan{Boot: stale, Mode: ModeOverlap, Check: func(string) error { |
| + if checks.Add(1) > 1 { |
| + return errors.New("superseded") |
| + } |
| + return nil |
| + }} |
| + } |
| + return WakePlan{Boot: next, Mode: ModeOverlap} |
| + } |
| +} |
| + |
| +// No hook (a box with no replica bucket): the wake still runs the file it |
| +// captured, by path, and records nothing. |
| +func TestANilHookRunsTheCapturedFileAndRecordsNothing(t *testing.T) { |
| a, b := identityActivator(t, "v1") |
| done, err := a.beginRequest(context.Background(), identityRoute(t, "one.example.com")) |
| if err != nil { |
| t.Fatal(err) |
| } |
| done() |
| - if b.startCount() != 1 || b.lastExeVersion() != "v1" { |
| - t.Fatalf("starts %d of %q: a nil BeforeExec changed the wake", b.startCount(), b.lastExeVersion()) |
| + if b.startCount() != 1 || b.lastExeVersion() != "v1" || b.liveStarts() != 0 { |
| + t.Fatalf("starts %d of %q (live-link %d), want the captured v1 by path", b.startCount(), b.lastExeVersion(), b.liveStarts()) |
| } |
| +} |
| |
| - // Ruling R-EXEC-NOVERSION: a link outside the version store still wakes |
| - // with no hook (no replica bucket), and is refused with one. |
| - a, b = identityActivator(t, "") |
| +// Rulings R-EXEC-NOVERSION and -2: a live link outside the version store |
| +// names no build. With no hook the wake is today's, in every layout: the |
| +// live-link seed, start and sidecar, nothing published or recorded. With a |
| +// hook it is refused. |
| +func TestAnUnversionedLinkWithNoHookWakesAsToday(t *testing.T) { |
| + a, b := sidecarActivator(t, "") |
| + b.fakeBackend.setIdentity("") |
| b.mu.Lock() |
| b.ident.unversioned = true |
| b.mu.Unlock() |
| - done, err = a.beginRequest(context.Background(), identityRoute(t, "two.example.com")) |
| + b.restoreFresh = true |
| + a.GenerationsFile = filepath.Join(t.TempDir(), "zdd", "generations.json") |
| + done, err := a.beginRequest(context.Background(), seeded(identityRoute(t, "two.example.com"))) |
| if err != nil { |
| t.Fatalf("an unversioned link with no hook: %v", err) |
| } |
| done() |
| - if x, _ := b.lastExe(); x.id.StagedPath != unversionedPath { |
| - t.Fatalf("ran %+v, want the resolved file by path", x.id) |
| + if got := identityEvents(b.fakeBackend); !slices.Equal(got, []string{"seed", "start:instance"}) { |
| + t.Fatalf("events %v, want today's live-link seed and start", got) |
| } |
| - a, b = identityActivator(t, "") |
| + if _, ok := b.lastExe(); ok || len(b.sidecarsOn()) != 0 || b.sidecarStartCount() != 1 { |
| + t.Fatalf("exe starts %v, generation sidecars %d, live-link sidecars %d; want today's wake", ok, len(b.sidecarsOn()), b.sidecarStartCount()) |
| + } |
| + if g, err := release.ReadGenerations(a.GenerationsFile); err == nil && g != nil && len(g.Paths["two.example.com"]) != 0 { |
| + t.Fatalf("published %v for a file no build names", g.Paths) |
| + } |
| + |
| + a, b = sidecarActivator(t, "") |
| + b.fakeBackend.setIdentity("") |
| b.mu.Lock() |
| b.ident.unversioned = true |
| b.mu.Unlock() |
| @@ -395,9 +461,103 @@ func TestANilHookIsToday(t *testing.T) { |
| if _, err := a.beginRequest(context.Background(), identityRoute(t, "two.example.com")); !errors.Is(err, errBuildStateUnwritten) { |
| t.Fatalf("err = %v, want an unversioned build refused while a build state is kept", err) |
| } |
| - if b.startCount() != 0 { |
| - t.Fatalf("%d starts of a build no build state names", b.startCount()) |
| + if b.startCount() != 0 || b.sidecarStartCount() != 0 { |
| + t.Fatalf("%d starts, %d sidecars of a build no build state names", b.startCount(), b.sidecarStartCount()) |
| + } |
| +} |
| + |
| +// rootLikeControl refuses a staged executable with no version, as root's |
| +// validateStagedExe answers 403, so a wake that sends one fails here as it |
| +// would on a box. |
| +type rootLikeControl struct{ identityControl } |
| + |
| +var errRootRefusesUnversioned = errors.New("instance-control: 403: the staged executable names no version") |
| + |
| +func (c rootLikeControl) StartExe(ctx context.Context, host string, role instancectl.Role, max int64, x instancectl.StagedExe) (string, error) { |
| + if x.Version == "" { |
| + c.rec("refused startexe") |
| + return "", errRootRefusesUnversioned |
| + } |
| + return c.identityControl.StartExe(ctx, host, role, max, x) |
| +} |
| +func (c rootLikeControl) SeedExe(ctx context.Context, host string, max int64, x instancectl.StagedExe) (instancectl.SeedResponse, error) { |
| + if x.Version == "" { |
| + c.rec("refused seedexe") |
| + return instancectl.SeedResponse{}, errRootRefusesUnversioned |
| + } |
| + return c.identityControl.SeedExe(ctx, host, max, x) |
| +} |
| +func (c rootLikeControl) StartGenExe(ctx context.Context, host string, n uint64, max int64, x instancectl.StagedExe) (string, error) { |
| + if x.Version == "" { |
| + c.rec("refused startgenexe") |
| + return "", errRootRefusesUnversioned |
| + } |
| + return c.identityControl.StartGenExe(ctx, host, n, max, x) |
| +} |
| +func (c rootLikeControl) StartSidecarExe(ctx context.Context, host string, max int64, x instancectl.StagedExe) (string, error) { |
| + if x.Version == "" { |
| + c.rec("refused startsidecarexe") |
| + return "", errRootRefusesUnversioned |
| + } |
| + return c.identityControl.StartSidecarExe(ctx, host, max, x) |
| +} |
| + |
| +// leaselessExec is a real ExecBackend with the bucket's lease and restore |
| +// faked, so a whole doWake runs over its real nested start and seed. |
| +type leaselessExec struct { |
| + *ExecBackend |
| + fb *fakeBackend |
| +} |
| + |
| +func (b *leaselessExec) Acquire(ctx context.Context, rt registry.Route) (Lease, error) { |
| + return b.fb.Acquire(ctx, rt) |
| +} |
| +func (b *leaselessExec) Restore(context.Context, registry.Route) (bool, error) { return true, nil } |
| + |
| +// Ruling R-EXEC-NOVERSION-2 on the real nested backend: an unversioned live |
| +// link with no hook seeds and starts through root's plain verbs, as today, |
| +// where the staged verbs would name no version and root would refuse them. |
| +func TestANestedUnversionedLinkWithNoHookWakesAsToday(t *testing.T) { |
| + ctl := &fakeControl{inv: "inv1", seed: instancectl.SeedResponse{InvocationID: "s1", Result: "success", Status: "0", Code: "exited"}} |
| + unit := &fakeUnit{} |
| + eb, rt := nestedBackend(t, ctl, unit) |
| + eb.Control = rootLikeControl{identityControl{ctl}} |
| + hand := filepath.Join(t.TempDir(), "app") |
| + if err := os.WriteFile(hand, []byte("#!/bin/sh\n"), 0o755); err != nil { |
| + t.Fatal(err) |
| + } |
| + repointLive(t, eb, rt.Host, hand) |
| + if id, err := eb.LiveIdentity(context.Background(), rt); err != nil || id.Version != "" { |
| + t.Fatalf("setup: live identity %+v, %v; want an unversioned file", id, err) |
| } |
| + rt.Seed = `["seed"]` |
| + // The seed unit's cgroup is empty after it (fakeUnit's zero state); the |
| + // run unit comes up inside root's start, where the app would bind. |
| + var stopServing func() |
| + ctl.onStart = func() { |
| + unit.set("active", "inv1", true, true) |
| + stopServing = serveOK(t, rt.Socket) |
| + } |
| + defer func() { |
| + if stopServing != nil { |
| + stopServing() |
| + } |
| + }() |
| + reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + a := New(Config{Bucket: "test-bucket", Idle: time.Hour}, &leaselessExec{ExecBackend: eb, fb: &fakeBackend{}}, reg) |
| + done, err := a.beginRequest(context.Background(), rt) |
| + if err != nil { |
| + t.Fatalf("a nested unversioned link with no hook did not wake: %v\nroot was asked:\n%s", err, ctl.joined()) |
| + } |
| + done() |
| + got := ctl.joined() |
| + if !strings.Contains(got, "seed "+rt.Host) || !strings.Contains(got, "start "+rt.Host+" run") || strings.Contains(got, "exe") { |
| + t.Fatalf("root was asked:\n%s\nwant today's plain seed and run start", got) |
| + } |
| + unit.set("inactive", "inv1", false, false) |
| } |
| |
| // A refused record starts nothing on any wake kind: no seed, no instance, |
| @@ -742,13 +902,7 @@ func TestARefusedSidecarRecordStartsNoSidecar(t *testing.T) { |
| pa, pb, _ := planActivator(t, WakePlan{}) |
| pb.fakeBackend.setIdentity("v1") |
| pb.sidecarStaged = true |
| - var plans atomic.Int32 |
| - pa.Planner = func(context.Context, registry.Route) WakePlan { |
| - if plans.Add(1) == 1 { |
| - return WakePlan{Boot: stale, Mode: ModeOverlap, Check: func(string) error { return errors.New("superseded") }} |
| - } |
| - return WakePlan{Boot: next, Mode: ModeOverlap} |
| - } |
| + pa.Planner = laterReplan(stale, next) |
| var n atomic.Int32 |
| pl := &execLog{fail: func(string) bool { return n.Add(1) == 3 }} |
| pa.BeforeExec = pl.hook |
| @@ -788,8 +942,8 @@ func TestASidecarOnAnUnstagedGenerationResolvesRecordsAndRunsByPath(t *testing.T |
| b.lastSidecar().kill() |
| a.sweepOnce() |
| waitFor(t, "the sidecar restart", func() bool { return len(b.sidecarsOn()) == 1 && a.sidecarSettled(host) }) |
| - if sc := b.sidecarsOn(); sc[0].StagedPath != fakeIdentity("v1").StagedPath || sc[0].Build != "v1" { |
| - t.Fatalf("restarted on %+v, want the resolved v1 by path", sc[0]) |
| + if sc := b.sidecarsOn(); sc[0].StagedPath != fakeIdentity("v1").StagedPath || sc[0].Build != "v1" || sc[0].Tier != cgroup.TierDefault { |
| + t.Fatalf("restarted on %+v, want the resolved v1 by path at its own tier (GenTier)", sc[0]) |
| } |
| if calls, _, _ := l.seen(); !slices.Equal(calls, []string{host + "@v1"}) { |
| t.Fatalf("records %v, want the resolved v1", calls) |
| @@ -822,3 +976,55 @@ func TestASidecarReplacementRecordsTheNewBuild(t *testing.T) { |
| t.Fatalf("replacement ran %q, want %q", paths[1], f.staged) |
| } |
| } |
| + |
| +// A sidecar restart asks ownership again after its record: a lease lost |
| +// while the build state was written execs nothing. |
| +func TestASidecarExecsOnlyIfStillOwnedAfterItsRecord(t *testing.T) { |
| + a, b := sidecarActivator(t, "v1") |
| + host := "one.example.com" |
| + var lose atomic.Bool |
| + l := &execLog{} |
| + a.BeforeExec = func(ctx context.Context, h, build string) error { |
| + if lose.Load() { |
| + b.lastRep().setErr(errors.New("lease lost")) |
| + } |
| + return l.hook(ctx, h, build) |
| + } |
| + done, err := a.beginRequest(context.Background(), identityRoute(t, host)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + done() |
| + lose.Store(true) |
| + crashSidecar(t, a, host) |
| + waitFor(t, "the restart's record", func() bool { |
| + calls, _, _ := l.seen() |
| + return len(calls) == 2 && a.sidecarSettled(host) |
| + }) |
| + if n := len(b.sidecarsOn()); n != 1 || b.sidecarStartCount() != 0 { |
| + t.Fatalf("%d generation sidecars, %d live-link: a sidecar exec'd after its lease was lost", n, b.sidecarStartCount()) |
| + } |
| +} |
| + |
| +// A worker wake whose token cannot be minted fails after step 2b published |
| +// its build: the wake's pending build is dropped with it, and the lease |
| +// goes back. |
| +func TestAWorkerTokenMintFailureDropsThePendingBuild(t *testing.T) { |
| + a, b, _, _, _ := stagedActivator(t) |
| + b.fakeBackend.setIdentity("v1") |
| + a.GenerationsFile = filepath.Join(t.TempDir(), "zdd", "generations.json") |
| + old := randRead |
| + randRead = func([]byte) (int, error) { return 0, errors.New("entropy pool exhausted") } |
| + defer func() { randRead = old }() |
| + host := "bot.x.internal" |
| + if err := a.WakeWorker(testWorkerRoute(host)); err == nil || !strings.Contains(err.Error(), "mint worker token") { |
| + t.Fatalf("WakeWorker = %v, want the mint failure", err) |
| + } |
| + waitFor(t, "the pending build to be dropped", func() bool { |
| + g, err := release.ReadGenerations(a.GenerationsFile) |
| + return err == nil && g != nil && !slices.Contains(g.Paths[host], fakeIdentity("v1").StagedPath) |
| + }) |
| + if n := b.releaseCount(); n != 1 || b.workerStarts() != 0 { |
| + t.Fatalf("releases %d, workers %d, want the lease back and nothing spawned", n, b.workerStarts()) |
| + } |
| +} |
amadan