| new file mode 100644 |
| index 000000000..4d0ebb0c3 |
| --- /dev/null |
| +++ b/internal/activator/exec_gate_test.go |
| @@ -0,0 +1,373 @@ |
| +package activator |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "os" |
| + "path/filepath" |
| + "slices" |
| + "strings" |
| + "sync" |
| + "sync/atomic" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/instancectl" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// launchBackend is the swap fixture's fakeSwapBackend (lease, restore, |
| +// raise, switch) with StartStaged on a REAL ExecBackend, flat or nested, so |
| +// a staged spawn runs the production preparation: the admin token, the |
| +// config read, the control socket's bind and, nested, the launch file and |
| +// the root start call. The activator's row is mapped onto the real |
| +// backend's row (rt) of the chosen layout. Its KillGen stops the |
| +// generation's handle, as execStagedBackend's does. |
| +type launchBackend struct { |
| + *fakeSwapBackend |
| + eb *ExecBackend |
| + rt registry.Route |
| + ctl *exeControl // nested only: what root was asked to start |
| + |
| + lmu sync.Mutex |
| + entered []uint64 // generation numbers that reached the real backend |
| + procs map[uint64]Proc |
| + builds []string // the builds the real backend returned a process for |
| +} |
| + |
| +func (b *launchBackend) StartStaged(ctx context.Context, _ registry.Route, g Gen) (Gen, Proc, error) { |
| + b.lmu.Lock() |
| + b.entered = append(b.entered, g.N) |
| + b.lmu.Unlock() |
| + got, p, err := b.eb.StartStaged(ctx, b.rt, g) |
| + if p != nil { |
| + b.lmu.Lock() |
| + b.procs[g.N] = p |
| + b.builds = append(b.builds, g.Build) |
| + b.lmu.Unlock() |
| + } |
| + return got, p, err |
| +} |
| + |
| +func (b *launchBackend) KillGen(ctx context.Context, _ registry.Route, n uint64) error { |
| + b.lmu.Lock() |
| + p := b.procs[n] |
| + b.lmu.Unlock() |
| + if p == nil { |
| + return nil |
| + } |
| + _ = p.Stop(ctx) |
| + if !p.Exited() { |
| + return ErrStillRunning |
| + } |
| + return nil |
| +} |
| + |
| +func (b *launchBackend) gens() []uint64 { |
| + b.lmu.Lock() |
| + defer b.lmu.Unlock() |
| + return slices.Clone(b.entered) |
| +} |
| + |
| +func (b *launchBackend) stopAll() { |
| + b.lmu.Lock() |
| + procs := make([]Proc, 0, len(b.procs)) |
| + for _, p := range b.procs { |
| + procs = append(procs, p) |
| + } |
| + b.lmu.Unlock() |
| + for _, p := range procs { |
| + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) |
| + _ = p.Stop(ctx) |
| + cancel() |
| + } |
| +} |
| + |
| +// launched is the evidence that a build's process was started: the real |
| +// backend returned a process for it; flat, the marker its script writes the |
| +// moment it runs; nested, root was asked to start the unit with it. |
| +func (b *launchBackend) launched(t *testing.T, id release.StagedIdentity, marker string) bool { |
| + t.Helper() |
| + b.lmu.Lock() |
| + started := slices.Contains(b.builds, id.Version) |
| + b.lmu.Unlock() |
| + if started { |
| + return true |
| + } |
| + if b.ctl != nil { |
| + b.ctl.mu.Lock() |
| + defer b.ctl.mu.Unlock() |
| + for _, x := range b.ctl.got { |
| + if x.Path == id.StagedPath { |
| + return true |
| + } |
| + } |
| + return false |
| + } |
| + _, err := os.Stat(marker) |
| + return err == nil |
| +} |
| + |
| +// prepGone checks that what generation n's preparation made is gone after |
| +// a refusal at the launch: its control socket, and nested, its launch file. |
| +func (b *launchBackend) prepGone(t *testing.T, n uint64) { |
| + t.Helper() |
| + var sock string |
| + p := b.eb.pathsFor(b.rt) |
| + switch p.layout { |
| + case LayoutNested: |
| + sock = nestedGenSocketIn(p.runDir, n) |
| + launch := filepath.Join(p.dataDir, instancectl.GenLaunchFileName(n)) |
| + if _, err := os.Stat(launch); !os.IsNotExist(err) { |
| + t.Fatalf("generation %d's launch file %s after a refusal at the launch: %v, want it removed", n, launch, err) |
| + } |
| + default: |
| + sock = b.eb.flatGenSocket(b.rt.Host, n) |
| + } |
| + if _, err := os.Stat(sock + ".control"); !os.IsNotExist(err) { |
| + t.Fatalf("generation %d's control socket after a refusal at the launch: %v, want it unbound", n, err) |
| + } |
| +} |
| + |
| +// newLaunchBackend builds the real backend for layout and wraps fb. |
| +func newLaunchBackend(t *testing.T, layout string, fb *fakeSwapBackend) *launchBackend { |
| + t.Helper() |
| + b := &launchBackend{fakeSwapBackend: fb, procs: map[uint64]Proc{}} |
| + switch layout { |
| + case "flat": |
| + eb := testExecBackend(t) |
| + eb.RunDirOverride = t.TempDir() |
| + host := "launch.example.com" |
| + stageLive(t, eb, host, "v0") |
| + b.eb, b.rt = eb, registry.Route{Host: host, Kind: "instance", Hibernate: true, |
| + DBPath: filepath.Join(eb.DataDir, host+".db"), Socket: filepath.Join(eb.RunDirOverride, host+".sock")} |
| + case "nested": |
| + eb, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{}) |
| + eb.GenContainment = true |
| + b.ctl = &exeControl{} |
| + eb.Control = b.ctl |
| + b.eb, b.rt = eb, rt |
| + default: |
| + t.Fatalf("layout %q", layout) |
| + } |
| + if got, _ := b.eb.Layout(b.rt); (layout == "flat") != (got == LayoutFlat) || (layout == "nested") != (got == LayoutNested) { |
| + t.Fatalf("fixture layout %v, want %s", got, layout) |
| + } |
| + t.Cleanup(b.stopAll) |
| + return b |
| +} |
| + |
| +// markerScript is a staged build that writes marker the moment it runs. |
| +func markerScript(t *testing.T, dir, ver, marker string) release.StagedIdentity { |
| + t.Helper() |
| + return stagedScript(t, dir, ver, "#!/bin/sh\necho ran > '"+marker+"'\nexec sleep 30\n") |
| +} |
| + |
| +// onCall runs f on the call numbered want (counting from 1) and on no |
| +// other. |
| +func onCall(want int32, f func()) func(string) { |
| + var n atomic.Int32 |
| + return func(string) { |
| + if n.Add(1) == want { |
| + f() |
| + } |
| + } |
| +} |
| + |
| +// Final review F1: every staged instance spawn's composed gate is asked at |
| +// the backend's final launch point, AFTER the backend's own preparation: |
| +// flat, after the token, the config read and the control socket's bind, |
| +// right before the process starts; nested, after the launch file, right |
| +// before root is asked to start the unit. A lease lost, a fence set or the |
| +// attempt superseded during that preparation starts nothing, removes what |
| +// the preparation made, records no execution evidence and is abandonment, |
| +// never a failure of the build. The four callers: a planned wake's B, its |
| +// fallback, a hold's pinned build and an overlap successor, on both |
| +// layouts. Before the fix the gate ran only before StartStaged, and each of |
| +// these launched. |
| +func TestTheExecGateIsAskedAtTheBackendsLaunchPoint(t *testing.T) { |
| + for _, layout := range []string{"flat", "nested"} { |
| + t.Run(layout+"/planned B, lease lost", func(t *testing.T) { |
| + b := newLaunchBackend(t, layout, &fakeSwapBackend{fakeBackend: &fakeBackend{}}) |
| + dir := t.TempDir() |
| + mark := filepath.Join(t.TempDir(), "v2.ran") |
| + next := markerScript(t, dir, "v2", mark) |
| + good := markerScript(t, dir, "v1", filepath.Join(t.TempDir(), "v1.ran")) |
| + a, evs := launchActivator(t, b, WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good}) |
| + b.eb.launchHook = onCall(1, func() { b.lastRep().setErr(errors.New("lease renewal failed")) }) |
| + _, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| + if !errors.Is(err, errSwapLeaseLost) || classifyStep(nil, err) == stepFailed { |
| + t.Fatalf("wake = %v, want the lost lease, abandonment", err) |
| + } |
| + gens := b.gens() |
| + if len(gens) != 1 { |
| + t.Fatalf("the backend was entered for generations %v, want B's alone (the refusal is inside it)", gens) |
| + } |
| + if b.launched(t, next, mark) { |
| + t.Fatal("B was launched after the lease was lost during its preparation") |
| + } |
| + b.prepGone(t, gens[0]) |
| + time.Sleep(50 * time.Millisecond) |
| + if k := evs.kinds(); len(k) != 0 { |
| + t.Fatalf("events %v: a refusal at the launch was recorded", k) |
| + } |
| + }) |
| + t.Run(layout+"/planned fallback, fenced", func(t *testing.T) { |
| + defer func(k time.Duration) { firstWakeKillBound = k }(firstWakeKillBound) |
| + firstWakeKillBound = 500 * time.Millisecond |
| + b := newLaunchBackend(t, layout, &fakeSwapBackend{fakeBackend: &fakeBackend{}}) |
| + dir := t.TempDir() |
| + markB, markGood := filepath.Join(t.TempDir(), "v2.ran"), filepath.Join(t.TempDir(), "v1.ran") |
| + next, good := markerScript(t, dir, "v2", markB), markerScript(t, dir, "v1", markGood) |
| + a, evs := launchActivator(t, b, WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good}) |
| + const host = "one.example.com" |
| + a.genHealth = func(context.Context, Gen) error { return errors.New("GET / answered 500") } |
| + b.eb.launchHook = onCall(2, func() { |
| + a.mu.Lock() |
| + a.fences = map[string]uint64{host: 1} |
| + a.mu.Unlock() |
| + }) |
| + if _, err := a.beginRequest(context.Background(), testRoute(host)); err == nil { |
| + t.Fatal("served after the fallback was refused") |
| + } |
| + if gens := b.gens(); len(gens) != 2 { |
| + t.Fatalf("the backend was entered for generations %v, want B's and the fallback's", gens) |
| + } |
| + waitFor(t, "B's launch evidence", func() bool { return b.launched(t, next, markB) }) |
| + time.Sleep(100 * time.Millisecond) |
| + if b.launched(t, good, markGood) { |
| + t.Fatal("the good build was launched on a host fenced during its preparation") |
| + } |
| + b.prepGone(t, b.gens()[1]) |
| + // B ran and failed: that is recorded. The refused fallback is not |
| + // a fallback that served. |
| + evs.wait(t, GenFailed) |
| + if k := evs.kinds(); slices.Contains(k, GenFellBack) { |
| + t.Fatalf("events %v: a fallback refused at its launch was reported as served", k) |
| + } |
| + }) |
| + t.Run(layout+"/hold's pinned build, lease lost", func(t *testing.T) { |
| + f := newSwapFixture(t) |
| + b := newLaunchBackend(t, layout, f.b) |
| + f.a.backend = b |
| + sink := &lockedSink{} |
| + f.a.Logs = sink |
| + mark := filepath.Join(t.TempDir(), "v2.ran") |
| + id := markerScript(t, t.TempDir(), "v2", mark) |
| + at := f.attempt(ModeHold) |
| + at.StagedPath, at.Digest = id.StagedPath, id.Digest |
| + b.eb.launchHook = onCall(1, func() { f.b.lastRep().setErr(errors.New("lease renewal failed")) }) |
| + res := f.a.Swap(context.Background(), at, f.newest) |
| + if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, errSwapLeaseLost.Error()) { |
| + t.Fatalf("Swap = %+v, want abandoned: the lost lease", res) |
| + } |
| + gens := b.gens() |
| + if len(gens) != 1 { |
| + t.Fatalf("the backend was entered for generations %v, want the pin's alone", gens) |
| + } |
| + if b.launched(t, id, mark) { |
| + t.Fatal("the hold's build was launched after the lease was lost during its preparation") |
| + } |
| + b.prepGone(t, gens[0]) |
| + if got := sink.errors(); len(got) != 0 { |
| + t.Fatalf("member error events %q for an abandoned hold", got) |
| + } |
| + }) |
| + t.Run(layout+"/overlap successor, superseded", func(t *testing.T) { |
| + f := newSwapFixture(t) |
| + b := newLaunchBackend(t, layout, f.b) |
| + f.a.backend = b |
| + mark := filepath.Join(t.TempDir(), "v2.ran") |
| + id := markerScript(t, t.TempDir(), "v2", mark) |
| + at := f.attempt(ModeOverlap) |
| + at.StagedPath, at.Digest = id.StagedPath, id.Digest |
| + b.eb.launchHook = onCall(1, func() { f.rev.Add(1) }) |
| + res := f.a.Swap(context.Background(), at, f.newest) |
| + if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, errSwapSuperseded.Error()) { |
| + t.Fatalf("Swap = %+v, want abandoned: superseded", res) |
| + } |
| + gens := b.gens() |
| + if len(gens) != 1 { |
| + t.Fatalf("the backend was entered for generations %v, want the successor's alone", gens) |
| + } |
| + if b.launched(t, id, mark) { |
| + t.Fatal("the successor was launched after its attempt was superseded during its preparation") |
| + } |
| + b.prepGone(t, gens[0]) |
| + if g, members := f.a.genStateForTest(f.host); g.N != f.incumbent.N || len(members) != 0 { |
| + t.Fatalf("current %+v, members %v: want the incumbent serving and an empty operation set", g, members) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// launchActivator is planActivator over a launchBackend. |
| +func launchActivator(t *testing.T, b *launchBackend, plan WakePlan) (*Service, *genLog) { |
| + t.Helper() |
| + reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + a := New(Config{Bucket: "test-bucket", Idle: time.Hour}, b, reg) |
| + a.gens = true |
| + evs := &genLog{} |
| + a.OnGen = evs.add |
| + a.Planner = func(context.Context, registry.Route) WakePlan { return plan } |
| + a.genHealth = func(context.Context, Gen) error { return nil } |
| + t.Cleanup(func() { a.Shutdown(context.Background()) }) |
| + return a, evs |
| +} |
| + |
| +// Merge triage T9: on a first wake whose health window is short (under the |
| +// verdict floor, so B would not be tried and the good build would boot in |
| +// its place), a refusal by B's own gate is abandonment: nothing boots, B is |
| +// not reported untried, and nothing is recorded. verdictFloorGate asks the |
| +// gate first, so a build held since the plan (or a lost lease, a fence) is |
| +// never mistaken for a lack of time. Here B's selection check passes at |
| +// checkPlan and refuses at B's exec. With the floor asked first, the good |
| +// build booted untried and the wake reported GenHealthy. |
| +func TestAGateRefusalOnAShortWindowAbandonsRatherThanGoingUntried(t *testing.T) { |
| + shrinkVerdictFloor(t, time.Hour) // every window is under the floor |
| + dir := t.TempDir() |
| + good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| + a, b, evs := planActivator(t, WakePlan{}) |
| + var asked atomic.Int32 |
| + a.Planner = func(context.Context, registry.Route) WakePlan { |
| + return WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good, |
| + Check: func(build string) error { |
| + if build == "v2" && asked.Add(1) >= 2 { |
| + return errors.New("v2 failed on a sibling") |
| + } |
| + return nil |
| + }} |
| + } |
| + _, err := a.beginRequest(context.Background(), testRoute("one.example.com")) |
| + if !errors.Is(err, errPlanStale) || errors.Is(err, errNoTimeForVerdict) { |
| + t.Fatalf("wake = %v, want the gate's refusal (errPlanStale), never a lack of time", err) |
| + } |
| + if got := classifyStep(nil, err); got != stepStale { |
| + t.Fatalf("classifyStep = %v, want stepStale: abandonment", got) |
| + } |
| + if gens := b.started(); len(gens) != 0 { |
| + t.Fatalf("spawned %+v: the good build booted untried after B's gate refused", gens) |
| + } |
| + time.Sleep(50 * time.Millisecond) |
| + if k := evs.kinds(); len(k) != 0 { |
| + t.Fatalf("events %v: an abandoned short-window wake reported something (an untried B reads as GenHealthy)", k) |
| + } |
| +} |
| + |
| +// The exec gate rides on the context through SplitBackend to its exec half, |
| +// which asks it at its launch (genbackend.go's contract): a refusal there |
| +// reaches the caller unchanged. |
| +func TestSplitBackendCarriesTheExecGate(t *testing.T) { |
| + exec := &recordingGenExec{fakeBackend: &fakeBackend{}} |
| + s := &SplitBackend{Exec: exec, Unit: &UnitBackend{}} |
| + refused := errors.New("refused at the launch") |
| + ctx := withExecGate(context.Background(), func() error { return refused }) |
| + _, p, err := s.StartStaged(ctx, registry.Route{Host: "e.example.com"}, Gen{N: 7, StagedPath: "/opt/carlos/versions/v2/app"}) |
| + if !errors.Is(err, refused) || p != nil || exec.started != 0 { |
| + t.Fatalf("StartStaged = %v, %v (started %d): want the gate's refusal and nothing started", p, err, exec.started) |
| + } |
| +} |