| 1 | //go:build linux |
| 2 | |
| 3 | package stopevidence |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "errors" |
| 8 | "net/http" |
| 9 | "net/http/httptest" |
| 10 | "os" |
| 11 | "path/filepath" |
| 12 | "reflect" |
| 13 | "sync/atomic" |
| 14 | "testing" |
| 15 | "time" |
| 16 | |
| 17 | "github.com/carlosframework/platform/internal/activator" |
| 18 | "github.com/carlosframework/platform/internal/cgroup" |
| 19 | "github.com/carlosframework/platform/internal/registry" |
| 20 | ) |
| 21 | |
| 22 | type evidenceProc struct{ exited atomic.Bool } |
| 23 | |
| 24 | func (p *evidenceProc) Stop(context.Context) error { p.exited.Store(true); return nil } |
| 25 | func (p *evidenceProc) Exited() bool { return p.exited.Load() } |
| 26 | |
| 27 | type evidenceRep struct{} |
| 28 | |
| 29 | func (evidenceRep) Err() error { return nil } |
| 30 | func (evidenceRep) Close(context.Context) error { return nil } |
| 31 | |
| 32 | type evidenceLease struct{} |
| 33 | |
| 34 | func (evidenceLease) Release(context.Context) error { return nil } |
| 35 | func (evidenceLease) Replicate(context.Context, string, time.Duration) (activator.Rep, error) { |
| 36 | return evidenceRep{}, nil |
| 37 | } |
| 38 | |
| 39 | type evidenceBackend struct{} |
| 40 | |
| 41 | func (evidenceBackend) Acquire(context.Context, registry.Route) (activator.Lease, error) { |
| 42 | return evidenceLease{}, nil |
| 43 | } |
| 44 | func (evidenceBackend) Restore(context.Context, registry.Route) (bool, error) { return false, nil } |
| 45 | func (evidenceBackend) Seed(context.Context, registry.Route) error { return nil } |
| 46 | func (evidenceBackend) Start(context.Context, registry.Route) (activator.Proc, string, error) { |
| 47 | return &evidenceProc{}, "test-build", nil |
| 48 | } |
| 49 | func (evidenceBackend) NextDue(context.Context, registry.Route) (int64, error) { return 0, nil } |
| 50 | func (evidenceBackend) HasSidecar(registry.Route) bool { return false } |
| 51 | func (evidenceBackend) StartSidecar(context.Context, registry.Route) (activator.Proc, error) { |
| 52 | return nil, errors.New("unexpected sidecar") |
| 53 | } |
| 54 | func (evidenceBackend) StartWorker(context.Context, registry.Route, map[string]string) (activator.Proc, string, error) { |
| 55 | return nil, "", errors.New("unexpected worker") |
| 56 | } |
| 57 | |
| 58 | func fenceFixture(t *testing.T, host string, generation uint64) *activator.StopEvidence { |
| 59 | t.Helper() |
| 60 | dir := t.TempDir() |
| 61 | reg, err := registry.Open(filepath.Join(dir, "registry.db")) |
| 62 | if err != nil { |
| 63 | t.Fatal(err) |
| 64 | } |
| 65 | t.Cleanup(func() { reg.Close() }) |
| 66 | a := activator.New(activator.Config{Bucket: "test", Idle: time.Hour}, evidenceBackend{}, reg) |
| 67 | t.Cleanup(func() { |
| 68 | ctx, cancel := context.WithTimeout(context.Background(), time.Second) |
| 69 | defer cancel() |
| 70 | a.Shutdown(ctx) |
| 71 | }) |
| 72 | rt := registry.Route{Host: host, Kind: "instance", Hibernate: true, Socket: filepath.Join(dir, "app.sock"), DBPath: filepath.Join(dir, "data.db")} |
| 73 | if err := reg.Upsert(rt); err != nil { |
| 74 | t.Fatal(err) |
| 75 | } |
| 76 | ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) |
| 77 | defer cancel() |
| 78 | called := false |
| 79 | w := httptest.NewRecorder() |
| 80 | r := httptest.NewRequest(http.MethodGet, "http://"+host+"/", nil).WithContext(ctx) |
| 81 | if !a.ServeHTTP(w, r, rt, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { called = true; w.WriteHeader(http.StatusNoContent) })) || !called { |
| 82 | t.Fatalf("wake failed: %d %s", w.Code, w.Body.String()) |
| 83 | } |
| 84 | proof, err := a.FenceAndStop(ctx, host, generation) |
| 85 | if err != nil || !proof.Matches(host, generation) { |
| 86 | t.Fatal("fence proof", err) |
| 87 | } |
| 88 | return proof |
| 89 | } |
| 90 | |
| 91 | type evidenceGroup struct { |
| 92 | populated bool |
| 93 | members []int |
| 94 | populationErr, membersErr error |
| 95 | } |
| 96 | |
| 97 | func (g *evidenceGroup) Populated() (bool, error) { return g.populated, g.populationErr } |
| 98 | func (g *evidenceGroup) Members() ([]int, error) { return g.members, g.membersErr } |
| 99 | |
| 100 | func TestCollectRequiresEveryCondition(t *testing.T) { |
| 101 | host := "shop.example" |
| 102 | proof := fenceFixture(t, host, 7) |
| 103 | failure := errors.New("read failed") |
| 104 | for _, tc := range []struct { |
| 105 | name string |
| 106 | change func(*testing.T, *Inputs, *evidenceGroup) |
| 107 | want error |
| 108 | }{ |
| 109 | {"success", func(t *testing.T, in *Inputs, g *evidenceGroup) {}, nil}, |
| 110 | {"no-fence", func(t *testing.T, in *Inputs, g *evidenceGroup) { in.Fence = nil }, ErrFence}, |
| 111 | {"empty-fence", func(t *testing.T, in *Inputs, g *evidenceGroup) { in.Fence = &activator.StopEvidence{} }, ErrFence}, |
| 112 | {"wrong-host", func(t *testing.T, in *Inputs, g *evidenceGroup) { in.Host = "other.example" }, ErrFence}, |
| 113 | {"wrong-generation", func(t *testing.T, in *Inputs, g *evidenceGroup) { in.Generation = 8 }, ErrFence}, |
| 114 | {"no-generation", func(t *testing.T, in *Inputs, g *evidenceGroup) { in.Generation = 0 }, ErrInvalidInput}, |
| 115 | {"no-group", func(t *testing.T, in *Inputs, g *evidenceGroup) { in.Cgroup = nil }, ErrNoCgroupEvidence}, |
| 116 | {"population", func(t *testing.T, in *Inputs, g *evidenceGroup) { g.populated = true }, ErrPopulated}, |
| 117 | {"members", func(t *testing.T, in *Inputs, g *evidenceGroup) { g.members = []int{55} }, ErrMembers}, |
| 118 | {"population-error", func(t *testing.T, in *Inputs, g *evidenceGroup) { g.populationErr = failure }, failure}, |
| 119 | {"members-error", func(t *testing.T, in *Inputs, g *evidenceGroup) { g.membersErr = failure }, failure}, |
| 120 | {"replicator-open", func(t *testing.T, in *Inputs, g *evidenceGroup) { in.ReplicatorsClosed = false }, ErrReplicatorsOpen}, |
| 121 | {"no-paths", func(t *testing.T, in *Inputs, g *evidenceGroup) { in.DBPaths = nil }, ErrInvalidInput}, |
| 122 | {"relative-path", func(t *testing.T, in *Inputs, g *evidenceGroup) { in.DBPaths = []string{"data.db"} }, ErrInvalidInput}, |
| 123 | {"proc-read-error", func(t *testing.T, in *Inputs, g *evidenceGroup) { in.ProcRoot = filepath.Join(in.ProcRoot, "missing") }, ErrIncomplete}, |
| 124 | {"db-holder", func(t *testing.T, in *Inputs, g *evidenceGroup) { addHolder(t, in, "") }, ErrFDHolders}, |
| 125 | {"wal-holder", func(t *testing.T, in *Inputs, g *evidenceGroup) { addHolder(t, in, "-wal") }, ErrFDHolders}, |
| 126 | {"shm-holder", func(t *testing.T, in *Inputs, g *evidenceGroup) { addHolder(t, in, "-shm") }, ErrFDHolders}, |
| 127 | } { |
| 128 | t.Run(tc.name, func(t *testing.T) { |
| 129 | g := &evidenceGroup{} |
| 130 | in := Inputs{Host: host, Generation: 7, Fence: proof, Cgroup: g, DBPaths: []string{filepath.Join(t.TempDir(), "data.db")}, ProcRoot: t.TempDir(), ReplicatorsClosed: true} |
| 131 | tc.change(t, &in, g) |
| 132 | e, err := Collect(t.Context(), in) |
| 133 | if tc.want != nil { |
| 134 | if !errors.Is(err, tc.want) || !reflect.DeepEqual(e, Evidence{}) { |
| 135 | t.Fatalf("evidence on failure: %+v %v", e, err) |
| 136 | } |
| 137 | return |
| 138 | } |
| 139 | if err != nil || e.Host != host || e.Generation != 7 || !e.FenceClosed || !e.SpawnAttemptsDrained || !e.CgroupUnpopulated || !e.CgroupMembersEmpty || !e.ReplicatorsClosed || !e.FDScanComplete || e.CheckedAt.IsZero() || len(e.CheckedPaths) != 3 { |
| 140 | t.Fatalf("incomplete evidence: %+v %v", e, err) |
| 141 | } |
| 142 | g.populated = true |
| 143 | if _, err := Collect(t.Context(), in); !errors.Is(err, ErrPopulated) { |
| 144 | t.Fatal("cached evidence", err) |
| 145 | } |
| 146 | }) |
| 147 | } |
| 148 | ctx, cancel := context.WithCancel(t.Context()) |
| 149 | cancel() |
| 150 | if e, err := Collect(ctx, Inputs{}); !errors.Is(err, context.Canceled) || !reflect.DeepEqual(e, Evidence{}) { |
| 151 | t.Fatal(e, err) |
| 152 | } |
| 153 | if err := Cleanup(t.Context(), nil); !errors.Is(err, ErrInvalidInput) { |
| 154 | t.Fatal(err) |
| 155 | } |
| 156 | if err := Cleanup(t.Context(), &cgroup.Cleanup{}); !errors.Is(err, cgroup.ErrStrictState) { |
| 157 | t.Fatal(err) |
| 158 | } |
| 159 | } |
| 160 | |
| 161 | func addHolder(t *testing.T, in *Inputs, suffix string) { |
| 162 | t.Helper() |
| 163 | dir := filepath.Join(in.ProcRoot, "55", "task", "55", "fd") |
| 164 | if err := os.MkdirAll(dir, 0700); err != nil { |
| 165 | t.Fatal(err) |
| 166 | } |
| 167 | if err := os.Symlink(in.DBPaths[0]+suffix, filepath.Join(dir, "4")); err != nil { |
| 168 | t.Fatal(err) |
| 169 | } |
| 170 | } |
| 171 | |