amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package host
2
3import (
4 "context"
5 "crypto/sha256"
6 "encoding/hex"
7 "errors"
8 "net"
9 "net/http"
10 "net/http/httptest"
11 "os"
12 "path/filepath"
13 "strings"
14 "sync"
15 "sync/atomic"
16 "testing"
17 "time"
18
19 "github.com/carlosframework/platform/internal/activator"
20 "github.com/carlosframework/platform/internal/cgroup"
21 "github.com/carlosframework/platform/internal/registry"
22 "github.com/carlosframework/platform/internal/release"
23)
24
25// Settings plan 4, Astra round 2 finding 1 (ruling R-PLANNED-RECOVERY): the
26// pre-exec write's revalidation admits the planner's checked recovery or
27// fallback onto the good build, for that transition only, and still writes
28// the record. These run a real activator wake (ServeHTTP, the planner, the
29// held wake check and every exec through startPinnedAt and step 2b) with the
30// real build-state hook (buildStates.beforeExec) and heldTarget.
31
32// plannedProc is a fake process: Stop ends it.
33type plannedProc struct{ exited atomic.Bool }
34
35func (p *plannedProc) Stop(context.Context) error { p.exited.Store(true); return nil }
36func (p *plannedProc) Exited() bool { return p.exited.Load() }
37
38type plannedLease struct{}
39
40func (plannedLease) Replicate(context.Context, string, time.Duration) (activator.Rep, error) {
41 return plannedRep{}, nil
42}
43func (plannedLease) Release(context.Context) error { return nil }
44
45type plannedRep struct{}
46
47func (plannedRep) Err() error { return nil }
48func (plannedRep) Close(context.Context) error { return nil }
49
50// plannedBackend is the smallest exec backend a planned wake runs on: a
51// lease and restore that always succeed, staged generations (a build in
52// fail exits during its health check, with its process; any other answers
53// HTTP 200 on a real socket), a generation kill, the live link's version,
54// and the executable identity step 2b records.
55type plannedBackend struct {
56 socket string
57 live string
58 fail map[string]bool
59
60 mu sync.Mutex
61 started []string
62 killed []uint64
63 procs map[uint64]*plannedProc
64}
65
66func (b *plannedBackend) Acquire(context.Context, registry.Route) (activator.Lease, error) {
67 return plannedLease{}, nil
68}
69func (b *plannedBackend) Restore(context.Context, registry.Route) (bool, error) { return false, nil }
70func (b *plannedBackend) Seed(context.Context, registry.Route) error { return nil }
71func (b *plannedBackend) Start(context.Context, registry.Route) (activator.Proc, string, error) {
72 return nil, "", errors.New("a planned wake never starts the live link")
73}
74func (b *plannedBackend) NextDue(context.Context, registry.Route) (int64, error) { return 0, nil }
75func (b *plannedBackend) HasSidecar(registry.Route) bool { return false }
76func (b *plannedBackend) StartSidecar(context.Context, registry.Route) (activator.Proc, error) {
77 return nil, errors.New("no sidecar")
78}
79func (b *plannedBackend) StartWorker(context.Context, registry.Route, map[string]string) (activator.Proc, string, error) {
80 return nil, "", errors.New("no worker")
81}
82
83func (b *plannedBackend) StartStaged(ctx context.Context, _ registry.Route, g activator.Gen) (activator.Gen, activator.Proc, error) {
84 // The exec seam, as ExecBackend's: the caller's gate and the wake's
85 // pending record (amadan #272), before anything starts.
86 if err := activator.RunExecGate(ctx); err != nil {
87 return g, nil, err
88 }
89 b.mu.Lock()
90 defer b.mu.Unlock()
91 b.started = append(b.started, g.Build)
92 p := &plannedProc{}
93 if b.procs == nil {
94 b.procs = map[uint64]*plannedProc{}
95 }
96 b.procs[g.N] = p
97 if b.fail[g.Build] {
98 return g, p, errors.New("the process exited during its health check")
99 }
100 g.Socket = b.socket
101 return g, p, nil
102}
103func (b *plannedBackend) GenTier(registry.Route, string) cgroup.Limit { return cgroup.TierDefault }
104func (b *plannedBackend) GenReady(registry.Route, activator.Gen, activator.Proc) {}
105func (b *plannedBackend) RaiseGenParent(context.Context, registry.Route, cgroup.Limit, cgroup.Limit) error {
106 return nil
107}
108func (b *plannedBackend) RestoreGenParent(context.Context, registry.Route, cgroup.Limit) error {
109 return nil
110}
111func (b *plannedBackend) GenSwitched(registry.Route, activator.Gen, activator.Proc) error { return nil }
112func (b *plannedBackend) KillGen(_ context.Context, _ registry.Route, n uint64) error {
113 b.mu.Lock()
114 defer b.mu.Unlock()
115 b.killed = append(b.killed, n)
116 if p := b.procs[n]; p != nil {
117 p.exited.Store(true)
118 }
119 return nil
120}
121func (b *plannedBackend) LiveVersion(string) (string, error) { return b.live, nil }
122func (b *plannedBackend) LiveIdentity(context.Context, registry.Route) (release.StagedIdentity, error) {
123 return release.StagedIdentity{}, errors.New("a planned wake never captures the live link")
124}
125func (b *plannedBackend) StartGenExe(context.Context, registry.Route, uint64, bool, release.StagedIdentity) (activator.Proc, activator.Gen, error) {
126 return nil, activator.Gen{}, errors.New("a planned wake never starts the live link")
127}
128func (b *plannedBackend) SeedExe(context.Context, registry.Route, release.StagedIdentity) error {
129 return nil
130}
131
132func (b *plannedBackend) builds() []string {
133 b.mu.Lock()
134 defer b.mu.Unlock()
135 return append([]string(nil), b.started...)
136}
137
138// healthySocket serves HTTP 200 on a unix socket under a short directory
139// (a socket path is bounded).
140func healthySocket(t *testing.T) string {
141 t.Helper()
142 dir, err := os.MkdirTemp("", "pr")
143 if err != nil {
144 t.Fatal(err)
145 }
146 t.Cleanup(func() { os.RemoveAll(dir) })
147 path := filepath.Join(dir, "s")
148 l, err := net.Listen("unix", path)
149 if err != nil {
150 t.Fatal(err)
151 }
152 srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })}
153 go srv.Serve(l) //nolint:errcheck
154 t.Cleanup(func() { srv.Close() })
155 return path
156}
157
158// plannedStaged writes a staged build and returns its identity (the
159// activator verifies the bytes before every staged exec).
160func plannedStaged(t *testing.T, dir, ver string) release.StagedIdentity {
161 t.Helper()
162 body := []byte("build " + ver)
163 path := filepath.Join(dir, ver)
164 if err := os.WriteFile(path, body, 0o755); err != nil {
165 t.Fatal(err)
166 }
167 sum := sha256.Sum256(body)
168 return release.StagedIdentity{Version: ver, StagedPath: path, Digest: hex.EncodeToString(sum[:])}
169}
170
171// plannedWake is a manual instance with good build A (GoodVersion), a
172// planner that answers plan, and the real build-state hook and held wake
173// check over b's store and registry. The activator leases at the hibernate
174// prefix of bucket "hib", which b reads there too.
175func plannedWake(t *testing.T, live string, fail map[string]bool, plan func(good, t release.StagedIdentity) activator.WakePlan) (*activator.Service, *plannedBackend, *buildStates, release.Store) {
176 t.Helper()
177 b, s, reg := testBuildStates(t)
178 b.prefix = func(h string) string { return activator.HibernatePrefix("", h) }
179 dir := t.TempDir()
180 good, next := plannedStaged(t, dir, "A"), plannedStaged(t, dir, "T")
181 if err := reg.SetGoodBuild(astraHost, registry.GoodBuild{}, registry.GoodBuild{Version: "A", Digest: good.Digest}); err != nil {
182 t.Fatal(err)
183 }
184 // Root's install stamp names the live build, as root's repoint leaves it.
185 if err := reg.SetVersions(astraHost, live, ""); err != nil {
186 t.Fatal(err)
187 }
188 setSettings(t, reg, release.UpdateSettings{Updates: release.UpdatesManual, Channel: "stable", InstallRequest: 1})
189 be := &plannedBackend{socket: healthySocket(t), live: live, fail: fail}
190 svc := activator.New(activator.Config{Bucket: "hib", Idle: time.Hour}, be, reg)
191 svc.Planner = func(context.Context, registry.Route) activator.WakePlan { return plan(good, next) }
192 svc.OnGen = func(activator.GenEvent) {}
193 svc.BeforeExec = b.beforeExec
194 svc.HeldTarget = b.heldTarget
195 t.Cleanup(func() { svc.Shutdown(context.Background()) })
196 return svc, be, b, s
197}
198
199// serve sends one request through the activator: it wakes the instance.
200func serve(t *testing.T, svc *activator.Service, reg *registry.Registry) *httptest.ResponseRecorder {
201 t.Helper()
202 rt, err := reg.Get(astraHost)
203 if err != nil || rt == nil {
204 t.Fatalf("route: %v", err)
205 }
206 rec := httptest.NewRecorder()
207 req := httptest.NewRequest(http.MethodGet, "http://"+astraHost+"/", nil)
208 proxy := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Write([]byte("served")) }) //nolint:errcheck
209 if !svc.ServeHTTP(rec, req, *rt, proxy) {
210 t.Fatal("the activator did not take the request")
211 }
212 return rec
213}
214
215func durable(t *testing.T, b *buildStates, s release.Store) *release.BuildState {
216 t.Helper()
217 st, _, err := getState(context.Background(), s, release.BuildStateKey(astraHost), astraHost)
218 if err != nil {
219 t.Fatal(err)
220 }
221 return st
222}
223
224// The overlap wake's fallback (Astra round 2 finding 1's scenario): a manual
225// instance with good build A and Update now resolved to T. Its first wake
226// records T (completing R1) and T fails its health check. The planner's
227// fallback A must record and run, though root has not reported T failed and
228// the durable state's last_executed is T.
229func TestTheOverlapFallbackAfterTFailsIsRecordedAndRuns(t *testing.T) {
230 svc, be, b, s := plannedWake(t, "T", map[string]bool{"T": true}, func(good, next release.StagedIdentity) activator.WakePlan {
231 return activator.WakePlan{Boot: next, Mode: activator.ModeOverlap, FirstWake: true, Fallback: &good}
232 })
233 if err := put(t, b, context.Background(), astraHost, release.BuildState{LastExecuted: "A"}); err != nil {
234 t.Fatal(err)
235 }
236 setCopy(t, b.reg, release.BuildStateCopy{Known: true, Replica: true, State: &release.BuildState{LastExecuted: "A"}})
237 decided(t, b, release.Consumed{Resolved: &release.ResolvedRequest{ID: 1, Target: "T"}, Reason: string(release.ReasonRequest), Desired: "T", Detail: "T"})
238
239 rec := serve(t, svc, b.reg)
240 if rec.Code != http.StatusOK || rec.Body.String() != "served" {
241 t.Fatalf("the wake was not served by the fallback: %d %q; generations %v", rec.Code, rec.Body.String(), be.builds())
242 }
243 if got := be.builds(); len(got) != 2 || got[0] != "T" || got[1] != "A" {
244 t.Fatalf("generations %v, want T then the good build A", got)
245 }
246 st := durable(t, b, s)
247 if st == nil || st.LastExecuted != "A" || st.Completed != 1 {
248 t.Fatalf("durable state %+v; want A recorded before it ran, R1 completed by T's run", st)
249 }
250}
251
252// A planned recovery boot onto the good build (decideWake's recovery: the
253// failure book blocks T) before root's consumed file names T failed: the
254// durable state's last_executed is T, and the planner chose A in T's place.
255// The record admits A for that transition and the wake runs it.
256func TestAPlannedRecoveryBeforeRootRecordsTheFailureRuns(t *testing.T) {
257 svc, be, b, s := plannedWake(t, "T", nil, func(good, _ release.StagedIdentity) activator.WakePlan {
258 return activator.WakePlan{Boot: good, Mode: activator.ModeOverlap}
259 })
260 tState := release.BuildState{LastExecuted: "T", Resolved: &release.ResolvedRequest{ID: 1, Target: "T"}, Completed: 1}
261 if err := put(t, b, context.Background(), astraHost, tState); err != nil {
262 t.Fatal(err)
263 }
264 setCopy(t, b.reg, release.BuildStateCopy{Known: true, Replica: true, State: &tState})
265 decided(t, b, release.Consumed{Resolved: &release.ResolvedRequest{ID: 1, Target: "T"}, Reason: string(release.ReasonRequest), Desired: "T", Detail: "T"})
266
267 rec := serve(t, svc, b.reg)
268 if rec.Code != http.StatusOK || rec.Body.String() != "served" {
269 t.Fatalf("the recovery boot was not served: %d %q; generations %v", rec.Code, rec.Body.String(), be.builds())
270 }
271 if got := be.builds(); len(got) != 1 || got[0] != "A" {
272 t.Fatalf("generations %v, want the good build A alone", got)
273 }
274 if st := durable(t, b, s); st == nil || st.LastExecuted != "A" || st.Completed != 1 {
275 t.Fatalf("durable state %+v; want A recorded, R1 still completed", st)
276 }
277}
278
279// The negative (GoodVersion is not exempt in general; Astra round 1 finding
280// 1): the wake decided on a stale copy that names the good build A, which is
281// installed, so the planner plans A as an ordinary wake, not in place of
282// anything. The durable state says the data last ran T. With no planner
283// selection the record refuses A and nothing runs. The same hook, called
284// directly with no selection, or with a selection for another transition,
285// refuses A too.
286func TestTheGoodBuildWithoutThePlannersSelectionIsRefused(t *testing.T) {
287 svc, be, b, s := plannedWake(t, "A", nil, func(good, _ release.StagedIdentity) activator.WakePlan {
288 return activator.WakePlan{Boot: good, Mode: activator.ModeOverlap}
289 })
290 if err := put(t, b, context.Background(), astraHost, release.BuildState{LastExecuted: "T"}); err != nil {
291 t.Fatal(err)
292 }
293 setCopy(t, b.reg, release.BuildStateCopy{Known: true, Replica: true, State: &release.BuildState{LastExecuted: "A"}})
294 decided(t, b, release.Consumed{Reason: string(release.ReasonManual), Desired: "A", Detail: "A"})
295
296 rec := serve(t, svc, b.reg)
297 if rec.Code == http.StatusOK {
298 t.Fatalf("the good build ran over T's data with no planner selection: %q; generations %v", rec.Body.String(), be.builds())
299 }
300 if got := be.builds(); len(got) != 0 {
301 t.Fatalf("generations %v, want none", got)
302 }
303 if st := durable(t, b, s); st == nil || st.LastExecuted != "T" {
304 t.Fatalf("durable state %+v; want T untouched", st)
305 }
306 ctx := context.Background()
307 if err := b.beforeExec(ctx, astraHost, "A"); err == nil || !strings.Contains(err.Error(), "T") {
308 t.Fatalf("beforeExec(A) with no selection = %v; want a stale refusal naming T", err)
309 }
310 other := activator.WithPlannedSelection(ctx, activator.PlannedSelection{Build: "A", InPlaceOf: "B"})
311 if err := b.beforeExec(other, astraHost, "A"); err == nil {
312 t.Fatal("beforeExec(A) with a selection made in place of another build was admitted")
313 }
314 wrong := activator.WithPlannedSelection(ctx, activator.PlannedSelection{Build: "C", InPlaceOf: "T"})
315 if err := b.beforeExec(wrong, astraHost, "A"); err == nil {
316 t.Fatal("beforeExec(A) with a selection of another build was admitted")
317 }
318 if st := durable(t, b, s); st == nil || st.LastExecuted != "T" {
319 t.Fatalf("durable state %+v; want T untouched", st)
320 }
321}
322