amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package host
2
3import (
4 "context"
5 "errors"
6 "os"
7 "path/filepath"
8 "strconv"
9 "sync"
10 "testing"
11
12 "github.com/carlosframework/platform/internal/activator"
13 "github.com/carlosframework/platform/internal/cgroup"
14 "github.com/carlosframework/platform/internal/hostbox"
15 "github.com/carlosframework/platform/internal/hostclaim"
16 execlease "github.com/carlosframework/platform/internal/hostexec"
17 "github.com/carlosframework/platform/internal/hostguard"
18 "github.com/carlosframework/platform/internal/instancectl"
19)
20
21type launchClaims struct {
22 mu sync.Mutex
23 obs hostclaim.Observed
24}
25
26func (c *launchClaims) Read(_ context.Context, host string) (hostclaim.Observed, error) {
27 c.mu.Lock()
28 defer c.mu.Unlock()
29 o := c.obs
30 if o.Claim != nil {
31 cl := *o.Claim
32 h := *cl.Holder
33 cl.Holder = &h
34 o.Claim = &cl
35 }
36 return o, nil
37}
38
39func (c *launchClaims) set(fn func(*hostclaim.Claim)) {
40 c.mu.Lock()
41 defer c.mu.Unlock()
42 cl := *c.obs.Claim
43 h := *cl.Holder
44 cl.Holder = &h
45 fn(&cl)
46 c.obs.Claim = &cl
47}
48
49type launchUnits struct {
50 mu sync.Mutex
51 starts []instancectl.ClaimUnit
52 stops []string
53 startErr error
54 beforeCheck func()
55 sameErr error
56 goneErr error
57 // goneAfterStartErr answers Gone only once a start was attempted.
58 goneAfterStartErr error
59 stopErr error
60 started instancectl.ClaimUnitStarted
61}
62
63func (u *launchUnits) Start(_ context.Context, in instancectl.ClaimUnit) (instancectl.ClaimUnitStarted, error) {
64 u.mu.Lock()
65 defer u.mu.Unlock()
66 u.starts = append(u.starts, in)
67 if u.startErr != nil {
68 return instancectl.ClaimUnitStarted{}, u.startErr
69 }
70 return u.started, nil
71}
72
73func (u *launchUnits) Stop(_ context.Context, host string) error {
74 u.mu.Lock()
75 defer u.mu.Unlock()
76 u.stops = append(u.stops, host)
77 return u.stopErr
78}
79
80func (u *launchUnits) Gone(context.Context, string, string) error {
81 u.mu.Lock()
82 defer u.mu.Unlock()
83 if u.goneAfterStartErr != nil && len(u.starts) > 0 {
84 return u.goneAfterStartErr
85 }
86 return u.goneErr
87}
88
89func (u *launchUnits) counts() (int, int) {
90 u.mu.Lock()
91 defer u.mu.Unlock()
92 return len(u.starts), len(u.stops)
93}
94
95type launchKernel struct {
96 path string
97 want hostbox.LaunchFile
98 verifyErr error
99 observed ClaimLaunchObserved
100 observeErr error
101 observe func(instancectl.ClaimUnitStarted) (ClaimLaunchObserved, error)
102 digest string
103}
104
105func (k *launchKernel) Verify(path string, want hostbox.LaunchFile) error {
106 if k.verifyErr != nil {
107 return k.verifyErr
108 }
109 if path != k.path || want != k.want {
110 return hostbox.ErrChanged
111 }
112 return nil
113}
114
115func (k *launchKernel) Observe(instancectl.ClaimUnitStarted) (ClaimLaunchObserved, error) {
116 if k.observe != nil {
117 return k.observe(instancectl.ClaimUnitStarted{})
118 }
119 return k.observed, k.observeErr
120}
121
122type launchFixture struct {
123 root *ClaimLaunchRoot
124 claims *launchClaims
125 units *launchUnits
126 kernel *launchKernel
127 peer StopPeer
128 req hostbox.RoleLaunchRequest
129 journal *hostbox.LaunchJournal
130 target hostguard.FenceTarget
131 actor *hostbox.Actor
132}
133
134func newLaunchFixture(t *testing.T, op hostclaim.Op) *launchFixture {
135 return newLaunchFixtureHost(t, op, "shop.example")
136}
137
138func newLaunchFixtureHost(t *testing.T, op hostclaim.Op, host string) *launchFixture {
139 t.Helper()
140 boot, err := execlease.BootID()
141 if err != nil {
142 t.Skip("root launch requires Linux")
143 }
144 start, err := claimConstructionProcessStart()
145 if err != nil {
146 t.Fatal(err)
147 }
148 data := t.TempDir()
149 if err := os.Mkdir(filepath.Join(data, "hostguard"), 0o770); err != nil {
150 t.Fatal(err)
151 }
152 claim := &hostclaim.Claim{V: 2, Host: host, Gen: 3, Holder: &hostclaim.Holder{Incarnation: "inc-1", Stamp: 11, Session: 13, WriterBox: "box-1"}, Op: op}
153 claims := &launchClaims{obs: hostclaim.Observed{Host: host, Claim: claim, ETag: "etag-1"}}
154 guard := hostguard.Guard{DataDir: data, LockDir: t.TempDir(), Claims: claims}
155 if err := guard.Activate(t.Context(), host, "box-1", nil); err != nil {
156 t.Fatal(err)
157 }
158 journal, err := hostbox.OpenLaunchJournal(filepath.Join(t.TempDir(), "launch.db"))
159 if err != nil {
160 t.Fatal(err)
161 }
162 t.Cleanup(func() { journal.Close() })
163 previousAgentRoot := claimContainmentAgentRoot
164 claimContainmentAgentRoot = func(context.Context) (string, error) { return "/sys/fs/cgroup/system.slice/carlos-edge.service", nil }
165 t.Cleanup(func() { claimContainmentAgentRoot = previousAgentRoot })
166 target := hostguard.FenceTarget{Region: "eu", Host: host, Incarnation: "inc-1", Stamp: 11, Session: 13, Op: op.ID, Phase: op.Phase, Box: "box-1", Enrollment: "enrollment-1", BootID: boot, ControllerID: "controller-1", ClaimETag: "etag-1", DirectoryRevision: "7", RosterRevision: 7}
167 if err = journal.Prelaunch(t.Context(), hostbox.LaunchInventory{ControllerPID: os.Getpid(), ProcessStart: start, Target: target, Groups: []hostbox.LaunchGroup{{RelativePath: filepath.Join("system.slice/carlos-edge.service", cgroup.ExecName(claimContainmentName(target))), Device: 1, Inode: 9}}, NoState: true}); err != nil {
168 t.Fatal(err)
169 }
170 actor := &hostbox.Actor{Box: "box-1", Enrollment: "enrollment-1", Revision: 7, Binding: "7", Region: "eu", Eligible: true}
171 exe := hostbox.LaunchFile{Device: 5, Inode: 6}
172 units := &launchUnits{started: instancectl.ClaimUnitStarted{Unit: instancectl.UnitName(instancectl.RoleRun, host), InvocationID: "unit-inv-1", ControlGroup: instancectl.ClaimUnitControlGroup(host), MainPID: 4242}}
173 kernel := &launchKernel{path: "/opt/carlos/versions/v1/app", want: exe, observed: ClaimLaunchObserved{ProcessStart: "ps-4242", TenantPID: 4243, TenantStart: "ps-4243", Group: hostbox.LaunchFile{Device: 3, Inode: 4}, Executable: exe}}
174 peer := StopPeer{UID: 1001, PID: os.Getpid(), ProcessStart: start, BootID: boot, ControllerID: "controller-1"}
175 root := &ClaimLaunchRoot{ValidateExecutable: func(string, string, string) error { return nil }, Guard: guard, Journal: journal, Units: units, Kernel: kernel, Actor: func(context.Context) (hostbox.Actor, error) { return *actor, nil }}
176 reservation := hostbox.LaunchReservation{ID: "attempt-1:main", Role: "main", InvocationID: "inv-1", Inputs: "inputs-1", Target: target, Group: hostbox.LaunchGroup{RelativePath: instancectl.ClaimUnitControlGroup(host)}, Executable: exe}
177 req := hostbox.RoleLaunchRequest{V: hostbox.RoleLaunchV, Op: hostbox.RoleLaunchStart, Reservation: &reservation, Executable: kernel.path, MemoryMax: cgroup.TierDefault.MemoryMax}
178 f := &launchFixture{root: root, claims: claims, units: units, kernel: kernel, peer: peer, req: req, journal: journal, target: target, actor: actor}
179 configureRev2Launch(t, f)
180 return f
181}
182
183func servingOp() hostclaim.Op { return hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box-1"} }
184
185func (f *launchFixture) noReservation(t *testing.T, name string) {
186 t.Helper()
187 if starts, _ := f.units.counts(); starts != 0 {
188 t.Fatalf("%s reached the unit start", name)
189 }
190 if _, err := f.journal.UnitReservation(t.Context(), f.target, f.req.Reservation.ID); err == nil {
191 t.Fatalf("%s journaled a reservation", name)
192 }
193}
194
195func TestClaimLaunchRootStartsExactlyTheReservedRole(t *testing.T) {
196 f := newLaunchFixture(t, servingOp())
197 resp, err := f.root.Serve(t.Context(), f.peer, f.req)
198 if err != nil {
199 t.Fatal(err)
200 }
201 group := hostbox.LaunchGroup{RelativePath: instancectl.ClaimUnitControlGroup(f.target.Host), Device: 3, Inode: 4, InvocationID: "unit-inv-1"}
202 if resp.V != hostbox.RoleLaunchV || resp.Protocol != hostbox.RoleLaunchProtocol || resp.Grant == nil || resp.Grant.Check(*f.req.Reservation) != nil || resp.Grant.Group != group || resp.Grant.PID != 4242 || resp.Grant.ProcessStart != "ps-4242" {
203 t.Fatalf("grant=%+v", resp)
204 }
205 if len(f.units.starts) != 1 || f.units.starts[0] != (instancectl.ClaimUnit{Host: f.target.Host, Executable: f.kernel.path, MemoryMax: cgroup.TierDefault.MemoryMax}) {
206 t.Fatalf("unit starts=%+v", f.units.starts)
207 }
208 got, err := f.journal.UnitReservation(t.Context(), f.target, f.req.Reservation.ID)
209 if err != nil || got.State != hostbox.UnitStarted || got.Group != group || got.PID != 4242 || got.ProcessStart != "ps-4242" || got.Executable != f.req.Reservation.Executable {
210 t.Fatalf("journal=%+v %v", got, err)
211 }
212 // A replay of the same request, after success, is refused without effect.
213 if _, err := f.root.Serve(t.Context(), f.peer, f.req); !errors.Is(err, hostbox.ErrChanged) {
214 t.Fatalf("replay: %v", err)
215 }
216 if starts, _ := f.units.counts(); starts != 1 {
217 t.Fatal("replay started a second unit")
218 }
219}
220
221func TestClaimLaunchRootSerializesConcurrentDuplicates(t *testing.T) {
222 f := newLaunchFixture(t, servingOp())
223 var wg sync.WaitGroup
224 errs := make(chan error, 4)
225 for range 4 {
226 wg.Add(1)
227 go func() { defer wg.Done(); _, err := f.root.Serve(t.Context(), f.peer, f.req); errs <- err }()
228 }
229 wg.Wait()
230 close(errs)
231 ok := 0
232 for err := range errs {
233 if err == nil {
234 ok++
235 }
236 }
237 if starts, _ := f.units.counts(); ok != 1 || starts != 1 {
238 t.Fatalf("successes=%d starts=%d", ok, starts)
239 }
240}
241
242func TestClaimLaunchRootTrustsOnlyItsOwnProviders(t *testing.T) {
243 for name, mutate := range map[string]func(*launchFixture){
244 "other controller": func(f *launchFixture) { f.peer.ControllerID = "controller-2" },
245 "other boot": func(f *launchFixture) { f.peer.BootID = "boot-2" },
246 "root uid": func(f *launchFixture) { f.peer.UID = 0 },
247 "not the record's pid": func(f *launchFixture) { f.peer.PID = 43 },
248 "restarted controller": func(f *launchFixture) { f.peer.ProcessStart = "start-43" },
249 "roster box": func(f *launchFixture) { f.actor.Box = "box-2" },
250 "roster enrollment": func(f *launchFixture) { f.actor.Enrollment = "enrollment-2" },
251 "box binding": func(f *launchFixture) { f.actor.Binding = "8" },
252 "roster region": func(f *launchFixture) { f.actor.Region = "us" },
253 "quarantined box": func(f *launchFixture) { f.actor.Quarantined = true },
254 "holder session": func(f *launchFixture) { f.claims.set(func(c *hostclaim.Claim) { c.Holder.Session = 14 }) },
255 "holder stamp": func(f *launchFixture) { f.claims.set(func(c *hostclaim.Claim) { c.Holder.Stamp = 12 }) },
256 "executor elsewhere": func(f *launchFixture) { f.claims.set(func(c *hostclaim.Claim) { c.Op.Exec = "box-2" }) },
257 "writer elsewhere": func(f *launchFixture) { f.claims.set(func(c *hostclaim.Claim) { c.Holder.WriterBox = "box-2" }) },
258 "box declared lost": func(f *launchFixture) { f.claims.set(func(c *hostclaim.Claim) { c.Lost = []string{"box-1"} }) },
259 "operation began": func(f *launchFixture) {
260 f.claims.set(func(c *hostclaim.Claim) { c.Op.ID, c.Op.Kind = "op-9", hostclaim.KindParking })
261 },
262 "build not verified": func(f *launchFixture) { f.kernel.verifyErr = hostbox.ErrChanged },
263 "build path differs": func(f *launchFixture) { f.req.Executable = "/opt/carlos/versions/v2/app" },
264 "tier not a platform": func(f *launchFixture) { f.req.MemoryMax = 12345 },
265 "reservation target op": func(f *launchFixture) { r := *f.req.Reservation; r.Target.Op = "op-1"; f.req.Reservation = &r },
266 } {
267 f := newLaunchFixture(t, servingOp())
268 mutate(f)
269 if _, err := f.root.Serve(t.Context(), f.peer, f.req); err == nil {
270 t.Fatalf("%s launched", name)
271 }
272 f.noReservation(t, name)
273 }
274}
275
276func TestClaimLaunchRootRefusesWithoutThePrelaunchRecord(t *testing.T) {
277 f := newLaunchFixture(t, servingOp())
278 r := *f.req.Reservation
279 r.Target.Session = 14
280 f.claims.set(func(c *hostclaim.Claim) { c.Holder.Session = 14 })
281 f.req.Reservation = &r
282 if _, err := f.root.Serve(t.Context(), f.peer, f.req); err == nil {
283 t.Fatal("launched a session with no prelaunch record")
284 }
285 if starts, _ := f.units.counts(); starts != 0 {
286 t.Fatal("unit started")
287 }
288}
289
290func TestClaimLaunchRootProbeIsBoundToTheOperationAndInstallStamp(t *testing.T) {
291 probing := hostclaim.Op{Kind: hostclaim.KindLanding, ID: "op-1", Phase: hostclaim.PhaseLandingProbing, Exec: "box-1", Journal: map[string]string{"install_stamp": "11"}}
292 f := newLaunchFixture(t, probing)
293 if _, err := f.root.Serve(t.Context(), f.peer, f.req); err != nil {
294 t.Fatalf("probe: %v", err)
295 }
296 for name, mutate := range map[string]func(*hostclaim.Claim){
297 "install stamp": func(c *hostclaim.Claim) { c.Op.Journal = map[string]string{"install_stamp": "12"} },
298 "probe passed": func(c *hostclaim.Claim) { c.Op.Phase = hostclaim.PhaseLandingProbed },
299 "other operation": func(c *hostclaim.Claim) { c.Op.ID = "op-2" },
300 "relocation no pin": func(c *hostclaim.Claim) { c.Op.Kind, c.Op.Journal = hostclaim.KindRelocating, nil },
301 } {
302 f := newLaunchFixture(t, probing)
303 f.claims.set(mutate)
304 if _, err := f.root.Serve(t.Context(), f.peer, f.req); err == nil {
305 t.Fatalf("%s launched", name)
306 }
307 f.noReservation(t, name)
308 }
309}
310
311func TestClaimLaunchRootRefusesAFencedSession(t *testing.T) {
312 probing := hostclaim.Op{Kind: hostclaim.KindLanding, ID: "op-1", Phase: hostclaim.PhaseLandingProbing, Exec: "box-1"}
313 f := newLaunchFixture(t, probing)
314 fence := f.target
315 fence.FenceID, fence.ObligationID = "fence-1", "obligation-1"
316 if _, err := f.root.Guard.BeginFence(t.Context(), fence); err != nil {
317 t.Fatal(err)
318 }
319 if _, err := f.root.Serve(t.Context(), f.peer, f.req); !errors.Is(err, hostguard.ErrFenced) {
320 t.Fatalf("fenced session launched: %v", err)
321 }
322 f.noReservation(t, "fenced session")
323}
324
325func TestClaimLaunchRootCleansUpAStartItCannotVouchFor(t *testing.T) {
326 f := newLaunchFixture(t, servingOp())
327 f.kernel.observed.Executable = hostbox.LaunchFile{Device: 5, Inode: 7}
328 if _, err := f.root.Serve(t.Context(), f.peer, f.req); err == nil {
329 t.Fatal("swapped build accepted")
330 }
331 if _, stops := f.units.counts(); stops != 1 {
332 t.Fatal("swapped build left running")
333 }
334 got, err := f.journal.UnitReservation(t.Context(), f.target, f.req.Reservation.ID)
335 if err != nil || got.State != hostbox.UnitSettled {
336 t.Fatalf("journal=%+v %v", got, err)
337 }
338 if _, err := f.root.Serve(t.Context(), f.peer, f.req); !errors.Is(err, hostbox.ErrChanged) {
339 t.Fatalf("spent reservation retried: %v", err)
340 }
341
342 // A failed start whose unit root cannot prove gone stays pending, so the
343 // stop journal keeps refusing a receipt.
344 f = newLaunchFixture(t, servingOp())
345 f.units.startErr = errors.New("start failed")
346 f.units.goneAfterStartErr = errors.New("still active")
347 if _, err := f.root.Serve(t.Context(), f.peer, f.req); err == nil {
348 t.Fatal("failed start accepted")
349 }
350 got, err = f.journal.UnitReservation(t.Context(), f.target, f.req.Reservation.ID)
351 if err != nil || got.State != hostbox.UnitIntended {
352 t.Fatalf("unproven unit settled: %+v %v", got, err)
353 }
354 sj, err := f.journal.ReadStopKernelJournal(t.Context(), f.target)
355 if err != nil || sj.Settled {
356 t.Fatalf("stop journal settled=%v %v", sj.Settled, err)
357 }
358}
359
360func TestClaimLaunchRootStopSettlesAndBurnsReservations(t *testing.T) {
361 f := newLaunchFixture(t, servingOp())
362 if _, err := f.root.Serve(t.Context(), f.peer, f.req); err != nil {
363 t.Fatal(err)
364 }
365 stop := hostbox.RoleLaunchRequest{V: hostbox.RoleLaunchV, Op: hostbox.RoleLaunchStop, Reservation: f.req.Reservation}
366 other := f.peer
367 other.ControllerID = "controller-2"
368 if _, err := f.root.Serve(t.Context(), other, stop); err == nil {
369 t.Fatal("another controller stopped the role")
370 }
371 f.units.goneErr = errors.New("still active")
372 if _, err := f.root.Serve(t.Context(), f.peer, stop); err == nil {
373 t.Fatal("stop reported success while the unit stayed")
374 }
375 f.units.goneErr = nil
376 if _, err := f.root.Serve(t.Context(), f.peer, stop); err != nil {
377 t.Fatal(err)
378 }
379 got, err := f.journal.UnitReservation(t.Context(), f.target, f.req.Reservation.ID)
380 if err != nil || got.State != hostbox.UnitSettled {
381 t.Fatalf("journal=%+v %v", got, err)
382 }
383 _, before := f.units.counts()
384 if _, err := f.root.Serve(t.Context(), f.peer, stop); err != nil {
385 t.Fatalf("idempotent stop: %v", err)
386 }
387 if _, after := f.units.counts(); after != before {
388 t.Fatal("settled stop touched the unit again")
389 }
390
391 // A stop for a reservation root never saw burns it.
392 g := newLaunchFixture(t, servingOp())
393 stop.Reservation = g.req.Reservation
394 if _, err := g.root.Serve(t.Context(), g.peer, stop); err != nil {
395 t.Fatal(err)
396 }
397 if _, err := g.root.Serve(t.Context(), g.peer, g.req); !errors.Is(err, hostbox.ErrChanged) {
398 t.Fatalf("start after stop: %v", err)
399 }
400 if starts, _ := g.units.counts(); starts != 0 {
401 t.Fatal("burnt reservation started")
402 }
403}
404
405func TestClaimLaunchRootCapabilityNeedsEveryProvider(t *testing.T) {
406 f := newLaunchFixture(t, servingOp())
407 q := hostbox.RoleLaunchRequest{V: hostbox.RoleLaunchV, Op: hostbox.RoleLaunchCapability}
408 resp, err := f.root.Serve(t.Context(), f.peer, q)
409 if err != nil || resp.Protocol != hostbox.RoleLaunchProtocol || resp.Grant != nil {
410 t.Fatalf("capability=%+v %v", resp, err)
411 }
412 for name, strip := range map[string]func(*ClaimLaunchRoot){
413 "local": func(r *ClaimLaunchRoot) { r.Local = nil },
414 "clock": func(r *ClaimLaunchRoot) { r.Clock = nil },
415 "data root": func(r *ClaimLaunchRoot) { r.DataRoot = "" },
416 "units": func(r *ClaimLaunchRoot) { r.Units = nil },
417 "kernel": func(r *ClaimLaunchRoot) { r.Kernel = nil },
418 "journal": func(r *ClaimLaunchRoot) { r.Journal = nil },
419 "actor": func(r *ClaimLaunchRoot) { r.Actor = nil },
420 "guard": func(r *ClaimLaunchRoot) { r.Guard.Claims = nil },
421 } {
422 g := newLaunchFixture(t, servingOp())
423 strip(g.root)
424 if _, err := g.root.Serve(t.Context(), g.peer, q); !errors.Is(err, activator.ErrClaimIsolationUnavailable) {
425 t.Fatalf("%s missing: %v", name, err)
426 }
427 if _, err := g.root.Serve(t.Context(), g.peer, g.req); !errors.Is(err, activator.ErrClaimIsolationUnavailable) {
428 t.Fatalf("%s missing start: %v", name, err)
429 }
430 }
431}
432
433func TestClaimUnitStopKernelRequiresTheUnitGone(t *testing.T) {
434 f := newLaunchFixture(t, servingOp())
435 if _, err := f.root.Serve(t.Context(), f.peer, f.req); err != nil {
436 t.Fatal(err)
437 }
438 inner := 0
439 k := claimUnitStopKernel{Journal: f.journal, Units: f.units, Inner: stopKernelFunc(func(_ context.Context, target hostguard.FenceTarget, _ StopInventory, _ DrainAcknowledgement) (StopKernelEvidence, error) {
440 inner++
441 j, err := f.journal.ReadStopKernelJournal(t.Context(), target)
442 if err != nil || !j.Settled {
443 t.Fatalf("inner verifier saw an unsettled unit: %+v %v", j, err)
444 }
445 return StopKernelEvidence{CgroupIdentities: []string{"cg"}}, nil
446 })}
447 f.units.goneErr = errors.New("still active")
448 if _, err := k.VerifyStopped(t.Context(), f.target, StopInventory{}, DrainAcknowledgement{}); err == nil || inner != 0 {
449 t.Fatalf("running unit verified: %v inner=%d", err, inner)
450 }
451 f.units.goneErr = nil
452 ev, err := k.VerifyStopped(t.Context(), f.target, StopInventory{}, DrainAcknowledgement{})
453 if err != nil || inner != 1 || len(ev.UnitInvocationIDs) != 1 || ev.UnitInvocationIDs[0] != "unit-inv-1" {
454 t.Fatalf("evidence=%+v %v inner=%d", ev, err, inner)
455 }
456}
457
458func TestClaimUnitStopKernelGuardsRecovery(t *testing.T) {
459 f := newLaunchFixture(t, servingOp())
460 if _, err := f.root.Serve(t.Context(), f.peer, f.req); err != nil {
461 t.Fatal(err)
462 }
463 called := 0
464 k := claimUnitStopKernel{Journal: f.journal, Units: f.units, Inner: recoveryKernelFunc(func(context.Context, hostguard.FenceTarget, StopInventory) (StopKernelEvidence, DrainAcknowledgement, error) {
465 called++
466 return StopKernelEvidence{}, DrainAcknowledgement{}, nil
467 })}
468 f.units.goneErr = errors.New("still active")
469 if _, _, err := k.RecoverStopped(t.Context(), f.target, StopInventory{}); err == nil || called != 0 {
470 t.Fatalf("recovery settled a running unit: %v called=%d", err, called)
471 }
472 f.units.goneErr = nil
473 ev, _, err := k.RecoverStopped(t.Context(), f.target, StopInventory{})
474 if err != nil || called != 1 || len(ev.UnitInvocationIDs) != 1 {
475 t.Fatalf("recovery=%+v %v called=%d", ev, err, called)
476 }
477 if _, ok := StopKernelVerifier(claimUnitStopKernel{Inner: stopKernelFunc(nil)}).(stopRecoveryKernel); !ok {
478 t.Fatal("wrapper hides recovery")
479 }
480 if _, _, err := (claimUnitStopKernel{Journal: f.journal, Units: f.units, Inner: stopKernelFunc(nil)}).RecoverStopped(t.Context(), f.target, StopInventory{}); !errors.Is(err, ErrClaimStopUnavailable) {
481 t.Fatalf("recovery without an inner recovery kernel: %v", err)
482 }
483}
484
485type recoveryKernelFunc func(context.Context, hostguard.FenceTarget, StopInventory) (StopKernelEvidence, DrainAcknowledgement, error)
486
487func (f recoveryKernelFunc) VerifyStopped(context.Context, hostguard.FenceTarget, StopInventory, DrainAcknowledgement) (StopKernelEvidence, error) {
488 return StopKernelEvidence{}, ErrClaimStopUnavailable
489}
490
491func (f recoveryKernelFunc) RecoverStopped(ctx context.Context, t hostguard.FenceTarget, in StopInventory) (StopKernelEvidence, DrainAcknowledgement, error) {
492 return f(ctx, t, in)
493}
494
495var _ = strconv.Itoa
496
497func (u *launchUnits) StartChecked(ctx context.Context, in instancectl.ClaimUnit, _ instancectl.Launch, check func() error) (instancectl.ClaimUnitStarted, error) {
498 if u.beforeCheck != nil {
499 u.beforeCheck()
500 }
501 if err := check(); err != nil {
502 return instancectl.ClaimUnitStarted{}, err
503 }
504 return u.Start(ctx, in)
505}
506func (u *launchUnits) Same(context.Context, instancectl.ClaimUnitStarted) error { return u.sameErr }
507func (k *launchKernel) VerifyDigest(_ context.Context, path string, want hostbox.LaunchFile, digest string) error {
508 if digest != k.digest {
509 return hostbox.ErrChanged
510 }
511 return k.Verify(path, want)
512}
513
514func (u *launchUnits) StartReserved(ctx context.Context, in instancectl.ClaimUnit, l instancectl.Launch, p instancectl.ClaimExecPermit, check func() error) (instancectl.ClaimUnitStarted, error) {
515 if !p.Valid() {
516 return instancectl.ClaimUnitStarted{}, instancectl.ErrClaimUnit
517 }
518 return u.StartChecked(ctx, in, l, check)
519}
520