amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package activator
2
3import (
4 "context"
5 "errors"
6 "fmt"
7 "path/filepath"
8 "slices"
9 "sync"
10 "sync/atomic"
11 "testing"
12 "time"
13
14 "github.com/carlosframework/platform/internal/registry"
15 "github.com/carlosframework/platform/internal/release"
16 "github.com/carlosframework/platform/internal/updatewords"
17)
18
19// Settings spec §4.3 rule 3, "The wake checks too" (settings plan 4 Task 7):
20// before a held instance spawns, the wake compares the installed build with
21// its target and never runs a different installed build.
22
23func heldOn(v string) func(string) HeldWake {
24 return func(string) HeldWake { return HeldWake{Target: v, Held: true} }
25}
26
27// shrinkWakeWait bounds a held wake's wait for root (wakeWaitBound, the
28// bound its waiters have).
29func shrinkWakeWait(t *testing.T, d time.Duration) {
30 t.Helper()
31 old := wakeWaitBound
32 wakeWaitBound = d
33 t.Cleanup(func() { wakeWaitBound = old })
34}
35
36// shrinkHeldPoll sets how often a held wake polls while it waits, for one
37// test. It asks for the answer again every heldReask polls, so a test that
38// waits for a re-ask waits heldReask*d.
39func shrinkHeldPoll(t *testing.T, d time.Duration) {
40 t.Helper()
41 old := heldPoll
42 heldPoll = d
43 t.Cleanup(func() { heldPoll = old })
44}
45
46// asks records RequestStage calls as host@build.
47type asks struct {
48 mu sync.Mutex
49 got []string
50 then func(host, build string)
51}
52
53func (r *asks) ask(host, build string) {
54 r.mu.Lock()
55 r.got = append(r.got, host+"@"+build)
56 r.mu.Unlock()
57 if r.then != nil {
58 r.then(host, build)
59 }
60}
61
62func (r *asks) seen() []string {
63 r.mu.Lock()
64 defer r.mu.Unlock()
65 return append([]string(nil), r.got...)
66}
67
68func (a *Service) entryGone(host string) bool {
69 a.mu.Lock()
70 defer a.mu.Unlock()
71 return a.entries[host] == nil
72}
73
74// Spec §8 "Returning residence": the wake never runs a stale link. It asks
75// root to install the target, waits for it, and runs exactly that.
76func TestAHeldWakeNeverRunsAStaleLink(t *testing.T) {
77 a, b := identityActivator(t, "v3") // an old live link left on this box
78 a.HeldTarget = heldOn("v1")
79 r := &asks{then: func(string, string) { b.setLive("v1") }} // root installs v1
80 a.RequestStage = r.ask
81 l := &execLog{}
82 a.BeforeExec = l.hook
83 done, err := a.beginRequest(context.Background(), identityRoute(t, "one.example.com"))
84 if err != nil {
85 t.Fatal(err)
86 }
87 done()
88 if got := r.seen(); !slices.Equal(got, []string{"one.example.com@v1"}) || b.lastExeVersion() != "v1" || b.startCount() != 1 {
89 t.Fatalf("asked %v, started %d of %q; want v1 asked for and started, never v3", got, b.startCount(), b.lastExeVersion())
90 }
91 if calls, _, _ := l.seen(); !slices.Equal(calls, []string{"one.example.com@v1"}) {
92 t.Fatalf("records %v, want v1", calls)
93 }
94}
95
96// The wait is bounded: a root that never installs the target fails the wake
97// within the waiters' bound, and nothing runs.
98func TestAHeldWakeWaitsOnlyWithinItsBound(t *testing.T) {
99 shrinkWakeWait(t, 300*time.Millisecond)
100 a, b := identityActivator(t, "v3")
101 a.HeldTarget = heldOn("v1")
102 r := &asks{}
103 a.RequestStage = r.ask
104 host := "one.example.com"
105 began := time.Now()
106 if _, err := a.beginRequest(context.Background(), identityRoute(t, host)); !errors.Is(err, errStaleLink) && !errors.Is(err, errWakeTimeout) {
107 t.Fatalf("err = %v; want the wake to fail, not run v3", err)
108 }
109 waitFor(t, "the held wake to end", func() bool { return a.entryGone(host) })
110 if took := time.Since(began); took > 5*time.Second {
111 t.Fatalf("the held wake waited %v, past its bound", took)
112 }
113 if b.startCount() != 0 || b.seedCount() != 0 {
114 t.Fatalf("started %d, seeded %d: the stale build ran", b.startCount(), b.seedCount())
115 }
116 if got := r.seen(); len(got) != 1 {
117 t.Fatalf("asked root %v, want once", got)
118 }
119}
120
121func TestAnUnknownBuildNeverWakes(t *testing.T) {
122 a, b := identityActivator(t, "v1")
123 a.HeldTarget = func(string) HeldWake { return HeldWake{Held: true, Unknown: true} }
124 a.RequestStage = func(string, string) { t.Error("asked root to install an unknown build") }
125 _, err := a.beginRequest(context.Background(), identityRoute(t, "one.example.com"))
126 if !errors.Is(err, errBuildUnknown) {
127 t.Fatalf("err = %v", err)
128 }
129 if acq, _, starts := b.counts(); acq != 0 || starts != 0 {
130 t.Fatalf("acquired %d, started %d: an unknown build woke", acq, starts)
131 }
132 if got, want := memberSafeWakeReason(err), updatewords.Text("settings.box.reason.unknown"); got != want {
133 t.Fatalf("member reads %q, want %q", got, want)
134 }
135}
136
137func TestAnAutoInstanceIsNotChecked(t *testing.T) {
138 a, b := identityActivator(t, "v1")
139 a.HeldTarget = func(string) HeldWake { return HeldWake{} }
140 a.RequestStage = func(string, string) { t.Error("asked root for an instance that follows its channel") }
141 done, err := a.beginRequest(context.Background(), identityRoute(t, "one.example.com"))
142 if err != nil {
143 t.Fatal(err)
144 }
145 done()
146 if b.startCount() != 1 || b.lastExeVersion() != "v1" {
147 t.Fatal("an auto instance's wake changed")
148 }
149}
150
151// Astra I3 #2: with CARLOS_SWAP off there is no planner. A target root
152// reports failed is never run, even while it is still the live link.
153func TestWithNoPlannerAFailedTargetIsRefusedAtTheLiveLink(t *testing.T) {
154 a, b := identityActivator(t, "v9") // the failed build is still installed; Planner nil
155 a.HeldTarget = func(string) HeldWake { return HeldWake{Target: "v9", Held: true, Failed: true} }
156 a.RequestStage = func(string, string) { t.Error("asked root to install a build it reports failed") }
157 _, err := a.beginRequest(context.Background(), identityRoute(t, "one.example.com"))
158 if !errors.Is(err, errTargetFailed) {
159 t.Fatalf("err = %v, want errTargetFailed", err)
160 }
161 if b.startCount() != 0 {
162 t.Fatal("a build root reported failed ran again")
163 }
164 if got, want := memberSafeWakeReason(err), fmt.Sprintf(updatewords.Text("settings.box.reason.failed"), "v9"); got != want {
165 t.Fatalf("member reads %q, want %q", got, want)
166 }
167}
168
169// Astra I3 #5: with no planner, a retained, verified staged copy of the
170// target runs directly, recorded first; root's repoint is not needed. Root
171// is still asked to install the target, without the wake waiting for it
172// (ruling R-RETAINED-NUDGE): here root never repoints, and the wake runs
173// the retained file at once.
174func TestWithNoPlannerARetainedCopyRunsWithoutARepoint(t *testing.T) {
175 shrinkWakeWait(t, 10*time.Second)
176 a, b := identityActivator(t, "v3") // stale v3 installed
177 retained := stagedID(t, t.TempDir(), "v1")
178 a.HeldTarget = func(string) HeldWake { return HeldWake{Target: "v1", Held: true, Staged: &retained} }
179 r := &asks{} // records, never repoints
180 a.RequestStage = r.ask
181 l := &execLog{spawns: b.startCount}
182 a.BeforeExec = l.hook
183 rt := identityRoute(t, "one.example.com")
184 began := time.Now()
185 done, err := a.beginRequest(context.Background(), rt)
186 if err != nil {
187 t.Fatalf("the wake waited for root instead of running the retained copy: %v", err)
188 }
189 done()
190 if took := time.Since(began); took >= time.Second {
191 t.Fatalf("the wake took %v: it waited for root", took)
192 }
193 if got := r.seen(); !slices.Equal(got, []string{"one.example.com@v1"}) {
194 t.Fatalf("asked root %v, want v1 asked for once, not waited on", got)
195 }
196 x, ok := b.lastExe()
197 calls, at, _ := l.seen()
198 if !ok || x.id != retained || !slices.Equal(calls, []string{"one.example.com@v1"}) || at[0] != 0 {
199 t.Fatalf("ran %+v, recorded %v at spawn counts %v; want the retained v1, recorded first", x, calls, at)
200 }
201 if c, _ := b.identityCounts(); c != 0 || b.liveStarts() != 0 {
202 t.Fatalf("%d live-link captures, %d live-link starts: the retained copy is the wake's identity", c, b.liveStarts())
203 }
204 if g, _ := a.entryGen(rt.Host); g.StagedPath != retained.StagedPath || g.Build != "v1" {
205 t.Fatalf("serving generation %+v, want the retained file's", g)
206 }
207}
208
209// Ruling R-RETAINED-NUDGE: a retained wake runs v1 while the live link
210// still names v3, which the sweep's convergence reads as drift and cycles
211// after ConvergeGrace. Root, asked at the wake, repoints inside the grace:
212// the drift is gone and the instance keeps serving. The control (root never
213// repoints) shows the sweep does cycle it.
214func TestARetainedWakeIsNotCycledOnceRootRepoints(t *testing.T) {
215 t.Parallel()
216 for _, repoint := range []bool{true, false} {
217 a, b, clk, _ := testActivator(t, time.Hour)
218 b.setIdentity("v3")
219 retained := stagedID(t, t.TempDir(), "v1")
220 a.HeldTarget = func(string) HeldWake { return HeldWake{Target: "v1", Held: true, Staged: &retained} }
221 r := &asks{}
222 a.RequestStage = r.ask
223 host := "one.example.com"
224 done, err := a.beginRequest(context.Background(), identityRoute(t, host))
225 if err != nil {
226 t.Fatal(err)
227 }
228 done()
229 if len(r.seen()) != 1 {
230 t.Fatalf("asked root %v, want once", r.seen())
231 }
232 a.sweepOnce() // the drift is seen: v1 runs, v3 is linked
233 clk.advance(ConvergeGrace / 2)
234 if repoint {
235 b.setLive("v1") // root's pass, asked at the wake, lands inside the grace
236 }
237 a.sweepOnce()
238 clk.advance(ConvergeGrace)
239 a.sweepOnce()
240 _, awake := a.entryGen(host)
241 if repoint && !awake {
242 t.Fatal("the retained wake was cycled although root repointed inside the grace")
243 }
244 if !repoint && awake {
245 t.Fatal("control: a retained wake on a link root never moves was not cycled; the sweep fixture proves nothing")
246 }
247 }
248}
249
250// Anchors D18: a retained copy is a staged file run that no plan or pin
251// published. The wake publishes it as pending (ruling R-COLDPUBLISH) before
252// it records, so root's sweep cannot remove the file between the capture
253// and the exec.
254//
255// The wake goroutine republishes the generations file after the waiting
256// request is let go (wake's tail), so the test waits for that goroutine to
257// end before t.TempDir's cleanup removes the file's directory; a write still
258// in flight there failed the cleanup with "directory not empty" (fix wave 1,
259// item 6, about 15 in 200 runs).
260func TestARetainedCopyIsPublishedBeforeItIsRecorded(t *testing.T) {
261 shrinkWakeWait(t, 2*time.Second)
262 a, b := identityActivator(t, "v3")
263 a.GenerationsFile = filepath.Join(t.TempDir(), "zdd", "generations.json")
264 ended := make(chan struct{}, 1)
265 a.wakeEnded = func(string) { ended <- struct{}{} }
266 t.Cleanup(func() { // before TempDir's own cleanup, which was registered first
267 select {
268 case <-ended:
269 case <-time.After(10 * time.Second):
270 t.Error("the wake goroutine did not end")
271 }
272 })
273 retained := stagedID(t, t.TempDir(), "v1")
274 a.HeldTarget = func(string) HeldWake { return HeldWake{Target: "v1", Held: true, Staged: &retained} }
275 names := func() bool {
276 g, err := release.ReadGenerations(a.GenerationsFile)
277 return err == nil && g != nil && slices.Contains(g.Paths["one.example.com"], retained.StagedPath)
278 }
279 var publishedFirst atomic.Bool
280 l := &execLog{}
281 a.BeforeExec = func(ctx context.Context, host, build string) error {
282 publishedFirst.Store(names())
283 return l.hook(ctx, host, build)
284 }
285 done, err := a.beginRequest(context.Background(), identityRoute(t, "one.example.com"))
286 if err != nil {
287 t.Fatal(err)
288 }
289 done()
290 if !publishedFirst.Load() {
291 t.Fatal("the build state named the retained copy before the generations file did")
292 }
293 waitForPublications(t, a.GenerationsFile, 2)
294 waitFor(t, "the serving generation to stay published", names)
295 if b.startCount() != 1 {
296 t.Fatalf("starts = %d", b.startCount())
297 }
298}
299
300// With a planner, a failed target is the planner's call (I2 #4): the wake is
301// not refused by the settings rule, and the planner's recovery boots.
302func TestWithAPlannerAFailedTargetDefersToIt(t *testing.T) {
303 good := stagedID(t, t.TempDir(), "v0")
304 a, b, _ := planActivator(t, WakePlan{Boot: good, Mode: ModeOverlap})
305 b.fakeBackend.setIdentity("v9")
306 a.HeldTarget = func(string) HeldWake { return HeldWake{Target: "v9", Held: true, Failed: true} }
307 a.RequestStage = func(string, string) { t.Error("asked root for a target the planner decides") }
308 done, err := a.beginRequest(context.Background(), identityRoute(t, "one.example.com"))
309 if err != nil {
310 t.Fatal(err)
311 }
312 done()
313 if gens := b.started(); len(gens) != 1 || gens[0].Build != "v0" {
314 t.Fatalf("started %+v, want the planner's recovery v0", gens)
315 }
316}
317
318// stampRoutes is the registry row the wake re-reads (RouteLookup); root's
319// repoint stamps Version, as adoption does.
320type stampRoutes struct {
321 mu sync.Mutex
322 rt registry.Route
323}
324
325func (s *stampRoutes) Get(string) (*registry.Route, error) {
326 s.mu.Lock()
327 defer s.mu.Unlock()
328 rt := s.rt
329 return &rt, nil
330}
331
332func (s *stampRoutes) stamp(v string) { s.mu.Lock(); s.rt.Version = v; s.mu.Unlock() }
333
334// Astra I2 #4: the held wake never pins its target itself. Generations are
335// on and the target v1 is staged on this box (the case where the I1 wake
336// check set e.pin and skipped the planner), but the failure book blocks v1
337// here: the planner, which sees root's install, boots the good build v0.
338func TestAHeldWakeGoesThroughTheFailureAwarePlan(t *testing.T) {
339 dir := t.TempDir()
340 good, target := stagedID(t, dir, "v0"), stagedID(t, dir, "v1")
341 a, b, _ := planActivator(t, WakePlan{}) // generations on, a recording GenBackend
342 routes := &stampRoutes{rt: identityRoute(t, "one.example.com")}
343 routes.rt.Version, routes.rt.GoodVersion = "v3", "v0"
344 a.Routes = routes
345 b.fakeBackend.setIdentity("v3")
346 var mu sync.Mutex
347 var planned []string
348 a.Planner = func(_ context.Context, rt registry.Route) WakePlan {
349 mu.Lock()
350 planned = append(planned, rt.Version)
351 mu.Unlock()
352 return WakePlan{Boot: good, Mode: ModeOverlap} // decideWake's recovery when Blocks(v1)
353 }
354 // The target's staged copy exists, but a planner is on: the wake asks
355 // root and lets the planner choose, never running the copy itself.
356 a.HeldTarget = func(string) HeldWake { return HeldWake{Target: "v1", Held: true, Staged: &target} }
357 a.RequestStage = func(host, build string) { b.fakeBackend.setLive(build); routes.stamp(build) }
358 done, err := a.beginRequest(context.Background(), routes.rt)
359 if err != nil {
360 t.Fatal(err)
361 }
362 done()
363 mu.Lock()
364 defer mu.Unlock()
365 if gens := b.started(); len(planned) != 1 || planned[0] != "v1" || len(gens) != 1 || gens[0].Build != "v0" {
366 t.Fatalf("planner saw %v, started %+v; want one plan on root's install v1, booting the good v0", planned, gens)
367 }
368 a.mu.Lock()
369 pinned := a.entries["one.example.com"] != nil && a.entries["one.example.com"].pin != nil
370 a.mu.Unlock()
371 if pinned {
372 t.Fatal("the held wake pinned its target")
373 }
374}
375
376// Astra I2 #4, the race half: root installs something other than the target
377// between the wait and the identity; the wake refuses rather than run it,
378// and records and publishes nothing.
379func TestAHeldWakeRefusesAnIdentityThatIsNotItsTarget(t *testing.T) {
380 a, b := identityActivator(t, "v3")
381 a.GenerationsFile = filepath.Join(t.TempDir(), "zdd", "generations.json")
382 a.HeldTarget = heldOn("v1")
383 a.RequestStage = func(string, string) { b.setLive("v1") }
384 b.mu.Lock()
385 b.ident.onLiveIdentity = func(f *identityFake) { f.live = "v2" } // a repoint after the wait
386 b.mu.Unlock()
387 l := &execLog{}
388 a.BeforeExec = l.hook
389 _, err := a.beginRequest(context.Background(), identityRoute(t, "one.example.com"))
390 if !errors.Is(err, errStaleLink) {
391 t.Fatalf("err = %v, want errStaleLink", err)
392 }
393 if b.startCount() != 0 || b.seedCount() != 0 {
394 t.Fatal("the wake ran a build that is not its target")
395 }
396 if calls, _, _ := l.seen(); len(calls) != 0 {
397 t.Fatalf("records %v: a refused identity was recorded", calls)
398 }
399 waitFor(t, "the lease to go back", func() bool { return b.releaseCount() == 1 })
400 if g, err := release.ReadGenerations(a.GenerationsFile); err == nil && g != nil && len(g.Paths["one.example.com"]) != 0 {
401 t.Fatalf("published %v for a refused identity", g.Paths)
402 }
403 if got := memberSafeWakeReason(err); got != memberSafeWakeReason(ErrFenced) {
404 t.Fatalf("member reads %q, want the fence's wording", got)
405 }
406}
407
408// Ruling R-EXEC-NOVERSION-2: a held wake never takes the unversioned
409// live-link fallback, because its target is a version.
410func TestAHeldWakeNeverRunsAnUnversionedLink(t *testing.T) {
411 a, b := identityActivator(t, "v1") // the link's version reads v1 ...
412 b.mu.Lock()
413 b.ident.unversioned = true // ... but the file it resolves to is outside the version store
414 b.mu.Unlock()
415 a.HeldTarget = heldOn("v1")
416 _, err := a.beginRequest(context.Background(), identityRoute(t, "one.example.com"))
417 if !errors.Is(err, errStaleLink) {
418 t.Fatalf("err = %v, want errStaleLink", err)
419 }
420 if b.startCount() != 0 || b.liveStarts() != 0 {
421 t.Fatalf("started %d (live-link %d): a held wake ran an unversioned file", b.startCount(), b.liveStarts())
422 }
423}
424
425// A hold swap's pinned wake was chosen by the swap from root's install
426// (e.pin): the held check does not run for it.
427func TestAHoldSwapsPinnedWakeIsNotChecked(t *testing.T) {
428 f := newSwapFixture(t)
429 f.a.HeldTarget = func(string) HeldWake {
430 t.Error("the held check ran for a hold swap's pinned wake")
431 return HeldWake{}
432 }
433 if res := f.a.Swap(context.Background(), f.attempt(ModeHold), f.newest); res.Outcome != SwapDone {
434 t.Fatalf("Swap = %+v", res)
435 }
436}
437
438// Ruling R-WAKEWORDS: members never see the held check's internal text.
439func TestTheHeldWakeRefusalsReadInTheMembersTerms(t *testing.T) {
440 cases := []struct {
441 err error
442 want string
443 }{
444 {fmt.Errorf("one.example.com: %w", errBuildUnknown), updatewords.Text("settings.box.reason.unknown")},
445 {fmt.Errorf("one.example.com: %w", &targetFailedError{host: "one.example.com", build: "v9"}), fmt.Sprintf(updatewords.Text("settings.box.reason.failed"), "v9")},
446 {fmt.Errorf("start one.example.com: %w: would run v2, wants v1", errStaleLink), memberSafeWakeReason(ErrFenced)},
447 }
448 for _, c := range cases {
449 if got := memberSafeWakeReason(c.err); got != c.want {
450 t.Errorf("memberSafeWakeReason(%v) = %q, want %q", c.err, got, c.want)
451 }
452 }
453 if !errors.Is(&targetFailedError{build: "v9"}, errTargetFailed) {
454 t.Fatal("a failed target is not errTargetFailed")
455 }
456}
457
458// heldIdentityOK: the target, or, under a plan that is not a first wake,
459// the good build the planner chose as recovery. Nothing else.
460func TestHeldIdentityOK(t *testing.T) {
461 rt := registry.Route{GoodVersion: "v0"}
462 plan := WakePlan{Boot: release.StagedIdentity{Version: "v0", StagedPath: "/x/v0"}}
463 first := plan
464 first.FirstWake = true
465 for _, c := range []struct {
466 target, id string
467 plan WakePlan
468 ok bool
469 }{
470 {"", "v7", WakePlan{}, true},
471 {"v1", "v1", WakePlan{}, true},
472 {"v1", "v2", WakePlan{}, false},
473 {"v1", "v0", WakePlan{}, false},
474 {"v1", "v0", plan, true},
475 {"v1", "v0", first, false},
476 {"v1", "", WakePlan{}, false},
477 } {
478 if got := heldIdentityOK(c.target, release.StagedIdentity{Version: c.id}, c.plan, rt); got != c.ok {
479 t.Errorf("heldIdentityOK(%q, %q, planned=%v first=%v) = %v, want %v", c.target, c.id, c.plan.planned(), c.plan.FirstWake, got, c.ok)
480 }
481 }
482}
483
484// The wait asks for the answer again: a changed one ends it at once, with
485// its own refusal, rather than at the bound onto a stale target.
486func TestAHeldWakeEndsItsWaitWhenTheAnswerChanges(t *testing.T) {
487 cases := []struct {
488 name string
489 next HeldWake
490 want error
491 }{
492 {"unknown", HeldWake{Held: true, Unknown: true}, errBuildUnknown},
493 {"failed", HeldWake{Target: "v1", Held: true, Failed: true}, errTargetFailed},
494 {"another target", HeldWake{Target: "v2", Held: true}, errStaleLink},
495 {"no longer held", HeldWake{}, errStaleLink},
496 }
497 for _, c := range cases {
498 t.Run(c.name, func(t *testing.T) {
499 shrinkWakeWait(t, 20*time.Second)
500 shrinkHeldPoll(t, 25*time.Millisecond) // the re-ask comes 8 polls in
501 a, b := identityActivator(t, "v3")
502 var asked atomic.Int32
503 a.HeldTarget = func(string) HeldWake {
504 if asked.Add(1) == 1 {
505 return HeldWake{Target: "v1", Held: true}
506 }
507 return c.next
508 }
509 a.RequestStage = func(string, string) {} // root never installs v1
510 began := time.Now()
511 _, err := a.beginRequest(context.Background(), identityRoute(t, "one.example.com"))
512 if !errors.Is(err, c.want) {
513 t.Fatalf("err = %v, want %v", err, c.want)
514 }
515 if took := time.Since(began); took >= 10*time.Second {
516 t.Fatalf("the wait ran %v: the changed answer did not end it", took)
517 }
518 if b.startCount() != 0 {
519 t.Fatal("a wake whose answer changed ran something")
520 }
521 })
522 }
523}
524
525// countingRoutes is a RouteLookup whose nth Get runs a hook.
526type countingRoutes struct {
527 rt registry.Route
528 n atomic.Int32
529 onGet func(n int32)
530}
531
532func (c *countingRoutes) Get(string) (*registry.Route, error) {
533 if c.onGet != nil {
534 c.onGet(c.n.Add(1))
535 }
536 rt := c.rt
537 return &rt, nil
538}
539
540// A held unit-backed row (systemd execs the live link; nothing can pin its
541// file): root installs the target, then the link moves after the wait. The
542// read just before the start refuses it, and nothing starts.
543func TestAHeldUnitRowRefusesALinkThatMovedAfterTheWait(t *testing.T) {
544 exec, unit := &fakeBackend{}, &fakeBackend{}
545 exec.setIdentity("v3") // live/<host>: SplitBackend.LiveVersion reads the exec half's link
546 split := &SplitBackend{Exec: exec, Unit: struct{ Backend }{unit}}
547 reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db"))
548 if err != nil {
549 t.Fatal(err)
550 }
551 a := New(Config{Bucket: "test-bucket", Idle: time.Hour}, split, reg)
552 ur := identityRoute(t, "u.example.com")
553 ur.Backing = registry.BackingUnit
554 routes := &countingRoutes{rt: ur}
555 // Get 1 is step 0's; Get 2 is the re-read after the wait.
556 routes.onGet = func(n int32) {
557 if n == 2 {
558 exec.setLive("v2")
559 }
560 }
561 a.Routes = routes
562 a.HeldTarget = heldOn("v1")
563 a.RequestStage = func(string, string) { exec.setLive("v1") }
564 _, err = a.beginRequest(context.Background(), ur)
565 if !errors.Is(err, errStaleLink) {
566 t.Fatalf("err = %v, want errStaleLink", err)
567 }
568 if unit.startCount() != 0 || exec.startCount() != 0 {
569 t.Fatalf("unit starts %d, exec starts %d: a moved link was started", unit.startCount(), exec.startCount())
570 }
571 waitFor(t, "the lease to go back", func() bool { return unit.releaseCount() == 1 })
572}
573
574// The wait's re-ask is bounded by the wait (Task 12a): an answer that is slow
575// to come back (the agent's refresh reads the bucket, up to its own 5 s
576// budget) never holds the wake past wakeWaitBound.
577func TestAHeldWakesReaskNeverOutlastsTheWait(t *testing.T) {
578 // The re-ask comes 8 polls (175 ms) into a 400 ms wait, and its answer
579 // takes 5 s: the wait must end at its own bound.
580 shrinkHeldPoll(t, 25*time.Millisecond)
581 shrinkWakeWait(t, 400*time.Millisecond)
582 a, b := identityActivator(t, "v3")
583 var asked atomic.Int32
584 a.HeldTarget = func(string) HeldWake {
585 if asked.Add(1) > 1 {
586 time.Sleep(5 * time.Second) // the refresh's own budget
587 }
588 return HeldWake{Target: "v1", Held: true}
589 }
590 a.RequestStage = func(string, string) {} // root never installs v1
591 // The check itself, timed: the request's own waiters give up at the same
592 // bound, so through beginRequest an overrun shows only as their timeout
593 // while the wake is still stuck in the re-ask.
594 began := time.Now()
595 _, _, err := a.wakeCheck(context.Background(), "one.example.com", true)
596 took := time.Since(began)
597 if !errors.Is(err, errStaleLink) {
598 t.Fatalf("err = %v, want errStaleLink at the bound", err)
599 }
600 if asked.Load() < 2 {
601 t.Fatalf("asked %d times: the wait never asked again", asked.Load())
602 }
603 if took >= 3*time.Second {
604 t.Fatalf("the wait ran %v, past its 0.4 s bound: the re-ask outlasted it", took)
605 }
606 if b.startCount() != 0 {
607 t.Fatal("a wake that never saw its target ran something")
608 }
609}
610
611// Astra round 1 finding 1: a wake whose build state could not be read for it
612// is refused before anything is acquired, and root is not asked.
613func TestAnUnreadBuildStateRefusesTheWake(t *testing.T) {
614 a, b := identityActivator(t, "v1")
615 a.HeldTarget = func(string) HeldWake { return HeldWake{Held: true, Unread: true} }
616 a.RequestStage = func(string, string) { t.Error("asked root for a wake whose build state is unread") }
617 _, err := a.beginRequest(context.Background(), identityRoute(t, "one.example.com"))
618 if !errors.Is(err, errBuildStateUnwritten) {
619 t.Fatalf("err = %v, want errBuildStateUnwritten", err)
620 }
621 if acq, _, starts := b.counts(); acq != 0 || starts != 0 {
622 t.Fatalf("acquired %d, started %d: a wake decided from a stale copy", acq, starts)
623 }
624}
625
626// Ruling R-HELD-PURE (Astra round 1 finding 4): with no decision from root
627// for the current settings the wake holds. Its build state's build runs at
628// once when the live link names it, and root is not asked.
629func TestAWakeRootHasNotDecidedForRunsItsInstalledBuild(t *testing.T) {
630 a, b := identityActivator(t, "v1")
631 a.HeldTarget = func(string) HeldWake { return HeldWake{Held: true, AskRoot: true, Target: "v1"} }
632 a.RequestStage = func(string, string) { t.Error("asked root although the build state's build is installed") }
633 done, err := a.beginRequest(context.Background(), identityRoute(t, "one.example.com"))
634 if err != nil {
635 t.Fatal(err)
636 }
637 done()
638 if b.lastExeVersion() != "v1" || b.startCount() != 1 {
639 t.Fatalf("started %d of %q, want v1", b.startCount(), b.lastExeVersion())
640 }
641}
642
643// Otherwise it never runs the live link it does not know it may run: it asks
644// root for a pass (no build named) and waits for root's decision, then
645// follows it: a hold on v1 runs v1 once root installs it; a decision that the
646// instance is not held is today's wake; no decision within the bound fails
647// the wake with nothing run.
648func TestAWakeRootHasNotDecidedForWaitsForRoot(t *testing.T) {
649 cases := []struct {
650 name string
651 decided HeldWake
652 install string
653 want string // the build started, "" for none
654 }{
655 {"held on its build", HeldWake{Held: true, Target: "v1"}, "v1", "v1"},
656 {"not held", HeldWake{}, "", "v3"},
657 {"never decides", HeldWake{Held: true, AskRoot: true, Target: "v1"}, "", ""},
658 }
659 for _, c := range cases {
660 t.Run(c.name, func(t *testing.T) {
661 // Root's answer is asked again 8 polls (200 ms) in; a wake root
662 // never decides for fails at the bound.
663 shrinkHeldPoll(t, 25*time.Millisecond)
664 shrinkWakeWait(t, time.Second)
665 a, b := identityActivator(t, "v3")
666 var asked atomic.Bool
667 a.HeldTarget = func(string) HeldWake {
668 if asked.Load() {
669 return c.decided
670 }
671 return HeldWake{Held: true, AskRoot: true, Target: "v1"}
672 }
673 r := &asks{then: func(host, build string) {
674 asked.Store(true) // root's pass decides
675 if c.install != "" && build == c.install {
676 b.setLive(c.install)
677 }
678 }}
679 a.RequestStage = r.ask
680 done, err := a.beginRequest(context.Background(), identityRoute(t, "one.example.com"))
681 if c.want == "" {
682 if !errors.Is(err, errStaleLink) && !errors.Is(err, errWakeTimeout) {
683 t.Fatalf("err = %v; want the wake to fail", err)
684 }
685 waitFor(t, "the wake to end", func() bool { return a.entryGone("one.example.com") })
686 if b.startCount() != 0 {
687 t.Fatalf("started %q with no decision from root", b.lastExeVersion())
688 }
689 return
690 }
691 if err != nil {
692 t.Fatal(err)
693 }
694 done()
695 if b.lastExeVersion() != c.want || b.startCount() != 1 {
696 t.Fatalf("started %d of %q, want %s", b.startCount(), b.lastExeVersion(), c.want)
697 }
698 if got := r.seen(); len(got) == 0 || got[0] != "one.example.com@" {
699 t.Fatalf("asked %v; want a pass asked for first, naming no build", got)
700 }
701 })
702 }
703}
704