amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package host
2
3import (
4 "context"
5 "errors"
6 "io"
7 "os"
8 "path/filepath"
9 "sync"
10 "testing"
11 "time"
12
13 "github.com/carlosframework/platform/internal/activator"
14 "github.com/carlosframework/platform/internal/domains"
15 "github.com/carlosframework/platform/internal/layout"
16 "github.com/carlosframework/platform/internal/provision"
17 "github.com/carlosframework/platform/internal/registry"
18 "github.com/carlosframework/platform/internal/release"
19 "github.com/carlosframework/platform/internal/seeds"
20)
21
22// seedRecordFixture writes an account, the shop app and one record of
23// shop-1.example.com carrying a seed in state, and returns the record.
24func seedRecordFixture(t *testing.T, deploy release.Store, kind, state string) provision.Record {
25 t.Helper()
26 ctx := context.Background()
27 sqid := domains.Sqid(2)
28 manifest := []byte(`{"id":2,"sqid":"` + sqid + `","name":"Shop","created_at":"2026-08-02T09:00:00Z"}`)
29 if _, err := deploy.Put(ctx, "accounts/"+sqid+"/account.json", manifest, ""); err != nil {
30 t.Fatal(err)
31 }
32 if err := provision.PutManifest(ctx, deploy, sqid, "shop", provision.Manifest{
33 Schema: 1, App: "shop", Domain: "example.com", Seed: []string{"seed"}}); err != nil {
34 t.Fatal(err)
35 }
36 id, err := seeds.NewID()
37 if err != nil {
38 t.Fatal(err)
39 }
40 rec := provision.Record{Schema: 1, Host: "shop-1.example.com", Slug: "shop-1", App: "shop", Account: sqid,
41 Channel: "stable", CreatedAt: time.Now().UTC(),
42 Seed: &provision.SeedSource{ID: id, Kind: kind, State: state, SHA256: "a", Bytes: 2, By: "t", At: time.Now().UTC()}}
43 if err := provision.CreateRecord(ctx, deploy, rec); err != nil {
44 t.Fatal(err)
45 }
46 return rec
47}
48
49func putSnapshot(t *testing.T, data release.Store, host, id string) string {
50 t.Helper()
51 k := seeds.SnapshotKey(host, id)
52 if _, err := data.Put(context.Background(), k, []byte("db"), ""); err != nil {
53 t.Fatal(err)
54 }
55 return k
56}
57
58func snapshotPresent(t *testing.T, data release.Store, key string) bool {
59 t.Helper()
60 _, _, err := data.Get(context.Background(), key)
61 if errors.Is(err, os.ErrNotExist) {
62 return false
63 }
64 if err != nil {
65 t.Fatal(err)
66 }
67 return true
68}
69
70func seedStateOf(t *testing.T, deploy release.Store, rec provision.Record) string {
71 t.Helper()
72 got, err := provision.GetRecord(context.Background(), deploy, rec.Account, rec.App, rec.Host)
73 if err != nil || got.Seed == nil {
74 t.Fatalf("record: %+v, %v", got, err)
75 }
76 return got.Seed.State
77}
78
79func TestSeedPlacedHookCASesThenDeletesOnlyThisSeed(t *testing.T) {
80 ctx := context.Background()
81 deploy, data := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir())
82 rec := seedRecordFixture(t, deploy, seeds.KindReplica, seeds.StateReady)
83 id := rec.Seed.ID
84 other, _ := seeds.NewID()
85 mine, theirs := putSnapshot(t, data, rec.Host, id), putSnapshot(t, data, rec.Host, other)
86
87 hook := seedPlacedHook(deploy, data)
88 if err := hook(ctx, rec.Account, rec.App, rec.Host, id); err != nil {
89 t.Fatal(err)
90 }
91 if st := seedStateOf(t, deploy, rec); st != seeds.StatePlaced {
92 t.Fatalf("state = %s, want placed", st)
93 }
94 if snapshotPresent(t, data, mine) {
95 t.Fatal("this seed's snapshot survived its placement")
96 }
97 if !snapshotPresent(t, data, theirs) {
98 t.Fatal("another seed's snapshot was deleted")
99 }
100
101 // Another id than the record's: refused, nothing deleted.
102 if err := hook(ctx, rec.Account, rec.App, rec.Host, other); !errors.Is(err, provision.ErrSeedConflict) {
103 t.Fatalf("placing another seed's id = %v, want ErrSeedConflict", err)
104 }
105 if !snapshotPresent(t, data, theirs) {
106 t.Fatal("a refused placement deleted a snapshot")
107 }
108
109 // An id that is not an id: refused before the record or the bucket.
110 if err := hook(ctx, rec.Account, rec.App, rec.Host, "../x"); err == nil {
111 t.Fatal("a malformed id was accepted")
112 }
113}
114
115// The hook runs twice on one finishing wake (Restore's finishPlacement and
116// doWake's step 5b), and two boxes can finish one placement. Each call is a
117// success, and a call that finds the record already placed with this id
118// still deletes the snapshot: the first call's delete may be the one that
119// failed.
120func TestSeedPlacedHookIsIdempotent(t *testing.T) {
121 ctx := context.Background()
122 deploy, data := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir())
123 rec := seedRecordFixture(t, deploy, seeds.KindReplica, seeds.StateReady)
124 key := putSnapshot(t, data, rec.Host, rec.Seed.ID)
125
126 boxA, boxB := seedPlacedHook(deploy, data), seedPlacedHook(deploy, data)
127 if err := boxA(ctx, rec.Account, rec.App, rec.Host, rec.Seed.ID); err != nil {
128 t.Fatal(err)
129 }
130 // The first call's delete lost: the snapshot is back, and the record
131 // already reads placed. The second call deletes it.
132 putSnapshot(t, data, rec.Host, rec.Seed.ID)
133 if err := boxA(ctx, rec.Account, rec.App, rec.Host, rec.Seed.ID); err != nil {
134 t.Fatalf("second call on one box: %v", err)
135 }
136 if snapshotPresent(t, data, key) {
137 t.Fatal("an already-placed record's call left the snapshot")
138 }
139 if err := boxB(ctx, rec.Account, rec.App, rec.Host, rec.Seed.ID); err != nil {
140 t.Fatalf("another box finishing the same placement: %v", err)
141 }
142 if st := seedStateOf(t, deploy, rec); st != seeds.StatePlaced {
143 t.Fatalf("state = %s, want placed", st)
144 }
145
146 // Two boxes at once, from ready: both succeed, one CAS wins, the
147 // snapshot goes.
148 deploy2, data2 := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir())
149 rec2 := seedRecordFixture(t, deploy2, seeds.KindReplica, seeds.StateReady)
150 key2 := putSnapshot(t, data2, rec2.Host, rec2.Seed.ID)
151 var wg sync.WaitGroup
152 errs := make([]error, 2)
153 for i := range errs {
154 wg.Add(1)
155 go func() {
156 defer wg.Done()
157 errs[i] = seedPlacedHook(deploy2, data2)(ctx, rec2.Account, rec2.App, rec2.Host, rec2.Seed.ID)
158 }()
159 }
160 wg.Wait()
161 for i, err := range errs {
162 if err != nil {
163 t.Fatalf("box %d: %v", i, err)
164 }
165 }
166 if st := seedStateOf(t, deploy2, rec2); st != seeds.StatePlaced {
167 t.Fatalf("state = %s, want placed", st)
168 }
169 if snapshotPresent(t, data2, key2) {
170 t.Fatal("the snapshot survived two finishing boxes")
171 }
172}
173
174// A refused CAS for any reason but "already placed with this id" deletes
175// nothing: the snapshot may be the only copy of the seed.
176func TestSeedPlacedHookRefusedDeletesNothing(t *testing.T) {
177 ctx := context.Background()
178 for name, tc := range map[string]struct{ kind, state string }{
179 "aborted": {seeds.KindReplica, seeds.StateAborted},
180 "rolled-back": {seeds.KindReplica, seeds.StateRolledBack},
181 "reserving": {seeds.KindReplica, seeds.StateReserving},
182 "import, ready": {seeds.KindImport, seeds.StateReady},
183 } {
184 t.Run(name, func(t *testing.T) {
185 deploy, data := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir())
186 rec := seedRecordFixture(t, deploy, tc.kind, tc.state)
187 key := putSnapshot(t, data, rec.Host, rec.Seed.ID)
188 if err := seedPlacedHook(deploy, data)(ctx, rec.Account, rec.App, rec.Host, rec.Seed.ID); !errors.Is(err, provision.ErrSeedConflict) {
189 t.Fatalf("err = %v, want ErrSeedConflict", err)
190 }
191 if !snapshotPresent(t, data, key) {
192 t.Fatal("a refused placement deleted the snapshot")
193 }
194 if st := seedStateOf(t, deploy, rec); st != tc.state {
195 t.Fatalf("state = %s, want %s unchanged", st, tc.state)
196 }
197 })
198 }
199 // No record at all: an error, nothing deleted.
200 deploy, data := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir())
201 id, _ := seeds.NewID()
202 key := putSnapshot(t, data, "shop-9.example.com", id)
203 if err := seedPlacedHook(deploy, data)(ctx, domains.Sqid(2), "shop", "shop-9.example.com", id); err == nil {
204 t.Fatal("placing a seed with no record succeeded")
205 }
206 if !snapshotPresent(t, data, key) {
207 t.Fatal("a placement with no record deleted the snapshot")
208 }
209 // A box that has not wired the stores records nothing.
210 if err := seedPlacedHook(nil, data)(ctx, domains.Sqid(2), "shop", "shop-9.example.com", id); err == nil {
211 t.Fatal("an unwired hook reported the seed placed")
212 }
213}
214
215func TestStoreSeedObjectsOpensTheSnapshotKey(t *testing.T) {
216 ctx := context.Background()
217 data := release.NewDirStore(t.TempDir())
218 id, _ := seeds.NewID()
219 if _, err := data.Put(ctx, seeds.SnapshotKey("shop-1.example.com", id), []byte("db-bytes"), ""); err != nil {
220 t.Fatal(err)
221 }
222 rc, err := storeSeedObjects{data}.Open(ctx, "shop-1.example.com", id)
223 if err != nil {
224 t.Fatal(err)
225 }
226 b, _ := io.ReadAll(rc)
227 rc.Close()
228 if string(b) != "db-bytes" {
229 t.Fatalf("read %q", b)
230 }
231 if _, err := (storeSeedObjects{data}).Open(ctx, "shop-1.example.com", "../x"); err == nil {
232 t.Fatal("a malformed id opened something")
233 }
234}
235
236func resetArmedSeedHooks(t *testing.T) {
237 t.Helper()
238 old := armedSeedHooks.Load()
239 armedSeedHooks.Store(nil)
240 t.Cleanup(func() { armedSeedHooks.Store(old) })
241}
242
243// wireSeedPlacement sets the claim flag whatever happens to the stores, and
244// arms the reader, the recorder and the provision pass only together, with
245// the data bucket always set on the pass's hooks.
246func TestWireSeedPlacement(t *testing.T) {
247 data, deploy := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir())
248 ok := func(s release.Store) func() (release.Store, error) {
249 return func() (release.Store, error) { return s, nil }
250 }
251 fail := func() (release.Store, error) { return nil, errors.New("no credentials") }
252
253 t.Run("armed", func(t *testing.T) {
254 resetArmedSeedHooks(t)
255 eb := &activator.ExecBackend{}
256 wireSeedPlacement(eb, true, ok(data), ok(deploy), seedBox{})
257 if !eb.ClaimArmed || eb.Seeds == nil || eb.OnSeedPlaced == nil {
258 t.Fatalf("ClaimArmed %v Seeds %v OnSeedPlaced set %v", eb.ClaimArmed, eb.Seeds, eb.OnSeedPlaced != nil)
259 }
260 h := armedSeedHooks.Load()
261 if h == nil || h.Data != release.Store(data) {
262 t.Fatalf("armed hooks = %+v, want the data bucket", h)
263 }
264 })
265 t.Run("claims off", func(t *testing.T) {
266 resetArmedSeedHooks(t)
267 eb := &activator.ExecBackend{ClaimArmed: true}
268 wireSeedPlacement(eb, false, ok(data), ok(deploy), seedBox{})
269 if eb.ClaimArmed {
270 t.Fatal("ClaimArmed set with claim control off")
271 }
272 })
273 for name, opens := range map[string][2]func() (release.Store, error){
274 "no data bucket": {fail, ok(deploy)},
275 "no record store": {ok(data), fail},
276 } {
277 t.Run(name, func(t *testing.T) {
278 resetArmedSeedHooks(t)
279 eb := &activator.ExecBackend{}
280 wireSeedPlacement(eb, true, opens[0], opens[1], seedBox{})
281 if !eb.ClaimArmed {
282 t.Fatal("ClaimArmed not set when a store failed")
283 }
284 if eb.Seeds != nil || eb.OnSeedPlaced != nil || armedSeedHooks.Load() != nil {
285 t.Fatal("seed placement half-armed")
286 }
287 })
288 }
289 t.Run("hooks without a data bucket are not armed", func(t *testing.T) {
290 resetArmedSeedHooks(t)
291 armSeedHooks(&provision.SeedHooks{})
292 if armedSeedHooks.Load() != nil {
293 t.Fatal("armed SeedHooks with a nil Data")
294 }
295 })
296}
297
298// reconcileSerialized hands the provision pass the armed hooks: with them
299// an aborted seed is finished (its snapshot deleted, the record rolled
300// back); without them it is left alone.
301func TestReconcileSerializedCarriesTheArmedSeedHooks(t *testing.T) {
302 t.Parallel()
303 ctx := context.Background()
304 deploy, data := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir())
305 rec := seedRecordFixture(t, deploy, seeds.KindReplica, seeds.StateAborted)
306 key := putSnapshot(t, data, rec.Host, rec.Seed.ID)
307
308 resetArmedSeedHooks(t)
309 if err := reconcileSerialized(ctx, deploy, restartTestRegistry(t), provision.ReconcileConfig{DataDir: t.TempDir()}); err != nil {
310 t.Fatal(err)
311 }
312 if st := seedStateOf(t, deploy, rec); st != seeds.StateAborted || !snapshotPresent(t, data, key) {
313 t.Fatalf("unarmed pass acted on the seed: state %s", st)
314 }
315
316 armSeedHooks(&provision.SeedHooks{Data: data})
317 if err := reconcileSerialized(ctx, deploy, restartTestRegistry(t), provision.ReconcileConfig{DataDir: t.TempDir()}); err != nil {
318 t.Fatal(err)
319 }
320 if st := seedStateOf(t, deploy, rec); st != seeds.StateRolledBack {
321 t.Fatalf("state = %s, want rolled-back", st)
322 }
323 if snapshotPresent(t, data, key) {
324 t.Fatal("the aborted seed's snapshot survived an armed pass")
325 }
326}
327
328// fakeFence is the activator's fence as the import rollback meets it: one
329// fence per host, an exclusive install that refuses any fence already
330// held, and a lift that takes only the generation it names.
331// FenceAndStopIfUnfenced answers err once it would install; nil and
332// ErrStopUnproven leave the fence installed, as the activator does.
333type fakeFence struct {
334 gens map[string]uint64
335 err error
336 quiesced bool
337 uncertain bool
338 calls int
339 deadline time.Duration // the time left on the last install's context
340 lifted []uint64
341}
342
343func (f *fakeFence) FenceAndStopIfUnfenced(ctx context.Context, host string, gen uint64) (*activator.StopEvidence, error) {
344 f.calls++
345 if d, ok := ctx.Deadline(); ok {
346 f.deadline = time.Until(d)
347 }
348 if f.gens[host] != 0 {
349 return nil, activator.ErrFenceHeld
350 }
351 if f.err == nil || errors.Is(f.err, activator.ErrStopUnproven) {
352 f.gens[host] = gen
353 }
354 return nil, f.err
355}
356
357func (f *fakeFence) LiftFence(host string, gen uint64) bool {
358 f.lifted = append(f.lifted, gen)
359 if gen == 0 || f.gens[host] != gen {
360 return false
361 }
362 delete(f.gens, host)
363 return true
364}
365func (f *fakeFence) FenceGeneration(host string) uint64 { return f.gens[host] }
366func (f *fakeFence) Quiesced(string) bool { return f.quiesced }
367func (f *fakeFence) StopUncertain(string) bool { return f.uncertain }
368
369func storeOpener(s release.Store) func() (release.Store, error) {
370 return func() (release.Store, error) { return s, nil }
371}
372
373// wiredSeedHooks arms seed placement as the agent does on a box without
374// claim control, with f as the activator and reg's generation source, and
375// returns the provision pass's hooks.
376func wiredSeedHooks(t *testing.T, f layout.Fence, reg *registry.Registry, data, deploy release.Store) *provision.SeedHooks {
377 t.Helper()
378 resetArmedSeedHooks(t)
379 eb := &activator.ExecBackend{NestedOK: func() error { return nil }}
380 wireSeedPlacement(eb, false, storeOpener(data), storeOpener(deploy), seedBox{
381 Fence: f, Prefix: "instances", Swept: func() bool { return true }, NextFenceGen: fenceGenerations(reg)})
382 h := armedSeedHooks.Load()
383 if h == nil {
384 t.Fatal("no seed hooks armed")
385 }
386 return h
387}
388
389// The rollback's step 2 (spec §5.6) by the layout pass's rule, and the
390// generation it is known by surviving an agent restart.
391func TestRollbackFenceAfterRestart(t *testing.T) {
392 ctx := context.Background()
393 const host = "shop-1.example.com"
394 swept := true
395 f := &fakeFence{gens: map[string]uint64{}, err: activator.ErrStopUnproven, quiesced: true}
396 fence := fenceForRollback(f, func() bool { return swept })
397 if ok, _ := fence(ctx, host, 7); !ok {
398 t.Fatal("cold, swept, quiesced: want proceed")
399 }
400 if f.deadline <= 0 || f.deadline > 30*time.Second {
401 t.Fatalf("the install ran with %v left, want a deadline of at most 30s", f.deadline)
402 }
403 // Ours already (an earlier pass installed it): proceed without asking.
404 f.err, f.calls = errors.New("must not be called"), 0
405 if ok, _ := fence(ctx, host, 7); !ok || f.calls != 0 {
406 t.Fatalf("our own fence: want proceed without an install (calls %d)", f.calls)
407 }
408 // Not swept at start: lift and defer.
409 f = &fakeFence{gens: map[string]uint64{}, err: activator.ErrStopUnproven, quiesced: true}
410 swept = false
411 fence = fenceForRollback(f, func() bool { return swept })
412 if ok, _ := fence(ctx, host, 8); ok || len(f.lifted) != 1 || f.lifted[0] != 8 || f.gens[host] != 0 {
413 t.Fatalf("not swept: proceed %v lifted %v fences %v", ok, f.lifted, f.gens)
414 }
415 // Swept, but not quiesced, or its stop uncertain: lift and defer.
416 for name, f := range map[string]*fakeFence{
417 "not quiesced": {gens: map[string]uint64{}, err: activator.ErrStopUnproven},
418 "stop uncertain": {gens: map[string]uint64{}, err: activator.ErrStopUnproven, quiesced: true, uncertain: true},
419 "another error": {gens: map[string]uint64{}, err: activator.ErrFenceNotInstalled, quiesced: true},
420 } {
421 fence := fenceForRollback(f, func() bool { return true })
422 if ok, err := fence(ctx, host, 9); ok || err == nil || f.gens[host] != 0 {
423 t.Fatalf("%s: proceed %v, %v, fences %v", name, ok, err, f.gens)
424 }
425 }
426 // Another operation's fence: defer and touch nothing.
427 f = &fakeFence{gens: map[string]uint64{host: 3}}
428 fence = fenceForRollback(f, func() bool { return true })
429 if ok, err := fence(ctx, host, 9); ok || !errors.Is(err, activator.ErrFenceHeld) || len(f.lifted) != 0 || f.gens[host] != 3 {
430 t.Fatalf("held: proceed %v, %v, lifted %v", ok, err, f.lifted)
431 }
432 // Generation 0 is no fence's: on a host with none, FenceGeneration's 0
433 // would otherwise read as "ours already" and proceed unfenced.
434 f = &fakeFence{gens: map[string]uint64{}}
435 fence = fenceForRollback(f, func() bool { return true })
436 if ok, err := fence(ctx, host, 0); ok || err == nil || f.calls != 0 {
437 t.Fatalf("generation 0: proceed %v, %v, installs %d", ok, err, f.calls)
438 }
439
440 // After an agent restart. The journal kept the generation the rollback
441 // was given before it; the activator's fences died with the process,
442 // and another operation, the layout pass, has fenced the host since.
443 // The resumed rollback must neither take that fence for its own nor
444 // lift it: provision lifts at the journal's generation on every path
445 // that does not proceed, the held-stale-journal lift
446 // (seedrollback.go, rollBackRoute's first check) among them.
447 path := filepath.Join(t.TempDir(), "registry.db")
448 reg, err := registry.Open(path)
449 if err != nil {
450 t.Fatal(err)
451 }
452 // The clock before the restart ran an hour ahead of the clock after it
453 // (a box that booted before NTP synced): the generation the journal
454 // stores is ahead of every wall-clock reading the restarted agent makes.
455 if _, err := reg.NextFenceGeneration(time.Now().Add(time.Hour)); err != nil {
456 t.Fatal(err)
457 }
458 data, deploy := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir())
459 before := wiredSeedHooks(t, &fakeFence{gens: map[string]uint64{}}, reg, data, deploy)
460 stored, err := before.NextFenceGen()
461 if err != nil || stored == 0 {
462 t.Fatalf("generation before the restart: %d, %v", stored, err)
463 }
464 if err := reg.Close(); err != nil {
465 t.Fatal(err)
466 }
467
468 reg, err = registry.Open(path)
469 if err != nil {
470 t.Fatal(err)
471 }
472 defer reg.Close()
473 other := newLayoutPass(reg, nil, t.TempDir(), nil, true).Generation()
474 if other <= stored {
475 t.Fatalf("the layout pass after the restart drew generation %d, not above the %d the journal holds", other, stored)
476 }
477 f = &fakeFence{gens: map[string]uint64{host: other}, err: activator.ErrStopUnproven, quiesced: true}
478 after := wiredSeedHooks(t, f, reg, data, deploy)
479 if ok, err := after.FenceForRollback(ctx, host, stored); ok || !errors.Is(err, activator.ErrFenceHeld) {
480 t.Fatalf("resumed under another operation's fence: proceed %v, %v", ok, err)
481 }
482 after.LiftFence(host, stored)
483 if f.gens[host] != other {
484 t.Fatalf("the resumed rollback lifted another operation's fence: fences %v", f.gens)
485 }
486 if next, err := after.NextFenceGen(); err != nil || next <= other {
487 t.Fatalf("a generation minted after the restart: %d (%v); before %d, the layout pass's %d", next, err, stored, other)
488 }
489 // Once the other operation lets go, the rollback fences at its own
490 // stored generation and proceeds, as on a cold host.
491 f.LiftFence(host, other)
492 if ok, err := after.FenceForRollback(ctx, host, stored); !ok || f.gens[host] != stored {
493 t.Fatalf("after the other fence lifted: proceed %v, %v, fences %v", ok, err, f.gens)
494 }
495}
496
497// The rollback's seven hooks are all or none (provision skips a rollback
498// missing any), so production wiring must set every one; with claim control
499// on, seeds are refused until #250 and none of the swap's or the
500// rollback's hooks is armed.
501func TestWireSeedPlacementArmsTheSwapAndTheRollback(t *testing.T) {
502 ctx := context.Background()
503 data, deploy := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir())
504 f := &fakeFence{gens: map[string]uint64{}}
505 reg := restartTestRegistry(t)
506 nestedErr := errors.New("probe")
507 box := seedBox{Fence: f, Prefix: "instances", Swept: func() bool { return true }, NextFenceGen: fenceGenerations(reg)}
508
509 rollbackHooks := func(h *provision.SeedHooks) map[string]bool {
510 return map[string]bool{
511 "FenceForRollback": h.FenceForRollback != nil, "LiftFence": h.LiftFence != nil,
512 "RemoveLocal": h.RemoveLocal != nil, "DeleteReplica": h.DeleteReplica != nil,
513 "NextFenceGen": h.NextFenceGen != nil, "LocalSeedID": h.LocalSeedID != nil,
514 "RetireKey": h.RetireKey != nil,
515 }
516 }
517
518 t.Run("armed", func(t *testing.T) {
519 resetArmedSeedHooks(t)
520 eb := &activator.ExecBackend{NestedOK: func() error { return nestedErr }}
521 wireSeedPlacement(eb, false, storeOpener(data), storeOpener(deploy), box)
522 h := armedSeedHooks.Load()
523 if h == nil || h.Data != release.Store(data) {
524 t.Fatalf("armed hooks = %+v", h)
525 }
526 for name, set := range rollbackHooks(h) {
527 if !set {
528 t.Errorf("the rollback hook %s is not wired: no rollback runs", name)
529 }
530 }
531 if h.NestedOK == nil || h.NestedOK() != nestedErr {
532 t.Error("NestedOK is not the exec backend's probe")
533 }
534 if h.LegacyOwners == nil {
535 t.Fatal("LegacyOwners is not wired")
536 }
537 if h.EnvBefore == nil {
538 t.Fatal("EnvBefore is not wired: no import swaps")
539 }
540 body := `{"v":1,"names":{"old.example.com":{"account":"bbw","app":"shop"}}}`
541 if _, err := deploy.Put(ctx, seeds.LegacyOwnersKey, []byte(body), ""); err != nil {
542 t.Fatal(err)
543 }
544 if o, err := h.LegacyOwners(ctx); err != nil || o == nil || o.Names["old.example.com"].App != "shop" {
545 t.Errorf("LegacyOwners read %+v, %v from the record store", o, err)
546 }
547 if g, err := h.NextFenceGen(); err != nil || g == 0 {
548 t.Errorf("NextFenceGen = %d, %v", g, err)
549 }
550 f.gens["shop-1.example.com"] = 41
551 h.LiftFence("shop-1.example.com", 41)
552 if f.gens["shop-1.example.com"] != 0 {
553 t.Error("LiftFence did not reach the activator")
554 }
555 })
556 t.Run("claim control on", func(t *testing.T) {
557 resetArmedSeedHooks(t)
558 eb := &activator.ExecBackend{NestedOK: func() error { return nil }}
559 wireSeedPlacement(eb, true, storeOpener(data), storeOpener(deploy), box)
560 h := armedSeedHooks.Load()
561 if h == nil || h.Data != release.Store(data) {
562 t.Fatalf("armed hooks = %+v, want the data bucket", h)
563 }
564 for name, set := range rollbackHooks(h) {
565 if set {
566 t.Errorf("the rollback hook %s is armed with claim control on", name)
567 }
568 }
569 if h.NestedOK != nil || h.LegacyOwners != nil || h.EnvBefore != nil {
570 t.Error("the swap's hooks are armed with claim control on")
571 }
572 })
573 t.Run("no activator fence", func(t *testing.T) {
574 resetArmedSeedHooks(t)
575 wireSeedPlacement(&activator.ExecBackend{}, false, storeOpener(data), storeOpener(deploy), seedBox{Prefix: "instances"})
576 h := armedSeedHooks.Load()
577 if h == nil {
578 t.Fatal("no hooks")
579 }
580 for name, set := range rollbackHooks(h) {
581 if set {
582 t.Errorf("the rollback hook %s is armed without a fence: half the rollback", name)
583 }
584 }
585 })
586}
587
588// The rollback's step 6 deletes the host's replica and nothing beside it:
589// not a host whose name only begins with this one, not another host, not
590// the seeds/ prefix.
591func TestDeleteReplicaDeletesOnlyTheHostsReplica(t *testing.T) {
592 ctx := context.Background()
593 data := release.NewDirStore(t.TempDir())
594 const host = "shop-1.example.com"
595 put := func(k string) {
596 if _, err := data.Put(ctx, k, []byte("x"), ""); err != nil {
597 t.Fatal(err)
598 }
599 }
600 mine := []string{"instances/" + host + "/generations/a/wal/1.ltx", "instances/" + host + "/lock.json"}
601 theirs := []string{"instances/" + host + ".au/lock.json", "instances/other.example.com/lock.json", "seeds/" + host + "/sd_aaaaaaaaaaaaaaaaaaaaaaaaaa/snapshot.db"}
602 for _, k := range append(append([]string{}, mine...), theirs...) {
603 put(k)
604 }
605 del := deleteReplica(data, "instances")
606 for _, bad := range []string{"", "..", "a/b", "."} {
607 if err := del(ctx, bad); err == nil {
608 t.Fatalf("host %q accepted", bad)
609 }
610 }
611 if err := del(ctx, host); err != nil {
612 t.Fatal(err)
613 }
614 for _, k := range mine {
615 if snapshotPresent(t, data, k) {
616 t.Errorf("%s survived", k)
617 }
618 }
619 for _, k := range theirs {
620 if !snapshotPresent(t, data, k) {
621 t.Errorf("%s was deleted", k)
622 }
623 }
624 if err := del(ctx, host); err != nil {
625 t.Fatalf("nothing there: %v", err)
626 }
627}
628