amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package provision
2
3import (
4 "context"
5 "errors"
6 "os"
7 "path/filepath"
8 "reflect"
9 "strings"
10 "testing"
11 "time"
12
13 "github.com/carlosframework/platform/internal/registry"
14 "github.com/carlosframework/platform/internal/release"
15 "github.com/carlosframework/platform/internal/seeds"
16)
17
18func seededRecord(t *testing.T, s release.Store, sqid, kind, state string) Record {
19 t.Helper()
20 rec := acmeRecord(sqid)
21 id, _ := seeds.NewID()
22 rec.Seed = &SeedSource{ID: id, Kind: kind, State: state, SHA256: "a", Bytes: 1, By: "t", At: time.Now()}
23 if err := CreateRecord(ctx, s, rec); err != nil {
24 t.Fatal(err)
25 }
26 return rec
27}
28
29func nestedSeedCfg(data release.Store) ReconcileConfig {
30 return ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data",
31 Layout: func(context.Context, release.Store) (string, error) { return release.LayoutModeNested, nil },
32 Seeds: &SeedHooks{Data: data}}
33}
34
35func TestReconcileMintsAFromSeedNestedWithoutTheSeedCommand(t *testing.T) {
36 s, sqid := setup(t) // the manifest declares Seed ["seed"]
37 rec := seededRecord(t, s, sqid, seeds.KindReplica, seeds.StateReady)
38 reg := regWith(t)
39 if err := Reconcile(ctx, s, reg, nestedSeedCfg(dirStore(t))); err != nil {
40 t.Fatal(err)
41 }
42 got := mustGet(t, reg, rec.Host)
43 if got.Seed != "" || got.DBPath != NestedDBPath("/data", rec.Host) || got.SeedSource != SeedColumn(rec) {
44 t.Fatalf("row = seed %q db %q col %q", got.Seed, got.DBPath, got.SeedSource)
45 }
46 // The record moves on (placed), and the next pass converges the column
47 // and still never stamps the seed command.
48 if _, err := SetRecordSeed(ctx, s, rec.Account, rec.App, rec.Host, SeedPlace(rec.Seed.ID)); err != nil {
49 t.Fatal(err)
50 }
51 if err := Reconcile(ctx, s, reg, nestedSeedCfg(dirStore(t))); err != nil {
52 t.Fatal(err)
53 }
54 got = mustGet(t, reg, rec.Host)
55 row, _ := seeds.ParseRowSeed(got.SeedSource)
56 if got.Seed != "" || row == nil || row.State != seeds.StatePlaced {
57 t.Fatalf("after placed: seed %q col %+v", got.Seed, row)
58 }
59}
60
61func TestReconcileDoesNotMintASeedFlatOrClaimed(t *testing.T) {
62 s, sqid := setup(t)
63 rec := seededRecord(t, s, sqid, seeds.KindReplica, seeds.StateReady)
64 for name, cfg := range map[string]ReconcileConfig{
65 "flat": {Region: "ie", Pools: []string{"tito"}, DataDir: "/data", Seeds: &SeedHooks{Data: dirStore(t)}},
66 // The claim hook admits the write, so only the seed dispatch can
67 // refuse it.
68 "claimed": func() ReconcileConfig {
69 c := nestedSeedCfg(dirStore(t))
70 c.ClaimEnabled = true
71 c.ClaimInstall = func(_ context.Context, _ ClaimRecord, write func(registry.ClaimOwnership) error) error {
72 return write(registry.ClaimOwnership{ClaimIncarnation: "inc", ClaimStamp: 1, ClaimOwner: "owner", ClaimRowID: "row"})
73 }
74 return c
75 }(),
76 "unwired": {Region: "ie", Pools: []string{"tito"}, DataDir: "/data",
77 Layout: func(context.Context, release.Store) (string, error) { return release.LayoutModeNested, nil }},
78 } {
79 reg := regWith(t)
80 if err := Reconcile(ctx, s, reg, cfg); err != nil {
81 t.Fatal(err)
82 }
83 if rt, _ := reg.Get(rec.Host); rt != nil {
84 t.Errorf("%s: minted %+v", name, rt)
85 }
86 }
87}
88
89func TestReconcileSkipsSeedsThatAreNotReady(t *testing.T) {
90 for _, c := range []struct{ kind, state string }{
91 {seeds.KindReplica, seeds.StateReserving},
92 {seeds.KindImport, seeds.StateAwaitingUpload},
93 {seeds.KindImport, seeds.StateUploaded},
94 {seeds.KindImport, seeds.StateRefused},
95 {seeds.KindReplica, seeds.StateRolledBack},
96 } {
97 s, sqid := setup(t)
98 rec := seededRecord(t, s, sqid, c.kind, c.state)
99 reg := regWith(t)
100 if err := Reconcile(ctx, s, reg, nestedSeedCfg(dirStore(t))); err != nil {
101 t.Fatal(err)
102 }
103 if rt, _ := reg.Get(rec.Host); rt != nil {
104 t.Errorf("%s %s: minted %+v", c.kind, c.state, rt)
105 }
106 }
107}
108
109func TestReconcileFinishesAnAbortedSeedWithNoJournal(t *testing.T) {
110 s, sqid := setup(t)
111 data := dirStore(t)
112 rec := seededRecord(t, s, sqid, seeds.KindReplica, seeds.StateAborted)
113 other, _ := seeds.NewID()
114 mine := seeds.SnapshotKey(rec.Host, rec.Seed.ID)
115 theirs := seeds.SnapshotKey(rec.Host, other)
116 for _, k := range []string{mine, theirs} {
117 if _, err := data.Put(ctx, k, []byte("db"), ""); err != nil {
118 t.Fatal(err)
119 }
120 }
121 reg := regWith(t)
122 if err := Reconcile(ctx, s, reg, nestedSeedCfg(data)); err != nil {
123 t.Fatal(err)
124 }
125 if _, _, err := data.Get(ctx, mine); !errors.Is(err, os.ErrNotExist) {
126 t.Fatalf("this seed's snapshot: %v, want it deleted", err)
127 }
128 if _, _, err := data.Get(ctx, theirs); err != nil {
129 t.Fatalf("another seed's snapshot was touched: %v", err)
130 }
131 got, err := GetRecord(ctx, s, rec.Account, rec.App, rec.Host)
132 if err != nil || got.Seed.State != seeds.StateRolledBack {
133 t.Fatalf("record = %+v, %v; want rolled-back", got.Seed, err)
134 }
135 // Idempotent: a second pass over a rolled-back record does nothing.
136 if err := Reconcile(ctx, s, reg, nestedSeedCfg(data)); err != nil {
137 t.Fatal(err)
138 }
139}
140
141// A record that would mint a flat row even on a nested box (it does not
142// hibernate, so the activator never manages it) would start without its
143// seed and without the seed command: it is not minted.
144func TestReconcileDoesNotMintASeedThatWouldNotHibernate(t *testing.T) {
145 s, sqid := setup(t)
146 rec := acmeRecord(sqid)
147 off := false
148 rec.Hibernate = &off
149 id, _ := seeds.NewID()
150 rec.Seed = &SeedSource{ID: id, Kind: seeds.KindReplica, State: seeds.StateReady, SHA256: "a", Bytes: 1, By: "t", At: time.Now()}
151 if err := CreateRecord(ctx, s, rec); err != nil {
152 t.Fatal(err)
153 }
154 reg := regWith(t)
155 if err := Reconcile(ctx, s, reg, nestedSeedCfg(dirStore(t))); err != nil {
156 t.Fatal(err)
157 }
158 if rt, _ := reg.Get(rec.Host); rt != nil {
159 t.Errorf("minted %+v", rt)
160 }
161}
162
163// A --from seed aborted after this box minted its row (a lost publish
164// answer): the row is removed first, the way a retired record's row is, and
165// the seed stays aborted with its snapshot. The next pass, which finds no
166// row, rolls it back.
167func TestReconcileRemovesAnAbortedFromSeedsRowBeforeRollingBack(t *testing.T) {
168 s, sqid := setup(t)
169 data := dirStore(t)
170 rec := seededRecord(t, s, sqid, seeds.KindReplica, seeds.StateReady)
171 snap := seeds.SnapshotKey(rec.Host, rec.Seed.ID)
172 if _, err := data.Put(ctx, snap, []byte("db"), ""); err != nil {
173 t.Fatal(err)
174 }
175 reg := regWith(t)
176 if err := Reconcile(ctx, s, reg, nestedSeedCfg(data)); err != nil {
177 t.Fatal(err)
178 }
179 mustGet(t, reg, rec.Host)
180 if _, err := SetRecordSeed(ctx, s, rec.Account, rec.App, rec.Host, SeedAbort(rec.Seed.ID)); err != nil {
181 t.Fatal(err)
182 }
183
184 if err := Reconcile(ctx, s, reg, nestedSeedCfg(data)); err != nil {
185 t.Fatal(err)
186 }
187 if rt, _ := reg.Get(rec.Host); rt != nil {
188 t.Fatalf("the aborted seed's row is still up: %+v", rt)
189 }
190 got, err := GetRecord(ctx, s, rec.Account, rec.App, rec.Host)
191 if err != nil || got.Seed.State != seeds.StateAborted {
192 t.Fatalf("record after the row's removal = %+v, %v; want still aborted", got.Seed, err)
193 }
194 if _, _, err := data.Get(ctx, snap); err != nil {
195 t.Fatalf("the snapshot went with the row: %v", err)
196 }
197
198 if err := Reconcile(ctx, s, reg, nestedSeedCfg(data)); err != nil {
199 t.Fatal(err)
200 }
201 got, err = GetRecord(ctx, s, rec.Account, rec.App, rec.Host)
202 if err != nil || got.Seed.State != seeds.StateRolledBack {
203 t.Fatalf("record on the pass with no row = %+v, %v; want rolled-back", got.Seed, err)
204 }
205 if _, _, err := data.Get(ctx, snap); !errors.Is(err, os.ErrNotExist) {
206 t.Fatalf("snapshot after the rollback: %v, want it deleted", err)
207 }
208 if rt, _ := reg.Get(rec.Host); rt != nil {
209 t.Fatalf("a rolled-back seed was minted again: %+v", rt)
210 }
211}
212
213// A row this pass has no authority over (not Provisioned: hand-added) is
214// left alone, and the seed stays aborted while it stands.
215func TestReconcileLeavesAnAbortedSeedsUnownedRow(t *testing.T) {
216 s, sqid := setup(t)
217 rec := seededRecord(t, s, sqid, seeds.KindReplica, seeds.StateAborted)
218 reg := regWith(t, registry.Route{Host: rec.Host, Socket: "/run/carlos/hand.sock", DBPath: "/data/hand.db",
219 App: rec.App, Account: rec.Account, Region: "ie", Pool: "tito"})
220 if err := Reconcile(ctx, s, reg, nestedSeedCfg(dirStore(t))); err != nil {
221 t.Fatal(err)
222 }
223 mustGet(t, reg, rec.Host)
224 got, err := GetRecord(ctx, s, rec.Account, rec.App, rec.Host)
225 if err != nil || got.Seed.State != seeds.StateAborted {
226 t.Fatalf("record = %+v, %v; want still aborted while the row stands", got.Seed, err)
227 }
228}
229
230// An import's row is the instance it replaces: an aborted import never
231// removes it, and rolls back as before (its swap's rollback is Plan 7's).
232func TestReconcileNeverRemovesAnAbortedImportsRow(t *testing.T) {
233 s, sqid := setup(t)
234 rec := seededRecord(t, s, sqid, seeds.KindImport, seeds.StateAborted)
235 reg := regWith(t, registry.Route{Host: rec.Host, Socket: DefaultSocket(rec.Host), DBPath: DefaultDBPath("/data", rec.Host),
236 App: rec.App, Account: rec.Account, Region: "ie", Pool: "tito", Provisioned: true})
237 if err := Reconcile(ctx, s, reg, nestedSeedCfg(dirStore(t))); err != nil {
238 t.Fatal(err)
239 }
240 mustGet(t, reg, rec.Host)
241 got, err := GetRecord(ctx, s, rec.Account, rec.App, rec.Host)
242 if err != nil || got.Seed.State != seeds.StateRolledBack {
243 t.Fatalf("record = %+v, %v; want rolled-back", got.Seed, err)
244 }
245}
246
247// diskBox is the rollback's seven box hooks over a real data directory: the
248// marker and the database are files, read and removed as the activator's
249// ReadSeedMarker and PurgeSeedLocal do (their own tests are the activator's
250// seedpurge_linux_test.go). It records the steps in order.
251type diskBox struct {
252 t *testing.T
253 s release.Store
254 rec Record
255 steps []string
256 proceed bool
257 gen uint64
258 fenced map[string]uint64
259 markErr error
260}
261
262func newDiskBox(t *testing.T, s release.Store, rec Record) *diskBox {
263 return &diskBox{t: t, s: s, rec: rec, proceed: true, fenced: map[string]uint64{}}
264}
265
266func (b *diskBox) hooks(h *SeedHooks) {
267 h.NextFenceGen = func() (uint64, error) { b.gen++; return b.gen, nil }
268 h.FenceForRollback = func(_ context.Context, host string, gen uint64) (bool, error) {
269 b.steps = append(b.steps, "fence")
270 if !b.proceed {
271 return false, nil
272 }
273 b.fenced[host] = gen
274 return true, nil
275 }
276 h.LiftFence = func(host string, gen uint64) {
277 b.steps = append(b.steps, "lift")
278 if b.fenced[host] == gen {
279 delete(b.fenced, host)
280 }
281 }
282 h.LocalSeedID = func(db string) (string, error) {
283 b.steps = append(b.steps, "read-marker")
284 if b.markErr != nil {
285 return "", b.markErr
286 }
287 m, err := os.ReadFile(db + ".seed")
288 if errors.Is(err, os.ErrNotExist) {
289 return "", nil
290 }
291 return string(m), err
292 }
293 h.RemoveLocal = func(rt registry.Route) error {
294 b.steps = append(b.steps, "remove-local")
295 if _, ok := b.fenced[rt.Host]; !ok {
296 b.t.Errorf("removed the local data of %s while it was not fenced and stopped", rt.Host)
297 }
298 // Before the rollback: the record is still aborted.
299 if got, err := GetRecord(ctx, b.s, b.rec.Account, b.rec.App, b.rec.Host); err != nil || got.Seed.State != seeds.StateAborted {
300 b.t.Errorf("removed the local data with the record at %+v, %v; want aborted", got.Seed, err)
301 }
302 for _, p := range []string{rt.DBPath, rt.DBPath + "-wal", rt.DBPath + ".seed"} {
303 if err := os.Remove(p); err != nil && !errors.Is(err, os.ErrNotExist) {
304 return err
305 }
306 }
307 return nil
308 }
309 h.DeleteReplica = func(context.Context, string) error {
310 b.t.Error("the no-journal path deleted a replica")
311 return nil
312 }
313 h.RetireKey = func(string, string, string) error {
314 b.t.Error("the no-journal path retired an instance key: only an import's rollback does")
315 return nil
316 }
317}
318
319// placeLocal writes a database and its <db>.seed marker naming id at db,
320// as a wake that placed the seed leaves them.
321func placeLocal(t *testing.T, db, id string) {
322 t.Helper()
323 if err := os.MkdirAll(filepath.Dir(db), 0o700); err != nil {
324 t.Fatal(err)
325 }
326 for p, body := range map[string]string{db: "the seed's data", db + "-wal": "wal", db + ".seed": id} {
327 if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
328 t.Fatal(err)
329 }
330 }
331}
332
333func diskSeedCfg(t *testing.T, data release.Store, box *diskBox) ReconcileConfig {
334 cfg := nestedSeedCfg(data)
335 cfg.DataDir = t.TempDir()
336 box.hooks(cfg.Seeds)
337 return cfg
338}
339
340func requireSeedState(t *testing.T, s release.Store, rec Record, state string) {
341 t.Helper()
342 got, err := GetRecord(ctx, s, rec.Account, rec.App, rec.Host)
343 if err != nil || got.Seed == nil || got.Seed.State != state {
344 t.Fatalf("record = %+v, %v; want %s", got.Seed, err, state)
345 }
346}
347
348// Final review F2: a --from seed whose publish landed but whose answer was
349// lost. A box minted its row and a wake placed its database; then the abort
350// won. The row goes first; the pass that finds no row removes the database
351// (its marker names this seed) under a fence, before the record is rolled
352// back. That pass runs as after an agent restart: nothing in memory, only
353// the marker on disk. A plain instance created at the host afterwards is
354// minted at the same path and finds no file there.
355func TestReconcileAbortedFromSeedRemovesItsLocalDatabase(t *testing.T) {
356 s, sqid := setup(t)
357 data := dirStore(t)
358 rec := seededRecord(t, s, sqid, seeds.KindReplica, seeds.StateReady)
359 box := newDiskBox(t, s, rec)
360 cfg := diskSeedCfg(t, data, box)
361 reg := regWith(t)
362 if err := Reconcile(ctx, s, reg, cfg); err != nil {
363 t.Fatal(err)
364 }
365 db := mustGet(t, reg, rec.Host).DBPath
366 if db != NestedDBPath(cfg.DataDir, rec.Host) {
367 t.Fatalf("minted at %s", db)
368 }
369 placeLocal(t, db, rec.Seed.ID)
370 if _, err := SetRecordSeed(ctx, s, rec.Account, rec.App, rec.Host, SeedAbort(rec.Seed.ID)); err != nil {
371 t.Fatal(err)
372 }
373 if err := Reconcile(ctx, s, reg, cfg); err != nil {
374 t.Fatal(err)
375 }
376 if rt, _ := reg.Get(rec.Host); rt != nil {
377 t.Fatalf("the aborted seed's row is still up: %+v", rt)
378 }
379 requireSeedState(t, s, rec, seeds.StateAborted)
380 if _, err := os.Stat(db); err != nil {
381 t.Fatalf("the database went with the row, before the no-row pass: %v", err)
382 }
383
384 restarted := newDiskBox(t, s, rec) // the agent restarted: no fences, no memory
385 restarted.hooks(cfg.Seeds)
386 if err := Reconcile(ctx, s, reg, cfg); err != nil {
387 t.Fatal(err)
388 }
389 requireSeedState(t, s, rec, seeds.StateRolledBack)
390 if want := []string{"read-marker", "fence", "read-marker", "remove-local", "lift"}; !reflect.DeepEqual(restarted.steps, want) {
391 t.Fatalf("steps = %q, want %q", restarted.steps, want)
392 }
393 if len(restarted.fenced) != 0 {
394 t.Fatalf("a fence was left: %v", restarted.fenced)
395 }
396 for _, p := range []string{db, db + "-wal", db + ".seed"} {
397 if _, err := os.Lstat(p); !errors.Is(err, os.ErrNotExist) {
398 t.Fatalf("%s survived the rollback: %v", p, err)
399 }
400 }
401
402 // The console's finish deletes the record; a plain instance follows.
403 if err := DeleteRecord(ctx, s, rec.Account, rec.App, rec.Host); err != nil {
404 t.Fatal(err)
405 }
406 plain := acmeRecord(sqid)
407 if err := CreateRecord(ctx, s, plain); err != nil {
408 t.Fatal(err)
409 }
410 if err := Reconcile(ctx, s, reg, cfg); err != nil {
411 t.Fatal(err)
412 }
413 next := mustGet(t, reg, plain.Host)
414 if next.SeedSource != "" {
415 t.Fatalf("the plain instance's row = %+v", next)
416 }
417 if _, err := os.Lstat(next.DBPath); !errors.Is(err, os.ErrNotExist) {
418 t.Fatalf("the plain instance at %s finds a database: %v", next.DBPath, err)
419 }
420}
421
422// The marker decides, as for an import whose row is gone (Plan 7): a
423// marker naming another seed, or none, rolls back with the file left; a
424// marker that cannot be read, or a fence that does not proceed, keeps the
425// seed aborted with everything in place.
426func TestReconcileAbortedFromSeedDecidesItsLocalDataFromTheMarker(t *testing.T) {
427 other, _ := seeds.NewID()
428 for _, c := range []struct {
429 name string
430 marker string // "" writes no marker; "this" names the seed
431 markErr error
432 noFence bool
433 state string
434 removed bool
435 steps []string
436 logged string
437 }{
438 {name: "this seed", marker: "this", state: seeds.StateRolledBack, removed: true,
439 steps: []string{"read-marker", "fence", "read-marker", "remove-local", "lift"}, logged: "its local data is removed"},
440 {name: "another seed", marker: other, state: seeds.StateRolledBack,
441 steps: []string{"read-marker"}, logged: "its seed marker names \"" + other + "\", not this seed"},
442 {name: "no marker", state: seeds.StateRolledBack,
443 steps: []string{"read-marker"}, logged: "its seed marker names \"\", not this seed"},
444 {name: "unreadable marker", marker: "this", markErr: errors.New("permission denied"), state: seeds.StateAborted,
445 steps: []string{"read-marker"}, logged: "the seed marker beside"},
446 {name: "not fenced", marker: "this", noFence: true, state: seeds.StateAborted,
447 steps: []string{"read-marker", "fence", "lift"}, logged: "is not fenced and stopped yet"},
448 } {
449 t.Run(c.name, func(t *testing.T) {
450 s, sqid := setup(t)
451 data := dirStore(t)
452 rec := seededRecord(t, s, sqid, seeds.KindReplica, seeds.StateAborted)
453 snap := seeds.SnapshotKey(rec.Host, rec.Seed.ID)
454 if _, err := data.Put(ctx, snap, []byte("db"), ""); err != nil {
455 t.Fatal(err)
456 }
457 box := newDiskBox(t, s, rec)
458 box.markErr, box.proceed = c.markErr, !c.noFence
459 cfg := diskSeedCfg(t, data, box)
460 db := NestedDBPath(cfg.DataDir, rec.Host)
461 switch c.marker {
462 case "":
463 placeLocal(t, db, "")
464 if err := os.Remove(db + ".seed"); err != nil {
465 t.Fatal(err)
466 }
467 case "this":
468 placeLocal(t, db, rec.Seed.ID)
469 default:
470 placeLocal(t, db, c.marker)
471 }
472 logged := captureLog(t)
473 if err := Reconcile(ctx, s, regWith(t), cfg); err != nil {
474 t.Fatal(err)
475 }
476 requireSeedState(t, s, rec, c.state)
477 if !reflect.DeepEqual(box.steps, c.steps) {
478 t.Fatalf("steps = %q, want %q", box.steps, c.steps)
479 }
480 if !strings.Contains(logged.String(), c.logged) {
481 t.Fatalf("not logged %q:\n%s", c.logged, logged)
482 }
483 if _, err := os.Lstat(db); errors.Is(err, os.ErrNotExist) != c.removed {
484 t.Fatalf("database at %s: %v; want removed %v", db, err, c.removed)
485 }
486 _, _, err := data.Get(ctx, snap)
487 if gone := errors.Is(err, os.ErrNotExist); gone != (c.state == seeds.StateRolledBack) {
488 t.Fatalf("snapshot: %v; want it deleted only with the rollback", err)
489 }
490 })
491 }
492}
493