amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1//go:build linux
2
3package activator
4
5import (
6 "bytes"
7 "context"
8 "crypto/sha256"
9 "database/sql"
10 "encoding/hex"
11 "errors"
12 "io"
13 "os"
14 "path/filepath"
15 "strings"
16 "testing"
17
18 "github.com/carlosframework/platform/internal/instancectl"
19 "github.com/carlosframework/platform/internal/provision"
20 "github.com/carlosframework/platform/internal/registry"
21 "github.com/carlosframework/platform/internal/seeds"
22)
23
24type fakeSeedObjects struct {
25 body map[string][]byte // id -> snapshot
26 err error
27 opens int
28 read int64 // bytes the fallback pulled from the object
29 // midway, when set, fails the read after the first 100 bytes.
30 midway error
31}
32
33func (f *fakeSeedObjects) Open(_ context.Context, host, id string) (io.ReadCloser, error) {
34 f.opens++
35 if f.err != nil {
36 return nil, f.err
37 }
38 b, ok := f.body[id]
39 if !ok {
40 return nil, os.ErrNotExist
41 }
42 var r io.Reader = bytes.NewReader(b)
43 if f.midway != nil {
44 r = io.MultiReader(bytes.NewReader(b[:100]), &failingReader{f.midway})
45 }
46 return io.NopCloser(&countingReader{r: r, n: &f.read}), nil
47}
48
49type failingReader struct{ err error }
50
51func (f *failingReader) Read([]byte) (int, error) { return 0, f.err }
52
53type countingReader struct {
54 r io.Reader
55 n *int64
56}
57
58func (c *countingReader) Read(p []byte) (int, error) {
59 n, err := c.r.Read(p)
60 *c.n += int64(n)
61 return n, err
62}
63
64// seededRoute is nestedBackend's row with a seed column for a snapshot
65// made by writeMinimalSQLite.
66func seededRoute(t *testing.T, b *ExecBackend, rt registry.Route, state string) (registry.Route, *fakeSeedObjects, string) {
67 t.Helper()
68 id, _ := seeds.NewID()
69 snap := filepath.Join(t.TempDir(), "snap.db")
70 if err := writeMinimalSQLite(snap); err != nil {
71 t.Fatal(err)
72 }
73 body, _ := os.ReadFile(snap)
74 sum := sha256.Sum256(body)
75 rt.SeedSource = seeds.RowSeed{ID: id, Kind: seeds.KindReplica, SHA256: hex.EncodeToString(sum[:]), Bytes: int64(len(body)), State: state}.Encode()
76 objs := &fakeSeedObjects{body: map[string][]byte{id: body}}
77 b.Seeds = objs
78 return rt, objs, id
79}
80
81func noOwnReplica(t *testing.T) {
82 t.Helper()
83 old := hasOwnReplica
84 hasOwnReplica = func(context.Context, string, string, string) (bool, error) { return false, nil }
85 t.Cleanup(func() { hasOwnReplica = old })
86}
87
88func TestSeededRestorePlacesTheSnapshot(t *testing.T) {
89 noOwnReplica(t)
90 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
91 rt, objs, id := seededRoute(t, b, rt, seeds.StateReady)
92 fresh, err := b.Restore(context.Background(), rt)
93 if err != nil || fresh {
94 t.Fatalf("Restore = fresh %v, %v; want placed, never fresh", fresh, err)
95 }
96 if objs.opens != 1 {
97 t.Fatalf("snapshot opened %d times", objs.opens)
98 }
99 marker, err := os.ReadFile(rt.DBPath + SeedMarkerSuffix)
100 if err != nil || strings.TrimSpace(string(marker)) != id {
101 t.Fatalf("marker = %q, %v; want the seed id", marker, err)
102 }
103 if err := checkSeedDB(context.Background(), rt.DBPath); err != nil {
104 t.Fatalf("the placed database does not check: %v", err)
105 }
106 // Nothing of the fetch is left beside the database.
107 ents, _ := os.ReadDir(filepath.Dir(rt.DBPath))
108 for _, e := range ents {
109 if strings.HasPrefix(e.Name(), ".seed-") {
110 t.Errorf("%s left behind", e.Name())
111 }
112 }
113 // A second wake (a crash after the rename, before the first upload)
114 // keeps the placed file and does not fetch again.
115 if fresh, err := b.Restore(context.Background(), rt); err != nil || fresh || objs.opens != 1 {
116 t.Fatalf("resume: fresh %v err %v opens %d", fresh, err, objs.opens)
117 }
118}
119
120func TestSeededRestorePurgesAnotherSeedsDatabase(t *testing.T) {
121 noOwnReplica(t)
122 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
123 rt, _, id := seededRoute(t, b, rt, seeds.StateSwapped)
124 os.MkdirAll(filepath.Dir(rt.DBPath), 0o700)
125 os.WriteFile(rt.DBPath, []byte("an aborted import's data"), 0o600)
126 os.WriteFile(rt.DBPath+"-wal", []byte("w"), 0o600)
127 os.WriteFile(rt.DBPath+"-shm", []byte("s"), 0o600)
128 os.WriteFile(rt.DBPath+SeedMarkerSuffix, []byte("sd_bbbbbbbbbbbbbbbbbbbbbbbbbb\n"), 0o600)
129 meta := filepath.Join(filepath.Dir(rt.DBPath), ".activator-"+filepath.Base(rt.DBPath)+"-litestream")
130 os.MkdirAll(meta, 0o700)
131 if _, err := b.Restore(context.Background(), rt); err != nil {
132 t.Fatal(err)
133 }
134 for _, p := range []string{rt.DBPath + "-wal", rt.DBPath + "-shm", meta} {
135 if _, err := os.Stat(p); !os.IsNotExist(err) {
136 t.Errorf("%s survived the purge", p)
137 }
138 }
139 if m, _ := os.ReadFile(rt.DBPath + SeedMarkerSuffix); strings.TrimSpace(string(m)) != id {
140 t.Fatalf("marker = %q, want %s", m, id)
141 }
142 if b, _ := os.ReadFile(rt.DBPath); bytes.Equal(b, []byte("an aborted import's data")) {
143 t.Fatal("the other seed's database stood in for this one")
144 }
145}
146
147func TestSeededRestorePurgesAnUnmarkedDatabase(t *testing.T) {
148 noOwnReplica(t)
149 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
150 rt, objs, _ := seededRoute(t, b, rt, seeds.StateReady)
151 os.MkdirAll(filepath.Dir(rt.DBPath), 0o700)
152 os.WriteFile(rt.DBPath, []byte("an unmarked database"), 0o600)
153 if fresh, err := b.Restore(context.Background(), rt); err != nil || fresh || objs.opens != 1 {
154 t.Fatalf("fresh %v err %v opens %d; want the unmarked database replaced by the snapshot", fresh, err, objs.opens)
155 }
156 if got, _ := os.ReadFile(rt.DBPath); !bytes.Equal(got, objs.body[mustSeedID(t, rt)]) {
157 t.Fatal("the unmarked database stood in for the seed")
158 }
159}
160
161func TestSeededRestoreRemovesASymlinkedDatabaseNotItsTarget(t *testing.T) {
162 noOwnReplica(t)
163 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
164 rt, _, id := seededRoute(t, b, rt, seeds.StateReady)
165 os.MkdirAll(filepath.Dir(rt.DBPath), 0o700)
166 outside := filepath.Join(t.TempDir(), "outside.db")
167 os.WriteFile(outside, []byte("not the instance's"), 0o600)
168 outsideMarker := filepath.Join(t.TempDir(), "marker")
169 os.WriteFile(outsideMarker, []byte(id+"\n"), 0o600)
170 if err := os.Symlink(outside, rt.DBPath); err != nil {
171 t.Fatal(err)
172 }
173 if err := os.Symlink(outsideMarker, rt.DBPath+SeedMarkerSuffix); err != nil {
174 t.Fatal(err)
175 }
176 if _, err := b.Restore(context.Background(), rt); err != nil {
177 t.Fatal(err)
178 }
179 if got, _ := os.ReadFile(outside); string(got) != "not the instance's" {
180 t.Fatalf("the link's target was touched: %q", got)
181 }
182 if fi, err := os.Lstat(rt.DBPath); err != nil || !fi.Mode().IsRegular() {
183 t.Fatalf("the database is not a placed file: %v %v", fi, err)
184 }
185 if err := checkSeedDB(context.Background(), rt.DBPath); err != nil {
186 t.Fatalf("the placed database does not check: %v", err)
187 }
188}
189
190func mustSeedID(t *testing.T, rt registry.Route) string {
191 t.Helper()
192 src, err := seeds.ParseRowSeed(rt.SeedSource)
193 if err != nil || src == nil {
194 t.Fatalf("seed column: %v", err)
195 }
196 return src.ID
197}
198
199func TestSeededRestoreRefusesABadSnapshot(t *testing.T) {
200 noOwnReplica(t)
201 for _, c := range []struct {
202 name string
203 mangle func(*fakeSeedObjects, string)
204 want string
205 }{
206 {"short", func(f *fakeSeedObjects, id string) { f.body[id] = f.body[id][:100] }, "length or sha256"},
207 {"long", func(f *fakeSeedObjects, id string) { f.body[id] = append(f.body[id], 'x') }, "length or sha256"},
208 {"wrong bytes", func(f *fakeSeedObjects, id string) { f.body[id] = bytes.Repeat([]byte{'x'}, len(f.body[id])) }, "length or sha256"},
209 {"unreadable", func(f *fakeSeedObjects, id string) { f.err = errors.New("bucket down") }, "could not be read"},
210 } {
211 t.Run(c.name, func(t *testing.T) {
212 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
213 rt, objs, id := seededRoute(t, b, rt, seeds.StateReady)
214 c.mangle(objs, id)
215 fresh, err := b.Restore(context.Background(), rt)
216 var spe *SeedPlacementError
217 if fresh || !errors.As(err, &spe) || !strings.Contains(spe.Sentence, c.want) {
218 t.Fatalf("Restore = fresh %v, %v; want a %q refusal", fresh, err, c.want)
219 }
220 if _, err := os.Stat(rt.DBPath); !os.IsNotExist(err) {
221 t.Fatal("a refused snapshot left a database at the row's path")
222 }
223 })
224 }
225}
226
227// The object is read through the recorded size and one byte more, never to
228// its end: an oversized object costs the box nothing beyond the record.
229func TestSeededRestoreReadsNoMoreThanTheRecordedSize(t *testing.T) {
230 noOwnReplica(t)
231 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
232 rt, objs, id := seededRoute(t, b, rt, seeds.StateReady)
233 want := int64(len(objs.body[id]))
234 objs.body[id] = append(objs.body[id], make([]byte, 4<<20)...)
235 var spe *SeedPlacementError
236 if _, err := b.Restore(context.Background(), rt); !errors.As(err, &spe) || !strings.Contains(spe.Sentence, "length or sha256") {
237 t.Fatalf("Restore = %v; want the length refusal", err)
238 }
239 if objs.read > want+1 {
240 t.Fatalf("read %d bytes of the object; the record says %d", objs.read, want)
241 }
242}
243
244func TestSeededRestoreRefusesANonDatabase(t *testing.T) {
245 noOwnReplica(t)
246 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
247 id, _ := seeds.NewID()
248 body := []byte("not sqlite at all, but the right length and hash")
249 sum := sha256.Sum256(body)
250 rt.SeedSource = seeds.RowSeed{ID: id, Kind: seeds.KindImport, SHA256: hex.EncodeToString(sum[:]), Bytes: int64(len(body)), State: seeds.StateSwapped}.Encode()
251 b.Seeds = &fakeSeedObjects{body: map[string][]byte{id: body}}
252 _, err := b.Restore(context.Background(), rt)
253 var spe *SeedPlacementError
254 if !errors.As(err, &spe) || !strings.Contains(spe.Sentence, "integrity check") {
255 t.Fatalf("Restore = %v; want the integrity refusal", err)
256 }
257}
258
259func TestSeededRestoreRefusesClaimAndFlat(t *testing.T) {
260 noOwnReplica(t)
261 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
262 rt, _, _ = seededRoute(t, b, rt, seeds.StateReady)
263 b.ClaimArmed = true
264 var spe *SeedPlacementError
265 if _, err := b.Restore(context.Background(), rt); !errors.As(err, &spe) || !strings.Contains(spe.Sentence, "claim control") {
266 t.Fatalf("claim-armed: %v", err)
267 }
268 b.ClaimArmed = false
269 flat := testExecBackend(t)
270 flat.Seeds = b.Seeds
271 frt := registry.Route{Host: rt.Host, Kind: "instance", Hibernate: true, SeedSource: rt.SeedSource,
272 Socket: filepath.Join(t.TempDir(), "x.sock")} // DBPath empty: the flat default
273 if _, err := flat.Restore(context.Background(), frt); !errors.As(err, &spe) || !strings.Contains(spe.Sentence, "nested") {
274 t.Fatalf("flat: %v", err)
275 }
276}
277
278func TestSeededRestoreFailsWhenTheReplicaCannotBeListed(t *testing.T) {
279 old := hasOwnReplica
280 hasOwnReplica = func(context.Context, string, string, string) (bool, error) {
281 return false, errors.New("listing failed")
282 }
283 defer func() { hasOwnReplica = old }()
284 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
285 rt, objs, _ := seededRoute(t, b, rt, seeds.StateReady)
286 if fresh, err := b.Restore(context.Background(), rt); err == nil || fresh || objs.opens != 0 {
287 t.Fatalf("Restore = fresh %v, %v, opens %d; want a failed wake and no fetch", fresh, err, objs.opens)
288 }
289}
290
291func TestSeededRestoreUsesItsOwnReplicaOnceItHasOne(t *testing.T) {
292 old := hasOwnReplica
293 hasOwnReplica = func(context.Context, string, string, string) (bool, error) { return true, nil }
294 defer func() { hasOwnReplica = old }()
295 oldRestore := restoreReplica
296 restored := 0
297 restoreReplica = func(_ context.Context, dbPath, _, _, _ string) (bool, error) {
298 restored++
299 return true, writeMinimalSQLite(dbPath)
300 }
301 defer func() { restoreReplica = oldRestore }()
302 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
303 rt, objs, _ := seededRoute(t, b, rt, seeds.StateReady)
304 if fresh, err := b.Restore(context.Background(), rt); err != nil || fresh || restored != 1 || objs.opens != 0 {
305 t.Fatalf("fresh %v err %v restored %d opens %d; want the own replica, not the seed", fresh, err, restored, objs.opens)
306 }
307}
308
309// A listing that saw a replica and a restore that then found none must not
310// come back fresh: fresh would boot the instance empty and run the
311// manifest's seed command over a seeded row.
312func TestSeededRestoreNeverReportsFreshWhenItsReplicaVanishes(t *testing.T) {
313 old := hasOwnReplica
314 hasOwnReplica = func(context.Context, string, string, string) (bool, error) { return true, nil }
315 defer func() { hasOwnReplica = old }()
316 oldRestore := restoreReplica
317 restoreReplica = func(context.Context, string, string, string, string) (bool, error) { return false, nil }
318 defer func() { restoreReplica = oldRestore }()
319 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
320 rt, objs, _ := seededRoute(t, b, rt, seeds.StateReady)
321 if fresh, err := b.Restore(context.Background(), rt); err == nil || fresh || objs.opens != 0 {
322 t.Fatalf("Restore = fresh %v, %v, opens %d; want a failed wake", fresh, err, objs.opens)
323 }
324}
325
326func TestMemberSafeWakeReasonPassesTheSeedSentence(t *testing.T) {
327 err := fmtWrap(seedRefusedFlat("shop.example.com"))
328 if got := memberSafeWakeReason(err); got != "wake failed: shop.example.com could not start from its seed: this box does not run instances nested." {
329 t.Fatalf("reason = %q", got)
330 }
331}
332
333func fmtWrap(err error) error { return errors.Join(errors.New("restore shop.example.com"), err) }
334
335// countOwnReplicaLists stubs hasOwnReplica and counts its calls.
336func countOwnReplicaLists(t *testing.T, has bool) *int {
337 t.Helper()
338 n := 0
339 old := hasOwnReplica
340 hasOwnReplica = func(context.Context, string, string, string) (bool, error) { n++; return has, nil }
341 t.Cleanup(func() { hasOwnReplica = old })
342 return &n
343}
344
345// A placed seed stays on the row for life. After a move, the destination's
346// replica is staged under moves/<host>/, which only the plain path's staging
347// fallback promotes: the wake must take it, never the deleted snapshot.
348func TestSeededRestoreOfAPlacedSeedPromotesAStagedReplica(t *testing.T) {
349 lists := countOwnReplicaLists(t, false)
350 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
351 rt, objs, _ := seededRoute(t, b, rt, seeds.StatePlaced)
352 b.ClaimArmed = true // placed: no claim refusal either
353 f := newFakeMirrorStore()
354 b.Mirror = f
355 f.put(b.Cfg.Bucket, MovesPrefix(rt.Host)+"generations/a/snapshot", "e1")
356 oldRestore := restoreReplica
357 var restoredFrom []string
358 restoreReplica = func(_ context.Context, dbPath, bucket, prefix, _ string) (bool, error) {
359 if got, _ := f.List(context.Background(), bucket, prefix); len(got) == 0 {
360 return false, nil
361 }
362 restoredFrom = append(restoredFrom, prefix)
363 return true, writeMinimalSQLite(dbPath)
364 }
365 defer func() { restoreReplica = oldRestore }()
366 os.MkdirAll(filepath.Dir(rt.DBPath), 0o700) // lease.Restore's own MkdirAll
367 fresh, err := b.Restore(context.Background(), rt)
368 if err != nil || fresh || objs.opens != 0 || *lists != 0 || len(restoredFrom) != 1 {
369 t.Fatalf("fresh %v err %v opens %d lists %d restored %v; want the promoted staged replica", fresh, err, objs.opens, *lists, restoredFrom)
370 }
371}
372
373func TestSeededRestoreOfAPlacedSeedKeepsTheWarmCopyWithoutListing(t *testing.T) {
374 lists := countOwnReplicaLists(t, true)
375 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
376 rt, objs, _ := seededRoute(t, b, rt, seeds.StatePlaced)
377 b.ClaimArmed = true
378 oldRestore := restoreReplica
379 restoreReplica = func(context.Context, string, string, string, string) (bool, error) {
380 t.Fatal("a warm copy was restored over")
381 return false, nil
382 }
383 defer func() { restoreReplica = oldRestore }()
384 os.MkdirAll(filepath.Dir(rt.DBPath), 0o700)
385 if err := writeMinimalSQLite(rt.DBPath); err != nil {
386 t.Fatal(err)
387 }
388 if fresh, err := b.Restore(context.Background(), rt); err != nil || fresh || objs.opens != 0 || *lists != 0 {
389 t.Fatalf("fresh %v err %v opens %d lists %d; want the warm copy, no listing, no fetch", fresh, err, objs.opens, *lists)
390 }
391}
392
393func TestSeededRestoreOfAPlacedSeedWithNothingToRestoreFails(t *testing.T) {
394 countOwnReplicaLists(t, false)
395 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
396 rt, objs, _ := seededRoute(t, b, rt, seeds.StatePlaced)
397 oldRestore := restoreReplica
398 restoreReplica = func(context.Context, string, string, string, string) (bool, error) { return false, nil }
399 defer func() { restoreReplica = oldRestore }()
400 if fresh, err := b.Restore(context.Background(), rt); err == nil || fresh || objs.opens != 0 {
401 t.Fatalf("fresh %v err %v opens %d; want a failed wake, never fresh", fresh, err, objs.opens)
402 }
403}
404
405// committedImport is a row whose import is committed: placed first, so
406// the host has a replica of its own (preflight M11).
407func committedImport(t *testing.T, b *ExecBackend, rt registry.Route) (registry.Route, *fakeSeedObjects) {
408 t.Helper()
409 rt, objs, _ := seededRoute(t, b, rt, seeds.StateCommitted)
410 col, err := seeds.ParseRowSeed(rt.SeedSource)
411 if err != nil {
412 t.Fatal(err)
413 }
414 col.Kind = seeds.KindImport
415 rt.SeedSource = col.Encode()
416 return rt, objs
417}
418
419// M11: a committed import restores its own replica, as placed does, and a
420// wake that finds none fails. It never boots empty: after commit the app
421// retires its old process, so an empty boot would lose the import.
422func TestSeededRestoreOfACommittedImportWithNothingToRestoreFails(t *testing.T) {
423 lists := countOwnReplicaLists(t, false)
424 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
425 rt, objs := committedImport(t, b, rt)
426 oldRestore := restoreReplica
427 restored := 0
428 restoreReplica = func(context.Context, string, string, string, string) (bool, error) { restored++; return false, nil }
429 defer func() { restoreReplica = oldRestore }()
430 if fresh, err := b.Restore(context.Background(), rt); err == nil || fresh || restored != 1 || objs.opens != 0 || *lists != 0 {
431 t.Fatalf("fresh %v err %v restored %d opens %d lists %d; want a failed wake, never fresh", fresh, err, restored, objs.opens, *lists)
432 }
433}
434
435func TestSeededRestoreOfACommittedImportRestoresItsOwnReplica(t *testing.T) {
436 lists := countOwnReplicaLists(t, false)
437 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
438 rt, objs := committedImport(t, b, rt)
439 b.ClaimArmed = true // as placed: no claim refusal for a restore of its own
440 oldRestore := restoreReplica
441 restoreReplica = func(_ context.Context, dbPath, _, _, _ string) (bool, error) {
442 return true, writeMinimalSQLite(dbPath)
443 }
444 defer func() { restoreReplica = oldRestore }()
445 os.MkdirAll(filepath.Dir(rt.DBPath), 0o700)
446 if fresh, err := b.Restore(context.Background(), rt); err != nil || fresh || objs.opens != 0 || *lists != 0 {
447 t.Fatalf("fresh %v err %v opens %d lists %d; want its own replica, no fetch", fresh, err, objs.opens, *lists)
448 }
449}
450
451// A crash after the marker and before the rename leaves a marker naming
452// this seed and no database: the snapshot is fetched again.
453func TestSeededRestoreRefetchesWhenOnlyTheMarkerSurvived(t *testing.T) {
454 noOwnReplica(t)
455 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
456 rt, objs, id := seededRoute(t, b, rt, seeds.StateReady)
457 os.MkdirAll(filepath.Dir(rt.DBPath), 0o700)
458 os.WriteFile(rt.DBPath+SeedMarkerSuffix, []byte(id+"\n"), 0o600)
459 if fresh, err := b.Restore(context.Background(), rt); err != nil || fresh || objs.opens != 1 {
460 t.Fatalf("fresh %v err %v opens %d; want one fetch", fresh, err, objs.opens)
461 }
462 if got, _ := os.ReadFile(rt.DBPath); !bytes.Equal(got, objs.body[id]) {
463 t.Fatal("the snapshot is not at the row's path")
464 }
465}
466
467func TestSeededRestoreLeavesAnUnplaceableSeedToThePlainPath(t *testing.T) {
468 for _, state := range []string{seeds.StateAborted, seeds.StateAwaitingUpload, seeds.StateRolledBack} {
469 t.Run(state, func(t *testing.T) {
470 lists := countOwnReplicaLists(t, false)
471 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
472 rt, objs, _ := seededRoute(t, b, rt, state)
473 oldRestore := restoreReplica
474 restored := 0
475 restoreReplica = func(context.Context, string, string, string, string) (bool, error) { restored++; return false, nil }
476 defer func() { restoreReplica = oldRestore }()
477 fresh, err := b.Restore(context.Background(), rt)
478 if err != nil || !fresh || restored != 1 || objs.opens != 0 || *lists != 0 {
479 t.Fatalf("fresh %v err %v restored %d opens %d lists %d; want the plain path", fresh, err, restored, objs.opens, *lists)
480 }
481 })
482 }
483}
484
485func TestSeededRestoreFailsOnAMalformedSeedColumn(t *testing.T) {
486 for _, col := range []string{`{"id":`, `{"id":"../x","kind":"replica","state":"ready"}`} {
487 lists := countOwnReplicaLists(t, false)
488 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
489 objs := &fakeSeedObjects{}
490 b.Seeds = objs
491 oldRestore := restoreReplica
492 restoreReplica = func(context.Context, string, string, string, string) (bool, error) { return false, nil }
493 rt.SeedSource = col
494 fresh, err := b.Restore(context.Background(), rt)
495 restoreReplica = oldRestore
496 if err == nil || fresh || objs.opens != 0 || *lists != 0 {
497 t.Fatalf("column %q: fresh %v err %v; want a failed wake", col, fresh, err)
498 }
499 }
500}
501
502func TestSeededRestoreTellsAReadFailureFromAWriteFailure(t *testing.T) {
503 noOwnReplica(t)
504 t.Run("read", func(t *testing.T) {
505 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
506 rt, objs, _ := seededRoute(t, b, rt, seeds.StateReady)
507 objs.midway = errors.New("connection reset")
508 var spe *SeedPlacementError
509 if _, err := b.Restore(context.Background(), rt); !errors.As(err, &spe) || !strings.Contains(spe.Sentence, "could not be read") {
510 t.Fatalf("Restore = %v; want SD-N16", err)
511 }
512 })
513 t.Run("write", func(t *testing.T) {
514 if os.Geteuid() == 0 {
515 t.Skip("root writes through a read-only directory")
516 }
517 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
518 rt, _, _ = seededRoute(t, b, rt, seeds.StateReady)
519 dir := filepath.Dir(rt.DBPath)
520 os.MkdirAll(dir, 0o700)
521 os.Chmod(dir, 0o500)
522 defer os.Chmod(dir, 0o700)
523 var spe *SeedPlacementError
524 if _, err := b.Restore(context.Background(), rt); !errors.As(err, &spe) || !strings.Contains(spe.Sentence, "could not be put in place") {
525 t.Fatalf("Restore = %v; want SD-N19", err)
526 }
527 })
528}
529
530// A same-uid process that swaps the fetched file around its check gets a
531// refusal, and nothing is left at the row's path to be kept as the seed.
532// Swapped before the check, the check cannot read the file it was given
533// (SQLite resolves the descriptor to the original's name, now gone);
534// swapped after it, the identity check refuses the entry at the temp name.
535func TestSeededRestoreRefusesASwappedTempFile(t *testing.T) {
536 noOwnReplica(t)
537 for _, c := range []struct {
538 name string
539 swapAfter bool
540 wantPhrases []string
541 }{
542 {"before the check", false, []string{"integrity check", "could not be put in place"}},
543 {"after the check", true, []string{"could not be put in place"}},
544 } {
545 t.Run(c.name, func(t *testing.T) {
546 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
547 rt, _, id := seededRoute(t, b, rt, seeds.StateReady)
548 dir := filepath.Dir(rt.DBPath)
549 other := filepath.Join(t.TempDir(), "other.db")
550 if err := writeMinimalSQLite(other); err != nil {
551 t.Fatal(err)
552 }
553 swap := func() {
554 tmp := filepath.Join(dir, ".seed-"+id+".tmp")
555 os.Remove(tmp)
556 os.Symlink(other, tmp)
557 }
558 old := checkSeedDB
559 readSwapped := false
560 checkSeedDB = func(ctx context.Context, path string) error {
561 if !c.swapAfter {
562 swap()
563 // The check is handed the fetched file itself, not a
564 // name the swap can redirect.
565 fi, err1 := os.Stat(path)
566 fo, err2 := os.Stat(other)
567 readSwapped = err1 == nil && err2 == nil && os.SameFile(fi, fo)
568 }
569 err := old(ctx, path)
570 if c.swapAfter {
571 swap()
572 }
573 return err
574 }
575 defer func() { checkSeedDB = old }()
576 var spe *SeedPlacementError
577 _, err := b.Restore(context.Background(), rt)
578 if readSwapped {
579 t.Fatal("the integrity check was pointed at the swapped-in file")
580 }
581 if !errors.As(err, &spe) {
582 t.Fatalf("Restore = %v; want a refusal", err)
583 }
584 matched := false
585 for _, w := range c.wantPhrases {
586 matched = matched || strings.Contains(spe.Sentence, w)
587 }
588 if !matched {
589 t.Fatalf("Restore = %q; want one of %q", spe.Sentence, c.wantPhrases)
590 }
591 for _, p := range []string{rt.DBPath, rt.DBPath + SeedMarkerSuffix} {
592 if _, err := os.Lstat(p); !os.IsNotExist(err) {
593 t.Errorf("%s exists after a refused swap", p)
594 }
595 }
596 if got, _ := os.ReadFile(other); len(got) == 0 {
597 t.Error("the swapped-in file's target was touched")
598 }
599 })
600 }
601}
602
603// Litestream's sources are WAL-mode databases, so a snapshot's header says
604// WAL. The check reads it through a descriptor and must still pass, and
605// leave nothing behind.
606func TestSeededRestorePlacesAWALModeSnapshot(t *testing.T) {
607 noOwnReplica(t)
608 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
609 snap := filepath.Join(t.TempDir(), "wal.db")
610 db, err := sql.Open("sqlite", snap)
611 if err != nil {
612 t.Fatal(err)
613 }
614 for _, q := range []string{"PRAGMA journal_mode=WAL", "create table t(x)", "insert into t values(1)", "PRAGMA wal_checkpoint(TRUNCATE)"} {
615 if _, err := db.Exec(q); err != nil {
616 t.Fatal(err)
617 }
618 }
619 db.Close()
620 body, _ := os.ReadFile(snap)
621 if body[18] != 2 || body[19] != 2 {
622 t.Fatalf("header bytes 18-19 = %d %d, want WAL", body[18], body[19])
623 }
624 id, _ := seeds.NewID()
625 sum := sha256.Sum256(body)
626 rt.SeedSource = seeds.RowSeed{ID: id, Kind: seeds.KindReplica, SHA256: hex.EncodeToString(sum[:]), Bytes: int64(len(body)), State: seeds.StateReady}.Encode()
627 b.Seeds = &fakeSeedObjects{body: map[string][]byte{id: body}}
628 if fresh, err := b.Restore(context.Background(), rt); err != nil || fresh {
629 t.Fatalf("Restore = fresh %v, %v", fresh, err)
630 }
631 ents, _ := os.ReadDir(filepath.Dir(rt.DBPath))
632 for _, e := range ents {
633 if strings.HasPrefix(e.Name(), ".seed-") {
634 t.Errorf("%s left behind", e.Name())
635 }
636 }
637}
638
639// syncRep is a Rep that can be asked for its first upload.
640type syncRep struct {
641 synced int
642 err error
643}
644
645func (r *syncRep) Err() error { return nil }
646func (r *syncRep) Close(context.Context) error { return nil }
647func (r *syncRep) SyncAndWait(context.Context) error { r.synced++; return r.err }
648
649// plainRep is a Rep that cannot be asked for its first upload.
650type plainRep struct{}
651
652func (plainRep) Err() error { return nil }
653func (plainRep) Close(context.Context) error { return nil }
654
655func withSeedState(t *testing.T, rt registry.Route, state string) registry.Route {
656 t.Helper()
657 src, err := seeds.ParseRowSeed(rt.SeedSource)
658 if err != nil || src == nil {
659 t.Fatalf("seed column %q: %v", rt.SeedSource, err)
660 }
661 src.State = state
662 rt.SeedSource = src.Encode()
663 return rt
664}
665
666func TestSeedPlacedWaitsForTheFirstUpload(t *testing.T) {
667 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
668 rt, _, id := seededRoute(t, b, rt, seeds.StateReady)
669 rt.Account, rt.App = "acct", "shop"
670 var calls []string
671 b.OnSeedPlaced = func(_ context.Context, acct, app, host, got string) error {
672 calls = append(calls, acct+"/"+app+"/"+host+"/"+got)
673 return nil
674 }
675 rep := &syncRep{}
676 if err := b.SeedPlaced(context.Background(), rt, rep); err != nil {
677 t.Fatal(err)
678 }
679 if rep.synced != 1 || len(calls) != 1 || calls[0] != "acct/shop/"+rt.Host+"/"+id {
680 t.Fatalf("synced %d, calls %q", rep.synced, calls)
681 }
682 // A failed first upload records nothing.
683 rep = &syncRep{err: errors.New("upload failed")}
684 calls = nil
685 if err := b.SeedPlaced(context.Background(), rt, rep); err == nil || len(calls) != 0 {
686 t.Fatalf("after a failed sync: err %v calls %q", err, calls)
687 }
688 // A replicator that cannot confirm the upload records nothing either:
689 // the snapshot must outlive every moment the replica might not exist.
690 if err := b.SeedPlaced(context.Background(), rt, plainRep{}); err == nil || len(calls) != 0 {
691 t.Fatalf("unconfirmable upload: err %v calls %q", err, calls)
692 }
693 // A recorder that fails is the caller's to log.
694 b.OnSeedPlaced = func(context.Context, string, string, string, string) error { return errors.New("conflict") }
695 if err := b.SeedPlaced(context.Background(), rt, &syncRep{}); err == nil {
696 t.Fatal("a failed record reported success")
697 }
698 b.OnSeedPlaced = nil
699 if err := b.SeedPlaced(context.Background(), rt, &syncRep{}); err == nil {
700 t.Fatal("a box with no recorder reported the seed recorded")
701 }
702 b.OnSeedPlaced = func(_ context.Context, acct, app, host, got string) error {
703 calls = append(calls, got)
704 return nil
705 }
706 // Placed already, not placeable, or no seed: nothing at all.
707 for name, r := range map[string]registry.Route{
708 "placed": withSeedState(t, rt, seeds.StatePlaced),
709 "aborted": withSeedState(t, rt, seeds.StateAborted),
710 "none": func() registry.Route { r := rt; r.SeedSource = ""; return r }(),
711 } {
712 rep = &syncRep{}
713 if err := b.SeedPlaced(context.Background(), r, rep); err != nil || rep.synced != 0 || len(calls) != 0 {
714 t.Fatalf("%s row: err %v synced %d calls %q", name, err, rep.synced, calls)
715 }
716 }
717}
718
719// Review Focus 5: the placed CAS failed after the first upload, so the row
720// still reads ready. The next wake finds the host's own replica and
721// finishes the placement; a failure there never fails the wake, and the
722// wake after that asks again.
723func TestSeedPlacedIsFinishedOnTheNextWake(t *testing.T) {
724 old := hasOwnReplica
725 hasOwnReplica = func(context.Context, string, string, string) (bool, error) { return true, nil }
726 defer func() { hasOwnReplica = old }()
727 oldRestore := restoreReplica
728 restoreReplica = func(_ context.Context, dbPath, _, _, _ string) (bool, error) { return true, writeMinimalSQLite(dbPath) }
729 defer func() { restoreReplica = oldRestore }()
730 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
731 rt, objs, id := seededRoute(t, b, rt, seeds.StateReady) // the CAS was lost last wake
732 rt.Account, rt.App = "acct", "shop"
733 var got []string
734 var recErr error
735 b.OnSeedPlaced = func(_ context.Context, acct, app, host, seed string) error {
736 got = append(got, acct+"/"+app+"/"+host+"/"+seed)
737 return recErr
738 }
739 want := "acct/shop/" + rt.Host + "/" + id
740 recErr = errors.New("record unreachable")
741 if fresh, err := b.Restore(context.Background(), rt); err != nil || fresh {
742 t.Fatalf("Restore with a failing recorder = fresh %v, %v; want a served wake", fresh, err)
743 }
744 recErr = nil
745 if fresh, err := b.Restore(context.Background(), rt); err != nil || fresh {
746 t.Fatalf("Restore = fresh %v, %v", fresh, err)
747 }
748 if len(got) != 2 || got[0] != want || got[1] != want {
749 t.Fatalf("finishPlacement calls = %q, want two for %s", got, want)
750 }
751 if objs.opens != 0 {
752 t.Fatalf("snapshot opened %d times; the own replica wins", objs.opens)
753 }
754 // No recorder on this box: the wake still serves.
755 b.OnSeedPlaced = nil
756 if _, err := b.Restore(context.Background(), rt); err != nil {
757 t.Fatal(err)
758 }
759}
760
761// Final review carry item: a placed or committed seed's own-replica
762// restore, or its warm copy, leaves a <db>.seed marker naming the seed, so
763// the rollback's missing-row purge and an aborted seed's local purge can
764// tell the database is the seed's. A marker naming another seed is
765// rewritten.
766func TestSeededRestoreOfItsOwnReplicaWritesTheMarker(t *testing.T) {
767 for _, c := range []struct {
768 name string
769 route func(t *testing.T, b *ExecBackend, rt registry.Route) registry.Route
770 warm bool
771 }{
772 {"committed import, restored", func(t *testing.T, b *ExecBackend, rt registry.Route) registry.Route {
773 rt, _ = committedImport(t, b, rt)
774 return rt
775 }, false},
776 {"placed seed, warm copy", func(t *testing.T, b *ExecBackend, rt registry.Route) registry.Route {
777 rt, _, _ = seededRoute(t, b, rt, seeds.StatePlaced)
778 return rt
779 }, true},
780 } {
781 t.Run(c.name, func(t *testing.T) {
782 countOwnReplicaLists(t, false)
783 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
784 rt = c.route(t, b, rt)
785 col, err := seeds.ParseRowSeed(rt.SeedSource)
786 if err != nil {
787 t.Fatal(err)
788 }
789 oldRestore := restoreReplica
790 restoreReplica = func(_ context.Context, dbPath, _, _, _ string) (bool, error) {
791 return true, writeMinimalSQLite(dbPath)
792 }
793 defer func() { restoreReplica = oldRestore }()
794 os.MkdirAll(filepath.Dir(rt.DBPath), 0o700)
795 if c.warm {
796 if err := writeMinimalSQLite(rt.DBPath); err != nil {
797 t.Fatal(err)
798 }
799 }
800 other, _ := seeds.NewID()
801 os.WriteFile(rt.DBPath+SeedMarkerSuffix, []byte(other+"\n"), 0o600)
802 if fresh, err := b.Restore(context.Background(), rt); err != nil || fresh {
803 t.Fatalf("fresh %v err %v", fresh, err)
804 }
805 if got, err := ReadSeedMarker(rt.DBPath); err != nil || got != col.ID {
806 t.Fatalf("marker after the restore = %q, %v; want %s", got, err, col.ID)
807 }
808 })
809 }
810}
811
812// Final review F2b: a database left at a route's path by another seed is
813// never this route's warm cache. Its <db>.seed marker names seed A; the
814// row has no seed, or seed B. The wake purges it and restores cold: the
815// row's own replica when there is one, fresh when the plain path allows
816// it. A marker naming the row's own seed keeps the file, as does no
817// marker; a marker that cannot be read fails the wake and touches nothing.
818func TestPlainRestoreNeverServesAnotherSeedsDatabase(t *testing.T) {
819 leftover := []byte("seed A's data")
820 for _, c := range []struct {
821 name string
822 seed func(t *testing.T, b *ExecBackend, rt registry.Route) (registry.Route, string) // the row, and the marker's seed
823 replica bool // the row has a replica of its own
824 unread bool // the marker cannot be read
825 kept bool
826 fresh bool
827 failed bool
828 }{
829 {name: "no seed on the row", seed: func(t *testing.T, _ *ExecBackend, rt registry.Route) (registry.Route, string) {
830 a, _ := seeds.NewID()
831 return rt, a
832 }, fresh: true},
833 {name: "the row names seed B", seed: func(t *testing.T, b *ExecBackend, rt registry.Route) (registry.Route, string) {
834 rt, _, _ = seededRoute(t, b, rt, seeds.StatePlaced)
835 a, _ := seeds.NewID()
836 return rt, a
837 }, replica: true},
838 {name: "the row's own seed", seed: func(t *testing.T, b *ExecBackend, rt registry.Route) (registry.Route, string) {
839 rt, _, id := seededRoute(t, b, rt, seeds.StatePlaced)
840 return rt, id
841 }, kept: true},
842 {name: "no marker", seed: func(t *testing.T, _ *ExecBackend, rt registry.Route) (registry.Route, string) {
843 return rt, ""
844 }, kept: true},
845 {name: "unreadable marker", seed: func(t *testing.T, _ *ExecBackend, rt registry.Route) (registry.Route, string) {
846 a, _ := seeds.NewID()
847 return rt, a
848 }, unread: true, failed: true},
849 } {
850 t.Run(c.name, func(t *testing.T) {
851 countOwnReplicaLists(t, c.replica)
852 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
853 rt, marker := c.seed(t, b, rt)
854 restored := 0
855 oldRestore := restoreReplica
856 restoreReplica = func(_ context.Context, dbPath, _, _, _ string) (bool, error) {
857 restored++
858 if !c.replica {
859 return false, nil
860 }
861 return true, writeMinimalSQLite(dbPath)
862 }
863 defer func() { restoreReplica = oldRestore }()
864 if err := os.MkdirAll(filepath.Dir(rt.DBPath), 0o700); err != nil {
865 t.Fatal(err)
866 }
867 if err := os.WriteFile(rt.DBPath, leftover, 0o600); err != nil {
868 t.Fatal(err)
869 }
870 if marker != "" {
871 if err := os.WriteFile(rt.DBPath+SeedMarkerSuffix, []byte(marker+"\n"), 0o600); err != nil {
872 t.Fatal(err)
873 }
874 }
875 if c.unread {
876 if err := os.Chmod(rt.DBPath+SeedMarkerSuffix, 0); err != nil {
877 t.Fatal(err)
878 }
879 t.Cleanup(func() { os.Chmod(rt.DBPath+SeedMarkerSuffix, 0o600) })
880 if _, err := ReadSeedMarker(rt.DBPath); err == nil {
881 t.Skip("this user reads a mode 0 file (root): the read error cannot be staged")
882 }
883 }
884 fresh, err := b.Restore(context.Background(), rt)
885 if (err != nil) != c.failed || fresh != c.fresh {
886 t.Fatalf("fresh %v err %v; want fresh %v failed %v", fresh, err, c.fresh, c.failed)
887 }
888 got, rerr := os.ReadFile(rt.DBPath)
889 switch {
890 case c.kept || c.failed:
891 if rerr != nil || !bytes.Equal(got, leftover) || restored != 0 {
892 t.Fatalf("database %q (%v), %d restores; want the file kept as it was and nothing restored", got, rerr, restored)
893 }
894 case bytes.Equal(got, leftover):
895 t.Fatal("the other seed's database was served")
896 case c.replica && (rerr != nil || restored != 1):
897 t.Fatalf("database %v, %d restores; want the row's own replica", rerr, restored)
898 }
899 if !c.kept && !c.failed {
900 if m, err := ReadSeedMarker(rt.DBPath); err != nil || m == marker {
901 t.Fatalf("marker after the purge = %q, %v; want seed A's gone", m, err)
902 }
903 }
904 })
905 }
906}
907
908// Re-review finding 8: a placement that crashed between its marker and its
909// rename leaves seed A's marker with no database. A plain instance later
910// created at the host must never be born beside it: its first wake removes
911// the marker (and any fetch leftover) before it boots fresh, the instance
912// writes, and its second wake keeps those writes as its warm cache.
913func TestPlainWakeOverAMarkerOnlyLeftoverKeepsTheReplacementsData(t *testing.T) {
914 countOwnReplicaLists(t, false)
915 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
916 oldRestore := restoreReplica
917 restoreReplica = func(context.Context, string, string, string, string) (bool, error) { return false, nil }
918 defer func() { restoreReplica = oldRestore }()
919 dir := filepath.Dir(rt.DBPath)
920 if err := os.MkdirAll(dir, 0o700); err != nil {
921 t.Fatal(err)
922 }
923 a, _ := seeds.NewID()
924 tmp := filepath.Join(dir, seedTempName(a))
925 for p, body := range map[string]string{rt.DBPath + SeedMarkerSuffix: a + "\n", tmp: "half a fetch"} {
926 if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
927 t.Fatal(err)
928 }
929 }
930 if fresh, err := b.Restore(context.Background(), rt); err != nil || !fresh {
931 t.Fatalf("first wake: fresh %v err %v; want fresh", fresh, err)
932 }
933 for _, p := range []string{rt.DBPath + SeedMarkerSuffix, tmp} {
934 if _, err := os.Lstat(p); !os.IsNotExist(err) {
935 t.Fatalf("%s survived the first wake: %v", p, err)
936 }
937 }
938 replacement := []byte("the replacement's own writes")
939 if err := os.WriteFile(rt.DBPath, replacement, 0o600); err != nil {
940 t.Fatal(err)
941 }
942 if fresh, err := b.Restore(context.Background(), rt); err != nil || fresh {
943 t.Fatalf("second wake: fresh %v err %v; want the warm cache", fresh, err)
944 }
945 if got, err := os.ReadFile(rt.DBPath); err != nil || !bytes.Equal(got, replacement) {
946 t.Fatalf("database after the second wake = %q, %v; want the replacement's writes", got, err)
947 }
948}
949
950// A marker that cannot be read fails the wake whether or not a database is
951// there, and nothing is created beside it.
952func TestPlainWakeFailsOnAnUnreadableMarkerAlone(t *testing.T) {
953 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
954 oldRestore := restoreReplica
955 restoreReplica = func(context.Context, string, string, string, string) (bool, error) {
956 t.Fatal("restored beside an unreadable marker")
957 return false, nil
958 }
959 defer func() { restoreReplica = oldRestore }()
960 os.MkdirAll(filepath.Dir(rt.DBPath), 0o700)
961 a, _ := seeds.NewID()
962 marker := rt.DBPath + SeedMarkerSuffix
963 if err := os.WriteFile(marker, []byte(a+"\n"), 0); err != nil {
964 t.Fatal(err)
965 }
966 t.Cleanup(func() { os.Chmod(marker, 0o600) })
967 if _, err := ReadSeedMarker(rt.DBPath); err == nil {
968 t.Skip("this user reads a mode 0 file (root): the read error cannot be staged")
969 }
970 if fresh, err := b.Restore(context.Background(), rt); err == nil || fresh {
971 t.Fatalf("fresh %v err %v; want a failed wake", fresh, err)
972 }
973 if _, err := os.Lstat(rt.DBPath); !os.IsNotExist(err) {
974 t.Fatalf("a database was created beside an unreadable marker: %v", err)
975 }
976}
977
978// Main's seed obligation, under the seed fallback: an earlier first-boot
979// seed command at this path whose processes are not proven gone left an
980// obligation beside the database. A seeded wake answers it before it
981// fetches, purges or places anything. While the proof fails the wake is
982// refused with ErrSeedUnproven and the file there is untouched; once it
983// holds, the leftover and the obligation go and the snapshot is placed.
984func TestSeededRestoreAnswersAnEarlierSeedsObligationFirst(t *testing.T) {
985 noOwnReplica(t)
986 b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
987 rt, objs, id := seededRoute(t, b, rt, seeds.StateReady)
988 ev := &switchableEvents{err: errors.New("cgroup.events: unreadable")}
989 b.CgroupEvents = ev.events
990 if err := os.MkdirAll(filepath.Dir(rt.DBPath), 0o700); err != nil {
991 t.Fatal(err)
992 }
993 if err := os.WriteFile(rt.DBPath, []byte("half"), 0o600); err != nil {
994 t.Fatal(err)
995 }
996 if err := recordSeedObligation(rt.Host, rt.DBPath, seedObligation{ownsDB: true, layout: LayoutNested}); err != nil {
997 t.Fatal(err)
998 }
999 ob := rt.DBPath + seedUnprovenSuffix
1000
1001 if fresh, err := b.Restore(context.Background(), rt); !errors.Is(err, ErrSeedUnproven) || fresh {
1002 t.Fatalf("Restore = %v, %v; want a refusal while the earlier seed is unproven", fresh, err)
1003 }
1004 if objs.opens != 0 {
1005 t.Fatalf("the snapshot was opened %d times under an unanswered obligation", objs.opens)
1006 }
1007 if got, _ := os.ReadFile(rt.DBPath); string(got) != "half" {
1008 t.Fatalf("database = %q after a refused wake, want it untouched", got)
1009 }
1010 if _, err := os.Stat(ob); err != nil {
1011 t.Fatalf("the obligation went with a proof that failed: %v", err)
1012 }
1013
1014 ev.set(nil)
1015 if fresh, err := b.Restore(context.Background(), rt); err != nil || fresh {
1016 t.Fatalf("Restore = %v, %v; want the snapshot placed once the proof holds", fresh, err)
1017 }
1018 if _, err := os.Stat(ob); !os.IsNotExist(err) {
1019 t.Fatalf("obligation still there (%v) after the proof held", err)
1020 }
1021 marker, err := os.ReadFile(rt.DBPath + SeedMarkerSuffix)
1022 if err != nil || strings.TrimSpace(string(marker)) != id || objs.opens != 1 {
1023 t.Fatalf("marker = %q, %v, opens %d; want this seed placed from one fetch", marker, err, objs.opens)
1024 }
1025 if err := checkSeedDB(context.Background(), rt.DBPath); err != nil {
1026 t.Fatalf("the placed database does not check: %v", err)
1027 }
1028}
1029
1030// Instance key provision under a seed (main's L21 key, minted create-only
1031// into <host>.env on every launch path): placing a snapshot writes no env
1032// file, and the seeded row's first launch mints a key of its own into its
1033// own <host>.env, as any other launch does, and keeps it afterwards. The
1034// row's seed column names no source host (seeds.RowSeed), so nothing of the
1035// source's environment can reach the new instance on the box.
1036func TestASeededRowMintsItsOwnInstanceKeyAtLaunch(t *testing.T) {
1037 noOwnReplica(t)
1038 b, rt := nestedBackend(t, &fakeControl{inv: "inv1"}, &fakeUnit{})
1039 rt, _, _ = seededRoute(t, b, rt, seeds.StateReady)
1040 if fresh, err := b.Restore(context.Background(), rt); err != nil || fresh {
1041 t.Fatalf("Restore = fresh %v, %v; want placed", fresh, err)
1042 }
1043 dir := provision.InstanceDir(b.DataDir, rt.Host)
1044 if _, err := os.Lstat(filepath.Join(dir, rt.Host+".env")); !os.IsNotExist(err) {
1045 t.Fatalf("placing the snapshot left an env file (%v): the key is the launch's to mint", err)
1046 }
1047 launch := func() string {
1048 t.Helper()
1049 _, ctl, logs, env, err := b.prepare(rt, instancectl.RoleRun)
1050 if err != nil {
1051 t.Fatalf("prepare: %v", err)
1052 }
1053 logs.Close()
1054 if ctl != nil {
1055 ctl.Close()
1056 }
1057 own, err := readEnvFileBeneath(dir, rt.Host+".env")
1058 if err != nil {
1059 t.Fatal(err)
1060 }
1061 if !wellFormedInstanceKey(own[InstanceKeyVar]) || env[InstanceKeyVar] != own[InstanceKeyVar] {
1062 t.Fatal("the seeded launch does not carry a key minted into its own env file")
1063 }
1064 return own[InstanceKeyVar]
1065 }
1066 first := launch()
1067 if again := launch(); again != first {
1068 t.Fatal("a second launch changed the seeded instance's key")
1069 }
1070}
1071