carlos / platform Public

Clone
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package host
2
3import (
4 "bytes"
5 "crypto/sha256"
6 "encoding/hex"
7 "encoding/json"
8 "errors"
9 "fmt"
10 "os"
11 "path/filepath"
12 "testing"
13
14 "github.com/carlosframework/platform/internal/appconfig"
15 "github.com/carlosframework/platform/internal/moves"
16 "github.com/carlosframework/platform/internal/provision"
17)
18
19// The Task 9 carries (Plan 5 Task 9a, M16): (b) a foreign database parked by
20// F's retirement, (c) a rendered config left behind when its ack fails.
21
22// parkedLocally lists the base names under the target's .parked/.
23func parkedLocally(t *testing.T, dir string) map[string][]byte {
24 t.Helper()
25 out := map[string][]byte{}
26 filepath.WalkDir(filepath.Join(dir, ".parked"), func(p string, d os.DirEntry, err error) error {
27 if err == nil && !d.IsDir() && d.Type().IsRegular() {
28 b, _ := os.ReadFile(p)
29 out[filepath.Base(p)] = b
30 }
31 return nil
32 })
33 return out
34}
35
36// cleanAfterRollback rolls the landing back, recovers the source and runs
37// the target's cleaning to cleaned.
38func (h *rollbackHarness) cleanAfterRollback(t *testing.T) {
39 t.Helper()
40 h.rollBack(t)
41 h.rollbackPass(t)
42 h.recovered(t)
43 for i := 0; i < 4 && h.rb(t).TargetPhase != provision.RbTargetCleaned; i++ {
44 h.rollbackPass(t)
45 }
46 if rb := h.rb(t); rb.TargetPhase != provision.RbTargetCleaned {
47 t.Fatalf("the target did not clean: %+v", rb)
48 }
49}
50
51// TestForeignDatabaseSurvivesRollbackRetirement (carry (b)): the landing
52// registers only what it placed at local, the files and the env; the
53// database and its marker are registered once the install proves them the
54// build's. A database that is not the build, standing at DBPath when a
55// landing stopped at local rolls back, is neither retired with F nor taken
56// by the moves-owned rename.
57func TestForeignDatabaseSurvivesRollbackRetirement(t *testing.T) {
58 t.Parallel()
59 h := &rollbackHarness{landingHarness: newLandingHarness(t)}
60 h.flip(t)
61 h.landAt = "after-inventory"
62 h.crashLand(t)
63 if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandLocal {
64 t.Fatalf("fixture: the landing is at %s", m.LandingPhase)
65 }
66 paths := h.claim(t).Inventory["uk-box"].Paths
67 if _, ok := paths[p1Host+".db"]; ok || len(paths) != 2 {
68 t.Fatalf("local registered %v, want the files and the env only", paths)
69 }
70 db := provision.DefaultDBPath(h.tdir, p1Host)
71 foreign := []byte("a database this landing never installed")
72 if err := os.WriteFile(db, foreign, 0o600); err != nil {
73 t.Fatal(err)
74 }
75 if err := os.WriteFile(db+".gen", []byte("someone-else\n"), 0o600); err != nil {
76 t.Fatal(err)
77 }
78 h.cleanAfterRollback(t)
79 if got, err := os.ReadFile(db); err != nil || !bytes.Equal(got, foreign) {
80 t.Fatalf("the foreign database: %q %v", got, err)
81 }
82 if _, err := os.Stat(db + ".gen"); err != nil {
83 t.Fatalf("the foreign marker: %v", err)
84 }
85 for name, body := range parkedLocally(t, h.tdir) {
86 if bytes.Equal(body, foreign) {
87 t.Fatalf("the foreign database was parked as %s", name)
88 }
89 }
90 if _, err := os.Lstat(filepath.Join(h.tdir, p1Host)); !errors.Is(err, os.ErrNotExist) {
91 t.Fatalf("the landing's files directory is still live: %v", err)
92 }
93}
94
95// TestInstalledDatabaseIsRegisteredAfterTheInstall (carry (b)): once
96// configured installs the build, the database and its marker are registered
97// under F, and a crash between that install and the record's stamp still
98// leaves them the op's by content: the cleaning parks the build.
99func TestInstalledDatabaseIsRegisteredAfterTheInstall(t *testing.T) {
100 t.Parallel()
101 h := &rollbackHarness{landingHarness: newLandingHarness(t)}
102 h.flip(t)
103 h.landAt = "after-db-rename"
104 h.crashLand(t)
105 if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandConfigured || (m.Landing != nil && m.Landing.InstalledDBSHA256 != "") {
106 t.Fatalf("fixture: %+v", m)
107 }
108 if _, ok := h.claim(t).Inventory["uk-box"].Paths[p1Host+".db"]; ok {
109 t.Fatal("the database was registered before the install finished")
110 }
111 db := provision.DefaultDBPath(h.tdir, p1Host)
112 built, err := os.ReadFile(db)
113 if err != nil {
114 t.Fatal(err)
115 }
116 h.cleanAfterRollback(t)
117 if _, err := os.Lstat(db); !errors.Is(err, os.ErrNotExist) {
118 t.Fatalf("the build is still at DBPath: %v", err)
119 }
120 if got, ok := parkedLocally(t, h.tdir)[p1Host+".db"]; !ok || !bytes.Equal(got, built) {
121 t.Fatal("the build was not parked")
122 }
123
124 h2 := newLandingHarness(t)
125 h2.flip(t)
126 h2.land(t)
127 h2.installed(t)
128 paths := h2.claim(t).Inventory["uk-box"].Paths
129 for _, p := range []string{p1Host + ".db", p1Host + ".db.gen", p1Host, p1Host + ".env"} {
130 if _, ok := paths[p]; !ok {
131 t.Fatalf("after configured the inventory is %v, missing %s", paths, p)
132 }
133 }
134}
135
136// TestFailedRenderAckIsParkedByCleaning (carry (c)): the held render wrote
137// <host>.config.env and its config ack failed. The journal recorded the
138// file's state before the render and the rendered sha256, so the target's
139// cleaning parks the file; a file whose content is not the render's is
140// left.
141func TestFailedRenderAckIsParkedByCleaning(t *testing.T) {
142 t.Parallel()
143 for _, tc := range []struct {
144 name string
145 replace []byte // written over the rendered file before the cleaning
146 parked bool
147 }{
148 {"the rendered file", nil, true},
149 {"another file at the path", []byte("SOMEONE=else\n"), false},
150 } {
151 t.Run(tc.name, func(t *testing.T) {
152 h := &rollbackHarness{landingHarness: newLandingHarness(t)}
153 h.flip(t)
154 path := filepath.Join(h.tdir, p1Host+".config.env")
155 rendered := []byte("SECRET=landing\n")
156 sum := sha256.Sum256(rendered)
157 ackFailed := errors.New("held config ack: the store refused the write")
158 h.renderHook = func(before func(string) error) (appconfig.HeldRender, error) {
159 if err := before(hex.EncodeToString(sum[:])); err != nil {
160 return appconfig.HeldRender{}, err
161 }
162 if err := os.WriteFile(path, rendered, 0o600); err != nil {
163 return appconfig.HeldRender{}, err
164 }
165 in := h.read(t, p1Host).Move.Inputs
166 return appconfig.HeldRender{Path: path, SHA256: hex.EncodeToString(sum[:]), BundleETags: in.BundleETags, OverlayETag: in.OverlayETag}, ackFailed
167 }
168 h.land(t)
169 if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandConfigured {
170 t.Fatalf("fixture: the landing is at %s", m.LandingPhase)
171 }
172 j := h.landJournal(t)
173 want := "sha256:" + hex.EncodeToString(sum[:])
174 if pre := journalledPreRender(t, j); pre.Before != "absent" || len(pre.Intended) != 1 || pre.Intended[0] != want || j.Steps[landStepRendered] != want {
175 t.Fatalf("the render's journal: pre %q rendered %q", j.Steps[landStepPreRender], j.Steps[landStepRendered])
176 }
177 if tc.replace != nil {
178 if err := os.WriteFile(path, tc.replace, 0o600); err != nil {
179 t.Fatal(err)
180 }
181 }
182 h.cleanAfterRollback(t)
183 _, err := os.Lstat(path)
184 if tc.parked {
185 if !errors.Is(err, os.ErrNotExist) {
186 t.Fatalf("the rendered config is still live: %v", err)
187 }
188 if got, ok := parkedLocally(t, h.tdir)[p1Host+".config.env"]; !ok || !bytes.Equal(got, rendered) {
189 t.Fatal("the rendered config was not parked")
190 }
191 return
192 }
193 if err != nil {
194 t.Fatalf("a file that is not the render's was taken: %v", err)
195 }
196 })
197 }
198}
199
200// TestReleaseTimeArchiveRefusalGoesStuck (M16): a target that served stops,
201// and its files directory holds something the archive can never carry. The
202// release records the stuck reason at releasing (not a retry forever), the
203// operator is paged once through moveCfg.Page, and once the cause is
204// removed a retry runs the release again.
205func TestReleaseTimeArchiveRefusalGoesStuck(t *testing.T) {
206 t.Parallel()
207 h := newRollbackHarness(t)
208 h.rollBack(t)
209 link := filepath.Join(h.tdir, p1Host, "link")
210 if err := os.Symlink("/etc/passwd", link); err != nil {
211 t.Fatal(err)
212 }
213 h.rollbackPass(t)
214 m := h.read(t, p1Host).Move
215 if m.State != provision.MoveStuck || m.StuckReason != provision.StuckReleaseTargetFiles || m.Rollback.TargetPhase != provision.RbTargetReleasing ||
216 m.Rollback.StuckAt != provision.RbSourceWaiting {
217 t.Fatalf("not stuck at releasing: %+v %+v", m, m.Rollback)
218 }
219 pagedOnce(t, h.pages, "uk-box", "target", provision.StuckReleaseTargetFiles)
220 // A stuck record is not the target's to work on, and pages no more.
221 h.rollbackPass(t)
222 if got := h.read(t, p1Host).Move; got.State != provision.MoveStuck || len(h.pages) != 1 {
223 t.Fatalf("a pass over the stuck record: %+v, %d pages", got, len(h.pages))
224 }
225 if err := os.Remove(link); err != nil {
226 t.Fatal(err)
227 }
228 h.retry(t)
229 h.rollbackPass(t)
230 if rb := h.rb(t); h.read(t, p1Host).Move.State != provision.MoveRollingBack || rb.TargetPhase == provision.RbTargetReleasing {
231 t.Fatalf("after the retry: %+v", rb)
232 }
233}
234
235// journalledPreRender decodes the landing journal's pre_render step.
236func journalledPreRender(t *testing.T, j *moves.Journal) preRender {
237 t.Helper()
238 var pre preRender
239 if err := json.Unmarshal([]byte(j.Steps[landStepPreRender]), &pre); err != nil {
240 t.Fatalf("pre_render %q: %v", j.Steps[landStepPreRender], err)
241 }
242 return pre
243}
244
245// TestRenderCrashBeforeItsRecordIsParkedByCleaning (fix round 1, I1): the
246// render journals the hash of the bytes it is about to place before it
247// writes them, so a crash after the file is written and before either the
248// ack or the rendered record lands still leaves the file the op's. The
249// cleaning parks it by that intended hash; other content at the path is
250// left.
251func TestRenderCrashBeforeItsRecordIsParkedByCleaning(t *testing.T) {
252 t.Parallel()
253 for _, tc := range []struct {
254 name string
255 replace []byte
256 parked bool
257 }{
258 {"the file the render was placing", nil, true},
259 {"other content at the path", []byte("SOMEONE=else\n"), false},
260 } {
261 t.Run(tc.name, func(t *testing.T) {
262 h := &rollbackHarness{landingHarness: newLandingHarness(t)}
263 h.flip(t)
264 path := filepath.Join(h.tdir, p1Host+".config.env")
265 rendered := []byte("SECRET=landing\n")
266 sum := sha256.Sum256(rendered)
267 crashed := errors.New("the agent died after the write")
268 h.renderHook = func(before func(string) error) (appconfig.HeldRender, error) {
269 if err := before(hex.EncodeToString(sum[:])); err != nil {
270 return appconfig.HeldRender{}, err
271 }
272 if err := os.WriteFile(path, rendered, 0o600); err != nil {
273 return appconfig.HeldRender{}, err
274 }
275 return appconfig.HeldRender{}, crashed // no ack, no answer
276 }
277 h.land(t)
278 j := h.landJournal(t)
279 if pre := journalledPreRender(t, j); len(pre.Intended) != 1 || pre.Intended[0] != "sha256:"+hex.EncodeToString(sum[:]) || pre.Before != "absent" {
280 t.Fatalf("pre_render %+v", pre)
281 }
282 if _, ok := j.Steps[landStepRendered]; ok {
283 t.Fatal("fixture: the rendered record landed")
284 }
285 if tc.replace != nil {
286 if err := os.WriteFile(path, tc.replace, 0o600); err != nil {
287 t.Fatal(err)
288 }
289 }
290 h.cleanAfterRollback(t)
291 _, err := os.Lstat(path)
292 if tc.parked {
293 if !errors.Is(err, os.ErrNotExist) {
294 t.Fatalf("the rendered config is still live: %v", err)
295 }
296 if got, ok := parkedLocally(t, h.tdir)[p1Host+".config.env"]; !ok || !bytes.Equal(got, rendered) {
297 t.Fatal("the rendered config was not parked")
298 }
299 return
300 }
301 if err != nil {
302 t.Fatalf("a file that is not the render's was taken: %v", err)
303 }
304 })
305 }
306}
307
308// TestRenderReplayKeepsEveryIntendedHash (fix round 2): render A is written
309// and the agent dies before the ack or the rendered record; the retry's
310// inputs have changed, so it renders B, journals B and fails to write it. A
311// is still the file on disk. The intended hashes are an append-only set
312// across the op's attempts, so A's stays recorded and the target's cleaning
313// still parks the file.
314func TestRenderReplayKeepsEveryIntendedHash(t *testing.T) {
315 t.Parallel()
316 h := &rollbackHarness{landingHarness: newLandingHarness(t)}
317 h.flip(t)
318 path := filepath.Join(h.tdir, p1Host+".config.env")
319 a, b := []byte("SECRET=a\n"), []byte("SECRET=b\n")
320 sumA, sumB := sha256.Sum256(a), sha256.Sum256(b)
321 attempt := 0
322 h.renderHook = func(before func(string) error) (appconfig.HeldRender, error) {
323 attempt++
324 if attempt == 1 {
325 if err := before(hex.EncodeToString(sumA[:])); err != nil {
326 return appconfig.HeldRender{}, err
327 }
328 if err := os.WriteFile(path, a, 0o600); err != nil {
329 return appconfig.HeldRender{}, err
330 }
331 return appconfig.HeldRender{}, errors.New("the agent died after writing A")
332 }
333 if err := before(hex.EncodeToString(sumB[:])); err != nil {
334 return appconfig.HeldRender{}, err
335 }
336 return appconfig.HeldRender{}, errors.New("writing B failed")
337 }
338 h.land(t)
339 h.land(t)
340 if attempt < 2 {
341 t.Fatalf("fixture: %d render attempts", attempt)
342 }
343 pre := journalledPreRender(t, h.landJournal(t))
344 wantA, wantB := "sha256:"+hex.EncodeToString(sumA[:]), "sha256:"+hex.EncodeToString(sumB[:])
345 if len(pre.Intended) != 2 || pre.Intended[0] != wantA || pre.Intended[1] != wantB {
346 t.Fatalf("intended %v; want [A B]", pre.Intended)
347 }
348 if got, err := os.ReadFile(path); err != nil || !bytes.Equal(got, a) {
349 t.Fatalf("fixture: the file is %q %v, want A", got, err)
350 }
351 h.cleanAfterRollback(t)
352 if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) {
353 t.Fatalf("A is still live: %v", err)
354 }
355 if got, ok := parkedLocally(t, h.tdir)[p1Host+".config.env"]; !ok || !bytes.Equal(got, a) {
356 t.Fatal("A was not parked")
357 }
358}
359
360// renderAttempt is one fake held render: it journals sum through the
361// installer's before-write hook, writes body to path when write is true,
362// and then dies (no ack, no answer) unless done is true.
363func renderAttempt(path string, body []byte, write, done bool, held appconfig.HeldRender) func(func(string) error) (appconfig.HeldRender, error) {
364 return func(before func(string) error) (appconfig.HeldRender, error) {
365 sum := sha256.Sum256(body)
366 if err := before(hex.EncodeToString(sum[:])); err != nil {
367 return appconfig.HeldRender{}, err
368 }
369 if !write {
370 return appconfig.HeldRender{}, errors.New("the write failed")
371 }
372 if err := os.WriteFile(path, body, 0o600); err != nil {
373 return appconfig.HeldRender{}, err
374 }
375 if !done {
376 return appconfig.HeldRender{}, errors.New("the agent died after the write")
377 }
378 held.Path, held.SHA256 = path, hex.EncodeToString(sum[:])
379 return held, nil
380 }
381}
382
383// TestRenderCrashAfterThePrunedJournal (fix round 4): render A is written,
384// then B is written; render C prunes the set to what is on disk, B, plus C,
385// journals it, and dies before writing. The file on disk is B, which the set
386// still names, so the cleaning parks it.
387func TestRenderCrashAfterThePrunedJournal(t *testing.T) {
388 t.Parallel()
389 h := &rollbackHarness{landingHarness: newLandingHarness(t)}
390 h.flip(t)
391 path := filepath.Join(h.tdir, p1Host+".config.env")
392 a, b, c := []byte("SECRET=a\n"), []byte("SECRET=b\n"), []byte("SECRET=c\n")
393 for _, step := range []func(func(string) error) (appconfig.HeldRender, error){
394 renderAttempt(path, a, true, false, appconfig.HeldRender{}),
395 renderAttempt(path, b, true, false, appconfig.HeldRender{}),
396 renderAttempt(path, c, false, false, appconfig.HeldRender{}),
397 } {
398 h.renderHook = step
399 h.land(t)
400 }
401 sumB, sumC := sha256.Sum256(b), sha256.Sum256(c)
402 pre := journalledPreRender(t, h.landJournal(t))
403 if want := []string{"sha256:" + hex.EncodeToString(sumB[:]), "sha256:" + hex.EncodeToString(sumC[:])}; len(pre.Intended) != 2 || pre.Intended[0] != want[0] || pre.Intended[1] != want[1] {
404 t.Fatalf("intended %v; want [B C]", pre.Intended)
405 }
406 if pre.Before != "absent" {
407 t.Fatalf("the earlier-state record changed: %q", pre.Before)
408 }
409 h.cleanAfterRollback(t)
410 if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) {
411 t.Fatalf("B is still live: %v", err)
412 }
413 if got, ok := parkedLocally(t, h.tdir)[p1Host+".config.env"]; !ok || !bytes.Equal(got, b) {
414 t.Fatal("B was not parked")
415 }
416}
417
418// TestReturnRenderRetriesStayBounded (fix round 4): a dirty return has no
419// deadline, so its held render may be retried any number of times. Forty
420// failed or changed attempts, half written and half not, keep the intended
421// set at most two hashes, always naming what is on disk; the next render
422// succeeds and the return completes with that content kept as the source's
423// config, still named.
424func TestReturnRenderRetriesStayBounded(t *testing.T) {
425 t.Parallel()
426 h := newReturnHarness(t, false)
427 path := filepath.Join(h.sdir, p1Host+".config.env")
428 for i := 0; i < 40; i++ {
429 h.renderHook = renderAttempt(path, []byte(fmt.Sprintf("SECRET=%d\n", i)), i%2 == 0, false, appconfig.HeldRender{})
430 if _, err := h.install(t); err == nil {
431 t.Fatalf("attempt %d: the render answered", i)
432 }
433 pre := journalledPreRender(t, h.returnJournal(t))
434 if len(pre.Intended) > 2 {
435 t.Fatalf("attempt %d: %d intended hashes", i, len(pre.Intended))
436 }
437 if got, err := fileSHA256(path); err == nil && !containsString(pre.Intended, got) {
438 t.Fatalf("attempt %d: the file on disk (%s) is not named by %v", i, got, pre.Intended)
439 }
440 }
441 final := []byte("SECRET=final\n")
442 h.renderHook = renderAttempt(path, final, true, true, h.render)
443 if _, err := h.install(t); err != nil {
444 t.Fatalf("the successful render: %v", err)
445 }
446 if rb := h.read(t, p1Host).Move.Rollback; rb.SourcePhase != provision.RbSourceRestored {
447 t.Fatalf("the return did not complete: %+v", rb)
448 }
449 got, err := os.ReadFile(path)
450 if err != nil || !bytes.Equal(got, final) {
451 t.Fatalf("the source's config is %q %v, want the final render", got, err)
452 }
453 sum := sha256.Sum256(final)
454 pre := journalledPreRender(t, h.returnJournal(t))
455 if len(pre.Intended) > 2 || !containsString(pre.Intended, "sha256:"+hex.EncodeToString(sum[:])) {
456 t.Fatalf("intended %v after the final render", pre.Intended)
457 }
458}
459
460// TestRenderTempIsRemovedByCleaning (fix round 5): the renderer writes the
461// rendered bytes to <host>.config.env.tmp and renames them over the path
462// (appconfig syncexec.go). A crash between the two leaves the temp holding
463// the rendered secrets, an unfinished write of this op's. The target's
464// cleaning removes it (never parks it: it is not live config), and leaves
465// <host>.config.env as it was before the render: absent, or an earlier file
466// that is not the render's, kept.
467func TestRenderTempIsRemovedByCleaning(t *testing.T) {
468 t.Parallel()
469 for _, earlier := range [][]byte{nil, []byte("EARLIER=1\n")} {
470 t.Run(fmt.Sprintf("earlier %v", earlier != nil), func(t *testing.T) {
471 h := &rollbackHarness{landingHarness: newLandingHarness(t)}
472 h.flip(t)
473 path := filepath.Join(h.tdir, p1Host+".config.env")
474 if earlier != nil {
475 if err := os.WriteFile(path, earlier, 0o600); err != nil {
476 t.Fatal(err)
477 }
478 }
479 rendered := []byte("SECRET=landing\n")
480 sum := sha256.Sum256(rendered)
481 h.renderHook = func(before func(string) error) (appconfig.HeldRender, error) {
482 if err := before(hex.EncodeToString(sum[:])); err != nil {
483 return appconfig.HeldRender{}, err
484 }
485 if err := os.WriteFile(path+".tmp", rendered, 0o600); err != nil {
486 return appconfig.HeldRender{}, err
487 }
488 return appconfig.HeldRender{}, errors.New("the agent died before the rename")
489 }
490 h.land(t)
491 if _, err := os.Stat(path + ".tmp"); err != nil {
492 t.Fatalf("fixture: no temp: %v", err)
493 }
494 h.cleanAfterRollback(t)
495 if _, err := os.Lstat(path + ".tmp"); !errors.Is(err, os.ErrNotExist) {
496 t.Fatalf("the renderer's temp is still there: %v", err)
497 }
498 got, err := os.ReadFile(path)
499 switch {
500 case earlier == nil && !errors.Is(err, os.ErrNotExist):
501 t.Fatalf("a config appeared: %q %v", got, err)
502 case earlier != nil && (err != nil || !bytes.Equal(got, earlier)):
503 t.Fatalf("the earlier config was not kept: %q %v", got, err)
504 }
505 for name, body := range parkedLocally(t, h.tdir) {
506 if bytes.Equal(body, rendered) {
507 t.Fatalf("the rendered secrets were parked as %s", name)
508 }
509 }
510 })
511 }
512}
513