amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package activator
2
3import (
4 "bytes"
5 "context"
6 "encoding/base64"
7 "errors"
8 "log"
9 "os"
10 "path/filepath"
11 "strings"
12 "sync"
13 "sync/atomic"
14 "syscall"
15 "testing"
16 "time"
17
18 "github.com/carlosframework/platform/internal/provision"
19 "github.com/carlosframework/platform/internal/registry"
20)
21
22// keyLines returns every CARLOS_INSTANCE_KEY value in raw, in order.
23func keyLines(raw []byte) []string {
24 var out []string
25 for _, line := range strings.Split(string(raw), "\n") {
26 if v, ok := strings.CutPrefix(line, InstanceKeyVar+"="); ok {
27 out = append(out, v)
28 }
29 }
30 return out
31}
32
33// syncProbe counts the key's syncs and fails them on demand. Set the
34// error fields only between calls.
35type syncProbe struct {
36 file, dir int
37 fileErr, dirErr error
38}
39
40func probeInstanceKeySync(t *testing.T) *syncProbe {
41 t.Helper()
42 p := &syncProbe{}
43 adminTokenMu.Lock()
44 old := instanceKeySync
45 instanceKeySync.file = func(f *os.File) error {
46 p.file++
47 if p.fileErr != nil {
48 return p.fileErr
49 }
50 return f.Sync()
51 }
52 instanceKeySync.dir = func(d string) error {
53 p.dir++
54 if p.dirErr != nil {
55 return p.dirErr
56 }
57 return syncDir(d)
58 }
59 adminTokenMu.Unlock()
60 t.Cleanup(func() {
61 adminTokenMu.Lock()
62 instanceKeySync = old
63 adminTokenMu.Unlock()
64 })
65 return p
66}
67
68func TestEnsureInstanceKeyMintsOnceInTheAgreedFormat(t *testing.T) {
69 p := filepath.Join(t.TempDir(), "shop.example.com.env")
70 got, err := ensureInstanceKeyNoConfig(p)
71 if err != nil || got != InstanceKeyMinted {
72 t.Fatalf("first ensure = %v, %v; want minted", got, err)
73 }
74 raw, _ := os.ReadFile(p)
75 keys := keyLines(raw)
76 if len(keys) != 1 {
77 t.Fatalf("file holds %d keys, want 1", len(keys))
78 }
79 dec, err := base64.StdEncoding.Strict().DecodeString(keys[0])
80 if err != nil || len(dec) != 32 || len(keys[0]) != 44 {
81 t.Fatalf("key is not 32 bytes in padded standard base64 (len %d, decode err %v)", len(keys[0]), err)
82 }
83 if !bytes.HasSuffix(raw, []byte("\n")) {
84 t.Fatal("the key line has no newline: the next append would join it")
85 }
86 if st, _ := os.Stat(p); st.Mode().Perm() != 0o600 {
87 t.Fatalf("mode = %o, want 600", st.Mode().Perm())
88 }
89 // The parser every launch path uses yields the exact characters minted.
90 env, _ := readEnvFile(p)
91 if env[InstanceKeyVar] != keys[0] {
92 t.Fatal("readEnvFile does not yield the minted key verbatim")
93 }
94 again, err := ensureInstanceKeyNoConfig(p)
95 if err != nil || again != InstanceKeyPresent {
96 t.Fatalf("second ensure = %v, %v; want present", again, err)
97 }
98 raw2, _ := os.ReadFile(p)
99 if !bytes.Equal(raw, raw2) {
100 t.Fatal("second ensure changed the file")
101 }
102}
103
104// Never overwrite: a key already there, of any shape, is kept byte for byte
105// and only reported.
106func TestEnsureInstanceKeyNeverRewritesAnExistingKey(t *testing.T) {
107 for _, existing := range []string{
108 "CARLOS_INSTANCE_KEY=hand-written-not-base64\n",
109 "A=1\nCARLOS_INSTANCE_KEY=" + strings.Repeat("A", 43) + "=", // well formed, no trailing newline
110 } {
111 p := filepath.Join(t.TempDir(), "shop.example.com.env")
112 if err := os.WriteFile(p, []byte(existing), 0o600); err != nil {
113 t.Fatal(err)
114 }
115 var buf bytes.Buffer
116 log.SetOutput(&buf)
117 got, err := ensureInstanceKeyNoConfig(p)
118 log.SetOutput(os.Stderr)
119 if err != nil || got != InstanceKeyPresent {
120 t.Fatalf("ensure over %q = %v, %v; want present", existing, got, err)
121 }
122 if raw, _ := os.ReadFile(p); string(raw) != existing {
123 t.Fatalf("file changed: %q -> %q", existing, raw)
124 }
125 if strings.Contains(buf.String(), "hand-written-not-base64") {
126 t.Fatal("the shape warning logged the key's value")
127 }
128 }
129}
130
131func TestEnsureInstanceKeyAppendsOnAFreshLineAndKeepsEveryOtherLine(t *testing.T) {
132 p := filepath.Join(t.TempDir(), "shop.example.com.env")
133 before := "CARLOS_ADMIN_TOKEN=" + strings.Repeat("a", 64) + "\n# comment\nDATABASE_URL=sqlite" // no trailing newline
134 if err := os.WriteFile(p, []byte(before), 0o644); err != nil {
135 t.Fatal(err)
136 }
137 if got, err := ensureInstanceKeyNoConfig(p); err != nil || got != InstanceKeyMinted {
138 t.Fatalf("ensure = %v, %v", got, err)
139 }
140 raw, _ := os.ReadFile(p)
141 if !strings.HasPrefix(string(raw), before+"\n"+InstanceKeyVar+"=") {
142 t.Fatalf("file = %q: the key must land on its own line after the untouched original", raw)
143 }
144 env, _ := readEnvFile(p)
145 if env["DATABASE_URL"] != "sqlite" || len(env["CARLOS_ADMIN_TOKEN"]) != 64 {
146 t.Fatal("an existing line was lost")
147 }
148 if st, _ := os.Stat(p); st.Mode().Perm() != 0o600 {
149 t.Fatalf("mode = %o, want the 0644 file tightened to 600", st.Mode().Perm())
150 }
151}
152
153// A key the app's configuration supplies is the app's key: minting one into
154// <host>.env would override it (mergeEnv) and change the key under its data.
155// The decision is the effective env's, so an empty CARLOS_INSTANCE_KEY= in
156// the own file does not hide the configured key (finding 4).
157func TestEnsureInstanceKeyDefersToTheRenderedConfiguration(t *testing.T) {
158 for _, own := range []string{"A=1\n", "A=1\n" + InstanceKeyVar + "=\n"} {
159 dir := t.TempDir()
160 p := filepath.Join(dir, "shop.example.com.env")
161 if err := os.WriteFile(filepath.Join(dir, "shop.example.com.config.env"), []byte(InstanceKeyVar+"=from-config\n"), 0o600); err != nil {
162 t.Fatal(err)
163 }
164 if err := os.WriteFile(p, []byte(own), 0o600); err != nil {
165 t.Fatal(err)
166 }
167 got, err := ensureInstanceKeyNoConfig(p)
168 if err != nil || got != InstanceKeyFromConfig {
169 t.Fatalf("own %q: ensure = %v, %v; want from-config", own, got, err)
170 }
171 if raw, _ := os.ReadFile(p); string(raw) != own {
172 t.Fatalf("own file changed: %q", raw)
173 }
174 }
175}
176
177func TestBackfillInstanceKeyAppendsOnlyToAnExistingEnvFile(t *testing.T) {
178 b := &ExecBackend{DataDir: t.TempDir()}
179 fresh := registry.Route{Host: "new.example.com", Socket: "/run/carlos/new.sock"}
180 got, err := b.BackfillInstanceKey(t.Context(), fresh)
181 if err != nil || got != InstanceKeyNoEnvFile {
182 t.Fatalf("backfill with no env file = %v, %v; want no-env-file", got, err)
183 }
184 if _, err := os.Lstat(filepath.Join(b.DataDir, fresh.Host+".env")); !os.IsNotExist(err) {
185 t.Fatalf("backfill created an env file: %v", err)
186 }
187 old := registry.Route{Host: "shop.example.com", Socket: "/run/carlos/shop.sock"}
188 envPath := filepath.Join(b.DataDir, old.Host+".env")
189 tok := "CARLOS_ADMIN_TOKEN=" + strings.Repeat("b", 64) + "\n"
190 if err := os.WriteFile(envPath, []byte(tok), 0o600); err != nil {
191 t.Fatal(err)
192 }
193 if got, err := b.BackfillInstanceKey(t.Context(), old); err != nil || got != InstanceKeyMinted {
194 t.Fatalf("backfill = %v, %v; want minted", got, err)
195 }
196 raw, _ := os.ReadFile(envPath)
197 if !strings.HasPrefix(string(raw), tok) || len(keyLines(raw)) != 1 {
198 t.Fatalf("file = %q", raw)
199 }
200 // Idempotent: a second backfill (the next agent start) writes nothing.
201 if got, err := b.BackfillInstanceKey(t.Context(), old); err != nil || got != InstanceKeyPresent {
202 t.Fatalf("second backfill = %v, %v; want present", got, err)
203 }
204 if raw2, _ := os.ReadFile(envPath); !bytes.Equal(raw, raw2) {
205 t.Fatal("second backfill changed the file")
206 }
207}
208
209// Review focus 1: a hand-edited file with two keys. Readers take the last
210// (parseEnvBytes), and the ensure writes nothing.
211func TestEnsureInstanceKeyLeavesAFileWithTwoKeysAlone(t *testing.T) {
212 p := filepath.Join(t.TempDir(), "shop.example.com.env")
213 body := InstanceKeyVar + "=first\n" + InstanceKeyVar + "=second\n"
214 if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
215 t.Fatal(err)
216 }
217 if got, err := ensureInstanceKeyNoConfig(p); err != nil || got != InstanceKeyPresent {
218 t.Fatalf("ensure = %v, %v; want present", got, err)
219 }
220 if raw, _ := os.ReadFile(p); string(raw) != body {
221 t.Fatalf("file changed: %q", raw)
222 }
223 if env, _ := readEnvFile(p); env[InstanceKeyVar] != "second" {
224 t.Fatalf("readers see %q, want the last line's value", env[InstanceKeyVar])
225 }
226}
227
228// Review focus 2: an empty value is no key. One is appended after it, and
229// readers take the appended one.
230func TestEnsureInstanceKeyTreatsAnEmptyValueAsAbsent(t *testing.T) {
231 p := filepath.Join(t.TempDir(), "shop.example.com.env")
232 if err := os.WriteFile(p, []byte(InstanceKeyVar+"=\n"), 0o600); err != nil {
233 t.Fatal(err)
234 }
235 if got, err := ensureInstanceKeyNoConfig(p); err != nil || got != InstanceKeyMinted {
236 t.Fatalf("ensure = %v, %v; want minted", got, err)
237 }
238 env, _ := readEnvFile(p)
239 if !wellFormedInstanceKey(env[InstanceKeyVar]) {
240 t.Fatal("readers do not see the appended key")
241 }
242}
243
244// Review focus 3: a seed, a visitor's wake and the backfill can ask at
245// once. Exactly one key is written.
246func TestEnsureInstanceKeyMintsOneKeyUnderConcurrentCallers(t *testing.T) {
247 p := filepath.Join(t.TempDir(), "shop.example.com.env")
248 var wg sync.WaitGroup
249 for i := 0; i < 8; i++ {
250 wg.Add(1)
251 go func() {
252 defer wg.Done()
253 if _, err := ensureInstanceKeyNoConfig(p); err != nil {
254 t.Error(err)
255 }
256 }()
257 }
258 wg.Wait()
259 if raw, _ := os.ReadFile(p); len(keyLines(raw)) != 1 {
260 t.Fatalf("%d keys written", len(keyLines(raw)))
261 }
262}
263
264// Review focus 5: an empty CARLOS_INSTANCE_KEY in the rendered
265// configuration supplies nothing, so the platform mints one.
266func TestEnsureInstanceKeyMintsWhenTheConfigurationValueIsEmpty(t *testing.T) {
267 dir := t.TempDir()
268 if err := os.WriteFile(filepath.Join(dir, "shop.example.com.config.env"), []byte(InstanceKeyVar+"=\n"), 0o600); err != nil {
269 t.Fatal(err)
270 }
271 if got, err := ensureInstanceKeyNoConfig(filepath.Join(dir, "shop.example.com.env")); err != nil || got != InstanceKeyMinted {
272 t.Fatalf("ensure = %v, %v; want minted", got, err)
273 }
274}
275
276// Finding 2: a mint whose file or directory sync fails reports failure,
277// and so does every retry until a sync succeeds. The retry finds the key
278// present, keeps its bytes, and still completes the chmod and both syncs
279// before it reports success.
280func TestEnsureInstanceKeyRedoesAFailedSyncBeforeReportingSuccess(t *testing.T) {
281 for _, tc := range []struct {
282 name string
283 fail func(*syncProbe, error)
284 }{
285 {"file sync", func(p *syncProbe, err error) { p.fileErr = err }},
286 {"directory sync", func(p *syncProbe, err error) { p.dirErr = err }},
287 } {
288 t.Run(tc.name, func(t *testing.T) {
289 probe := probeInstanceKeySync(t)
290 p := filepath.Join(t.TempDir(), "shop.example.com.env")
291 tc.fail(probe, errors.New("injected: EIO"))
292 if _, err := ensureInstanceKeyNoConfig(p); err == nil {
293 t.Fatal("a mint whose sync failed reported success")
294 }
295 raw, _ := os.ReadFile(p)
296 if len(keyLines(raw)) != 1 {
297 t.Fatalf("the failed mint left %d keys, want its 1", len(keyLines(raw)))
298 }
299 if _, err := ensureInstanceKeyNoConfig(p); err == nil {
300 t.Fatal("a retry reported success while the sync still fails")
301 }
302 if err := os.Chmod(p, 0o644); err != nil {
303 t.Fatal(err)
304 }
305 tc.fail(probe, nil)
306 before := *probe
307 got, err := ensureInstanceKeyNoConfig(p)
308 if err != nil || got != InstanceKeyPresent {
309 t.Fatalf("retry = %v, %v; want present", got, err)
310 }
311 if probe.file != before.file+1 || probe.dir != before.dir+1 {
312 t.Fatalf("the retry synced the file %d and the directory %d times, want once each", probe.file-before.file, probe.dir-before.dir)
313 }
314 if raw2, _ := os.ReadFile(p); !bytes.Equal(raw, raw2) {
315 t.Fatal("a retry rewrote the key")
316 }
317 if st, _ := os.Stat(p); st.Mode().Perm() != 0o600 {
318 t.Fatalf("mode = %o, want the present path to set 600", st.Mode().Perm())
319 }
320 })
321 }
322}
323
324// Ruling 26: a replica row is never minted a key, on any path; the only
325// key it may hold is its home's (replicaKeyGate).
326func TestReplicaRowsGetNoPlatformKey(t *testing.T) {
327 b := &ExecBackend{DataDir: t.TempDir()}
328 rt := registry.Route{Host: "shop-au.example.com", Kind: "instance", Socket: "/run/carlos/shop-au.sock", Replica: true}
329 envPath := filepath.Join(b.DataDir, rt.Host+".env")
330 if err := os.WriteFile(envPath, []byte("CARLOS_ADMIN_TOKEN=t\n"), 0o600); err != nil {
331 t.Fatal(err)
332 }
333 if got, err := b.ensureInstanceKeyFor(t.Context(), rt, b.pathsFor(rt)); err != nil || got != InstanceKeyReplica {
334 t.Fatalf("launch ensure = %v, %v; want replica", got, err)
335 }
336 if got, err := b.BackfillInstanceKey(t.Context(), rt); err != nil || got != InstanceKeyReplica {
337 t.Fatalf("backfill = %v, %v; want replica", got, err)
338 }
339 if raw, _ := os.ReadFile(envPath); len(keyLines(raw)) != 0 {
340 t.Fatal("a replica row was given a platform key")
341 }
342}
343
344// The flat ensure never follows a symlink planted at <host>.env (spec §7.2,
345// the hazard TestEnsureAdminTokenRefusesToChmodThroughASymlink guards). On
346// every path, mint, existing key, backfill and the claim settle, it refuses
347// and leaves the link's target's mode and bytes exactly as they were.
348func TestEnsureInstanceKeyRefusesToFollowASymlink(t *testing.T) {
349 wellFormed := InstanceKeyVar + "=" + strings.Repeat("A", 43) + "=\n"
350 for _, tc := range []struct {
351 name string
352 target string // the target's contents
353 mode os.FileMode
354 call func(b *ExecBackend, rt registry.Route, link string) error
355 }{
356 {"mint", "A=1\n", 0o644, func(_ *ExecBackend, _ registry.Route, link string) error {
357 _, err := ensureInstanceKeyNoConfig(link)
358 return err
359 }},
360 {"existing key", wellFormed, 0o644, func(_ *ExecBackend, _ registry.Route, link string) error {
361 _, err := ensureInstanceKeyNoConfig(link)
362 return err
363 }},
364 {"backfill mint", "A=1\n", 0o644, func(b *ExecBackend, rt registry.Route, _ string) error {
365 _, err := b.BackfillInstanceKey(t.Context(), rt)
366 return err
367 }},
368 {"backfill existing key", wellFormed, 0o644, func(b *ExecBackend, rt registry.Route, _ string) error {
369 _, err := b.BackfillInstanceKey(t.Context(), rt)
370 return err
371 }},
372 {"claim settle", wellFormed, 0o600, func(b *ExecBackend, rt registry.Route, _ string) error {
373 return b.settleClaimInstanceEnv(rt)
374 }},
375 } {
376 t.Run(tc.name, func(t *testing.T) {
377 b := &ExecBackend{DataDir: t.TempDir()}
378 rt := registry.Route{Host: "shop.example.com", Socket: "/run/carlos/shop.sock"}
379 target := filepath.Join(t.TempDir(), "someone-elses-file")
380 if err := os.WriteFile(target, []byte(tc.target), tc.mode); err != nil {
381 t.Fatal(err)
382 }
383 if err := os.Chmod(target, tc.mode); err != nil {
384 t.Fatal(err)
385 }
386 link := filepath.Join(b.DataDir, rt.Host+".env")
387 if err := os.Symlink(target, link); err != nil {
388 t.Skipf("this filesystem will not make symlinks: %v", err)
389 }
390 err := tc.call(b, rt, link)
391 if !errors.Is(err, errOwnEnvSymlink) {
392 t.Fatalf("err = %v, want errOwnEnvSymlink", err)
393 }
394 if strings.Contains(err.Error(), strings.Repeat("A", 43)) {
395 t.Fatal("the refusal put the key in its error")
396 }
397 st, err := os.Stat(target)
398 if err != nil {
399 t.Fatal(err)
400 }
401 if st.Mode().Perm() != tc.mode {
402 t.Fatalf("mode of the symlink's TARGET = %o, want the untouched %o", st.Mode().Perm(), tc.mode)
403 }
404 if raw, _ := os.ReadFile(target); string(raw) != tc.target {
405 t.Fatalf("the symlink's TARGET changed: %q -> %q", tc.target, raw)
406 }
407 })
408 }
409}
410
411// A dangling link is refused too: the mint's O_CREATE must not create the
412// file it points at.
413func TestEnsureInstanceKeyRefusesADanglingSymlink(t *testing.T) {
414 dir := t.TempDir()
415 target := filepath.Join(t.TempDir(), "not-yet")
416 link := filepath.Join(dir, "shop.example.com.env")
417 if err := os.Symlink(target, link); err != nil {
418 t.Skipf("this filesystem will not make symlinks: %v", err)
419 }
420 if _, err := ensureInstanceKeyNoConfig(link); !errors.Is(err, errOwnEnvSymlink) {
421 t.Fatalf("err = %v, want errOwnEnvSymlink", err)
422 }
423 if _, err := os.Lstat(target); !os.IsNotExist(err) {
424 t.Fatalf("the mint created the link's target: %v", err)
425 }
426}
427
428// The refusal is the open's own, not a check before it: the helper refuses
429// a link for every flag set the ensure uses, so no Lstat-to-open window
430// exists on unix.
431func TestOpenOwnEnvNoFollowRefusesALinkForEveryFlagSet(t *testing.T) {
432 dir := t.TempDir()
433 target := filepath.Join(dir, "target")
434 if err := os.WriteFile(target, []byte("A=1\n"), 0o644); err != nil {
435 t.Fatal(err)
436 }
437 link := filepath.Join(dir, "h.env")
438 if err := os.Symlink(target, link); err != nil {
439 t.Skipf("this filesystem will not make symlinks: %v", err)
440 }
441 for _, flags := range []int{os.O_RDONLY, os.O_APPEND | os.O_WRONLY, os.O_APPEND | os.O_WRONLY | os.O_CREATE} {
442 perm := os.FileMode(0)
443 if flags&os.O_CREATE != 0 {
444 perm = 0o600
445 }
446 f, err := openOwnEnvNoFollow(link, flags, perm)
447 if err == nil {
448 f.Close()
449 t.Fatalf("flags %#x: opened through a symlink", flags)
450 }
451 if !errors.Is(err, errOwnEnvSymlink) {
452 t.Fatalf("flags %#x: err = %v, want errOwnEnvSymlink", flags, err)
453 }
454 }
455 // An ordinary file still opens.
456 f, err := openOwnEnvNoFollow(target, os.O_RDONLY, 0)
457 if err != nil {
458 t.Fatal(err)
459 }
460 f.Close()
461}
462
463// setInstanceKeySeams swaps the append and the before-open hook for one
464// test, under adminTokenMu as their readers require. A nil argument
465// leaves that seam as it is.
466func setInstanceKeySeams(t *testing.T, write func(*os.File, []byte) (int, error), beforeOpen func(string)) {
467 t.Helper()
468 adminTokenMu.Lock()
469 oldWrite, oldHook := instanceKeyWrite, instanceKeyHooks.beforeOpen
470 if write != nil {
471 instanceKeyWrite = write
472 }
473 if beforeOpen != nil {
474 instanceKeyHooks.beforeOpen = beforeOpen
475 }
476 adminTokenMu.Unlock()
477 t.Cleanup(func() {
478 adminTokenMu.Lock()
479 instanceKeyWrite, instanceKeyHooks.beforeOpen = oldWrite, oldHook
480 adminTokenMu.Unlock()
481 })
482}
483
484// A mint whose append fails part way (a full disk) is undone: the file is
485// byte for byte what it was, so no short key line survives to be read as
486// present, and the next call mints a full key.
487func TestEnsureInstanceKeyUndoesAPartialAppend(t *testing.T) {
488 for _, tc := range []struct {
489 name string
490 before *string // nil: no file before the call
491 write func(f *os.File, b []byte) (int, error)
492 }{
493 {"nothing written, ENOSPC", ptr("A=1\n"), func(*os.File, []byte) (int, error) { return 0, syscall.ENOSPC }},
494 {"short key line, ENOSPC", ptr("A=1\n"), func(f *os.File, b []byte) (int, error) {
495 n, _ := f.Write(b[:len(InstanceKeyVar)+5])
496 return n, syscall.ENOSPC
497 }},
498 {"short after the fresh-line prefix, ENOSPC", ptr("A=1"), func(f *os.File, b []byte) (int, error) {
499 n, _ := f.Write(b[:3])
500 return n, syscall.ENOSPC
501 }},
502 {"short with no error", ptr("A=1\n"), func(f *os.File, b []byte) (int, error) {
503 return f.Write(b[:len(b)-1])
504 }},
505 {"new file, short key line", nil, func(f *os.File, b []byte) (int, error) {
506 n, _ := f.Write(b[:10])
507 return n, syscall.ENOSPC
508 }},
509 } {
510 t.Run(tc.name, func(t *testing.T) {
511 probe := probeInstanceKeySync(t)
512 p := filepath.Join(t.TempDir(), "shop.example.com.env")
513 if tc.before != nil {
514 if err := os.WriteFile(p, []byte(*tc.before), 0o600); err != nil {
515 t.Fatal(err)
516 }
517 }
518 setInstanceKeySeams(t, tc.write, nil)
519 if _, err := ensureInstanceKeyNoConfig(p); err == nil {
520 t.Fatal("a mint whose append failed reported success")
521 }
522 raw, err := os.ReadFile(p)
523 if err != nil {
524 t.Fatal(err)
525 }
526 want := ""
527 if tc.before != nil {
528 want = *tc.before
529 }
530 if string(raw) != want {
531 t.Fatalf("after the failed append the file is %q, want it undone to %q", raw, want)
532 }
533 if probe.file != 1 {
534 t.Fatalf("the undo synced the file %d times, want once", probe.file)
535 }
536 setInstanceKeySeams(t, func(f *os.File, b []byte) (int, error) { return f.Write(b) }, nil)
537 got, err := ensureInstanceKeyNoConfig(p)
538 if err != nil || got != InstanceKeyMinted {
539 t.Fatalf("next ensure = %v, %v; want minted", got, err)
540 }
541 raw, _ = os.ReadFile(p)
542 keys := keyLines(raw)
543 if len(keys) != 1 || !wellFormedInstanceKey(keys[0]) {
544 t.Fatalf("file = %q: want one full key", raw)
545 }
546 if !strings.HasPrefix(string(raw), want) {
547 t.Fatalf("file = %q: the original bytes were lost", raw)
548 }
549 })
550 }
551}
552
553func ptr(s string) *string { return &s }
554
555// The append open refuses a link on its own, not only because the read
556// before it did: a link swapped in after a clean read (the hook) is
557// refused on the mint, the existing-key and the backfill paths, and its
558// target is left exactly as it was.
559func TestEnsureInstanceKeyOpenRefusesALinkSwappedInAfterTheRead(t *testing.T) {
560 wellFormed := InstanceKeyVar + "=" + strings.Repeat("A", 43) + "=\n"
561 for _, tc := range []struct {
562 name string
563 own string
564 call func(b *ExecBackend, rt registry.Route, path string) error
565 }{
566 {"mint", "A=1\n", func(_ *ExecBackend, _ registry.Route, path string) error {
567 _, err := ensureInstanceKeyNoConfig(path)
568 return err
569 }},
570 {"existing key", wellFormed, func(_ *ExecBackend, _ registry.Route, path string) error {
571 _, err := ensureInstanceKeyNoConfig(path)
572 return err
573 }},
574 {"backfill mint", "A=1\n", func(b *ExecBackend, rt registry.Route, _ string) error {
575 _, err := b.BackfillInstanceKey(t.Context(), rt)
576 return err
577 }},
578 } {
579 t.Run(tc.name, func(t *testing.T) {
580 b := &ExecBackend{DataDir: t.TempDir()}
581 rt := registry.Route{Host: "shop.example.com", Socket: "/run/carlos/shop.sock"}
582 path := filepath.Join(b.DataDir, rt.Host+".env")
583 if err := os.WriteFile(path, []byte(tc.own), 0o600); err != nil {
584 t.Fatal(err)
585 }
586 target := filepath.Join(t.TempDir(), "someone-elses-file")
587 const targetBody = "B=2\n"
588 if err := os.WriteFile(target, []byte(targetBody), 0o644); err != nil {
589 t.Fatal(err)
590 }
591 if err := os.Chmod(target, 0o644); err != nil {
592 t.Fatal(err)
593 }
594 swapped := 0
595 setInstanceKeySeams(t, nil, func(p string) {
596 if p != path {
597 return
598 }
599 swapped++
600 if err := os.Remove(p); err != nil {
601 t.Error(err)
602 }
603 if err := os.Symlink(target, p); err != nil {
604 t.Error(err)
605 }
606 })
607 err := tc.call(b, rt, path)
608 if swapped != 1 {
609 t.Fatalf("the hook ran %d times, want once", swapped)
610 }
611 if !errors.Is(err, errOwnEnvSymlink) {
612 t.Fatalf("err = %v, want errOwnEnvSymlink", err)
613 }
614 st, err := os.Stat(target)
615 if err != nil {
616 t.Fatal(err)
617 }
618 if st.Mode().Perm() != 0o644 {
619 t.Fatalf("mode of the symlink's TARGET = %o, want the untouched 644", st.Mode().Perm())
620 }
621 if raw, _ := os.ReadFile(target); string(raw) != targetBody {
622 t.Fatalf("the symlink's TARGET changed: %q", raw)
623 }
624 })
625 }
626}
627
628// A refusal names the path once.
629func TestEnsureInstanceKeySymlinkErrorNamesThePathOnce(t *testing.T) {
630 dir := t.TempDir()
631 target := filepath.Join(dir, "target")
632 if err := os.WriteFile(target, []byte("A=1\n"), 0o644); err != nil {
633 t.Fatal(err)
634 }
635 link := filepath.Join(dir, "shop.example.com.env")
636 if err := os.Symlink(target, link); err != nil {
637 t.Skipf("this filesystem will not make symlinks: %v", err)
638 }
639 _, err := ensureInstanceKeyNoConfig(link)
640 if !errors.Is(err, errOwnEnvSymlink) {
641 t.Fatalf("err = %v, want errOwnEnvSymlink", err)
642 }
643 if n := strings.Count(err.Error(), link); n != 1 {
644 t.Fatalf("error %q names the path %d times, want once", err, n)
645 }
646}
647
648// startRecorded starts host on b with the recording binary and returns the
649// environment the child saw, or nil when no child ran (none recorded within a
650// second, or none is left running that could). The route's socket is a temp
651// path, so the row classifies as foreign; startRecordedRoute takes the row as
652// given.
653func startRecorded(t *testing.T, b *ExecBackend, host string) (map[string]string, error) {
654 t.Helper()
655 return startRecordedRoute(t, b, registry.Route{Host: host, Kind: "instance", Socket: tmpSock(t)})
656}
657
658func startRecordedRoute(t *testing.T, b *ExecBackend, rt registry.Route) (map[string]string, error) {
659 t.Helper()
660 _, envPath := stageRecordingLiveFullEnv(t, b, rt.Host, "v1")
661 ctx, cancel := recordedCtx(2*time.Second, envPath)
662 defer cancel()
663 proc, _, err := b.Start(ctx, rt) // the fake binary never binds: the health wait fails, as in the sibling tests
664 if proc != nil {
665 defer proc.Stop(context.Background())
666 }
667 raw := awaitChildFile(envPath, time.Second)
668 if raw == "" {
669 return nil, err
670 }
671 child := map[string]string{}
672 for _, line := range strings.Split(raw, "\n") {
673 if k, v, ok := strings.Cut(line, "="); ok {
674 child[k] = v
675 }
676 }
677 return child, err
678}
679
680// Start hands the child the key it minted, and a second Start (a restart, a
681// redeploy) hands it the same one.
682func TestExecBackendStartHandsTheChildTheMintedInstanceKey(t *testing.T) {
683 b := testExecBackend(t)
684 host := "shop.example.com"
685 first, _ := startRecorded(t, b, host)
686 minted, err := readEnvFile(filepath.Join(b.DataDir, host+".env"))
687 if err != nil {
688 t.Fatal(err)
689 }
690 if first == nil || !wellFormedInstanceKey(minted[InstanceKeyVar]) || first[InstanceKeyVar] != minted[InstanceKeyVar] {
691 t.Fatal("the child was not handed the key minted into <host>.env")
692 }
693 if second, _ := startRecorded(t, b, host); second == nil || second[InstanceKeyVar] != minted[InstanceKeyVar] {
694 t.Fatal("a second start handed the child a different key")
695 }
696}
697
698// Task 3 review (M4): the tests above launch through the foreign arm, since
699// a temp socket is neither default. A row that classifies as flat goes
700// through the same spawn body, and this pins it end to end: Start mints
701// into <DataDir>/<host>.env and hands the child that key.
702func TestExecBackendStartHandsAFlatRowsChildTheMintedInstanceKey(t *testing.T) {
703 b := testExecBackend(t)
704 b.RunDirOverride = shortRunDir(t)
705 host := "shop.example.com"
706 rt := flatGenRoute(b, host)
707 if l, why := b.Layout(rt); l != LayoutFlat {
708 t.Fatalf("fixture classifies %q (%s), want flat", l, why)
709 }
710 child, _ := startRecordedRoute(t, b, rt)
711 minted, err := readEnvFile(filepath.Join(b.DataDir, host+".env"))
712 if err != nil {
713 t.Fatal(err)
714 }
715 if child == nil || !wellFormedInstanceKey(minted[InstanceKeyVar]) || child[InstanceKeyVar] != minted[InstanceKeyVar] {
716 t.Fatal("a flat row's child was not handed the key minted into <host>.env")
717 }
718}
719
720// Finding 2 through a launch: while the key's sync fails, Start fails and
721// no child runs. Once the sync succeeds, the child is handed the key the
722// failed attempt wrote, never a second one.
723func TestExecBackendStartStartsNoChildUntilTheKeyIsSynced(t *testing.T) {
724 for _, tc := range []struct {
725 name string
726 fail func(*syncProbe, error)
727 }{
728 {"file sync", func(p *syncProbe, err error) { p.fileErr = err }},
729 {"directory sync", func(p *syncProbe, err error) { p.dirErr = err }},
730 } {
731 t.Run(tc.name, func(t *testing.T) {
732 probe := probeInstanceKeySync(t)
733 b := testExecBackend(t)
734 host := "shop.example.com"
735 tc.fail(probe, errors.New("injected: EIO"))
736 for i := 0; i < 2; i++ {
737 child, err := startRecorded(t, b, host)
738 if err == nil || !strings.Contains(err.Error(), "instance key for "+host) {
739 t.Fatalf("start %d with a failing sync: err = %v, want the instance key's error", i, err)
740 }
741 if child != nil {
742 t.Fatalf("start %d ran a child before the key was durable", i)
743 }
744 }
745 written, _ := readEnvFile(filepath.Join(b.DataDir, host+".env"))
746 tc.fail(probe, nil)
747 child, _ := startRecorded(t, b, host)
748 if child == nil || child[InstanceKeyVar] == "" || child[InstanceKeyVar] != written[InstanceKeyVar] {
749 t.Fatal("after the sync succeeded the child was not handed the key the failed attempt wrote")
750 }
751 })
752 }
753}
754
755// Finding 4 through a launch: <host>.env and <host>.config.env together.
756// An empty override never hands the child an empty key while the
757// configuration supplies one, and no key is minted over the configured one.
758// With neither supplying one, the child gets the minted key.
759func TestExecBackendStartNeverHandsTheChildAnEmptyKeyWhileOneExists(t *testing.T) {
760 tok := "CARLOS_ADMIN_TOKEN=" + strings.Repeat("c", 64) + "\n"
761 for _, tc := range []struct {
762 name, own, config string
763 configured bool
764 }{
765 {"empty override, configured key", tok + InstanceKeyVar + "=\n", InstanceKeyVar + "=from-config\n", true},
766 {"no override, configured key", tok, InstanceKeyVar + "=from-config\n", true},
767 {"empty override, empty configuration", tok + InstanceKeyVar + "=\n", InstanceKeyVar + "=\n", false},
768 } {
769 t.Run(tc.name, func(t *testing.T) {
770 b := testExecBackend(t)
771 host := "shop.example.com"
772 own := filepath.Join(b.DataDir, host+".env")
773 if err := os.WriteFile(own, []byte(tc.own), 0o600); err != nil {
774 t.Fatal(err)
775 }
776 if err := os.WriteFile(filepath.Join(b.DataDir, host+".config.env"), []byte(tc.config), 0o600); err != nil {
777 t.Fatal(err)
778 }
779 child, _ := startRecorded(t, b, host)
780 if child == nil {
781 t.Fatal("no child ran")
782 }
783 raw, _ := os.ReadFile(own)
784 if tc.configured {
785 if child[InstanceKeyVar] != "from-config" {
786 t.Fatalf("the child was handed %q, want the configured key", child[InstanceKeyVar])
787 }
788 if string(raw) != tc.own {
789 t.Fatal("a key was minted over the configured one")
790 }
791 return
792 }
793 env, _ := readEnvFile(own)
794 if !wellFormedInstanceKey(child[InstanceKeyVar]) || child[InstanceKeyVar] != env[InstanceKeyVar] {
795 t.Fatal("the child was not handed the minted key")
796 }
797 })
798 }
799}
800
801// Coverage (Astra round 1): the first process of a fresh flat instance is
802// often its seed. The seed mints the key, sees it, and the first start
803// hands the instance the same one.
804func TestExecBackendSeedMintsTheKeyForAFreshFlatInstance(t *testing.T) {
805 b := testExecBackend(t)
806 host := "seeded.example.com"
807 _, envPath := stageRecordingLiveFullEnv(t, b, host, "v1")
808 if err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`}); err != nil {
809 t.Fatalf("Seed: %v", err)
810 }
811 seed := map[string]string{}
812 for _, line := range strings.Split(waitForFile(t, envPath, time.Second), "\n") {
813 if k, v, ok := strings.Cut(line, "="); ok {
814 seed[k] = v
815 }
816 }
817 own, _ := readEnvFile(filepath.Join(b.DataDir, host+".env"))
818 if !wellFormedInstanceKey(own[InstanceKeyVar]) || seed[InstanceKeyVar] != own[InstanceKeyVar] {
819 t.Fatal("the seed did not mint the key and see it")
820 }
821 if child, _ := startRecorded(t, b, host); child == nil || child[InstanceKeyVar] != own[InstanceKeyVar] {
822 t.Fatal("the first start handed the instance a different key from its seed's")
823 }
824}
825
826// Coverage (Astra round 1): a hibernation wake restores the database and
827// leaves <host>.env alone, so the start after the restore hands the
828// instance the key it had.
829func TestHibernationRestoreKeepsTheInstanceKey(t *testing.T) {
830 b := testExecBackend(t)
831 host := "sleepy.example.com"
832 if child, _ := startRecorded(t, b, host); child == nil {
833 t.Fatal("no child ran")
834 }
835 envPath := filepath.Join(b.DataDir, host+".env")
836 before, _ := os.ReadFile(envPath)
837 key := parseEnvBytes(before)[InstanceKeyVar]
838 if !wellFormedInstanceKey(key) {
839 t.Fatal("the first start minted no key")
840 }
841 old := restoreReplica
842 restoreReplica = func(_ context.Context, dbPath, bucket, prefix, region string) (bool, error) {
843 return true, writeMinimalSQLite(dbPath)
844 }
845 t.Cleanup(func() { restoreReplica = old })
846 _ = os.Remove(filepath.Join(b.DataDir, host+".db")) // hibernated: the database is gone
847 rt := registry.Route{Host: host, Kind: "instance", Socket: tmpSock(t)}
848 if fresh, err := b.Restore(context.Background(), rt); err != nil || fresh {
849 t.Fatalf("Restore = fresh %v, %v; want a restored database", fresh, err)
850 }
851 if after, _ := os.ReadFile(envPath); !bytes.Equal(before, after) {
852 t.Fatal("the restore changed <host>.env")
853 }
854 if child, _ := startRecorded(t, b, host); child == nil || child[InstanceKeyVar] != key {
855 t.Fatal("the start after a restore handed the instance a different key")
856 }
857}
858
859// refuseAfterFirstClassification makes b classify a row as it does today
860// the first time it is asked and as layout (fenced or conflict) every time
861// after: a launch dispatcher's own check passes, and the launch body's
862// pathsFor sees a layout journal or a colliding peer that appeared since.
863func refuseAfterFirstClassification(b *ExecBackend, layout Layout) {
864 var calls atomic.Int32
865 switch layout {
866 case LayoutFenced:
867 b.LayoutFacts.Journal = func(string) bool { return calls.Add(1) > 1 }
868 case LayoutConflict:
869 b.LayoutFacts.SameID = func(a, b string) bool { return true }
870 b.LayoutFacts.Peers = func() ([]registry.Route, error) {
871 if calls.Add(1) == 1 {
872 return nil, nil
873 }
874 return []registry.Route{{Host: "other.example.com", Kind: "instance", Hibernate: true, Socket: "/nonexistent/other.sock"}}, nil
875 }
876 }
877}
878
879// Review (Task 2): the mint refuses a fenced or conflicting row on the
880// paths the launch body hands it, not only on the dispatcher's earlier
881// classification. A row that becomes fenced or conflicting between Start's
882// or Seed's check and the body's own pathsFor fails the launch with the
883// key's error, gets no key, and runs no child.
884func TestLaunchBodiesMintNoKeyForARowRefusedSinceTheDispatch(t *testing.T) {
885 for _, layout := range []Layout{LayoutFenced, LayoutConflict} {
886 for _, launch := range []string{"start", "seed"} {
887 t.Run(string(layout)+"/"+launch, func(t *testing.T) {
888 b := testExecBackend(t)
889 host := "shop.example.com"
890 _, envPath := stageRecordingLiveFullEnv(t, b, host, "v1")
891 refuseAfterFirstClassification(b, layout)
892 rt := registry.Route{Host: host, Kind: "instance", Socket: tmpSock(t), Seed: `["seed"]`}
893 ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
894 defer cancel()
895 var err error
896 if launch == "start" {
897 var proc Proc
898 proc, _, err = b.Start(ctx, rt)
899 if proc != nil {
900 defer proc.Stop(context.Background())
901 }
902 } else {
903 err = b.Seed(ctx, rt)
904 }
905 if err == nil || !strings.Contains(err.Error(), "instance key for "+host) || !strings.Contains(err.Error(), "not minted") {
906 t.Fatalf("%s of a row %s since the dispatch: err = %v, want the key's refusal", launch, layout, err)
907 }
908 // The host is named once, by the wrapper. (A fenced reason
909 // also names it inside the journal path, so the count is
910 // of the "<host>: " prefix.)
911 if n := strings.Count(err.Error(), host+": "); n != 1 {
912 t.Fatalf("the refusal names the host %d times: %v", n, err)
913 }
914 raw, _ := os.ReadFile(filepath.Join(b.DataDir, host+".env"))
915 if n := len(keyLines(raw)); n != 0 {
916 t.Fatalf("a %s row was minted %d key(s)", layout, n)
917 }
918 if _, serr := os.Stat(envPath); !errors.Is(serr, os.ErrNotExist) {
919 t.Fatalf("a child ran for a %s row (stat %v)", layout, serr)
920 }
921 })
922 }
923 }
924}
925
926// The refusal itself, for every layout pathsFor can report it on: a flat
927// row and a nested one, each fenced and conflicting. Nothing is written in
928// the data directory or the instance directory. (A nested row's prepare
929// cannot reach the mint in a test once fenced: its fenced paths name
930// /run/carlos, which prepare creates first. prepareAt hands the mint its
931// paths unchanged, so this is the refusal it gets.)
932func TestEnsureInstanceKeyForRefusesFencedAndConflictRows(t *testing.T) {
933 for _, nested := range []bool{false, true} {
934 for _, layout := range []Layout{LayoutFenced, LayoutConflict} {
935 name := "flat"
936 if nested {
937 name = "nested"
938 }
939 t.Run(name+"/"+string(layout), func(t *testing.T) {
940 var b *ExecBackend
941 var rt registry.Route
942 if nested {
943 b, rt = nestedBackend(t, &fakeControl{inv: "inv1"}, &fakeUnit{})
944 } else {
945 b = testExecBackend(t)
946 rt = registry.Route{Host: "shop.example.com", Kind: "instance", Hibernate: true,
947 DBPath: provision.DefaultDBPath(b.DataDir, "shop.example.com"), Socket: filepath.Join(provision.SocketDir, "shop.example.com.sock")}
948 }
949 if l, _ := b.Layout(rt); (nested && l != LayoutNested) || (!nested && l != LayoutFlat) {
950 t.Fatalf("fixture classifies as %s", l)
951 }
952 refuseAfterFirstClassification(b, layout)
953 _, _ = b.Layout(rt) // the dispatcher's check
954 p := b.pathsFor(rt)
955 if p.layout != layout {
956 t.Fatalf("paths classify as %s, want %s", p.layout, layout)
957 }
958 if _, err := b.ensureInstanceKeyFor(t.Context(), rt, p); err == nil || !strings.Contains(err.Error(), "not minted") {
959 t.Fatalf("ensure for a %s row: err = %v, want a refusal", layout, err)
960 } else if strings.HasPrefix(err.Error(), rt.Host) {
961 t.Fatalf("the refusal names the host itself (%v); its callers do", err)
962 }
963 for _, f := range []string{
964 filepath.Join(b.DataDir, rt.Host+".env"),
965 filepath.Join(provision.InstanceDir(b.DataDir, rt.Host), rt.Host+".env"),
966 } {
967 if _, err := os.Lstat(f); !errors.Is(err, os.ErrNotExist) {
968 t.Fatalf("%s exists after a refused ensure (lstat %v)", f, err)
969 }
970 }
971 })
972 }
973 }
974}
975
976// The backfill refuses the same rows (instancekey.go BackfillInstanceKey):
977// an existing env file without a key is left byte for byte.
978func TestBackfillInstanceKeyRefusesFencedAndConflictRows(t *testing.T) {
979 for _, layout := range []Layout{LayoutFenced, LayoutConflict} {
980 t.Run(string(layout), func(t *testing.T) {
981 b := testExecBackend(t)
982 host := "shop.example.com"
983 rt := registry.Route{Host: host, Kind: "instance", Hibernate: true,
984 DBPath: provision.DefaultDBPath(b.DataDir, host), Socket: filepath.Join(provision.SocketDir, host+".sock")}
985 own := filepath.Join(b.DataDir, host+".env")
986 tok := "CARLOS_ADMIN_TOKEN=" + strings.Repeat("d", 64) + "\n"
987 if err := os.WriteFile(own, []byte(tok), 0o600); err != nil {
988 t.Fatal(err)
989 }
990 refuseAfterFirstClassification(b, layout)
991 _, _ = b.Layout(rt)
992 if _, err := b.BackfillInstanceKey(t.Context(), rt); err == nil || !strings.Contains(err.Error(), "not backfilled") {
993 t.Fatalf("backfill of a %s row: err = %v, want a refusal", layout, err)
994 } else if strings.HasPrefix(err.Error(), host) {
995 t.Fatalf("the refusal names the host itself (%v); the agent's backfill does", err)
996 }
997 if raw, _ := os.ReadFile(own); string(raw) != tok {
998 t.Fatalf("the backfill changed a %s row's env file", layout)
999 }
1000 })
1001 }
1002}
1003
1004// v0.26.1 hotfix: the agent's backfill now runs beside the activator
1005// rather than before it, so a backfill and a start's own mint can reach
1006// the same host at once. Both read the files and ask a slow bucket outside
1007// adminTokenMu, and both are held there until each has done so, so each
1008// decides from a first read that found no key. They still decide and
1009// append under the lock, create-only, so the host ends with exactly one
1010// key, appended whole after the bytes that were there, and exactly one of
1011// the two reports it minted.
1012func TestBackfillAndAStartsMintRacingOnOneHostWriteOneKey(t *testing.T) {
1013 const rounds = 40
1014 tok := "CARLOS_ADMIN_TOKEN=" + strings.Repeat("b", 64) + "\n"
1015 for i := range rounds {
1016 b := &ExecBackend{DataDir: t.TempDir()}
1017 b.ConfiguredKey = func(ctx context.Context, _ registry.Route) (string, error) {
1018 select {
1019 case <-time.After(5 * time.Millisecond): // a slow bucket that sets no key
1020 return "", nil
1021 case <-ctx.Done():
1022 return "", ctx.Err()
1023 }
1024 }
1025 rt := registry.Route{Host: "shop.example.com", Kind: "instance", Status: "ready", Socket: "/run/carlos/shop.sock"}
1026 own := filepath.Join(b.DataDir, rt.Host+".env")
1027 if err := os.WriteFile(own, []byte(tok), 0o600); err != nil {
1028 t.Fatal(err)
1029 }
1030 // Both callers pass phase one before either takes the lock.
1031 var phase sync.WaitGroup
1032 phase.Add(2)
1033 instanceKeyHooks.betweenPhases = func(string) { phase.Done(); phase.Wait() }
1034 var wg sync.WaitGroup
1035 var backfilled, started InstanceKeyOutcome
1036 var berr, serr error
1037 wg.Add(2)
1038 go func() {
1039 defer wg.Done()
1040 backfilled, berr = b.BackfillInstanceKey(t.Context(), rt)
1041 }()
1042 go func() {
1043 defer wg.Done()
1044 started, serr = b.ensureInstanceKeyFor(t.Context(), rt, b.pathsFor(rt))
1045 }()
1046 wg.Wait()
1047 instanceKeyHooks.betweenPhases = nil
1048 if berr != nil || serr != nil {
1049 t.Fatalf("round %d: backfill %v, %v; start %v, %v", i, backfilled, berr, started, serr)
1050 }
1051 minted := 0
1052 for _, o := range []InstanceKeyOutcome{backfilled, started} {
1053 switch o {
1054 case InstanceKeyMinted:
1055 minted++
1056 case InstanceKeyPresent:
1057 default:
1058 t.Fatalf("round %d: outcome %v, want minted or present", i, o)
1059 }
1060 }
1061 raw, _ := os.ReadFile(own)
1062 keys := keyLines(raw)
1063 if minted != 1 || len(keys) != 1 || !wellFormedInstanceKey(keys[0]) || string(raw) != tok+InstanceKeyVar+"="+keys[0]+"\n" {
1064 t.Fatalf("round %d: %d minted, %d keys, file %d bytes; want one whole key appended once", i, minted, len(keys), len(raw))
1065 }
1066 }
1067}
1068