amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1//go:build linux
2
3package hostbox
4
5import (
6 "bytes"
7 "context"
8 "errors"
9 "os"
10 "path/filepath"
11 "testing"
12 "time"
13
14 "golang.org/x/sys/unix"
15
16 "github.com/carlosframework/platform/internal/hostclaim"
17 "github.com/carlosframework/platform/internal/hostguard"
18 "github.com/carlosframework/platform/internal/provision"
19 "github.com/carlosframework/platform/internal/registry"
20)
21
22// intendedFixture is a prior installation whose PrepareState persisted the
23// state-dir intent and then failed or crashed before its final inventory write.
24type intendedFixture struct {
25 handover ColdHandover
26 prior hostguard.FenceTarget
27 next ColdSuccessor
28 inventory *StopInventoryDB
29 reg *registry.Registry
30 data string
31 stateRel string
32 state string
33 id string
34}
35
36func newIntendedFixture(t *testing.T, edit func(*StopInventoryEntry)) intendedFixture {
37 t.Helper()
38 t.Setenv("CARLOS_HOST_CLAIM", "1")
39 source, store, initial := workFixture(t)
40 ctx := t.Context()
41 admit := func(old hostclaim.Observed, generation string) hostclaim.Observed {
42 next, err := hostclaim.Update(ctx, store, old, hostclaim.AdmitWriter(hostclaim.Admission{ObservedOpID: old.Claim.Op.ID, ObservedPhase: old.Claim.Op.Phase, ObservedSession: old.Claim.Holder.Session, ObservedGeneration: old.Claim.Holder.Generation, Box: "box", Region: "ie", Mode: "cold", NewGeneration: generation}))
43 if err != nil {
44 t.Fatal(err)
45 }
46 return next
47 }
48 old := admit(initial, "first")
49 if err := source.Guard.Activate(ctx, old.Host, "box", nil); err != nil {
50 t.Fatal(err)
51 }
52 prior := hostguard.FenceTarget{FenceID: "fence", Region: "ie", Host: old.Host, Incarnation: old.Claim.Holder.Incarnation, Stamp: old.Claim.Holder.Stamp, Session: old.Claim.Holder.Session, Box: "box", Enrollment: "enrollment", BootID: "boot", ControllerID: "controller", ClaimETag: old.ETag, DirectoryRevision: "1", Op: old.Claim.Op.ID, Phase: old.Claim.Op.Phase}
53 if _, err := source.Guard.BeginFence(ctx, prior); err != nil {
54 t.Fatal(err)
55 }
56 if err := source.Guard.CompleteFence(ctx, prior, []byte("verified-root-receipt"), time.Now().UTC(), func(context.Context, hostguard.FenceTarget, []byte, time.Time) error { return nil }); err != nil {
57 t.Fatal(err)
58 }
59 base := t.TempDir()
60 reg, err := registry.Open(filepath.Join(base, "registry.db"))
61 if err != nil {
62 t.Fatal(err)
63 }
64 t.Cleanup(func() { reg.Close() })
65 inventory, err := OpenStopInventoryDB(filepath.Join(base, "inventory.db"))
66 if err != nil {
67 t.Fatal(err)
68 }
69 t.Cleanup(func() { inventory.Close() })
70 launches, err := OpenLaunchJournal(filepath.Join(base, "launches.db"))
71 if err != nil {
72 t.Fatal(err)
73 }
74 t.Cleanup(func() { launches.Close() })
75 owner := registry.ClaimOwnership{ClaimIncarnation: prior.Incarnation, ClaimStamp: prior.Stamp, ClaimOwner: "old-owner", ClaimRowID: "old-route"}
76 data := source.Guard.DataDir
77 dbPath := provision.NestedDBPath(data, old.Host)
78 statePath := filepath.Dir(dbPath)
79 if err = os.MkdirAll(filepath.Dir(statePath), 0700); err != nil {
80 t.Fatal(err)
81 }
82 stateRel, err := filepath.Rel(data, statePath)
83 if err != nil {
84 t.Fatal(err)
85 }
86 if err = reg.Upsert(registry.Route{Host: old.Host, Kind: "instance", Socket: provision.NestedSocket(old.Host), DBPath: dbPath, ClaimOwnership: owner}); err != nil {
87 t.Fatal(err)
88 }
89 if err = reg.UpsertTimer(registry.Timer{Host: old.Host, Name: "daily", Kind: "cron", Spec: "0 0 * * *", NextAt: 123, Attempts: 2}); err != nil {
90 t.Fatal(err)
91 }
92 // The intent PrepareState writes before Mkdirat: typed, no identity.
93 state := StopInventoryEntry{Kind: "path", ArtifactID: "state-dir", ManagedRoot: "data", RelativePath: stateRel, Ownership: owner.ClaimOwner, Materialization: "intended", Type: unix.S_IFDIR}
94 if edit != nil {
95 edit(&state)
96 }
97 in := StopInventory{Complete: true, Host: old.Host, Incarnation: prior.Incarnation, InstallationStamp: prior.Stamp, Box: "box", Enrollment: "enrollment", DirectoryRevision: "1", Entries: []StopInventoryEntry{
98 {Kind: "route", ArtifactID: owner.ClaimRowID, ManagedRoot: "registry", RelativePath: old.Host, RowIdentity: owner.ClaimRowID, Ownership: owner.ClaimOwner, Materialization: "materialized"},
99 state,
100 }}
101 if err = inventory.WriteStopInventory(ctx, in); err != nil {
102 t.Fatal(err)
103 }
104 successor := admit(old, "second")
105 next := ColdSuccessor{Host: old.Host, Incarnation: prior.Incarnation, Stamp: successor.Claim.Holder.Stamp, Session: successor.Claim.Holder.Session, Generation: successor.Claim.Holder.Generation, Box: "box", Enrollment: "enrollment", DirectoryRevision: "1", ETag: successor.ETag, Op: successor.Claim.Op.ID, Phase: successor.Claim.Op.Phase}
106 handover := ColdHandover{Guard: source.Guard, Claims: store, Registry: reg, Inventory: inventory, Launches: launches, Roots: map[string]string{"data": data}, JournalDir: filepath.Join(base, "handover"), VerifyReceipt: func(context.Context, hostguard.FenceReceipt) error { return nil }}
107 return intendedFixture{handover: handover, prior: prior, next: next, inventory: inventory, reg: reg, data: data, stateRel: stateRel, state: statePath, id: "handover-" + ownershipFor(next.Host, next.Incarnation, next.Stamp, next.Box)}
108}
109
110func (f intendedFixture) parked(retirement, name string) string {
111 return filepath.Join(f.data, ".parked", f.prior.Incarnation, f.prior.Host, f.id+retirement, "data", name)
112}
113
114func (f intendedFixture) intendedJournal() string {
115 return filepath.Join(f.handover.JournalDir, "retired", f.id+"-intended.json")
116}
117
118func (f intendedFixture) obstructMainRetirement(t *testing.T) string {
119 t.Helper()
120 blocked := filepath.Join(f.handover.JournalDir, "retired", f.id+"-old.json")
121 if err := os.MkdirAll(blocked, 0700); err != nil {
122 t.Fatal(err)
123 }
124 return blocked
125}
126
127func (f intendedFixture) requirePriorRoute(t *testing.T) {
128 t.Helper()
129 if route, err := f.reg.Get(f.prior.Host); err != nil || route == nil || route.ClaimStamp != f.prior.Stamp || route.ClaimOwner != "old-owner" {
130 t.Fatalf("refusal changed prior route: %+v %v", route, err)
131 }
132}
133
134func (f intendedFixture) requireUnmoved(t *testing.T) {
135 t.Helper()
136 f.requirePriorRoute(t)
137 if _, err := os.Lstat(f.intendedJournal()); !errors.Is(err, os.ErrNotExist) {
138 t.Fatalf("refusal recorded an intended capture: %v", err)
139 }
140}
141
142func (f intendedFixture) requireComplete(t *testing.T) {
143 t.Helper()
144 if _, err := os.Lstat(f.state); !errors.Is(err, os.ErrNotExist) {
145 t.Fatalf("intended state dir remains: %v", err)
146 }
147 if route, err := f.reg.Get(f.prior.Host); err != nil || route == nil || route.ClaimStamp != f.next.Stamp {
148 t.Fatalf("route not handed to successor: %+v %v", route, err)
149 }
150}
151
152// Mkdirat failed (ENOSPC/EACCES) after the intent was durable: nothing exists
153// to move, so the next admission's handover must complete.
154func TestColdHandoverResumesAbsentIntendedStateDir(t *testing.T) {
155 for name, typ := range map[string]uint32{"typed": unix.S_IFDIR, "untyped": 0} {
156 t.Run(name, func(t *testing.T) {
157 f := newIntendedFixture(t, func(e *StopInventoryEntry) { e.Type = typ })
158 ctx := t.Context()
159 if err := f.handover.Run(ctx, f.prior, f.next); err != nil {
160 t.Fatalf("absent intended path stranded handover: %v", err)
161 }
162 f.requireComplete(t)
163 if _, err := os.Lstat(f.intendedJournal()); !errors.Is(err, os.ErrNotExist) {
164 t.Fatalf("absent intent captured an artifact: %v", err)
165 }
166 if _, err := os.Lstat(f.parked("-intended", f.stateRel)); !errors.Is(err, os.ErrNotExist) {
167 t.Fatalf("absent intent parked something: %v", err)
168 }
169 if err := f.handover.Run(ctx, f.prior, f.next); err != nil {
170 t.Fatal(err)
171 }
172 if err := f.handover.Recover(ctx); err != nil {
173 t.Fatal(err)
174 }
175 f.requireComplete(t)
176 })
177 }
178}
179
180// Crash after Mkdirat and before the materialized write: the empty directory
181// is parked by its captured inode, never unlinked, and replay never recaptures.
182func TestColdHandoverParksTypedEmptyIntendedStateDirAndReplays(t *testing.T) {
183 f := newIntendedFixture(t, nil)
184 ctx := t.Context()
185 if err := os.Mkdir(f.state, 0700); err != nil {
186 t.Fatal(err)
187 }
188 created, err := os.Lstat(f.state)
189 if err != nil {
190 t.Fatal(err)
191 }
192 // Fail the main retirement after the intended park, as an interrupted run.
193 blocked := f.obstructMainRetirement(t)
194 if err = f.handover.Run(ctx, f.prior, f.next); err == nil {
195 t.Fatalf("obstructed main retirement: %v", err)
196 }
197 f.requirePriorRoute(t)
198 parked := f.parked("-intended", f.stateRel)
199 got, err := os.Lstat(parked)
200 if err != nil || !os.SameFile(created, got) {
201 t.Fatalf("intended dir not parked by inode: %v", err)
202 }
203 if _, err = os.Lstat(f.state); !errors.Is(err, os.ErrNotExist) {
204 t.Fatalf("intended dir still at source: %v", err)
205 }
206 body, err := os.ReadFile(f.intendedJournal())
207 if err != nil {
208 t.Fatal(err)
209 }
210 var journal RetirementJournal
211 if err = decodeStrict(body, &journal); err != nil || !journal.Complete || len(journal.Inventory.Artifacts) != 1 || journal.Target == nil || *journal.Target != f.prior {
212 t.Fatalf("intended capture not journalled under the exact prior: %+v %v", journal, err)
213 }
214
215 // A same-named directory appearing after the durable decision is unknown.
216 if err = os.Mkdir(f.state, 0700); err != nil {
217 t.Fatal(err)
218 }
219 if err = os.Remove(blocked); err != nil {
220 t.Fatal(err)
221 }
222 if err = f.handover.Run(ctx, f.prior, f.next); !errors.Is(err, ErrChanged) {
223 t.Fatalf("late same-named directory accepted: %v", err)
224 }
225 if _, err = os.Lstat(f.state); err != nil {
226 t.Fatalf("late directory touched: %v", err)
227 }
228 if err = os.Remove(f.state); err != nil {
229 t.Fatal(err)
230 }
231
232 if err = f.handover.Recover(ctx); err != nil {
233 t.Fatal(err)
234 }
235 f.requireComplete(t)
236 if got, err = os.Lstat(parked); err != nil || !os.SameFile(created, got) {
237 t.Fatalf("parked intended dir not preserved: %v", err)
238 }
239 if again, err := os.ReadFile(f.intendedJournal()); err != nil || !bytes.Equal(again, body) {
240 t.Fatalf("replay rewrote intended capture: %v", err)
241 }
242 mainBody, err := os.ReadFile(filepath.Join(f.handover.JournalDir, "retired", f.id+"-old.json"))
243 if err != nil {
244 t.Fatal(err)
245 }
246 var rows RetirementJournal
247 if err = decodeStrict(mainBody, &rows); err != nil || !rows.Complete || len(rows.Archives) == 0 {
248 t.Fatalf("row archives missing: %v", err)
249 }
250
251 for range 2 {
252 if err = f.handover.Run(ctx, f.prior, f.next); err != nil {
253 t.Fatal(err)
254 }
255 if err = f.handover.Recover(ctx); err != nil {
256 t.Fatal(err)
257 }
258 }
259 if got, err = os.Lstat(parked); err != nil || !os.SameFile(created, got) {
260 t.Fatalf("replay disturbed parked intended dir: %v", err)
261 }
262 if again, err := os.ReadFile(filepath.Join(f.handover.JournalDir, "retired", f.id+"-old.json")); err != nil || !bytes.Equal(again, mainBody) {
263 t.Fatalf("replay rewrote row archives: %v", err)
264 }
265}
266
267// Ownership of a present intended name is only provable for an empty
268// directory of the recorded type with no successor claim on it. Every other
269// shape refuses before any artifact moves, and keeps refusing.
270func TestColdHandoverRefusesUnprovableIntendedStateDir(t *testing.T) {
271 cases := []struct {
272 name string
273 edit func(*StopInventoryEntry)
274 setup func(*testing.T, intendedFixture)
275 want error
276 check func(*testing.T, intendedFixture)
277 }{
278 {
279 name: "nonempty",
280 setup: func(t *testing.T, f intendedFixture) {
281 if err := os.Mkdir(f.state, 0700); err != nil {
282 t.Fatal(err)
283 }
284 if err := os.WriteFile(filepath.Join(f.state, "data.db"), []byte("unknown"), 0600); err != nil {
285 t.Fatal(err)
286 }
287 },
288 want: ErrChanged,
289 check: func(t *testing.T, f intendedFixture) {
290 if body, err := os.ReadFile(filepath.Join(f.state, "data.db")); err != nil || string(body) != "unknown" {
291 t.Fatalf("nonempty intended dir touched: %v", err)
292 }
293 },
294 },
295 {
296 name: "symlink",
297 setup: func(t *testing.T, f intendedFixture) {
298 if err := os.Mkdir(filepath.Join(f.data, "elsewhere"), 0700); err != nil {
299 t.Fatal(err)
300 }
301 if err := os.Symlink("elsewhere", f.state); err != nil {
302 t.Fatal(err)
303 }
304 },
305 want: ErrChanged,
306 check: func(t *testing.T, f intendedFixture) {
307 if info, err := os.Lstat(f.state); err != nil || info.Mode()&os.ModeSymlink == 0 {
308 t.Fatalf("symlink touched: %v", err)
309 }
310 if _, err := os.Lstat(filepath.Join(f.data, "elsewhere")); err != nil {
311 t.Fatalf("symlink target touched: %v", err)
312 }
313 },
314 },
315 {
316 name: "regular file",
317 setup: func(t *testing.T, f intendedFixture) {
318 if err := os.WriteFile(f.state, []byte("unknown"), 0600); err != nil {
319 t.Fatal(err)
320 }
321 },
322 want: ErrChanged,
323 check: func(t *testing.T, f intendedFixture) {
324 if body, err := os.ReadFile(f.state); err != nil || string(body) != "unknown" {
325 t.Fatalf("regular file touched: %v", err)
326 }
327 },
328 },
329 {
330 name: "untyped intent",
331 edit: func(e *StopInventoryEntry) { e.Type = 0 },
332 want: ErrChanged,
333 },
334 {
335 name: "wrong recorded type",
336 edit: func(e *StopInventoryEntry) { e.Type = unix.S_IFREG },
337 want: ErrChanged,
338 },
339 {
340 name: "intent with identity",
341 edit: func(e *StopInventoryEntry) { e.Device, e.Inode = 1, 1 },
342 want: ErrTypedInventory,
343 },
344 {
345 name: "successor state",
346 setup: func(t *testing.T, f intendedFixture) {
347 if err := os.Mkdir(f.state, 0700); err != nil {
348 t.Fatal(err)
349 }
350 successor := StopInventory{Complete: true, Host: f.next.Host, Incarnation: f.next.Incarnation, InstallationStamp: f.next.Stamp, Box: f.next.Box, Enrollment: f.next.Enrollment, DirectoryRevision: f.next.DirectoryRevision, Entries: []StopInventoryEntry{{Kind: "path", ArtifactID: "successor-state", ManagedRoot: "data", RelativePath: f.stateRel, Ownership: "successor-owner", Materialization: "intended", Type: unix.S_IFDIR}}}
351 if err := f.inventory.WriteStopInventory(t.Context(), successor); err != nil {
352 t.Fatal(err)
353 }
354 },
355 want: ErrChanged,
356 },
357 }
358 for _, tc := range cases {
359 t.Run(tc.name, func(t *testing.T) {
360 f := newIntendedFixture(t, tc.edit)
361 setup := tc.setup
362 if setup == nil {
363 setup = func(t *testing.T, f intendedFixture) {
364 if err := os.Mkdir(f.state, 0700); err != nil {
365 t.Fatal(err)
366 }
367 }
368 }
369 setup(t, f)
370 before, err := os.Lstat(f.state)
371 if err != nil {
372 t.Fatal(err)
373 }
374 for range 2 {
375 if err = f.handover.Run(t.Context(), f.prior, f.next); !errors.Is(err, tc.want) {
376 t.Fatalf("got %v, want %v", err, tc.want)
377 }
378 f.requireUnmoved(t)
379 if after, err := os.Lstat(f.state); err != nil || !os.SameFile(before, after) {
380 t.Fatalf("refused intended name changed: %v", err)
381 }
382 if _, err = os.Lstat(f.parked("-intended", f.stateRel)); !errors.Is(err, os.ErrNotExist) {
383 t.Fatalf("refused intended name parked: %v", err)
384 }
385 }
386 if tc.check != nil {
387 tc.check(t, f)
388 }
389 })
390 }
391}
392
393func TestColdHandoverAbsentIntentionDecisionSurvivesInterruptedRetirement(t *testing.T) {
394 f := newIntendedFixture(t, nil)
395 ctx := t.Context()
396 blocked := f.obstructMainRetirement(t)
397 if err := f.handover.Run(ctx, f.prior, f.next); err == nil {
398 t.Fatalf("first retirement: %v", err)
399 }
400 f.requirePriorRoute(t)
401 if err := os.Remove(blocked); err != nil {
402 t.Fatal(err)
403 }
404 if err := os.Mkdir(f.state, 0700); err != nil {
405 t.Fatal(err)
406 }
407 unknown, err := os.Lstat(f.state)
408 if err != nil {
409 t.Fatal(err)
410 }
411 if err := f.handover.Recover(ctx); !errors.Is(err, ErrChanged) {
412 t.Fatalf("accepted newly appearing intended path: %v", err)
413 }
414 got, err := os.Lstat(f.state)
415 if err != nil || !os.SameFile(unknown, got) {
416 t.Fatalf("unknown path moved: %v", err)
417 }
418 f.requireUnmoved(t)
419 if err := os.Remove(f.state); err != nil {
420 t.Fatal(err)
421 }
422 if err := f.handover.Recover(ctx); err != nil {
423 t.Fatal(err)
424 }
425 f.requireComplete(t)
426}
427
428func TestColdHandoverEmptyDirectoryUsesCapturedIdentity(t *testing.T) {
429 root := t.TempDir()
430 p := TypedParker{Roots: map[string]string{"data": root}, ParkedRoot: root, Host: "host", Incarnation: "inc", Retirement: "retire"}
431 path := filepath.Join(root, "state")
432 if err := os.Mkdir(path, 0700); err != nil {
433 t.Fatal(err)
434 }
435 a, err := p.CaptureArtifact("data", "state", "owner", ArtifactPath)
436 if err != nil {
437 t.Fatal(err)
438 }
439 if err := p.VerifyEmptyDirectory(a); err != nil {
440 t.Fatal(err)
441 }
442 if err := os.Rename(path, filepath.Join(root, "captured")); err != nil {
443 t.Fatal(err)
444 }
445 if err := os.Mkdir(path, 0700); err != nil {
446 t.Fatal(err)
447 }
448 if err := p.VerifyEmptyDirectory(a); !errors.Is(err, ErrChanged) {
449 t.Fatalf("substitute empty directory accepted: %v", err)
450 }
451 if err := os.Remove(path); err != nil {
452 t.Fatal(err)
453 }
454 if err := os.Symlink("captured", path); err != nil {
455 t.Fatal(err)
456 }
457 if err := p.VerifyEmptyDirectory(a); err == nil {
458 t.Fatal("followed symlink to captured directory")
459 }
460 if err := os.Remove(path); err != nil {
461 t.Fatal(err)
462 }
463 if err := os.Rename(filepath.Join(root, "captured"), path); err != nil {
464 t.Fatal(err)
465 }
466 if err := os.WriteFile(filepath.Join(path, "unknown"), nil, 0600); err != nil {
467 t.Fatal(err)
468 }
469 if err := p.VerifyEmptyDirectory(a); !errors.Is(err, ErrChanged) {
470 t.Fatalf("nonempty directory accepted: %v", err)
471 }
472}
473
474func TestClaimReviewHandoverRefusesCompletedSuccessorProducer(t *testing.T) {
475 f := newIntendedFixture(t, nil)
476 target := f.prior
477 target.Stamp = f.next.Stamp
478 target.Session = f.next.Session
479 j := f.handover.Launches
480 if err := j.Prelaunch(t.Context(), LaunchInventory{ControllerPID: 123, ProcessStart: "456", Target: target, Groups: []LaunchGroup{{RelativePath: "host", Device: 1, Inode: 1}}, NoState: true}); err != nil {
481 t.Fatal(err)
482 }
483 if err := j.RecordProducer(t.Context(), target, "delivery"); err != nil {
484 t.Fatal(err)
485 }
486 if _, err := j.CompleteProducer(t.Context(), target, "delivery", ProducerCompletion{}); err != nil {
487 t.Fatal(err)
488 }
489 if err := f.handover.Run(t.Context(), f.prior, f.next); !errors.Is(err, ErrChanged) {
490 t.Fatalf("handover allowed completed successor: %v", err)
491 }
492 f.requireUnmoved(t)
493}
494
495func TestPlan9Fix2ColdHandoverRefusesForeignInventory(t *testing.T) {
496 for _, root := range []string{"opt", "units", "data"} {
497 t.Run(root, func(t *testing.T) {
498 f := newIntendedFixture(t, nil)
499 victim := filepath.Join(f.data, "foreign")
500 if err := os.WriteFile(victim, []byte("root-only"), 0600); err != nil {
501 t.Fatal(err)
502 }
503 f.handover.Roots[root] = f.data
504 parker := TypedParker{Roots: f.handover.Roots, ParkedRoot: f.data, Incarnation: f.prior.Incarnation, Host: f.prior.Host, Retirement: "capture"}
505 a, err := parker.CaptureArtifact(root, "foreign", "old-owner", ArtifactPath)
506 if err != nil {
507 t.Fatal(err)
508 }
509 in, err := f.inventory.ReadExactStopInventory(t.Context(), f.prior)
510 if err != nil {
511 t.Fatal(err)
512 }
513 in.Entries = append(in.Entries, StopInventoryEntry{Kind: "path", ManagedRoot: root, RelativePath: a.Name, ArtifactID: "foreign", Ownership: a.Owner, Device: a.Dev, Inode: a.Ino, Type: a.Type, Materialization: "materialized"})
514 if err = f.inventory.WriteStopInventory(t.Context(), in); err != nil {
515 t.Fatal(err)
516 }
517 if err = f.handover.Run(t.Context(), f.prior, f.next); !errors.Is(err, ErrTypedInventory) {
518 t.Fatalf("foreign inventory: %v", err)
519 }
520 if raw, err := os.ReadFile(victim); err != nil || string(raw) != "root-only" {
521 t.Fatalf("foreign file moved: %v", err)
522 }
523 })
524 }
525}
526