amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1//go:build unix
2
3package hostguard
4
5import (
6 "context"
7 "errors"
8 "fmt"
9 "os"
10 "path/filepath"
11 "reflect"
12 "strings"
13 "testing"
14
15 "github.com/carlosframework/platform/internal/controlname"
16 "github.com/carlosframework/platform/internal/hostclaim"
17 "github.com/carlosframework/platform/internal/release"
18 "golang.org/x/sys/unix"
19)
20
21type readerFunc func(context.Context, string) (hostclaim.Observed, error)
22
23func (f readerFunc) Read(ctx context.Context, host string) (hostclaim.Observed, error) {
24 return f(ctx, host)
25}
26
27type guardFixture struct {
28 g Guard
29 obs hostclaim.Observed
30 err error
31 reads int
32}
33
34func newGuardFixture(t *testing.T) *guardFixture {
35 t.Helper()
36 f := &guardFixture{obs: hostclaim.Observed{
37 Host: "shop.example", ETag: "one",
38 Claim: &hostclaim.Claim{V: 2, Host: "shop.example", Gen: 1,
39 Holder: &hostclaim.Holder{Incarnation: "inc", Stamp: 1},
40 Op: hostclaim.Op{Kind: hostclaim.KindServing, Exec: "uk"}},
41 }}
42 f.g = Guard{DataDir: recordFixture(t), LockDir: t.TempDir(), Claims: readerFunc(func(ctx context.Context, host string) (hostclaim.Observed, error) {
43 f.reads++
44 return f.obs, f.err
45 })}
46 return f
47}
48
49func allow(*hostclaim.Claim) error { return nil }
50
51func TestWithRefusals(t *testing.T) {
52 failure := errors.New("read or identity failure")
53 for _, tc := range []struct {
54 name string
55 change func(*testing.T, *guardFixture, *Captured)
56 check func(*hostclaim.Claim) error
57 want error
58 }{
59 {"read", func(t *testing.T, f *guardFixture, c *Captured) { f.err = failure }, allow, failure},
60 {"absent", func(t *testing.T, f *guardFixture, c *Captured) { f.obs.Absent = true }, allow, ErrNoClaim},
61 {"nil-claim", func(t *testing.T, f *guardFixture, c *Captured) { f.obs.Claim = nil }, allow, ErrMalformedClaim},
62 {"wrong-observed-host", func(t *testing.T, f *guardFixture, c *Captured) { f.obs.Host = "else.example" }, allow, ErrMalformedClaim},
63 {"wrong-claim-host", func(t *testing.T, f *guardFixture, c *Captured) { f.obs.Claim.Host = "else.example" }, allow, ErrMalformedClaim},
64 {"nil-holder", func(t *testing.T, f *guardFixture, c *Captured) { f.obs.Claim.Holder = nil }, allow, ErrNoHolder},
65 {"claim-incarnation", func(t *testing.T, f *guardFixture, c *Captured) { f.obs.Claim.Holder.Incarnation = "other" }, allow, ErrClaimIncarnation},
66 {"check", func(t *testing.T, f *guardFixture, c *Captured) {}, func(*hostclaim.Claim) error { return failure }, failure},
67 {"nil-check", func(t *testing.T, f *guardFixture, c *Captured) {}, nil, ErrCheckRequired},
68 {"missing-record", func(t *testing.T, f *guardFixture, c *Captured) {
69 if err := os.Remove(filepath.Join(f.g.DataDir, "hostguard", f.obs.Host+".json")); err != nil {
70 t.Fatal(err)
71 }
72 }, allow, ErrNoRecord},
73 {"record-incarnation", func(t *testing.T, f *guardFixture, c *Captured) {
74 r := record{V: 1, Host: f.obs.Host, Incarnation: "other", ActiveStamp: 1}
75 if err := writeRecord(f.g.DataDir, r); err != nil {
76 t.Fatal(err)
77 }
78 }, allow, ErrRecordIncarnation},
79 {"inactive", func(t *testing.T, f *guardFixture, c *Captured) {
80 if err := f.g.RetireInstallation(t.Context(), f.obs.Host, allow); err != nil {
81 t.Fatal(err)
82 }
83 }, allow, ErrInactive},
84 {"lower", func(t *testing.T, f *guardFixture, c *Captured) { c.Stamp = 0 }, allow, ErrStampMismatch},
85 {"higher", func(t *testing.T, f *guardFixture, c *Captured) { c.Stamp = 2 }, allow, ErrStampMismatch},
86 {"retired", func(t *testing.T, f *guardFixture, c *Captured) {
87 f.obs.Claim.Holder.Stamp = 3
88 if err := f.g.Activate(t.Context(), f.obs.Host, "uk", nil); err != nil {
89 t.Fatal(err)
90 }
91 }, allow, ErrStampMismatch},
92 {"malformed-record", func(t *testing.T, f *guardFixture, c *Captured) {
93 if err := os.WriteFile(filepath.Join(f.g.DataDir, "hostguard", f.obs.Host+".json"), []byte(`{`), 0660); err != nil {
94 t.Fatal(err)
95 }
96 }, allow, ErrMalformedRecord},
97 } {
98 t.Run(tc.name, func(t *testing.T) {
99 f := newGuardFixture(t)
100 if err := f.g.Activate(t.Context(), f.obs.Host, "uk", nil); err != nil {
101 t.Fatal(err)
102 }
103 captured, err := Capture(f.g.DataDir, f.obs.Host)
104 if err != nil {
105 t.Fatal(err)
106 }
107 tc.change(t, f, &captured)
108 called := false
109 err = f.g.With(t.Context(), "shop.example", captured, tc.check, func() error { called = true; return nil })
110 if !errors.Is(err, tc.want) || called {
111 t.Fatalf("err=%v called=%v want=%v", err, called, tc.want)
112 }
113 })
114 }
115}
116
117func TestActivationAndRetirement(t *testing.T) {
118 f := newGuardFixture(t)
119 ctx, host := t.Context(), f.obs.Host
120 if err := f.g.Activate(ctx, host, "uk", nil); err != nil {
121 t.Fatal(err)
122 }
123 before, err := os.ReadFile(filepath.Join(f.g.DataDir, "hostguard", host+".json"))
124 if err != nil {
125 t.Fatal(err)
126 }
127 if err := f.g.Activate(ctx, host, "uk", nil); err != nil {
128 t.Fatal(err)
129 }
130 after, err := os.ReadFile(filepath.Join(f.g.DataDir, "hostguard", host+".json"))
131 if err != nil || string(before) != string(after) {
132 t.Fatal("equal stamp changed record", err)
133 }
134 for s := uint64(2); s <= 40; s++ {
135 f.obs.Claim.Holder.Stamp = s
136 if err := f.g.Activate(ctx, host, "uk", nil); err != nil {
137 t.Fatal(err)
138 }
139 }
140 r, err := readRecord(f.g.DataDir, host)
141 if err != nil || r.ActiveStamp != 40 || len(r.RetiredStamps) != 32 || r.RetiredStamps[0] != 8 || r.RetiredStamps[31] != 39 {
142 t.Fatalf("%+v %v", r, err)
143 }
144 if err := f.g.RetireInstallation(ctx, host, allow); err != nil {
145 t.Fatal(err)
146 }
147 if err := f.g.RetireInstallation(ctx, host, allow); err != nil {
148 t.Fatal(err)
149 }
150 for _, tc := range []struct {
151 stamp uint64
152 want error
153 }{{40, ErrRetiredStamp}, {1, ErrOldStamp}} {
154 f.obs.Claim.Holder.Stamp = tc.stamp
155 if err := f.g.Activate(ctx, host, "uk", nil); !errors.Is(err, tc.want) {
156 t.Fatal(err)
157 }
158 }
159 f.obs.Claim.Holder.Incarnation = "replacement"
160 f.obs.Claim.Holder.Stamp = 50
161 if err := f.g.Activate(ctx, host, "uk", nil); err != nil {
162 t.Fatal(err)
163 }
164 r, err = readRecord(f.g.DataDir, host)
165 if err != nil || r.Incarnation != "replacement" || r.ActiveStamp != 50 || len(r.RetiredStamps) != 0 {
166 t.Fatalf("%+v %v", r, err)
167 }
168 if err := os.Remove(filepath.Join(f.g.DataDir, "hostguard", host+".json")); err != nil {
169 t.Fatal(err)
170 }
171 if err := f.g.Activate(ctx, host, "uk", nil); err != nil {
172 t.Fatal("lost record recovery", err)
173 }
174}
175
176func TestLifecycleRefusals(t *testing.T) {
177 failure := errors.New("unavailable")
178 for _, tc := range []struct {
179 name string
180 change func(*guardFixture)
181 want error
182 }{
183 {"read", func(f *guardFixture) { f.err = failure }, failure},
184 {"absent", func(f *guardFixture) { f.obs.Absent = true }, ErrNoClaim},
185 {"holder", func(f *guardFixture) { f.obs.Claim.Holder = nil }, ErrNoHolder},
186 {"reader", func(f *guardFixture) { f.g.Claims = nil }, ErrNoClaims},
187 } {
188 t.Run(tc.name, func(t *testing.T) {
189 f := newGuardFixture(t)
190 tc.change(f)
191 if err := f.g.Activate(t.Context(), f.obs.Host, "uk", nil); !errors.Is(err, tc.want) {
192 t.Fatal(err)
193 }
194 if err := f.g.RetireInstallation(t.Context(), f.obs.Host, allow); !errors.Is(err, tc.want) {
195 t.Fatal(err)
196 }
197 })
198 }
199 f := newGuardFixture(t)
200 ctx, host := t.Context(), f.obs.Host
201 for _, box := range []string{"", "us"} {
202 if err := f.g.Activate(ctx, host, box, nil); !errors.Is(err, ErrNotParticipant) {
203 t.Fatal(err)
204 }
205 }
206 if err := f.g.Activate(ctx, host, "us", func(*hostclaim.Claim) bool { return false }); !errors.Is(err, ErrNotParticipant) {
207 t.Fatal(err)
208 }
209 f.obs.Claim.Op.Kind, f.obs.Claim.Op.Phase = hostclaim.KindMoving, "landing"
210 participant := func(c *hostclaim.Claim) bool { return c.Op.Kind == hostclaim.KindMoving && c.Op.Phase == "landing" }
211 if err := f.g.Activate(ctx, host, "us", participant); err != nil {
212 t.Fatal(err)
213 }
214 f.obs.Claim.Holder.Stamp = 0
215 if err := f.g.Activate(ctx, host, "uk", nil); !errors.Is(err, ErrMalformedClaim) {
216 t.Fatal(err)
217 }
218 f.obs.Claim.Holder.Stamp = 1
219 if err := f.g.RetireInstallation(ctx, host, nil); !errors.Is(err, ErrCheckRequired) {
220 t.Fatal(err)
221 }
222 if err := f.g.RetireInstallation(ctx, host, func(*hostclaim.Claim) error { return failure }); !errors.Is(err, failure) {
223 t.Fatal(err)
224 }
225 f.obs.Claim.Holder.Incarnation = "other"
226 if err := f.g.RetireInstallation(ctx, host, allow); !errors.Is(err, ErrRecordIncarnation) {
227 t.Fatal(err)
228 }
229 f.obs.Claim.Holder.Incarnation = "inc"
230 if err := os.Remove(filepath.Join(f.g.DataDir, "hostguard", host+".json")); err != nil {
231 t.Fatal(err)
232 }
233 if err := f.g.RetireInstallation(ctx, host, allow); !errors.Is(err, ErrNoRecord) {
234 t.Fatal(err)
235 }
236}
237
238func TestLifecycleRecordReadFailures(t *testing.T) {
239 for _, tc := range []struct {
240 name string
241 directory bool
242 want error
243 }{
244 {"malformed", false, ErrMalformedRecord}, {"read-error", true, nil},
245 } {
246 t.Run(tc.name, func(t *testing.T) {
247 f := newGuardFixture(t)
248 host := f.obs.Host
249 if err := f.g.Activate(t.Context(), host, "uk", nil); err != nil {
250 t.Fatal(err)
251 }
252 path := filepath.Join(f.g.DataDir, "hostguard", host+".json")
253 if tc.directory {
254 if err := os.Remove(path); err != nil {
255 t.Fatal(err)
256 }
257 if err := os.Mkdir(path, 0700); err != nil {
258 t.Fatal(err)
259 }
260 } else if err := os.WriteFile(path, []byte(`{`), 0660); err != nil {
261 t.Fatal(err)
262 }
263 for _, operation := range []func() error{
264 func() error { return f.g.Activate(t.Context(), host, "uk", nil) },
265 func() error { return f.g.RetireInstallation(t.Context(), host, allow) },
266 } {
267 err := operation()
268 if err == nil || errors.Is(err, ErrNoRecord) || (tc.want != nil && !errors.Is(err, tc.want)) {
269 t.Fatal("read failure acted", err)
270 }
271 }
272 })
273 }
274 f := newGuardFixture(t)
275 f.obs.Claim.Holder.Stamp = 3
276 if err := f.g.Activate(t.Context(), f.obs.Host, "uk", nil); err != nil {
277 t.Fatal(err)
278 }
279 f.obs.Claim.Holder.Stamp = 2
280 if err := f.g.Activate(t.Context(), f.obs.Host, "uk", nil); !errors.Is(err, ErrOldStamp) {
281 t.Fatal("lower than active stamp", err)
282 }
283 f.obs.Claim.Holder.Incarnation = ""
284 if err := f.g.Activate(t.Context(), f.obs.Host, "uk", nil); !errors.Is(err, ErrMalformedClaim) {
285 t.Fatal("empty incarnation", err)
286 }
287}
288
289func TestFenceStampRoundTrip(t *testing.T) {
290 f := newGuardFixture(t)
291 ctx, host := t.Context(), f.obs.Host
292 if err := f.g.Activate(ctx, host, "uk", nil); err != nil {
293 t.Fatal(err)
294 }
295 old, err := Capture(f.g.DataDir, host)
296 if err != nil {
297 t.Fatal(err)
298 }
299 f.obs.Claim.Op = hostclaim.Op{Kind: hostclaim.KindMoving, ID: "away", Exec: "us"}
300 f.obs.Claim.Holder.Stamp = 2
301 if err := f.g.RetireInstallation(ctx, host, func(c *hostclaim.Claim) error {
302 if c.Op.ID != "away" || c.Op.Exec != "us" {
303 return errors.New("wrong move")
304 }
305 return nil
306 }); err != nil {
307 t.Fatal(err)
308 }
309 f.obs.Claim.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "uk"}
310 f.obs.Claim.Holder.Stamp = 3
311 if err := f.g.Activate(ctx, host, "uk", nil); err != nil {
312 t.Fatal(err)
313 }
314 writes := 0
315 write := func() error { writes++; return nil }
316 if err := f.g.With(ctx, host, old, allow, write); !errors.Is(err, ErrStampMismatch) {
317 t.Fatal(err)
318 }
319 current, err := Capture(f.g.DataDir, host)
320 if err != nil {
321 t.Fatal(err)
322 }
323 if err := f.g.With(ctx, host, current, allow, write); err != nil {
324 t.Fatal(err)
325 }
326 if writes != 1 {
327 t.Fatal(writes)
328 }
329 r, err := readRecord(f.g.DataDir, host)
330 if err != nil || !reflect.DeepEqual(r.RetiredStamps, []uint64{1}) {
331 t.Fatalf("%+v %v", r, err)
332 }
333}
334
335func TestStoreReaderUsesHostclaimRead(t *testing.T) {
336 s := release.NewDirStore(t.TempDir())
337 r := StoreReader{Store: s}
338 obs, err := r.Read(t.Context(), "shop.example")
339 if err != nil || !obs.Absent || obs.Host != "shop.example" {
340 t.Fatalf("%+v %v", obs, err)
341 }
342 if _, err := s.Put(t.Context(), "control/hosts/shop.example.json", []byte(`{"unknown":true}`), ""); err != nil {
343 t.Fatal(err)
344 }
345 if _, err := r.Read(t.Context(), "shop.example"); !errors.Is(err, hostclaim.ErrMalformed) {
346 t.Fatal(err)
347 }
348}
349
350func TestLifecycleRetrySyncsRecord(t *testing.T) {
351 for _, retired := range []bool{false, true} {
352 name := "activate"
353 if retired {
354 name = "retire"
355 }
356 t.Run(name, func(t *testing.T) {
357 f := newGuardFixture(t)
358 host := f.obs.Host
359 operation := func() error { return f.g.Activate(t.Context(), host, "uk", nil) }
360 wantStamp := uint64(1)
361 if retired {
362 if err := operation(); err != nil {
363 t.Fatal(err)
364 }
365 operation = func() error { return f.g.RetireInstallation(t.Context(), host, allow) }
366 wantStamp = 0
367 }
368 failure := errors.New("injected fsync failure")
369 failKind := "directory"
370 var calls []string
371 f.g.openRecords = func(path string) (*recordDir, error) {
372 d, err := openRecordDir(path)
373 if err != nil {
374 return nil, err
375 }
376 d.fsync = func(fd int) error {
377 var st unix.Stat_t
378 if err := unix.Fstat(fd, &st); err != nil {
379 return err
380 }
381 kind := "file"
382 if st.Mode&unix.S_IFMT == unix.S_IFDIR {
383 kind = "directory"
384 }
385 calls = append(calls, kind)
386 if kind == failKind {
387 return failure
388 }
389 return unix.Fsync(fd)
390 }
391 return d, nil
392 }
393 if err := operation(); !errors.Is(err, failure) {
394 t.Fatalf("post-rename failure: %v", err)
395 }
396 if !reflect.DeepEqual(calls, []string{"file", "directory"}) {
397 t.Fatalf("initial syncs: %v", calls)
398 }
399 captured, err := Capture(f.g.DataDir, host)
400 if err != nil || captured.Stamp != wantStamp {
401 t.Fatalf("rename did not publish: %+v %v", captured, err)
402 }
403 defer unchangedFile(t, filepath.Join(f.g.DataDir, "hostguard", host+".json"))()
404 for _, kind := range []string{"file", "directory", ""} {
405 failKind, calls = kind, nil
406 err := operation()
407 if kind == "" {
408 if err != nil {
409 t.Fatal(err)
410 }
411 } else if !errors.Is(err, failure) {
412 t.Fatalf("retry acknowledged %s sync failure: %v", kind, err)
413 }
414 want := []string{"file", "directory"}
415 if kind == "file" {
416 want = []string{"file"}
417 }
418 if !reflect.DeepEqual(calls, want) {
419 t.Fatalf("retry syncs: %v want %v", calls, want)
420 }
421 }
422 })
423 }
424}
425
426func TestGuardHostnameLengthBoundary(t *testing.T) {
427 for _, length := range []int{221, 222, 250, 251, 252, 253, 254} {
428 t.Run(fmt.Sprint(length), func(t *testing.T) {
429 host := strings.Repeat(strings.Repeat("a", 63)+".", 3) + strings.Repeat("b", length-192)
430 if controlname.ValidHostShape(host) != (length <= 253) {
431 t.Fatalf("unexpected hostname validation for %d bytes", length)
432 }
433 f := newGuardFixture(t)
434 f.obs.Host, f.obs.Claim.Host = host, host
435 if length > 250 {
436 want := ErrHostTooLong
437 if length > 253 {
438 want = ErrInvalidHost
439 }
440 for name, operation := range map[string]func() error{
441 "capture": func() error { _, err := Capture(f.g.DataDir, host); return err },
442 "activate": func() error { return f.g.Activate(t.Context(), host, "uk", nil) },
443 "retire": func() error { return f.g.RetireInstallation(t.Context(), host, allow) },
444 "with": func() error {
445 return f.g.With(t.Context(), host, Captured{Incarnation: "inc", Stamp: 1}, allow, func() error { t.Fatal("callback ran"); return nil })
446 },
447 } {
448 if err := operation(); !errors.Is(err, want) {
449 t.Errorf("%s: got %v, want %v", name, err, want)
450 }
451 }
452 if f.reads != 0 {
453 t.Fatalf("read claim before rejecting hostname: %d", f.reads)
454 }
455 for _, dir := range []string{f.g.LockDir, filepath.Join(f.g.DataDir, "hostguard")} {
456 files, err := os.ReadDir(dir)
457 if err != nil || len(files) != 0 {
458 t.Fatalf("refusal created files: %v %v", files, err)
459 }
460 }
461 return
462 }
463 if err := f.g.Activate(t.Context(), host, "uk", nil); err != nil {
464 t.Fatal(err)
465 }
466 captured, err := Capture(f.g.DataDir, host)
467 if err != nil || captured != (Captured{Incarnation: "inc", Stamp: 1}) {
468 t.Fatal(captured, err)
469 }
470 if err := f.g.Activate(t.Context(), host, "uk", nil); err != nil {
471 t.Fatal(err)
472 }
473 called := false
474 if err := f.g.With(t.Context(), host, captured, allow, func() error { called = true; return nil }); err != nil || !called {
475 t.Fatal(called, err)
476 }
477 for i := 0; i < 2; i++ {
478 if err := f.g.RetireInstallation(t.Context(), host, allow); err != nil {
479 t.Fatal(err)
480 }
481 }
482 if got, err := Capture(f.g.DataDir, host); err != nil || got.Stamp != 0 {
483 t.Fatal(got, err)
484 }
485 if err := f.g.With(t.Context(), host, captured, allow, func() error { t.Fatal("retired callback ran"); return nil }); !errors.Is(err, ErrInactive) {
486 t.Fatal(err)
487 }
488 f.obs.Claim.Holder.Stamp = 2
489 if err := f.g.Activate(t.Context(), host, "uk", nil); err != nil {
490 t.Fatal(err)
491 }
492 for _, path := range []string{filepath.Join(f.g.DataDir, "hostguard", host+".json"), filepath.Join(f.g.LockDir, host+".lock")} {
493 if _, err := os.Stat(path); err != nil {
494 t.Fatal(err)
495 }
496 }
497 files, err := os.ReadDir(filepath.Join(f.g.DataDir, "hostguard"))
498 if err != nil || len(files) != 1 {
499 t.Fatalf("temporary files remain: %v %v", files, err)
500 }
501 })
502 }
503}
504