| 1 | package lbhealth |
| 2 | |
| 3 | import ( |
| 4 | "net/http" |
| 5 | "net/http/httptest" |
| 6 | "reflect" |
| 7 | "strings" |
| 8 | "testing" |
| 9 | "time" |
| 10 | ) |
| 11 | |
| 12 | var t0 = time.Date(2026, 10, 16, 12, 0, 0, 0, time.UTC) |
| 13 | |
| 14 | // healthy is a state with every condition met at t0. |
| 15 | func healthy() *State { |
| 16 | s := &State{} |
| 17 | s.SetHosting(true) |
| 18 | s.MapLoaded() |
| 19 | s.KeyLoaded() |
| 20 | s.CertsLoaded() |
| 21 | s.EdgeBeat(t0) |
| 22 | s.RefreshAttempted(t0) |
| 23 | s.SetActivator(func() time.Time { return t0 }) |
| 24 | return s |
| 25 | } |
| 26 | |
| 27 | func TestFailingNamesEachCondition(t *testing.T) { |
| 28 | if got := healthy().Failing(t0); len(got) != 0 { |
| 29 | t.Fatalf("healthy box fails %v", got) |
| 30 | } |
| 31 | if got := (&State{}).Failing(t0); !reflect.DeepEqual(got, []string{CondHosting, CondForwarding, CondCerts, CondEdge, CondActivator, CondRefresh}) { |
| 32 | t.Fatalf("zero state fails %v", got) |
| 33 | } |
| 34 | var nilState *State |
| 35 | if got := nilState.Failing(t0); len(got) != 6 { |
| 36 | t.Fatalf("nil state fails %v", got) |
| 37 | } |
| 38 | cases := []struct { |
| 39 | name string |
| 40 | spoil func(*State) |
| 41 | now time.Time |
| 42 | want string |
| 43 | }{ |
| 44 | {"joining", func(s *State) { s.SetHosting(false) }, t0, CondHosting}, |
| 45 | {"map never loaded", func(s *State) { s.mapLoaded.Store(false) }, t0, CondForwarding}, |
| 46 | {"key never loaded", func(s *State) { s.keyLoaded.Store(false) }, t0, CondForwarding}, |
| 47 | {"no certificate cache", func(s *State) { s.certs.Store(false) }, t0, CondCerts}, |
| 48 | {"edge beat stale", func(s *State) { s.EdgeBeat(t0.Add(-61 * time.Second)) }, t0, CondEdge}, |
| 49 | {"activator beat stale", func(s *State) { s.SetActivator(func() time.Time { return t0.Add(-61 * time.Second) }) }, t0, CondActivator}, |
| 50 | {"no activator", func(s *State) { s.SetActivator(nil) }, t0, CondActivator}, |
| 51 | {"refresh stuck", func(s *State) { s.RefreshAttempted(t0.Add(-5*time.Minute - time.Second)) }, t0, CondRefresh}, |
| 52 | } |
| 53 | for _, c := range cases { |
| 54 | t.Run(c.name, func(t *testing.T) { |
| 55 | s := healthy() |
| 56 | c.spoil(s) |
| 57 | if got := s.Failing(c.now); !reflect.DeepEqual(got, []string{c.want}) { |
| 58 | t.Fatalf("Failing = %v, want [%s]", got, c.want) |
| 59 | } |
| 60 | }) |
| 61 | } |
| 62 | // The windows are inclusive: a beat exactly 60s old and a refresh |
| 63 | // exactly 5 minutes old still count. |
| 64 | s := healthy() |
| 65 | s.EdgeBeat(t0.Add(-HeartbeatWindow)) |
| 66 | s.RefreshAttempted(t0.Add(-RefreshWindow)) |
| 67 | if got := s.Failing(t0); len(got) != 0 { |
| 68 | t.Fatalf("boundary fails %v", got) |
| 69 | } |
| 70 | } |
| 71 | |
| 72 | // A bucket outage after arming: refreshes keep failing, but each failed |
| 73 | // attempt is still a completed attempt, and the latches hold. The box |
| 74 | // stays healthy (A2-3, A3-3). |
| 75 | func TestBucketOutageAfterArmingStaysHealthy(t *testing.T) { |
| 76 | s := healthy() |
| 77 | for i := 1; i <= 20; i++ { |
| 78 | now := t0.Add(time.Duration(i) * 30 * time.Second) |
| 79 | s.RefreshAttempted(now) // a failed attempt: no latch is re-set |
| 80 | s.EdgeBeat(now) |
| 81 | s.SetActivator(func() time.Time { return now }) |
| 82 | if got := s.Failing(now); len(got) != 0 { |
| 83 | t.Fatalf("after %d failed refreshes: %v", i, got) |
| 84 | } |
| 85 | } |
| 86 | } |
| 87 | |
| 88 | func TestServe(t *testing.T) { |
| 89 | get := func(s *State, method, target, host string) (*httptest.ResponseRecorder, bool) { |
| 90 | w := httptest.NewRecorder() |
| 91 | r := httptest.NewRequest(method, target, nil) |
| 92 | r.Host = host |
| 93 | return w, Serve(w, r, s, t0) |
| 94 | } |
| 95 | w, ok := get(healthy(), http.MethodGet, Path, "10.0.1.7") |
| 96 | if !ok || w.Code != http.StatusOK || w.Body.String() != "ok\n" || w.Header().Get("Cache-Control") != "no-store" { |
| 97 | t.Fatalf("healthy GET: ok=%v %d %q %v", ok, w.Code, w.Body.String(), w.Header()) |
| 98 | } |
| 99 | w, ok = get(healthy(), http.MethodHead, Path, "any.example.com") |
| 100 | if !ok || w.Code != http.StatusOK || w.Body.Len() != 0 { |
| 101 | t.Fatalf("healthy HEAD: ok=%v %d %q", ok, w.Code, w.Body.String()) |
| 102 | } |
| 103 | joining := healthy() |
| 104 | joining.SetHosting(false) |
| 105 | w, ok = get(joining, http.MethodGet, Path, "10.0.1.7") |
| 106 | if !ok || w.Code != http.StatusServiceUnavailable || !strings.Contains(w.Body.String(), CondHosting) { |
| 107 | t.Fatalf("joining GET: ok=%v %d %q", ok, w.Code, w.Body.String()) |
| 108 | } |
| 109 | for _, c := range []struct{ method, target string }{ |
| 110 | {http.MethodPost, Path}, |
| 111 | {http.MethodGet, Path + "/"}, |
| 112 | {http.MethodGet, "/.carlos/lb-healthz"}, |
| 113 | {http.MethodGet, "/"}, |
| 114 | } { |
| 115 | if _, ok := get(healthy(), c.method, c.target, "10.0.1.7"); ok { |
| 116 | t.Fatalf("%s %s answered", c.method, c.target) |
| 117 | } |
| 118 | } |
| 119 | } |
| 120 | |
| 121 | func TestNilStateIgnoresUpdates(t *testing.T) { |
| 122 | var s *State |
| 123 | s.SetHosting(true) |
| 124 | s.MapLoaded() |
| 125 | s.KeyLoaded() |
| 126 | s.CertsLoaded() |
| 127 | s.EdgeBeat(t0) |
| 128 | s.RefreshAttempted(t0) |
| 129 | s.SetActivator(func() time.Time { return t0 }) |
| 130 | if s.Hosting() { |
| 131 | t.Fatal("nil state hosting") |
| 132 | } |
| 133 | } |
| 134 | |
amadan