| index fdaee8be5..58355ab35 100644 |
| --- a/internal/edge/lbhealth_test.go |
| +++ b/internal/edge/lbhealth_test.go |
| @@ -5,6 +5,8 @@ import ( |
| "errors" |
| "net/http" |
| "net/http/httptest" |
| + "os" |
| + "path/filepath" |
| "strings" |
| "testing" |
| "time" |
| @@ -12,6 +14,7 @@ import ( |
| "golang.org/x/crypto/acme/autocert" |
| |
| "github.com/carlosframework/platform/internal/lbhealth" |
| + "github.com/carlosframework/platform/internal/registry" |
| "github.com/carlosframework/platform/internal/release" |
| ) |
| |
| @@ -166,12 +169,20 @@ func TestHomeMapFeedsTheHealthState(t *testing.T) { |
| t.Fatalf("a failed attempt dropped forwarding or went unstamped: %s", got) |
| } |
| |
| - // A stuck attempt stamps nothing. |
| + // A stuck attempt stamps nothing, at entry or otherwise: the last stamp |
| + // is the failed attempt above, taken more than a second before t0, so |
| + // a minute and a bit short of the window after t0 it is already stale. |
| + // A stamp taken at entry would be fresh here. |
| s.down = false |
| s.block, s.entered = make(chan struct{}), make(chan struct{}, 1) |
| + time.Sleep(1200 * time.Millisecond) |
| + t0 := time.Now() |
| done := make(chan struct{}) |
| go func() { _ = m.Refresh(context.Background()); close(done) }() |
| <-s.entered |
| + if got := strings.Join(h.Failing(t0.Add(lbhealth.RefreshWindow-time.Second)), ","); !strings.Contains(got, lbhealth.CondRefresh) { |
| + t.Fatalf("a refresh stuck inside its attempt was stamped at entry: %s", got) |
| + } |
| stuckAt := time.Now() |
| if got := strings.Join(h.Failing(stuckAt.Add(lbhealth.RefreshWindow+time.Second)), ","); !strings.Contains(got, lbhealth.CondRefresh) { |
| t.Fatalf("a refresh stuck for longer than the window still counts: %s", got) |
| @@ -229,3 +240,120 @@ func TestRegistryWatchBeats(t *testing.T) { |
| time.Sleep(50 * time.Millisecond) |
| } |
| } |
| + |
| +// A certificate mirror whose first list fails has loaded nothing: the |
| +// certificate condition stays unlatched, and a later good refresh latches it. |
| +func TestCertMirrorFailedRefreshLeavesCertsUnlatched(t *testing.T) { |
| + inner := meshTestStore(t) |
| + s := &blockingStore{Store: inner, down: true} |
| + h := &lbhealth.State{} |
| + m := newCertMirror(s, autocert.DirCache(t.TempDir())) |
| + m.health = h |
| + if err := m.Refresh(context.Background()); err == nil { |
| + t.Fatal("refresh against a down bucket succeeded") |
| + } |
| + if !strings.Contains(strings.Join(h.Failing(time.Now()), ","), lbhealth.CondCerts) { |
| + t.Fatal("a failed refresh latched the certificate condition") |
| + } |
| + s.down = false |
| + if err := m.Refresh(context.Background()); err != nil { |
| + t.Fatal(err) |
| + } |
| + if strings.Contains(strings.Join(h.Failing(time.Now()), ","), lbhealth.CondCerts) { |
| + t.Fatal("the next good refresh did not latch") |
| + } |
| +} |
| + |
| +// failingFingerprint is a registry whose file stops being a database after |
| +// it opened (a directory stands where it was), so Fingerprint errors on every call. |
| +func failingFingerprint(t *testing.T) *registry.Registry { |
| + t.Helper() |
| + path := filepath.Join(t.TempDir(), "registry.db") |
| + reg, err := registry.Open(path) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, f := range []string{path, path + "-wal", path + "-shm"} { |
| + _ = os.Remove(f) |
| + } |
| + if err := os.Mkdir(path, 0o700); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := reg.Fingerprint(); err == nil { |
| + t.Fatal("setup: Fingerprint still works") |
| + } |
| + return reg |
| +} |
| + |
| +// A registry poll that fails does not beat. |
| +func TestRegistryWatchDoesNotBeatOnFingerprintError(t *testing.T) { |
| + e, err := New(openReg(t), nil) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(e.probers.Stop) |
| + e.reg = failingFingerprint(t) |
| + h := &lbhealth.State{} |
| + e.lbHealth = h |
| + stop := make(chan struct{}) |
| + t.Cleanup(func() { close(stop) }) |
| + go e.Watch(stop) |
| + time.Sleep(5 * time.Second) |
| + if !strings.Contains(strings.Join(h.Failing(time.Now()), ","), lbhealth.CondEdge) { |
| + t.Fatal("a failing Fingerprint beat") |
| + } |
| +} |
| + |
| +// wedgedProbe runs an edge over a state that is healthy at setup (every beat |
| +// taken then), so only the edge's beat can still move, and reports what is |
| +// failing a little before the window ends. |
| +func wedgedProbe(t *testing.T, wedge bool) (failing string) { |
| + t.Helper() |
| + old := servingProbeTimeout |
| + servingProbeTimeout = 100 * time.Millisecond |
| + t.Cleanup(func() { servingProbeTimeout = old }) |
| + e, err := New(openReg(t), nil) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(e.probers.Stop) |
| + e.EnableCache(NewCache(10, 1<<20)) |
| + h := healthyState() |
| + h.SetActivator(time.Now) // the activator is not what this test wedges |
| + e.lbHealth = h |
| + if wedge { |
| + // A request path deadlocked inside the cache: every cacheable GET |
| + // takes this lock. Released at cleanup so the probe goroutines end. |
| + e.cache.mu.Lock() |
| + t.Cleanup(e.cache.mu.Unlock) |
| + } |
| + stop := make(chan struct{}) |
| + t.Cleanup(func() { close(stop) }) |
| + go e.Watch(stop) |
| + time.Sleep(5 * time.Second) |
| + // Set the clock 57s ahead: a beat from the last tick or two is still |
| + // inside the 60s window, one from before the wedge (setup, 5s+ ago) is not. |
| + now := time.Now().Add(lbhealth.HeartbeatWindow - 3*time.Second) |
| + got := h.Failing(now) |
| + w := httptest.NewRecorder() |
| + lbhealth.Serve(w, httptest.NewRequest(http.MethodGet, lbhealth.Path, nil), h, now) |
| + if wedge != (w.Code == http.StatusServiceUnavailable) { |
| + t.Fatalf("wedge=%v: the health answer was %d %q", wedge, w.Code, w.Body.String()) |
| + } |
| + return strings.Join(got, ",") |
| +} |
| + |
| +// A deadlocked request path keeps the registry poll running, but the edge |
| +// stops beating, and the health answer turns 503 naming the edge once the |
| +// last beat ages out. |
| +func TestEdgeBeatStopsWhenTheServingPathIsWedged(t *testing.T) { |
| + if got := wedgedProbe(t, true); !strings.Contains(got, lbhealth.CondEdge) { |
| + t.Fatalf("a wedged serving path kept beating: %q", got) |
| + } |
| +} |
| + |
| +func TestEdgeBeatContinuesWhenTheServingPathIsFree(t *testing.T) { |
| + if got := wedgedProbe(t, false); strings.Contains(got, lbhealth.CondEdge) { |
| + t.Fatalf("a free serving path stopped beating: %q", got) |
| + } |
| +} |
amadan