amadan
carlos / todomvc-bakeoff Public
feat: ramps that end in errors report where errors started
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
5 files changed,
+36
−6
harness/harness_test.go+8 −0harness/report/charts.go+2 −1harness/report/report.go+8 −3harness/report/report_test.go+3 −2harness/sweep.go+15 −0
diff --git a/harness/harness_test.go b/harness/harness_test.go| index 0b43b70..2790349 100644 |
| --- a/harness/harness_test.go |
| +++ b/harness/harness_test.go |
| @@ -308,3 +308,11 @@ func TestKneeIgnoresAOneStepBlip(t *testing.T) { |
| t.Fatalf("got %+v, want 16: the fewest connections from which no more load adds 10%%", k) |
| } |
| } |
| + |
| +func TestKneeStopsAtErrors(t *testing.T) { |
| + sweep := []LoadResult{lr(16, 400, 60, 0), lr(32, 800, 70, 0), lr(64, 1300, 90, 0), lr(128, 1360, 370, 0.40)} |
| + k := Knee(sweep) |
| + if !k.Found || k.Concurrency != 64 || k.ErrorsAt != 128 || k.PeakRPS != 1300 { |
| + t.Fatalf("got %+v: want slow-down 64 (last clean step), errors from 128, peak from clean steps only", k) |
| + } |
| +} |
diff --git a/harness/report/charts.go b/harness/report/charts.go| index 2940782..db01a41 100644 |
| --- a/harness/report/charts.go |
| +++ b/harness/report/charts.go |
| @@ -14,6 +14,7 @@ type Row struct { |
| Value float64 |
| Parts []float64 // stacked charts only |
| Missing string |
| + Suffix string // appended to the value label |
| } |
| // Chart is one horizontal bar chart, apps in fixed report order. |
| @@ -81,7 +82,7 @@ func (c Chart) SVG() template.HTML { |
| } else if w > 0 { |
| fmt.Fprintf(&s, `<path class="s1" d="%s"><title>%s</title></path>`, barPath(labelW, float64(by), w, true), tip) |
| } |
| - fmt.Fprintf(&s, `<text class="value" x="%.1f" y="%d">%s</text>`, float64(labelW)+w+6, mid+5, html.EscapeString(c.Format(r.Value))) |
| + fmt.Fprintf(&s, `<text class="value" x="%.1f" y="%d">%s</text>`, float64(labelW)+w+6, mid+5, html.EscapeString(c.Format(r.Value)+r.Suffix)) |
| } |
| s.WriteString(`</svg>`) |
| return template.HTML(s.String()) |
diff --git a/harness/report/report.go b/harness/report/report.go| index cbfe986..cc23029 100644 |
| --- a/harness/report/report.go |
| +++ b/harness/report/report.go |
| @@ -79,7 +79,12 @@ func kneeChart(rs []b.AppResult, scenario, title, note string) Chart { |
| return float64(k.Concurrency), k.Found |
| }) |
| for i, r := range rs { |
| - if k := knee(r, scenario); live(r) && !k.Found && k.MaxTested > 0 { |
| + k := knee(r, scenario) |
| + switch { |
| + case !live(r): |
| + case k.ErrorsAt > 0: |
| + c.Rows[i].Suffix = " · errors from " + num(float64(k.ErrorsAt)) |
| + case !k.Found && k.MaxTested > 0: |
| c.Rows[i].Missing = "still scaling at " + num(float64(k.MaxTested)) |
| } |
| } |
| @@ -131,10 +136,10 @@ func charts(rs []b.AppResult) []Chart { |
| func(r b.AppResult) (float64, bool) { return pos(medianOf(r.Wake)) }), |
| chart(rs, "Peak reads per second", "GET / on a 50-item list, ramping from 1 to 1,024 concurrent connections, 10 s per step.", num, |
| func(r b.AppResult) (float64, bool) { return pos(knee(r, "read").PeakRPS) }), |
| - kneeChart(rs, "read", "Reads: where it slows down", "The most concurrent connections at which doubling the load still raised throughput by 10% or more."), |
| + kneeChart(rs, "read", "Reads: where it slows down", "The fewest concurrent connections from which more load never raised throughput by 10% or more. Errors: the first step where more than 1% of requests failed."), |
| chart(rs, "Peak toggles per second", "POST toggling one item, ramping the same way.", num, |
| func(r b.AppResult) (float64, bool) { return pos(knee(r, "toggle").PeakRPS) }), |
| - kneeChart(rs, "toggle", "Toggles: where it slows down", "The most concurrent connections at which doubling the load still raised throughput by 10% or more."), |
| + kneeChart(rs, "toggle", "Toggles: where it slows down", "The fewest concurrent connections from which more load never raised throughput by 10% or more. Errors: the first step where more than 1% of requests failed."), |
| chart(rs, "Binary size", "The linux-arm64 binary that ships.", mb, |
| func(r b.AppResult) (float64, bool) { return pos(float64(r.Build.Bytes)) }), |
| chart(rs, "Cold build", "Every library compiled from already-downloaded sources.", secs, |
diff --git a/harness/report/report_test.go b/harness/report/report_test.go| index f72bfb9..68ebed4 100644 |
| --- a/harness/report/report_test.go |
| +++ b/harness/report/report_test.go |
| @@ -16,7 +16,8 @@ func fixture() []b.AppResult { |
| TTFBEmpty: b.Stats{N: 30, P50: 41.2}, TTFB50: b.Stats{N: 30, P50: 45.9}, |
| Wake: []float64{120, 140, 130}, |
| Load: []b.LoadResult{{Scenario: "read", Concurrency: 1, RPS: 20}, {Scenario: "read", Concurrency: 2, RPS: 40}, |
| - {Scenario: "read", Concurrency: 4, RPS: 41}, {Scenario: "toggle", Concurrency: 1, RPS: 15}}, |
| + {Scenario: "read", Concurrency: 4, RPS: 41}, {Scenario: "toggle", Concurrency: 1, RPS: 15}, |
| + {Scenario: "toggle", Concurrency: 2, RPS: 30}, {Scenario: "toggle", Concurrency: 4, RPS: 31, ErrorRate: 0.4}}, |
| Weight: b.Weight{HTMLBytes: 9000, JSBytes: 2400}, |
| Code: b.CodeStats{Server: 300, Templates: 80, JS: 120, Tests: 200, Deps: 1}, |
| Build: b.Build{ColdSeconds: 40, WarmSeconds: 2, EditSeconds: 1.1, Bytes: 9_400_000, Host: "docs-t1"}, |
| @@ -36,7 +37,7 @@ func TestRenderAllApps(t *testing.T) { |
| out := buf.String() |
| for _, want := range []string{ |
| "Go", "Mojo", "single-file binaries only", "https://todo-go.bfj.oncarlos.com", "<svg", |
| - "<p>Nine apps.</p>", `<a href="https://example.com">link</a>`, "Clear & tidy <ok>", "9.4 MB", "2 connections", "still scaling at 1", |
| + "<p>Nine apps.</p>", `<a href="https://example.com">link</a>`, "Clear & tidy <ok>", "9.4 MB", "2 connections", "2 connections · errors from 4", |
| } { |
| if !strings.Contains(out, want) { |
| t.Errorf("output lacks %q", want) |
diff --git a/harness/sweep.go b/harness/sweep.go| index c2aefc9..42bf3fe 100644 |
| --- a/harness/sweep.go |
| +++ b/harness/sweep.go |
| @@ -17,13 +17,24 @@ type KneeResult struct { |
| PeakRPS float64 // the most requests per second any level reached |
| P50Ms float64 // median latency at Concurrency |
| MaxTested int |
| + ErrorsAt int // the first level with more than 1% errors, if any |
| } |
| // Knee finds where the app slows down: the fewest connections from which |
| // no further load in the sweep raised throughput by 10% or more. A single |
| // noisy step can't end the search early, because later growth resets it. |
| +// A step with more than 1% errors ends the sweep: the app failed there |
| +// rather than slowed, so the last clean step is where it stopped coping, |
| +// and peak throughput counts clean steps only. |
| func Knee(sweep []LoadResult) KneeResult { |
| var k KneeResult |
| + for i, r := range sweep { |
| + if r.ErrorRate > maxErrorRate { |
| + k.ErrorsAt = r.Concurrency |
| + sweep = sweep[:i] |
| + break |
| + } |
| + } |
| for _, r := range sweep { |
| if r.RPS > k.PeakRPS { |
| k.PeakRPS = r.RPS |
| @@ -46,6 +57,10 @@ func Knee(sweep []LoadResult) KneeResult { |
| break |
| } |
| } |
| + if !k.Found && k.ErrorsAt > 0 && len(sweep) > 0 { |
| + last := sweep[len(sweep)-1] |
| + k.Found, k.Concurrency, k.P50Ms = true, last.Concurrency, last.P50Ms |
| + } |
| return k |
| } |