amadan
carlos / todomvc-bakeoff Public
feat: keep each load step's status and error breakdown; probe command
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
3 files changed,
+52
−5
harness/cmd/bakeoff/main.go+37 −1harness/harness_test.go+3 −0harness/load.go+12 −4
diff --git a/harness/cmd/bakeoff/main.go b/harness/cmd/bakeoff/main.go| index 4ebe6e7..0726f7d 100644 |
| --- a/harness/cmd/bakeoff/main.go |
| +++ b/harness/cmd/bakeoff/main.go |
| @@ -5,11 +5,13 @@ |
| package main |
| import ( |
| + "encoding/json" |
| "errors" |
| "flag" |
| "fmt" |
| "io" |
| "os" |
| + "os/exec" |
| "path/filepath" |
| "strconv" |
| "strings" |
| @@ -78,6 +80,8 @@ func main() { |
| err = renderReport(*file) |
| case "findings": |
| err = findings(*slug, *file) |
| + case "probe": |
| + err = probe(*url, *levels, *dur) |
| case "record": |
| if rec.URL = *recURL; rec.URL == "" && strings.HasPrefix(rec.Status, "LIVE") { |
| rec.URL = *url |
| @@ -179,7 +183,11 @@ func load(slug, url, levels string, d, rest time.Duration) error { |
| if err != nil { |
| return err |
| } |
| - fmt.Printf("%s %s c=%d: %.0f rps, p50 %.1fms, p99 %.1fms, errors %.2f%%\n", slug, sc, c, r.RPS, r.P50Ms, r.P99Ms, r.ErrorRate*100) |
| + fmt.Printf("%s %s c=%d: %.0f rps, p50 %.1fms, p99 %.1fms, errors %.2f%%", slug, sc, c, r.RPS, r.P50Ms, r.P99Ms, r.ErrorRate*100) |
| + if r.ErrorRate > 0 { |
| + fmt.Printf(" status %v errors %v", r.Status, r.Errors) |
| + } |
| + fmt.Println() |
| sweep = append(sweep, r) |
| if b.SweepShouldStop(sweep) { |
| break |
| @@ -387,3 +395,31 @@ func findings(slug, file string) error { |
| } |
| return nil |
| } |
| + |
| +// probe runs one toggle step per level and prints oha's raw status and |
| +// error breakdown, to diagnose failures a ramp reports. |
| +func probe(url, levels string, d time.Duration) error { |
| + s, err := seeded(url, 50) |
| + if err != nil { |
| + return err |
| + } |
| + ids, err := s.IDs("/") |
| + if err != nil { |
| + return err |
| + } |
| + for _, l := range strings.Split(levels, ",") { |
| + out, err := exec.Command("oha", "--no-tui", "--output-format", "json", "-z", d.String(), "-c", strings.TrimSpace(l), |
| + "--redirect", "0", "-H", "Cookie: "+s.CookieHeader(), "-H", "Origin: "+url, |
| + "-m", "POST", "-T", "application/x-www-form-urlencoded", "-d", "filter=all", url+"/todos/"+ids[0]+"/toggle").Output() |
| + if err != nil { |
| + return err |
| + } |
| + var o struct { |
| + Status map[string]int `json:"statusCodeDistribution"` |
| + Errors map[string]int `json:"errorDistribution"` |
| + } |
| + json.Unmarshal(out, &o) |
| + fmt.Printf("c=%s status %v errors %v\n", l, o.Status, o.Errors) |
| + } |
| + return nil |
| +} |
diff --git a/harness/harness_test.go b/harness/harness_test.go| index 2790349..f336385 100644 |
| --- a/harness/harness_test.go |
| +++ b/harness/harness_test.go |
| @@ -73,6 +73,9 @@ func TestParseOhaCountsErrors(t *testing.T) { |
| if r.P50Ms != 1 || r.P99Ms != 3 { |
| t.Fatalf("want ms, got %+v", r) |
| } |
| + if r.Status["500"] != 10 || r.Errors["connection refused"] != 10 || r.Errors["aborted due to deadline"] != 0 { |
| + t.Fatalf("want the raw breakdown kept (minus deadline aborts), got %v %v", r.Status, r.Errors) |
| + } |
| } |
| func TestParseClaudeLog(t *testing.T) { |
diff --git a/harness/load.go b/harness/load.go| index 07f00e1..f48bf69 100644 |
| --- a/harness/load.go |
| +++ b/harness/load.go |
| @@ -14,6 +14,8 @@ type LoadResult struct { |
| Concurrency int |
| RPS, P50Ms, P95Ms, P99Ms float64 |
| ErrorRate float64 |
| + Status map[string]int `json:",omitempty"` // responses by status code |
| + Errors map[string]int `json:",omitempty"` // transport errors by message |
| } |
| // ParseOha reads oha's JSON output. Any status other than ok, and any |
| @@ -38,16 +40,22 @@ func ParseOha(b []byte, ok int) (LoadResult, error) { |
| bad += n |
| } |
| } |
| + errs := map[string]int{} |
| for msg, n := range o.Errors { |
| if msg != "aborted due to deadline" { |
| bad += n |
| + errs[msg] = n |
| } |
| } |
| r := LoadResult{ |
| - RPS: o.Summary.RequestsPerSec, |
| - P50Ms: o.Latency["p50"] * 1000, |
| - P95Ms: o.Latency["p95"] * 1000, |
| - P99Ms: o.Latency["p99"] * 1000, |
| + RPS: o.Summary.RequestsPerSec, |
| + P50Ms: o.Latency["p50"] * 1000, |
| + P95Ms: o.Latency["p95"] * 1000, |
| + P99Ms: o.Latency["p99"] * 1000, |
| + Status: o.Status, |
| + } |
| + if len(errs) > 0 { |
| + r.Errors = errs |
| } |
| if good+bad > 0 { |
| r.ErrorRate = float64(bad) / float64(good+bad) |