carlos / todomvc-bakeoff Public

Clone
git clone https://amadan.net/carlos/todomvc-bakeoff

Plain git — no account needed to clone.

Download

Download this file

1package bakeoff
2
3import (
4 "net/http/httptest"
5 "os"
6 "os/exec"
7 "path/filepath"
8 "strings"
9 "testing"
10 "time"
11
12 "bakeoff/harness/refapp/ref"
13)
14
15func refServer(t *testing.T) *httptest.Server {
16 t.Helper()
17 srv := httptest.NewUnstartedServer(nil)
18 srv.Start()
19 srv.Config.Handler = ref.NewApp(srv.URL, "../shared")
20 t.Cleanup(srv.Close)
21 return srv
22}
23
24func TestSeed(t *testing.T) {
25 srv := refServer(t)
26 s, err := NewSession(srv.URL)
27 if err != nil {
28 t.Fatal(err)
29 }
30 ids, err := Seed(s, 50)
31 if err != nil {
32 t.Fatal(err)
33 }
34 if len(ids) != 50 {
35 t.Fatalf("want 50 ids, got %d", len(ids))
36 }
37 done, err := s.IDs("/completed")
38 if err != nil {
39 t.Fatal(err)
40 }
41 if len(done) != 25 {
42 t.Fatalf("want 25 completed, got %d", len(done))
43 }
44 if s.ToggleID != ids[0] {
45 t.Fatalf("Seed must remember the toggle target: got %q, want %q", s.ToggleID, ids[0])
46 }
47 if !strings.Contains(s.CookieHeader(), "=") {
48 t.Fatalf("no cookie: %q", s.CookieHeader())
49 }
50}
51
52func TestParseOha(t *testing.T) {
53 b, _ := os.ReadFile("testdata/oha.json")
54 r, err := ParseOha(b, 200)
55 if err != nil {
56 t.Fatal(err)
57 }
58 if r.RPS <= 0 || r.P99Ms < r.P50Ms || r.P50Ms <= 0 {
59 t.Fatalf("bad result %+v", r)
60 }
61 if r.ErrorRate != 0 {
62 t.Fatalf("deadline aborts must not count as errors, got %v", r.ErrorRate)
63 }
64}
65
66func TestParseOhaCountsErrors(t *testing.T) {
67 b := []byte(`{"summary":{"requestsPerSec":10},"latencyPercentiles":{"p50":0.001,"p95":0.002,"p99":0.003},
68 "statusCodeDistribution":{"303":80,"500":10},"errorDistribution":{"connection refused":10,"aborted due to deadline":3}}`)
69 r, err := ParseOha(b, 303)
70 if err != nil {
71 t.Fatal(err)
72 }
73 if r.ErrorRate != 0.2 {
74 t.Fatalf("want 0.2, got %v", r.ErrorRate)
75 }
76 if r.P50Ms != 1 || r.P99Ms != 3 {
77 t.Fatalf("want ms, got %+v", r)
78 }
79 if r.Status["500"] != 10 || r.Errors["connection refused"] != 10 || r.Errors["aborted due to deadline"] != 0 {
80 t.Fatalf("want the raw breakdown kept (minus deadline aborts), got %v %v", r.Status, r.Errors)
81 }
82}
83
84func TestParseClaudeLog(t *testing.T) {
85 f, _ := os.Open("testdata/claude.jsonl")
86 defer f.Close()
87 tk, err := ParseClaudeLog(f)
88 if err != nil {
89 t.Fatal(err)
90 }
91 if tk.Output != 4 || tk.CacheWrite != 32396 || tk.Turns != 1 || tk.CostUSD < 0.25 || tk.Seconds <= 0 {
92 t.Fatalf("bad tokens %+v", tk)
93 }
94}
95
96func TestParseCodexLog(t *testing.T) {
97 f, _ := os.Open("testdata/codex.jsonl")
98 defer f.Close()
99 tk, err := ParseCodexLog(f)
100 if err != nil {
101 t.Fatal(err)
102 }
103 if tk.Input != 16532 || tk.CacheRead != 12032 || tk.Output != 5 || tk.Turns != 1 {
104 t.Fatalf("bad tokens %+v", tk)
105 }
106}
107
108func TestCountCode(t *testing.T) {
109 c, err := CountCode("testdata/codetree")
110 if err != nil {
111 t.Fatal(err)
112 }
113 want := CodeStats{Server: 5, Templates: 5, JS: 4, Tests: 2, Deps: 2}
114 if c != want {
115 t.Fatalf("got %+v, want %+v", c, want)
116 }
117}
118
119func TestPageWeight(t *testing.T) {
120 srv := refServer(t)
121 s, _ := NewSession(srv.URL)
122 w, err := PageWeight(s, "/")
123 if err != nil {
124 t.Fatal(err)
125 }
126 js, _ := os.ReadFile("refapp/ref/app.js")
127 if w.JSBytes != len(js) || w.HTMLBytes <= 0 {
128 t.Fatalf("got %+v, want JSBytes %d", w, len(js))
129 }
130}
131
132func TestMeasureTTFB(t *testing.T) {
133 srv := refServer(t)
134 s, _ := NewSession(srv.URL)
135 st, err := MeasureTTFB(s, "/", 1, 5)
136 if err != nil {
137 t.Fatal(err)
138 }
139 if st.N != 5 || st.Min > st.P50 || st.P50 > st.Max || st.Min <= 0 {
140 t.Fatalf("bad stats %+v", st)
141 }
142}
143
144func TestBuildStats(t *testing.T) {
145 dir := t.TempDir()
146 for _, f := range []string{"Makefile", "src.go"} {
147 b, _ := os.ReadFile("testdata/buildtree/" + f)
148 os.WriteFile(filepath.Join(dir, f), b, 0o644)
149 }
150 b, err := BuildStats(dir, false, 1, 2)
151 if err != nil {
152 t.Fatal(err)
153 }
154 if !(b.ColdSeconds > b.WarmSeconds && b.WarmSeconds > b.EditSeconds && b.EditSeconds > 0) {
155 t.Fatalf("want cold > warm > edit > 0, got %+v", b)
156 }
157 if b.Bytes != 6 {
158 t.Fatalf("want 6 bytes, got %d", b.Bytes)
159 }
160 if _, err := os.Stat(filepath.Join(dir, ".unbumped")); err != nil {
161 t.Fatal("unbump was not run")
162 }
163 last, _ := os.ReadFile(filepath.Join(dir, "dist", "history"))
164 if !strings.Contains(string(last), "// bakeoff edit ") {
165 t.Fatalf("the edit build must compile a unique comment line, built:\n%s", last)
166 }
167 src, _ := os.ReadFile(filepath.Join(dir, "src.go"))
168 if string(src) != "package main\n\nconst marker = \"steady\"\n" {
169 t.Fatalf("source not restored:\n%s", src)
170 }
171 if b.ColdRuns != 1 || b.Runs != 2 {
172 t.Fatalf("want 1 cold run and 2 warm/edit runs recorded, got %+v", b)
173 }
174}
175
176func TestUpdateRoundTrip(t *testing.T) {
177 dir := t.TempDir()
178 if err := UpdateIn(dir, "rust", func(r *AppResult) { r.Language = "Rust"; r.Wake = append(r.Wake, 12.5) }); err != nil {
179 t.Fatal(err)
180 }
181 if err := UpdateIn(dir, "rust", func(r *AppResult) { r.Wake = append(r.Wake, 13) }); err != nil {
182 t.Fatal(err)
183 }
184 r, err := LoadIn(dir, "rust")
185 if err != nil {
186 t.Fatal(err)
187 }
188 if r.Slug != "rust" || r.Language != "Rust" || len(r.Wake) != 2 {
189 t.Fatalf("got %+v", r)
190 }
191}
192
193func TestRunLoadAgainstRefapp(t *testing.T) {
194 if testing.Short() {
195 t.Skip("runs oha")
196 }
197 if _, err := exec.LookPath("oha"); err != nil {
198 t.Skip("oha not installed")
199 }
200 srv := refServer(t)
201 s, _ := NewSession(srv.URL)
202 if _, err := Seed(s, 10); err != nil {
203 t.Fatal(err)
204 }
205 for _, sc := range []string{"read", "toggle", "read-after"} {
206 r, err := RunLoad(s, sc, 2, time.Second)
207 if err != nil {
208 t.Fatal(err)
209 }
210 if r.RPS <= 0 || r.ErrorRate != 0 || r.Scenario != sc || r.Concurrency != 2 {
211 t.Fatalf("%s: bad result %+v", sc, r)
212 }
213 }
214}
215
216func TestParsePlaywright(t *testing.T) {
217 f, _ := os.Open("testdata/playwright.json")
218 defer f.Close()
219 got, err := ParsePlaywright(f)
220 if err != nil {
221 t.Fatal(err)
222 }
223 if got["js-on"] != (Behaviour{Passed: 1, Failed: 1}) || got["js-off"] != (Behaviour{Failed: 1, Skipped: 1}) {
224 t.Fatalf("got %+v (flaky counts as failed)", got)
225 }
226}
227
228func TestSourceFiles(t *testing.T) {
229 got, err := SourceFiles("testdata/codetree")
230 if err != nil {
231 t.Fatal(err)
232 }
233 want := []string{"main.go", "src/app.nim", "src/page.nimf", "static/app.js", "views/index.html"}
234 if strings.Join(got, ",") != strings.Join(want, ",") {
235 t.Fatalf("got %v, want %v", got, want)
236 }
237}
238
239func TestMedianReadability(t *testing.T) {
240 r, err := MedianReadability([]string{
241 `{"naming":4,"structure":3,"traceability":5,"errors":2,"comments":3,"overall":7,"notes":"a"}`,
242 "```json\n{\"naming\":2,\"structure\":4,\"traceability\":4,\"errors\":3,\"comments\":3,\"overall\":6,\"notes\":\"b\"}\n```",
243 `{"naming":5,"structure":5,"traceability":3,"errors":4,"comments":2,"overall":8,"notes":"c"}`,
244 })
245 if err != nil {
246 t.Fatal(err)
247 }
248 if r.Naming != 4 || r.Structure != 4 || r.Traceability != 4 || r.Errors != 3 || r.Comments != 3 || r.Overall != 7 || r.Notes != "a" {
249 t.Fatalf("got %+v (notes come from the run whose overall is the median)", r)
250 }
251}
252
253func TestParseFindings(t *testing.T) {
254 got := ParseFindings("1. [spec] [a.go:1](/x/a.go:1): first\n continues here.\n\n2. [security] b: second\n")
255 if len(got) != 2 || got[0] != "[spec] [a.go:1](/x/a.go:1): first continues here." || got[1] != "[security] b: second" {
256 t.Fatalf("got %q", got)
257 }
258 if len(ParseFindings("No findings.")) != 0 {
259 t.Fatal("No findings. must parse to none")
260 }
261 if !HasSecurity(got) || HasSecurity(got[:1]) {
262 t.Fatal("HasSecurity wrong")
263 }
264}
265
266func TestTokensSince(t *testing.T) {
267 build := Tokens{Input: 82, Output: 56663, CacheRead: 4280334, CacheWrite: 132243, Turns: 46, Seconds: 693, CostUSD: 3.05}
268 fixCumulative := Tokens{Input: 106, Output: 65224, CacheRead: 6096946, CacheWrite: 145431, Turns: 12, Seconds: 183, CostUSD: 3.69}
269 got := fixCumulative.Since(build)
270 want := Tokens{Input: 24, Output: 8561, CacheRead: 1816612, CacheWrite: 13188, Turns: 12, Seconds: 183}
271 if got.Input != want.Input || got.Output != want.Output || got.CacheRead != want.CacheRead || got.CacheWrite != want.CacheWrite ||
272 got.Turns != 12 || got.Seconds != 183 || got.CostUSD < 0.63 || got.CostUSD > 0.65 {
273 t.Fatalf("got %+v", got)
274 }
275}
276
277func TestApplyRecordKeepsUnsetFields(t *testing.T) {
278 r := AppResult{Language: "Go", Status: "LIVE", URL: "u", Version: "v1"}
279 ApplyRecord(&r, Record{Version: "v2"})
280 if r.Language != "Go" || r.Status != "LIVE" || r.URL != "u" || r.Version != "v2" {
281 t.Fatalf("got %+v", r)
282 }
283}
284
285func TestParseClaudeLogSumsTurnsAcrossResults(t *testing.T) {
286 f, _ := os.Open("testdata/claude-multi.jsonl")
287 defer f.Close()
288 tk, err := ParseClaudeLog(f)
289 if err != nil {
290 t.Fatal(err)
291 }
292 if tk.Turns != 5 || tk.Seconds != 1.5 || tk.Output != 140 || tk.CacheRead != 30 || tk.CostUSD != 1.5 {
293 t.Fatalf("got %+v (usage and cost are cumulative: take the last; turns and time are per result: sum)", tk)
294 }
295}
296
297// lr is a recorded ramp step; like real ones it carries a status
298// breakdown (completed responses), which is how a stall is told apart.
299func lr(c int, rps, p99, errs float64) LoadResult {
300 return LoadResult{Scenario: "read", Concurrency: c, RPS: rps, P99Ms: p99, ErrorRate: errs,
301 Status: map[string]int{"200": int(rps*10) + 1}}
302}
303
304func TestKneeIsFirstDoublingWithLittleGain(t *testing.T) {
305 sweep := []LoadResult{lr(1, 20, 60, 0), lr(2, 40, 60, 0), lr(4, 79, 61, 0), lr(8, 150, 70, 0), lr(16, 160, 130, 0), lr(32, 162, 260, 0)}
306 k := Knee(sweep)
307 if !k.Found || k.Concurrency != 8 || k.PeakRPS != 162 {
308 t.Fatalf("got %+v, want slowdown after 8 (16 added under 10%%), peak 162", k)
309 }
310 if k := Knee(sweep[:4]); k.Found || k.PeakRPS != 150 || k.MaxTested != 8 {
311 t.Fatalf("got %+v, want not found within 8", k)
312 }
313}
314
315func TestSweepStopsOnErrorsOrFlatThroughput(t *testing.T) {
316 if !SweepShouldStop([]LoadResult{lr(1, 20, 60, 0), lr(2, 40, 60, 0.02)}) {
317 t.Fatal("should stop on >1% errors")
318 }
319 if !SweepShouldStop([]LoadResult{lr(4, 100, 60, 0), lr(8, 105, 90, 0), lr(16, 108, 180, 0)}) {
320 t.Fatal("should stop after two flat doublings")
321 }
322 if SweepShouldStop([]LoadResult{lr(4, 100, 60, 0), lr(8, 105, 90, 0), lr(16, 190, 90, 0)}) {
323 t.Fatal("one flat step then growth: keep going")
324 }
325}
326
327func TestKneeIgnoresAOneStepBlip(t *testing.T) {
328 sweep := []LoadResult{lr(1, 58, 29, 0), lr(2, 108, 38, 0), lr(4, 98, 66, 0), lr(8, 184, 77, 0), lr(16, 314, 97, 0), lr(32, 325, 294, 0), lr(64, 324, 640, 0)}
329 if k := Knee(sweep); !k.Found || k.Concurrency != 16 || k.PeakRPS != 325 {
330 t.Fatalf("got %+v, want 16: the fewest connections from which no more load adds 10%%", k)
331 }
332}
333
334func TestKneeStopsAtErrors(t *testing.T) {
335 sweep := []LoadResult{lr(16, 400, 60, 0), lr(32, 800, 70, 0), lr(64, 1300, 90, 0), lr(128, 1360, 370, 0.40)}
336 k := Knee(sweep)
337 if k.ErrorsStatus != "" {
338 t.Fatalf("no breakdown recorded: status should be empty, got %q", k.ErrorsStatus)
339 }
340 if !k.Found || k.Concurrency != 64 || k.ErrorsAt != 128 || k.PeakRPS != 1300 {
341 t.Fatalf("got %+v: want slow-down 64 (last clean step), errors from 128, peak from clean steps only", k)
342 }
343}
344
345func TestApplyRecordAppendsNoteOnce(t *testing.T) {
346 r := AppResult{}
347 ApplyRecord(&r, Record{Note: "exits under load"})
348 ApplyRecord(&r, Record{Note: "exits under load"})
349 if len(r.Notes) != 1 || r.Notes[0] != "exits under load" {
350 t.Fatalf("got %q", r.Notes)
351 }
352}
353
354func TestAsleepFromLogs(t *testing.T) {
355 logs := `2026-10-24 09:50:52 [platform] todo-zig-2.bfj.oncarlos.com instance stopping
3562026-10-24 09:50:53 [platform] todo-zig-2.bfj.oncarlos.com instance exited (exit 0)
3572026-10-24 10:01:00 [platform] todo-zig.bfj.oncarlos.com instance woke in 1.1s (spawn 1s) (version x)`
358 if !AsleepFromLogs(logs, "todo-zig-2.bfj.oncarlos.com") {
359 t.Fatal("last event for the host is an exit: asleep")
360 }
361 logs += "\n2026-10-24 10:36:41 [platform] todo-zig-2.bfj.oncarlos.com instance woke in 1.156s (spawn 1.037s) (version 73268e0)"
362 if AsleepFromLogs(logs, "todo-zig-2.bfj.oncarlos.com") {
363 t.Fatal("last event is a wake: awake")
364 }
365 if AsleepFromLogs("", "todo-zig-2.bfj.oncarlos.com") {
366 t.Fatal("no events: unknown, so not asleep")
367 }
368}
369
370func TestParseProcStatus(t *testing.T) {
371 status := "Name:\tapp\nVmHWM:\t 20480 kB\nVmRSS:\t 12288 kB\nThreads:\t4\n"
372 rss, hwm, err := ParseProcStatus(status)
373 if err != nil || rss != 12288 || hwm != 20480 {
374 t.Fatalf("got rss %d hwm %d err %v", rss, hwm, err)
375 }
376}
377
378func TestMeasureMemoryOfRefapp(t *testing.T) {
379 if testing.Short() {
380 t.Skip("builds and loads the reference app")
381 }
382 if _, err := exec.LookPath("oha"); err != nil {
383 t.Skip("oha not installed")
384 }
385 bin := filepath.Join(t.TempDir(), "refapp")
386 if out, err := exec.Command("go", "build", "-o", bin, "./refapp").CombinedOutput(); err != nil {
387 t.Fatalf("%v\n%s", err, out)
388 }
389 shared, _ := filepath.Abs("../shared")
390 m, err := MeasureMemory(bin, []string{"--shared", shared}, time.Second)
391 if err != nil {
392 t.Fatal(err)
393 }
394 if m.IdleKB <= 0 || m.PeakKB < m.IdleKB || m.AfterKB <= 0 {
395 t.Fatalf("implausible %+v", m)
396 }
397}
398
399func TestKneeNamesTheFailingStatus(t *testing.T) {
400 bad := lr(128, 1360, 370, 0.40)
401 bad.Status = map[string]int{"200": 600, "503": 400}
402 k := Knee([]LoadResult{lr(32, 800, 70, 0), lr(64, 1300, 90, 0), bad})
403 if k.ErrorsStatus != "503" {
404 t.Fatalf("want 503, got %q", k.ErrorsStatus)
405 }
406 bad.Status = map[string]int{"200": 600, "502": 300, "503": 100}
407 if k := Knee([]LoadResult{lr(64, 1300, 90, 0), bad}); k.ErrorsStatus != "502" {
408 t.Fatalf("want the most common failing status 502, got %q", k.ErrorsStatus)
409 }
410}
411
412func TestKneeTreatsAStalledStepAsFailure(t *testing.T) {
413 stall := lr(32, 3.2, 0, 0)
414 stall.Status = nil
415 ok := lr(16, 500, 40, 0)
416 ok.Status = map[string]int{"200": 5000}
417 k := Knee([]LoadResult{lr(8, 300, 30, 0), ok, stall, lr(64, 700, 90, 0)})
418 if k.ErrorsAt != 32 || k.ErrorsStatus != "stalled" || k.PeakRPS != 500 || k.Concurrency != 16 {
419 t.Fatalf("a step that completed nothing is a failure, got %+v", k)
420 }
421}
422
423func TestParseOhaNoCompletionsIsAllErrors(t *testing.T) {
424 b := []byte(`{"summary":{"requestsPerSec":3.2},"latencyPercentiles":{},"statusCodeDistribution":{},"errorDistribution":{"aborted due to deadline":32}}`)
425 r, err := ParseOha(b, 200)
426 if err != nil {
427 t.Fatal(err)
428 }
429 if r.ErrorRate != 1 || r.RPS != 0 {
430 t.Fatalf("no completed response: want error rate 1 and 0 rps, got %+v", r)
431 }
432}
433