amadan
carlos / todomvc-bakeoff Public
feat: measurement harness
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
33 files changed,
+1627
−219
harness/apps.go+32 −0harness/behaviour.go+51 −0harness/buildstats.go+87 −0harness/cmd/bakeoff/main.go+251 −0harness/codestats.go+93 −0harness/harness_test.go+207 −0harness/load.go+85 −0harness/pageweight.go+33 −0harness/refapp/main.go+5 −217harness/refapp/{ => ref}/app.js+0 −0harness/refapp/{ => ref}/page.html.tmpl+0 −0harness/refapp/ref/ref.go+222 −0harness/refapp/{refapp_test.go => ref/ref_test.go}+2 −2harness/results.go+73 −0harness/session.go+137 −0harness/testdata/buildtree/Makefile+11 −0harness/testdata/claude.jsonl+2 −0harness/testdata/codetree/DEPS+2 −0harness/testdata/codetree/README.md+1 −0harness/testdata/codetree/go.sum+1 −0harness/testdata/codetree/main.go+4 −0harness/testdata/codetree/main_test.go+3 −0harness/testdata/codetree/static/app.js+5 −0harness/testdata/codetree/static/index.css+1 −0harness/testdata/codetree/static/van-1.5.2.min.js+1 −0harness/testdata/codetree/vendor/x.go+1 −0harness/testdata/codetree/views/index.html+3 −0harness/testdata/codex.jsonl+4 −0harness/testdata/oha.json+113 −0harness/testdata/playwright.json+4 −0harness/tokens.go+111 −0harness/ttfb.go+62 −0harness/wake.go+20 −0
diff --git a/harness/apps.go b/harness/apps.go| new file mode 100644 |
| index 0000000..c1a6f61 |
| --- /dev/null |
| +++ b/harness/apps.go |
| @@ -0,0 +1,32 @@ |
| +package bakeoff |
| + |
| +// App is one contestant. |
| +type App struct{ Slug, Language string } |
| + |
| +// Apps lists the contestants in report order. |
| +var Apps = []App{ |
| + {"go", "Go"}, |
| + {"go-rastrillo", "Go + Rastrillo"}, |
| + {"rust", "Rust"}, |
| + {"zig", "Zig"}, |
| + {"ocaml", "OCaml"}, |
| + {"haskell", "Haskell"}, |
| + {"swift", "Swift"}, |
| + {"mojo", "Mojo"}, |
| + {"spinel", "Spinel"}, |
| +} |
| + |
| +// Language names a slug's setup, or returns the slug itself. |
| +func Language(slug string) string { |
| + for _, a := range Apps { |
| + if a.Slug == slug { |
| + return a.Language |
| + } |
| + } |
| + return slug |
| +} |
| + |
| +// AppName and Host are the slug's Carloku app and hostname. |
| +func AppName(slug string) string { return "todo-" + slug } |
| +func Host(slug string) string { return AppName(slug) + ".bfj.oncarlos.com" } |
| +func URL(slug string) string { return "https://" + Host(slug) } |
diff --git a/harness/behaviour.go b/harness/behaviour.go| new file mode 100644 |
| index 0000000..25a5559 |
| --- /dev/null |
| +++ b/harness/behaviour.go |
| @@ -0,0 +1,51 @@ |
| +package bakeoff |
| + |
| +import ( |
| + "encoding/json" |
| + "io" |
| +) |
| + |
| +// ParsePlaywright counts the shared suite's results per project from |
| +// Playwright's JSON reporter. A flaky test (passed only on retry) counts |
| +// as failed: the suite runs without retries, so it shouldn't happen. |
| +func ParsePlaywright(r io.Reader) (map[string]Behaviour, error) { |
| + type test struct { |
| + Project string `json:"projectName"` |
| + Status string `json:"status"` |
| + } |
| + type suite struct { |
| + Specs []struct { |
| + Tests []test `json:"tests"` |
| + } `json:"specs"` |
| + Suites []suite `json:"suites"` |
| + } |
| + var report struct { |
| + Suites []suite `json:"suites"` |
| + } |
| + if err := json.NewDecoder(r).Decode(&report); err != nil { |
| + return nil, err |
| + } |
| + out := map[string]Behaviour{} |
| + var walk func([]suite) |
| + walk = func(ss []suite) { |
| + for _, s := range ss { |
| + for _, sp := range s.Specs { |
| + for _, t := range sp.Tests { |
| + b := out[t.Project] |
| + switch t.Status { |
| + case "expected": |
| + b.Passed++ |
| + case "skipped": |
| + b.Skipped++ |
| + default: |
| + b.Failed++ |
| + } |
| + out[t.Project] = b |
| + } |
| + } |
| + walk(s.Suites) |
| + } |
| + } |
| + walk(report.Suites) |
| + return out, nil |
| +} |
diff --git a/harness/buildstats.go b/harness/buildstats.go| new file mode 100644 |
| index 0000000..96e3735 |
| --- /dev/null |
| +++ b/harness/buildstats.go |
| @@ -0,0 +1,87 @@ |
| +package bakeoff |
| + |
| +import ( |
| + "fmt" |
| + "os" |
| + "os/exec" |
| + "path/filepath" |
| + "sort" |
| + "time" |
| +) |
| + |
| +// Build is the three build times for the shipped arm64 binary, as medians |
| +// in seconds, and its size. |
| +type Build struct { |
| + ColdSeconds, WarmSeconds, EditSeconds float64 |
| + Bytes int64 |
| + Host string |
| +} |
| + |
| +func mk(dir string, targets ...string) error { |
| + cmd := exec.Command("make", targets...) |
| + cmd.Dir = dir |
| + if out, err := cmd.CombinedOutput(); err != nil { |
| + return fmt.Errorf("make %v: %w\n%s", targets, err, out) |
| + } |
| + return nil |
| +} |
| + |
| +func timed(dir string, targets ...string) (float64, error) { |
| + start := time.Now() |
| + err := mk(dir, targets...) |
| + return time.Since(start).Seconds(), err |
| +} |
| + |
| +func median(v []float64) float64 { |
| + sort.Float64s(v) |
| + return v[len(v)/2] |
| +} |
| + |
| +// BuildStats measures cold (clean-deps; COLD=1 for Go, which passes -a), |
| +// warm (clean) and one-line-change (bump) builds, runs times each. |
| +func BuildStats(dir string, goCold bool, runs int) (Build, error) { |
| + coldTarget := []string{"build-arm64"} |
| + if goCold { |
| + coldTarget = append(coldTarget, "COLD=1") |
| + } |
| + var cold, warm, edit []float64 |
| + for i := 0; i < runs; i++ { |
| + if err := mk(dir, "clean", "clean-deps"); err != nil { |
| + return Build{}, err |
| + } |
| + s, err := timed(dir, coldTarget...) |
| + if err != nil { |
| + return Build{}, err |
| + } |
| + cold = append(cold, s) |
| + |
| + if err := mk(dir, "clean"); err != nil { |
| + return Build{}, err |
| + } |
| + if s, err = timed(dir, "build-arm64"); err != nil { |
| + return Build{}, err |
| + } |
| + warm = append(warm, s) |
| + |
| + if err := mk(dir, "bump"); err != nil { |
| + return Build{}, err |
| + } |
| + s, err = timed(dir, "build-arm64") |
| + if uerr := mk(dir, "unbump"); err == nil { |
| + err = uerr |
| + } |
| + if err != nil { |
| + return Build{}, err |
| + } |
| + edit = append(edit, s) |
| + } |
| + if err := mk(dir, "build-arm64"); err != nil { |
| + return Build{}, err |
| + } |
| + fi, err := os.Stat(filepath.Join(dir, "dist", "app-linux-arm64")) |
| + if err != nil { |
| + return Build{}, err |
| + } |
| + host, _ := os.Hostname() |
| + return Build{ColdSeconds: median(cold), WarmSeconds: median(warm), EditSeconds: median(edit), Bytes: fi.Size(), Host: host}, nil |
| +} |
diff --git a/harness/cmd/bakeoff/main.go b/harness/cmd/bakeoff/main.go| new file mode 100644 |
| index 0000000..5ad5a20 |
| --- /dev/null |
| +++ b/harness/cmd/bakeoff/main.go |
| @@ -0,0 +1,251 @@ |
| +// Command bakeoff measures one app and records the result in |
| +// results/<slug>.json. Run it from the repo root. |
| +// |
| +// bakeoff <ttfb|wake|load|weight|code|build|tokens|behaviour> --slug <s> [flags] |
| +package main |
| + |
| +import ( |
| + "errors" |
| + "flag" |
| + "fmt" |
| + "io" |
| + "os" |
| + "path/filepath" |
| + "strconv" |
| + "strings" |
| + "time" |
| + |
| + b "bakeoff/harness" |
| +) |
| + |
| +func main() { |
| + if len(os.Args) < 2 { |
| + usage() |
| + } |
| + cmd := os.Args[1] |
| + fs := flag.NewFlagSet(cmd, flag.ExitOnError) |
| + slug := fs.String("slug", "", "app slug, e.g. rust") |
| + url := fs.String("url", "", "app base URL (default https://todo-<slug>.bfj.oncarlos.com)") |
| + results := fs.String("results", "results", "results directory") |
| + dir := fs.String("dir", "", "app directory (default apps/<slug>)") |
| + runs := fs.Int("runs", 3, "build: runs per mode") |
| + dur := fs.Duration("duration", 30*time.Second, "load: length of each run") |
| + rest := fs.Duration("rest", 60*time.Second, "load: pause between runs") |
| + levels := fs.String("levels", "1,16,64", "load: concurrency levels") |
| + phase := fs.String("phase", "", "tokens: build|fix|security|review") |
| + logs := fs.String("logs", "builders/logs", "tokens: log directory") |
| + file := fs.String("file", "", "behaviour: Playwright JSON report") |
| + fs.Parse(os.Args[2:]) |
| + if *slug == "" { |
| + usage() |
| + } |
| + b.ResultsDir = *results |
| + if *url == "" { |
| + *url = b.URL(*slug) |
| + } |
| + if *dir == "" { |
| + *dir = filepath.Join("apps", *slug) |
| + } |
| + var err error |
| + switch cmd { |
| + case "ttfb": |
| + err = ttfb(*slug, *url) |
| + case "wake": |
| + err = wake(*slug, *url) |
| + case "load": |
| + err = load(*slug, *url, *levels, *dur, *rest) |
| + case "weight": |
| + err = weight(*slug, *url) |
| + case "code": |
| + err = code(*slug, *dir) |
| + case "build": |
| + err = build(*slug, *dir, *runs) |
| + case "tokens": |
| + err = tokens(*slug, *logs, *phase) |
| + case "behaviour": |
| + err = behaviour(*slug, *file) |
| + default: |
| + usage() |
| + } |
| + if err != nil { |
| + fmt.Fprintf(os.Stderr, "bakeoff %s %s: %v\n", cmd, *slug, err) |
| + os.Exit(1) |
| + } |
| +} |
| + |
| +func usage() { |
| + fmt.Fprintln(os.Stderr, "usage: bakeoff <ttfb|wake|load|weight|code|build|tokens|behaviour> --slug <s> [flags]") |
| + os.Exit(2) |
| +} |
| + |
| +func seeded(url string, n int) (*b.Session, error) { |
| + s, err := b.NewSession(url) |
| + if err != nil { |
| + return nil, err |
| + } |
| + if n > 0 { |
| + _, err = b.Seed(s, n) |
| + } |
| + return s, err |
| +} |
| + |
| +func ttfb(slug, url string) error { |
| + empty, err := seeded(url, 0) |
| + if err != nil { |
| + return err |
| + } |
| + e, err := b.MeasureTTFB(empty, "/", 3, 30) |
| + if err != nil { |
| + return err |
| + } |
| + full, err := seeded(url, 50) |
| + if err != nil { |
| + return err |
| + } |
| + f, err := b.MeasureTTFB(full, "/", 3, 30) |
| + if err != nil { |
| + return err |
| + } |
| + fmt.Printf("%s ttfb empty p50 %.1fms, 50 items p50 %.1fms\n", slug, e.P50, f.P50) |
| + return b.Update(slug, func(r *b.AppResult) { r.TTFBEmpty, r.TTFB50 = e, f }) |
| +} |
| + |
| +// wake takes one sample if the instance is asleep, and exits 3 if it is |
| +// awake, so a polling loop can tell "not asleep yet" from failure. |
| +func wake(slug, url string) error { |
| + asleep, err := b.IsAsleep(b.AppName(slug), b.Host(slug)) |
| + if err != nil { |
| + return err |
| + } |
| + if !asleep { |
| + fmt.Printf("%s is awake; no sample\n", slug) |
| + os.Exit(3) |
| + } |
| + // A fresh visitor, without first fetching / (that would wake it). |
| + s, err := b.SessionNoFetch(url) |
| + if err != nil { |
| + return err |
| + } |
| + ms, err := b.MeasureWake(s, "/") |
| + if err != nil { |
| + return err |
| + } |
| + fmt.Printf("%s wake %.1fms\n", slug, ms) |
| + return b.Update(slug, func(r *b.AppResult) { r.Wake = append(r.Wake, ms) }) |
| +} |
| + |
| +func load(slug, url, levels string, d, rest time.Duration) error { |
| + s, err := seeded(url, 50) |
| + if err != nil { |
| + return err |
| + } |
| + var out []b.LoadResult |
| + first := true |
| + for _, sc := range []string{"read", "toggle"} { |
| + for _, l := range strings.Split(levels, ",") { |
| + c, err := strconv.Atoi(strings.TrimSpace(l)) |
| + if err != nil { |
| + return err |
| + } |
| + if !first { |
| + time.Sleep(rest) |
| + } |
| + first = false |
| + r, err := b.RunLoad(s, sc, c, d) |
| + if err != nil { |
| + return err |
| + } |
| + fmt.Printf("%s %s c=%d: %.0f rps, p50 %.1fms, p99 %.1fms, errors %.2f%%\n", slug, sc, c, r.RPS, r.P50Ms, r.P99Ms, r.ErrorRate*100) |
| + out = append(out, r) |
| + } |
| + } |
| + return b.Update(slug, func(r *b.AppResult) { r.Load = out }) |
| +} |
| + |
| +func weight(slug, url string) error { |
| + s, err := seeded(url, 50) |
| + if err != nil { |
| + return err |
| + } |
| + w, err := b.PageWeight(s, "/") |
| + if err != nil { |
| + return err |
| + } |
| + fmt.Printf("%s page weight (50 items): html %d bytes, js %d bytes\n", slug, w.HTMLBytes, w.JSBytes) |
| + return b.Update(slug, func(r *b.AppResult) { r.Weight = w }) |
| +} |
| + |
| +func code(slug, dir string) error { |
| + c, err := b.CountCode(dir) |
| + if err != nil { |
| + return err |
| + } |
| + fmt.Printf("%s code: %+v\n", slug, c) |
| + return b.Update(slug, func(r *b.AppResult) { r.Code = c }) |
| +} |
| + |
| +func build(slug, dir string, runs int) error { |
| + st, err := b.BuildStats(dir, strings.HasPrefix(slug, "go"), runs) |
| + if err != nil { |
| + return err |
| + } |
| + fmt.Printf("%s build: cold %.1fs, warm %.1fs, edit %.1fs, %d bytes on %s\n", slug, st.ColdSeconds, st.WarmSeconds, st.EditSeconds, st.Bytes, st.Host) |
| + return b.Update(slug, func(r *b.AppResult) { r.Build = st }) |
| +} |
| + |
| +func tokens(slug, logs, phase string) error { |
| + parse := b.ParseClaudeLog |
| + if phase == "review" { |
| + parse = b.ParseCodexLog |
| + } else if phase != "build" && phase != "fix" && phase != "security" { |
| + return errors.New("--phase must be build, fix, security or review") |
| + } |
| + files, _ := filepath.Glob(filepath.Join(logs, slug+"."+phase+"*.jsonl")) |
| + if len(files) == 0 { |
| + return fmt.Errorf("no %s logs for %s in %s", phase, slug, logs) |
| + } |
| + var total b.Tokens |
| + for _, f := range files { |
| + fh, err := os.Open(f) |
| + if err != nil { |
| + return err |
| + } |
| + t, err := parse(fh) |
| + fh.Close() |
| + if err != nil { |
| + return fmt.Errorf("%s: %w", f, err) |
| + } |
| + total = total.Add(t) |
| + } |
| + fmt.Printf("%s %s tokens: %+v\n", slug, phase, total) |
| + return b.Update(slug, func(r *b.AppResult) { |
| + switch phase { |
| + case "build": |
| + r.TokensBuild = total |
| + case "fix": |
| + r.TokensFix = total |
| + case "security": |
| + r.TokensSecurity = total |
| + case "review": |
| + r.TokensReview = total |
| + } |
| + }) |
| +} |
| + |
| +func behaviour(slug, file string) error { |
| + var in io.Reader = os.Stdin |
| + if file != "" { |
| + f, err := os.Open(file) |
| + if err != nil { |
| + return err |
| + } |
| + defer f.Close() |
| + in = f |
| + } |
| + got, err := b.ParsePlaywright(in) |
| + if err != nil { |
| + return err |
| + } |
| + fmt.Printf("%s behaviour: js-on %+v, js-off %+v\n", slug, got["js-on"], got["js-off"]) |
| + return b.Update(slug, func(r *b.AppResult) { r.BehaviourJSOn, r.BehaviourJSOff = got["js-on"], got["js-off"] }) |
| +} |
diff --git a/harness/codestats.go b/harness/codestats.go| new file mode 100644 |
| index 0000000..7d85213 |
| --- /dev/null |
| +++ b/harness/codestats.go |
| @@ -0,0 +1,93 @@ |
| +package bakeoff |
| + |
| +import ( |
| + "bufio" |
| + "io/fs" |
| + "os" |
| + "path/filepath" |
| + "regexp" |
| + "strings" |
| +) |
| + |
| +// CodeStats counts non-blank lines by role, plus direct dependencies. |
| +type CodeStats struct{ Server, Templates, JS, Tests, Deps int } |
| + |
| +var ( |
| + skipDirs = map[string]bool{"dist": true, "vendor": true, ".cache": true, "node_modules": true, ".git": true, "_build": true, "dist-newstyle": true, "target": true, "zig-out": true, ".zig-cache": true, ".build": true} |
| + skipExts = map[string]bool{".css": true, ".md": true, ".sum": true, ".lock": true, ".json": true, ".db": true, ".png": true, ".ico": true, ".svg": true} |
| + sourceExts = map[string]bool{".go": true, ".rs": true, ".zig": true, ".ml": true, ".mli": true, ".hs": true, ".swift": true, ".mojo": true, ".🔥": true, ".rb": true, ".c": true, ".h": true, ".js": true, ".mjs": true, ".html": true, ".tmpl": true, ".erb": true, ".mustache": true, ".eex": true, ".sql": true} |
| + templateExt = map[string]bool{".html": true, ".tmpl": true, ".erb": true, ".mustache": true, ".eex": true} |
| + testName = regexp.MustCompile(`(_test\.|\.test\.|_spec\.|Spec\.hs$|Test\.hs$|Tests\.swift$)`) |
| + vanName = regexp.MustCompile(`^van[-.]`) |
| +) |
| + |
| +// CountCode classifies every source file under dir. Build output, vendored |
| +// code, lockfiles, CSS and docs are skipped; DEPS is counted as Deps. |
| +func CountCode(dir string) (CodeStats, error) { |
| + var c CodeStats |
| + err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { |
| + if err != nil { |
| + return err |
| + } |
| + rel, _ := filepath.Rel(dir, p) |
| + if d.IsDir() { |
| + if p != dir && skipDirs[d.Name()] { |
| + return filepath.SkipDir |
| + } |
| + return nil |
| + } |
| + if rel == "DEPS" { |
| + n, err := lines(p) |
| + c.Deps = n |
| + return err |
| + } |
| + ext := strings.ToLower(filepath.Ext(p)) |
| + if skipExts[ext] || !sourceExts[ext] || vanName.MatchString(d.Name()) { |
| + return nil |
| + } |
| + n, err := lines(p) |
| + if err != nil { |
| + return err |
| + } |
| + parts := strings.Split(filepath.ToSlash(rel), "/") |
| + inDir := func(names ...string) bool { |
| + for _, seg := range parts[:len(parts)-1] { |
| + for _, n := range names { |
| + if seg == n { |
| + return true |
| + } |
| + } |
| + } |
| + return false |
| + } |
| + switch { |
| + case inDir("test", "tests") || testName.MatchString(d.Name()): |
| + c.Tests += n |
| + case ext == ".js" || ext == ".mjs": |
| + c.JS += n |
| + case templateExt[ext] || inDir("templates", "views"): |
| + c.Templates += n |
| + default: |
| + c.Server += n |
| + } |
| + return nil |
| + }) |
| + return c, err |
| +} |
| + |
| +func lines(path string) (int, error) { |
| + f, err := os.Open(path) |
| + if err != nil { |
| + return 0, err |
| + } |
| + defer f.Close() |
| + n := 0 |
| + sc := bufio.NewScanner(f) |
| + sc.Buffer(make([]byte, 1<<20), 1<<24) |
| + for sc.Scan() { |
| + if strings.TrimSpace(sc.Text()) != "" { |
| + n++ |
| + } |
| + } |
| + return n, sc.Err() |
| +} |
diff --git a/harness/harness_test.go b/harness/harness_test.go| new file mode 100644 |
| index 0000000..0c7ba3b |
| --- /dev/null |
| +++ b/harness/harness_test.go |
| @@ -0,0 +1,207 @@ |
| +package bakeoff |
| + |
| +import ( |
| + "net/http/httptest" |
| + "os" |
| + "os/exec" |
| + "path/filepath" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "bakeoff/harness/refapp/ref" |
| +) |
| + |
| +func refServer(t *testing.T) *httptest.Server { |
| + t.Helper() |
| + srv := httptest.NewUnstartedServer(nil) |
| + srv.Start() |
| + srv.Config.Handler = ref.NewApp(srv.URL, "../shared") |
| + t.Cleanup(srv.Close) |
| + return srv |
| +} |
| + |
| +func TestSeed(t *testing.T) { |
| + srv := refServer(t) |
| + s, err := NewSession(srv.URL) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + ids, err := Seed(s, 50) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(ids) != 50 { |
| + t.Fatalf("want 50 ids, got %d", len(ids)) |
| + } |
| + done, err := s.IDs("/completed") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(done) != 25 { |
| + t.Fatalf("want 25 completed, got %d", len(done)) |
| + } |
| + if !strings.Contains(s.CookieHeader(), "=") { |
| + t.Fatalf("no cookie: %q", s.CookieHeader()) |
| + } |
| +} |
| + |
| +func TestParseOha(t *testing.T) { |
| + b, _ := os.ReadFile("testdata/oha.json") |
| + r, err := ParseOha(b, 200) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if r.RPS <= 0 || r.P99Ms < r.P50Ms || r.P50Ms <= 0 { |
| + t.Fatalf("bad result %+v", r) |
| + } |
| + if r.ErrorRate != 0 { |
| + t.Fatalf("deadline aborts must not count as errors, got %v", r.ErrorRate) |
| + } |
| +} |
| + |
| +func TestParseOhaCountsErrors(t *testing.T) { |
| + b := []byte(`{"summary":{"requestsPerSec":10},"latencyPercentiles":{"p50":0.001,"p95":0.002,"p99":0.003}, |
| + "statusCodeDistribution":{"303":80,"500":10},"errorDistribution":{"connection refused":10,"aborted due to deadline":3}}`) |
| + r, err := ParseOha(b, 303) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if r.ErrorRate != 0.2 { |
| + t.Fatalf("want 0.2, got %v", r.ErrorRate) |
| + } |
| + if r.P50Ms != 1 || r.P99Ms != 3 { |
| + t.Fatalf("want ms, got %+v", r) |
| + } |
| +} |
| + |
| +func TestParseClaudeLog(t *testing.T) { |
| + f, _ := os.Open("testdata/claude.jsonl") |
| + defer f.Close() |
| + tk, err := ParseClaudeLog(f) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if tk.Output != 4 || tk.CacheWrite != 32396 || tk.Turns != 1 || tk.CostUSD < 0.25 || tk.Seconds <= 0 { |
| + t.Fatalf("bad tokens %+v", tk) |
| + } |
| +} |
| + |
| +func TestParseCodexLog(t *testing.T) { |
| + f, _ := os.Open("testdata/codex.jsonl") |
| + defer f.Close() |
| + tk, err := ParseCodexLog(f) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if tk.Input != 16532 || tk.CacheRead != 12032 || tk.Output != 5 || tk.Turns != 1 { |
| + t.Fatalf("bad tokens %+v", tk) |
| + } |
| +} |
| + |
| +func TestCountCode(t *testing.T) { |
| + c, err := CountCode("testdata/codetree") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + want := CodeStats{Server: 3, Templates: 2, JS: 4, Tests: 2, Deps: 2} |
| + if c != want { |
| + t.Fatalf("got %+v, want %+v", c, want) |
| + } |
| +} |
| + |
| +func TestPageWeight(t *testing.T) { |
| + srv := refServer(t) |
| + s, _ := NewSession(srv.URL) |
| + w, err := PageWeight(s, "/") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + js, _ := os.ReadFile("refapp/ref/app.js") |
| + if w.JSBytes != len(js) || w.HTMLBytes <= 0 { |
| + t.Fatalf("got %+v, want JSBytes %d", w, len(js)) |
| + } |
| +} |
| + |
| +func TestMeasureTTFB(t *testing.T) { |
| + srv := refServer(t) |
| + s, _ := NewSession(srv.URL) |
| + st, err := MeasureTTFB(s, "/", 1, 5) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if st.N != 5 || st.Min > st.P50 || st.P50 > st.Max || st.Min <= 0 { |
| + t.Fatalf("bad stats %+v", st) |
| + } |
| +} |
| + |
| +func TestBuildStats(t *testing.T) { |
| + dir := t.TempDir() |
| + mk, _ := os.ReadFile("testdata/buildtree/Makefile") |
| + os.WriteFile(filepath.Join(dir, "Makefile"), mk, 0o644) |
| + b, err := BuildStats(dir, false, 1) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if !(b.ColdSeconds > b.WarmSeconds && b.WarmSeconds > b.EditSeconds && b.EditSeconds > 0) { |
| + t.Fatalf("want cold > warm > edit > 0, got %+v", b) |
| + } |
| + if b.Bytes != 6 { |
| + t.Fatalf("want 6 bytes, got %d", b.Bytes) |
| + } |
| + if _, err := os.Stat(filepath.Join(dir, ".unbumped")); err != nil { |
| + t.Fatal("unbump was not run") |
| + } |
| +} |
| + |
| +func TestUpdateRoundTrip(t *testing.T) { |
| + dir := t.TempDir() |
| + if err := UpdateIn(dir, "rust", func(r *AppResult) { r.Language = "Rust"; r.Wake = append(r.Wake, 12.5) }); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := UpdateIn(dir, "rust", func(r *AppResult) { r.Wake = append(r.Wake, 13) }); err != nil { |
| + t.Fatal(err) |
| + } |
| + r, err := LoadIn(dir, "rust") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if r.Slug != "rust" || r.Language != "Rust" || len(r.Wake) != 2 { |
| + t.Fatalf("got %+v", r) |
| + } |
| +} |
| + |
| +func TestRunLoadAgainstRefapp(t *testing.T) { |
| + if testing.Short() { |
| + t.Skip("runs oha") |
| + } |
| + if _, err := exec.LookPath("oha"); err != nil { |
| + t.Skip("oha not installed") |
| + } |
| + srv := refServer(t) |
| + s, _ := NewSession(srv.URL) |
| + if _, err := Seed(s, 10); err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, sc := range []string{"read", "toggle"} { |
| + r, err := RunLoad(s, sc, 2, time.Second) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if r.RPS <= 0 || r.ErrorRate != 0 || r.Scenario != sc || r.Concurrency != 2 { |
| + t.Fatalf("%s: bad result %+v", sc, r) |
| + } |
| + } |
| +} |
| + |
| +func TestParsePlaywright(t *testing.T) { |
| + f, _ := os.Open("testdata/playwright.json") |
| + defer f.Close() |
| + got, err := ParsePlaywright(f) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if got["js-on"] != (Behaviour{Passed: 1, Failed: 1}) || got["js-off"] != (Behaviour{Failed: 1, Skipped: 1}) { |
| + t.Fatalf("got %+v (flaky counts as failed)", got) |
| + } |
| +} |
diff --git a/harness/load.go b/harness/load.go| new file mode 100644 |
| index 0000000..07f00e1 |
| --- /dev/null |
| +++ b/harness/load.go |
| @@ -0,0 +1,85 @@ |
| +package bakeoff |
| + |
| +import ( |
| + "encoding/json" |
| + "fmt" |
| + "os/exec" |
| + "strconv" |
| + "time" |
| +) |
| + |
| +// LoadResult is one oha run. |
| +type LoadResult struct { |
| + Scenario string |
| + Concurrency int |
| + RPS, P50Ms, P95Ms, P99Ms float64 |
| + ErrorRate float64 |
| +} |
| + |
| +// ParseOha reads oha's JSON output. Any status other than ok, and any |
| +// transport error except oha's own end-of-run deadline aborts, is an error. |
| +func ParseOha(b []byte, ok int) (LoadResult, error) { |
| + var o struct { |
| + Summary struct { |
| + RequestsPerSec float64 `json:"requestsPerSec"` |
| + } `json:"summary"` |
| + Latency map[string]float64 `json:"latencyPercentiles"` |
| + Status map[string]int `json:"statusCodeDistribution"` |
| + Errors map[string]int `json:"errorDistribution"` |
| + } |
| + if err := json.Unmarshal(b, &o); err != nil { |
| + return LoadResult{}, err |
| + } |
| + var good, bad int |
| + for code, n := range o.Status { |
| + if code == strconv.Itoa(ok) { |
| + good += n |
| + } else { |
| + bad += n |
| + } |
| + } |
| + for msg, n := range o.Errors { |
| + if msg != "aborted due to deadline" { |
| + bad += n |
| + } |
| + } |
| + r := LoadResult{ |
| + RPS: o.Summary.RequestsPerSec, |
| + P50Ms: o.Latency["p50"] * 1000, |
| + P95Ms: o.Latency["p95"] * 1000, |
| + P99Ms: o.Latency["p99"] * 1000, |
| + } |
| + if good+bad > 0 { |
| + r.ErrorRate = float64(bad) / float64(good+bad) |
| + } |
| + return r, nil |
| +} |
| + |
| +// RunLoad drives the app with oha for d at the given concurrency. |
| +// "read" is GET / (200); "toggle" is POST /todos/<first id>/toggle (303). |
| +func RunLoad(s *Session, scenario string, concurrency int, d time.Duration) (LoadResult, error) { |
| + args := []string{"--no-tui", "--output-format", "json", "-z", d.String(), "-c", strconv.Itoa(concurrency), |
| + "--redirect", "0", "-H", "Cookie: " + s.CookieHeader(), "-H", "Origin: " + s.origin()} |
| + ok := 200 |
| + target := s.Base.String() + "/" |
| + switch scenario { |
| + case "read": |
| + case "toggle": |
| + ids, err := s.IDs("/") |
| + if err != nil || len(ids) == 0 { |
| + return LoadResult{}, fmt.Errorf("toggle needs a seeded list: %v", err) |
| + } |
| + ok = 303 |
| + target = s.Base.String() + "/todos/" + ids[0] + "/toggle" |
| + args = append(args, "-m", "POST", "-T", "application/x-www-form-urlencoded", "-d", "filter=all") |
| + default: |
| + return LoadResult{}, fmt.Errorf("unknown scenario %q", scenario) |
| + } |
| + out, err := exec.Command("oha", append(args, target)...).Output() |
| + if err != nil { |
| + return LoadResult{}, fmt.Errorf("oha: %w", err) |
| + } |
| + r, err := ParseOha(out, ok) |
| + r.Scenario, r.Concurrency = scenario, concurrency |
| + return r, err |
| +} |
diff --git a/harness/pageweight.go b/harness/pageweight.go| new file mode 100644 |
| index 0000000..ebec910 |
| --- /dev/null |
| +++ b/harness/pageweight.go |
| @@ -0,0 +1,33 @@ |
| +package bakeoff |
| + |
| +import ( |
| + "net/url" |
| + "regexp" |
| +) |
| + |
| +// Weight is what a first visit to a page downloads, apart from the shared |
| +// CSS every app serves identically. |
| +type Weight struct{ HTMLBytes, JSBytes int } |
| + |
| +var scriptSrc = regexp.MustCompile(`<script\b[^>]*\bsrc="([^"]+)"`) |
| + |
| +// PageWeight sums the page's HTML and every same-origin script it loads. |
| +func PageWeight(s *Session, path string) (Weight, error) { |
| + html, err := s.get(path) |
| + if err != nil { |
| + return Weight{}, err |
| + } |
| + w := Weight{HTMLBytes: len(html)} |
| + for _, m := range scriptSrc.FindAllStringSubmatch(html, -1) { |
| + u, err := s.Base.Parse(m[1]) |
| + if err != nil || u.Host != s.Base.Host { |
| + continue |
| + } |
| + js, err := s.get((&url.URL{Path: u.Path, RawQuery: u.RawQuery}).String()) |
| + if err != nil { |
| + return Weight{}, err |
| + } |
| + w.JSBytes += len(js) |
| + } |
| + return w, nil |
| +} |
diff --git a/harness/refapp/main.go b/harness/refapp/main.go| index 3a695d6..826fb29 100644 |
| --- a/harness/refapp/main.go |
| +++ b/harness/refapp/main.go |
| @@ -1,228 +1,16 @@ |
| -// Command refapp is the bakeoff's reference fixture: a minimal, in-memory |
| -// implementation of CONTRACT.md used only to prove the behaviour suite and |
| -// the harness before any builder starts. It is not a contestant, and |
| -// builders never see it. |
| +// Command refapp serves the bakeoff's reference fixture (package ref). |
| +// It is not a contestant, and builders never see it. |
| package main |
| import ( |
| - "crypto/rand" |
| - _ "embed" |
| - "encoding/hex" |
| "flag" |
| - "html/template" |
| "log" |
| "net" |
| "net/http" |
| "os" |
| - "path/filepath" |
| - "strconv" |
| - "strings" |
| - "sync" |
| -) |
| - |
| -//go:embed page.html.tmpl |
| -var pageSrc string |
| - |
| -//go:embed app.js |
| -var appJS []byte |
| - |
| -var page = template.Must(template.New("page").Parse(pageSrc)) |
| - |
| -const cookieName = "todo" |
| - |
| -type item struct { |
| - ID int |
| - Title string |
| - Completed bool |
| -} |
| - |
| -type list struct { |
| - next int |
| - items []item |
| -} |
| - |
| -type app struct { |
| - origin string |
| - shared string |
| - mu sync.Mutex |
| - lists map[string]*list |
| -} |
| - |
| -// NewApp returns the reference app's handler. origin is the app's own |
| -// origin for the Origin check; shared is the directory holding base.css |
| -// and index.css. |
| -func NewApp(origin, shared string) http.Handler { |
| - a := &app{origin: origin, shared: shared, lists: map[string]*list{}} |
| - mux := http.NewServeMux() |
| - mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.Write([]byte("ok")) }) |
| - mux.HandleFunc("GET /api/version", func(w http.ResponseWriter, _ *http.Request) { w.Write([]byte("refapp")) }) |
| - mux.HandleFunc("GET /static/{file}", a.static) |
| - mux.HandleFunc("GET /{$}", a.show("all")) |
| - mux.HandleFunc("GET /active", a.show("active")) |
| - mux.HandleFunc("GET /completed", a.show("completed")) |
| - mux.HandleFunc("POST /todos", a.act(func(l *list, r *http.Request) int { |
| - if t := strings.TrimSpace(r.FormValue("title")); t != "" { |
| - l.next++ |
| - l.items = append(l.items, item{ID: l.next, Title: t}) |
| - } |
| - return 0 |
| - })) |
| - mux.HandleFunc("POST /todos/toggle-all", a.act(func(l *list, _ *http.Request) int { |
| - all := true |
| - for _, it := range l.items { |
| - all = all && it.Completed |
| - } |
| - for i := range l.items { |
| - l.items[i].Completed = !all |
| - } |
| - return 0 |
| - })) |
| - mux.HandleFunc("POST /todos/clear-completed", a.act(func(l *list, _ *http.Request) int { |
| - kept := l.items[:0] |
| - for _, it := range l.items { |
| - if !it.Completed { |
| - kept = append(kept, it) |
| - } |
| - } |
| - l.items = kept |
| - return 0 |
| - })) |
| - mux.HandleFunc("POST /todos/{id}/{op}", a.act(func(l *list, r *http.Request) int { |
| - i := l.find(r.PathValue("id")) |
| - if i < 0 { |
| - return http.StatusNotFound |
| - } |
| - switch r.PathValue("op") { |
| - case "toggle": |
| - l.items[i].Completed = !l.items[i].Completed |
| - case "edit": |
| - if t := strings.TrimSpace(r.FormValue("title")); t != "" { |
| - l.items[i].Title = t |
| - } else { |
| - l.items = append(l.items[:i], l.items[i+1:]...) |
| - } |
| - case "destroy": |
| - l.items = append(l.items[:i], l.items[i+1:]...) |
| - default: |
| - return http.StatusNotFound |
| - } |
| - return 0 |
| - })) |
| - return mux |
| -} |
| - |
| -func (l *list) find(id string) int { |
| - n, err := strconv.Atoi(id) |
| - if err != nil { |
| - return -1 |
| - } |
| - for i, it := range l.items { |
| - if it.ID == n { |
| - return i |
| - } |
| - } |
| - return -1 |
| -} |
| - |
| -func (a *app) static(w http.ResponseWriter, r *http.Request) { |
| - switch f := r.PathValue("file"); f { |
| - case "base.css", "index.css": |
| - w.Header().Set("Content-Type", "text/css; charset=utf-8") |
| - http.ServeFile(w, r, filepath.Join(a.shared, f)) |
| - case "app.js": |
| - w.Header().Set("Content-Type", "text/javascript; charset=utf-8") |
| - w.Write(appJS) |
| - default: |
| - http.NotFound(w, r) |
| - } |
| -} |
| - |
| -// listFor returns the caller's list, minting a fresh one (and its cookie) |
| -// when the cookie is missing or unknown. Caller holds a.mu. |
| -func (a *app) listFor(w http.ResponseWriter, r *http.Request) *list { |
| - if c, err := r.Cookie(cookieName); err == nil { |
| - if l, ok := a.lists[c.Value]; ok { |
| - return l |
| - } |
| - } |
| - b := make([]byte, 16) |
| - rand.Read(b) |
| - key := hex.EncodeToString(b) |
| - l := &list{} |
| - a.lists[key] = l |
| - http.SetCookie(w, &http.Cookie{Name: cookieName, Value: key, Path: "/", HttpOnly: true, |
| - SameSite: http.SameSiteLaxMode, Secure: strings.HasPrefix(a.origin, "https://")}) |
| - return l |
| -} |
| - |
| -func filterPath(f string) string { |
| - switch f { |
| - case "active", "completed": |
| - return "/" + f |
| - } |
| - return "/" |
| -} |
| - |
| -func (a *app) act(f func(*list, *http.Request) int) http.HandlerFunc { |
| - return func(w http.ResponseWriter, r *http.Request) { |
| - if o := r.Header.Get("Origin"); o != "" && o != a.origin { |
| - http.Error(w, "cross-site request refused", http.StatusForbidden) |
| - return |
| - } |
| - r.ParseForm() |
| - a.mu.Lock() |
| - status := f(a.listFor(w, r), r) |
| - a.mu.Unlock() |
| - if status != 0 { |
| - http.Error(w, http.StatusText(status), status) |
| - return |
| - } |
| - filter := r.FormValue("filter") |
| - if r.Header.Get("X-Requested-With") == "fetch" { |
| - a.render(w, r, filter) |
| - return |
| - } |
| - http.Redirect(w, r, filterPath(filter), http.StatusSeeOther) |
| - } |
| -} |
| - |
| -func (a *app) show(filter string) http.HandlerFunc { |
| - return func(w http.ResponseWriter, r *http.Request) { a.render(w, r, filter) } |
| -} |
| - |
| -type view struct { |
| - Filter, Path string |
| - Items []item |
| - Total, Left, Done int |
| - AllDone bool |
| - EditID int |
| -} |
| -func (a *app) render(w http.ResponseWriter, r *http.Request, filter string) { |
| - if filter != "active" && filter != "completed" { |
| - filter = "all" |
| - } |
| - a.mu.Lock() |
| - l := a.listFor(w, r) |
| - v := view{Filter: filter, Path: filterPath(filter), Total: len(l.items)} |
| - for _, it := range l.items { |
| - if it.Completed { |
| - v.Done++ |
| - } else { |
| - v.Left++ |
| - } |
| - if filter == "all" || (filter == "completed") == it.Completed { |
| - v.Items = append(v.Items, it) |
| - } |
| - } |
| - a.mu.Unlock() |
| - v.AllDone = v.Total > 0 && v.Left == 0 |
| - v.EditID, _ = strconv.Atoi(r.URL.Query().Get("edit")) |
| - w.Header().Set("Content-Type", "text/html; charset=utf-8") |
| - if err := page.Execute(w, v); err != nil { |
| - log.Print(err) |
| - } |
| -} |
| + "bakeoff/harness/refapp/ref" |
| +) |
| func main() { |
| listen := flag.String("listen", "", "TCP address, e.g. 127.0.0.1:8080") |
| @@ -247,5 +35,5 @@ func main() { |
| if err != nil { |
| log.Fatal(err) |
| } |
| - log.Fatal(http.Serve(ln, NewApp(origin, *shared))) |
| + log.Fatal(http.Serve(ln, ref.NewApp(origin, *shared))) |
| } |
diff --git a/harness/refapp/app.js b/harness/refapp/ref/app.js| similarity index 100% |
| rename from harness/refapp/app.js |
| rename to harness/refapp/ref/app.js |
diff --git a/harness/refapp/page.html.tmpl b/harness/refapp/ref/page.html.tmpl| similarity index 100% |
| rename from harness/refapp/page.html.tmpl |
| rename to harness/refapp/ref/page.html.tmpl |
diff --git a/harness/refapp/ref/ref.go b/harness/refapp/ref/ref.go| new file mode 100644 |
| index 0000000..1d1907b |
| --- /dev/null |
| +++ b/harness/refapp/ref/ref.go |
| @@ -0,0 +1,222 @@ |
| +// Package ref is the bakeoff's reference fixture: a minimal, in-memory |
| +// implementation of CONTRACT.md used only to prove the behaviour suite and |
| +// the harness before any builder starts. It is not a contestant, and |
| +// builders never see it. |
| +package ref |
| + |
| +import ( |
| + "crypto/rand" |
| + _ "embed" |
| + "encoding/hex" |
| + "html/template" |
| + "log" |
| + "net/http" |
| + "path/filepath" |
| + "strconv" |
| + "strings" |
| + "sync" |
| +) |
| + |
| +//go:embed page.html.tmpl |
| +var pageSrc string |
| + |
| +//go:embed app.js |
| +var appJS []byte |
| + |
| +var page = template.Must(template.New("page").Parse(pageSrc)) |
| + |
| +const cookieName = "todo" |
| + |
| +type item struct { |
| + ID int |
| + Title string |
| + Completed bool |
| +} |
| + |
| +type list struct { |
| + next int |
| + items []item |
| +} |
| + |
| +type app struct { |
| + origin string |
| + shared string |
| + mu sync.Mutex |
| + lists map[string]*list |
| +} |
| + |
| +// NewApp returns the reference app's handler. origin is the app's own |
| +// origin for the Origin check; shared is the directory holding base.css |
| +// and index.css. |
| +func NewApp(origin, shared string) http.Handler { |
| + a := &app{origin: origin, shared: shared, lists: map[string]*list{}} |
| + mux := http.NewServeMux() |
| + mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.Write([]byte("ok")) }) |
| + mux.HandleFunc("GET /api/version", func(w http.ResponseWriter, _ *http.Request) { w.Write([]byte("refapp")) }) |
| + mux.HandleFunc("GET /static/{file}", a.static) |
| + mux.HandleFunc("GET /{$}", a.show("all")) |
| + mux.HandleFunc("GET /active", a.show("active")) |
| + mux.HandleFunc("GET /completed", a.show("completed")) |
| + mux.HandleFunc("POST /todos", a.act(func(l *list, r *http.Request) int { |
| + if t := strings.TrimSpace(r.FormValue("title")); t != "" { |
| + l.next++ |
| + l.items = append(l.items, item{ID: l.next, Title: t}) |
| + } |
| + return 0 |
| + })) |
| + mux.HandleFunc("POST /todos/toggle-all", a.act(func(l *list, _ *http.Request) int { |
| + all := true |
| + for _, it := range l.items { |
| + all = all && it.Completed |
| + } |
| + for i := range l.items { |
| + l.items[i].Completed = !all |
| + } |
| + return 0 |
| + })) |
| + mux.HandleFunc("POST /todos/clear-completed", a.act(func(l *list, _ *http.Request) int { |
| + kept := l.items[:0] |
| + for _, it := range l.items { |
| + if !it.Completed { |
| + kept = append(kept, it) |
| + } |
| + } |
| + l.items = kept |
| + return 0 |
| + })) |
| + mux.HandleFunc("POST /todos/{id}/{op}", a.act(func(l *list, r *http.Request) int { |
| + i := l.find(r.PathValue("id")) |
| + if i < 0 { |
| + return http.StatusNotFound |
| + } |
| + switch r.PathValue("op") { |
| + case "toggle": |
| + l.items[i].Completed = !l.items[i].Completed |
| + case "edit": |
| + if t := strings.TrimSpace(r.FormValue("title")); t != "" { |
| + l.items[i].Title = t |
| + } else { |
| + l.items = append(l.items[:i], l.items[i+1:]...) |
| + } |
| + case "destroy": |
| + l.items = append(l.items[:i], l.items[i+1:]...) |
| + default: |
| + return http.StatusNotFound |
| + } |
| + return 0 |
| + })) |
| + return mux |
| +} |
| + |
| +func (l *list) find(id string) int { |
| + n, err := strconv.Atoi(id) |
| + if err != nil { |
| + return -1 |
| + } |
| + for i, it := range l.items { |
| + if it.ID == n { |
| + return i |
| + } |
| + } |
| + return -1 |
| +} |
| + |
| +func (a *app) static(w http.ResponseWriter, r *http.Request) { |
| + switch f := r.PathValue("file"); f { |
| + case "base.css", "index.css": |
| + w.Header().Set("Content-Type", "text/css; charset=utf-8") |
| + http.ServeFile(w, r, filepath.Join(a.shared, f)) |
| + case "app.js": |
| + w.Header().Set("Content-Type", "text/javascript; charset=utf-8") |
| + w.Write(appJS) |
| + default: |
| + http.NotFound(w, r) |
| + } |
| +} |
| + |
| +// listFor returns the caller's list, minting a fresh one (and its cookie) |
| +// when the cookie is missing or unknown. Caller holds a.mu. |
| +func (a *app) listFor(w http.ResponseWriter, r *http.Request) *list { |
| + if c, err := r.Cookie(cookieName); err == nil { |
| + if l, ok := a.lists[c.Value]; ok { |
| + return l |
| + } |
| + } |
| + b := make([]byte, 16) |
| + rand.Read(b) |
| + key := hex.EncodeToString(b) |
| + l := &list{} |
| + a.lists[key] = l |
| + http.SetCookie(w, &http.Cookie{Name: cookieName, Value: key, Path: "/", HttpOnly: true, |
| + SameSite: http.SameSiteLaxMode, Secure: strings.HasPrefix(a.origin, "https://")}) |
| + return l |
| +} |
| + |
| +func filterPath(f string) string { |
| + switch f { |
| + case "active", "completed": |
| + return "/" + f |
| + } |
| + return "/" |
| +} |
| + |
| +func (a *app) act(f func(*list, *http.Request) int) http.HandlerFunc { |
| + return func(w http.ResponseWriter, r *http.Request) { |
| + if o := r.Header.Get("Origin"); o != "" && o != a.origin { |
| + http.Error(w, "cross-site request refused", http.StatusForbidden) |
| + return |
| + } |
| + r.ParseForm() |
| + a.mu.Lock() |
| + status := f(a.listFor(w, r), r) |
| + a.mu.Unlock() |
| + if status != 0 { |
| + http.Error(w, http.StatusText(status), status) |
| + return |
| + } |
| + filter := r.FormValue("filter") |
| + if r.Header.Get("X-Requested-With") == "fetch" { |
| + a.render(w, r, filter) |
| + return |
| + } |
| + http.Redirect(w, r, filterPath(filter), http.StatusSeeOther) |
| + } |
| +} |
| + |
| +func (a *app) show(filter string) http.HandlerFunc { |
| + return func(w http.ResponseWriter, r *http.Request) { a.render(w, r, filter) } |
| +} |
| + |
| +type view struct { |
| + Filter, Path string |
| + Items []item |
| + Total, Left, Done int |
| + AllDone bool |
| + EditID int |
| +} |
| + |
| +func (a *app) render(w http.ResponseWriter, r *http.Request, filter string) { |
| + if filter != "active" && filter != "completed" { |
| + filter = "all" |
| + } |
| + a.mu.Lock() |
| + l := a.listFor(w, r) |
| + v := view{Filter: filter, Path: filterPath(filter), Total: len(l.items)} |
| + for _, it := range l.items { |
| + if it.Completed { |
| + v.Done++ |
| + } else { |
| + v.Left++ |
| + } |
| + if filter == "all" || (filter == "completed") == it.Completed { |
| + v.Items = append(v.Items, it) |
| + } |
| + } |
| + a.mu.Unlock() |
| + v.AllDone = v.Total > 0 && v.Left == 0 |
| + v.EditID, _ = strconv.Atoi(r.URL.Query().Get("edit")) |
| + w.Header().Set("Content-Type", "text/html; charset=utf-8") |
| + if err := page.Execute(w, v); err != nil { |
| + log.Print(err) |
| + } |
| +} |
diff --git a/harness/refapp/refapp_test.go b/harness/refapp/ref/ref_test.go| similarity index 97% |
| rename from harness/refapp/refapp_test.go |
| rename to harness/refapp/ref/ref_test.go |
| index 3715d21..4745871 100644 |
| --- a/harness/refapp/refapp_test.go |
| +++ b/harness/refapp/ref/ref_test.go |
| @@ -1,4 +1,4 @@ |
| -package main |
| +package ref |
| import ( |
| "io" |
| @@ -15,7 +15,7 @@ func newTestServer(t *testing.T) *httptest.Server { |
| t.Helper() |
| srv := httptest.NewUnstartedServer(nil) |
| srv.Start() |
| - srv.Config.Handler = NewApp(srv.URL, "../../shared") |
| + srv.Config.Handler = NewApp(srv.URL, "../../../shared") |
| t.Cleanup(srv.Close) |
| return srv |
| } |
diff --git a/harness/results.go b/harness/results.go| new file mode 100644 |
| index 0000000..274b9fb |
| --- /dev/null |
| +++ b/harness/results.go |
| @@ -0,0 +1,73 @@ |
| +package bakeoff |
| + |
| +import ( |
| + "encoding/json" |
| + "errors" |
| + "io/fs" |
| + "os" |
| + "path/filepath" |
| +) |
| + |
| +// ResultsDir is where results/<slug>.json live, relative to the repo root. |
| +var ResultsDir = "results" |
| + |
| +// Readability is Fable's blind rubric score (per-field median of 3 runs). |
| +type Readability struct { |
| + Naming, Structure, Traceability, Errors, Comments, Overall float64 |
| + Notes string |
| +} |
| + |
| +// Behaviour is one project's run of the shared suite. |
| +type Behaviour struct{ Passed, Failed, Skipped int } |
| + |
| +// AppResult is everything the report shows for one app. |
| +type AppResult struct { |
| + Slug, Language, Status, URL, Version, Stack string |
| + BehaviourJSOn, BehaviourJSOff Behaviour |
| + TTFBEmpty, TTFB50 Stats |
| + Wake []float64 |
| + Load []LoadResult |
| + Weight Weight |
| + Code CodeStats |
| + Build Build |
| + TokensBuild, TokensFix, TokensSecurity Tokens |
| + TokensReview Tokens |
| + Readability Readability |
| + ReviewFindings []string |
| + Notes []string |
| +} |
| + |
| +// LoadIn reads dir/<slug>.json, or returns an empty result for slug. |
| +func LoadIn(dir, slug string) (AppResult, error) { |
| + r := AppResult{Slug: slug} |
| + b, err := os.ReadFile(filepath.Join(dir, slug+".json")) |
| + if errors.Is(err, fs.ErrNotExist) { |
| + return r, nil |
| + } |
| + if err != nil { |
| + return r, err |
| + } |
| + err = json.Unmarshal(b, &r) |
| + return r, err |
| +} |
| + |
| +// UpdateIn applies f to dir/<slug>.json and writes it back. |
| +func UpdateIn(dir, slug string, f func(*AppResult)) error { |
| + r, err := LoadIn(dir, slug) |
| + if err != nil { |
| + return err |
| + } |
| + f(&r) |
| + r.Slug = slug |
| + b, err := json.MarshalIndent(r, "", " ") |
| + if err != nil { |
| + return err |
| + } |
| + if err := os.MkdirAll(dir, 0o755); err != nil { |
| + return err |
| + } |
| + return os.WriteFile(filepath.Join(dir, slug+".json"), append(b, '\n'), 0o644) |
| +} |
| + |
| +// Update applies f to results/<slug>.json. |
| +func Update(slug string, f func(*AppResult)) error { return UpdateIn(ResultsDir, slug, f) } |
diff --git a/harness/session.go b/harness/session.go| new file mode 100644 |
| index 0000000..375742c |
| --- /dev/null |
| +++ b/harness/session.go |
| @@ -0,0 +1,137 @@ |
| +// Package bakeoff is the bakeoff's measurement harness: it drives any app |
| +// that keeps CONTRACT.md and records what it measures in results/<slug>.json. |
| +package bakeoff |
| + |
| +import ( |
| + "fmt" |
| + "io" |
| + "net/http" |
| + "net/http/cookiejar" |
| + "net/url" |
| + "regexp" |
| + "strings" |
| + "time" |
| +) |
| + |
| +// Session is one visitor: a cookie jar (so one list), an Origin header |
| +// matching the app, and a client that never follows redirects. |
| +type Session struct { |
| + Base *url.URL |
| + Client *http.Client |
| +} |
| + |
| +// SessionNoFetch starts a visitor for the app at base without making any |
| +// request (so a sleeping instance stays asleep until the caller's first one). |
| +func SessionNoFetch(base string) (*Session, error) { |
| + u, err := url.Parse(strings.TrimRight(base, "/")) |
| + if err != nil { |
| + return nil, err |
| + } |
| + jar, _ := cookiejar.New(nil) |
| + return &Session{Base: u, Client: &http.Client{ |
| + Jar: jar, |
| + Timeout: 60 * time.Second, |
| + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, |
| + }}, nil |
| +} |
| + |
| +// NewSession starts a visitor for the app at base and fetches / once so the |
| +// app sets its list cookie. |
| +func NewSession(base string) (*Session, error) { |
| + s, err := SessionNoFetch(base) |
| + if err != nil { |
| + return nil, err |
| + } |
| + if _, err := s.get("/"); err != nil { |
| + return nil, err |
| + } |
| + return s, nil |
| +} |
| + |
| +func (s *Session) origin() string { return s.Base.Scheme + "://" + s.Base.Host } |
| + |
| +func (s *Session) get(path string) (string, error) { |
| + res, err := s.Client.Get(s.Base.String() + path) |
| + if err != nil { |
| + return "", err |
| + } |
| + defer res.Body.Close() |
| + b, err := io.ReadAll(res.Body) |
| + if res.StatusCode != 200 { |
| + return "", fmt.Errorf("GET %s: %s", path, res.Status) |
| + } |
| + return string(b), err |
| +} |
| + |
| +func (s *Session) post(path string, form url.Values) error { |
| + req, _ := http.NewRequest("POST", s.Base.String()+path, strings.NewReader(form.Encode())) |
| + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") |
| + req.Header.Set("Origin", s.origin()) |
| + res, err := s.Client.Do(req) |
| + if err != nil { |
| + return err |
| + } |
| + io.Copy(io.Discard, res.Body) |
| + res.Body.Close() |
| + if res.StatusCode != http.StatusSeeOther { |
| + return fmt.Errorf("POST %s: %s, want 303", path, res.Status) |
| + } |
| + return nil |
| +} |
| + |
| +// Add creates one item. |
| +func (s *Session) Add(title string) error { |
| + return s.post("/todos", url.Values{"title": {title}, "filter": {"all"}}) |
| +} |
| + |
| +// Toggle flips one item. |
| +func (s *Session) Toggle(id string) error { |
| + return s.post("/todos/"+id+"/toggle", url.Values{"filter": {"all"}}) |
| +} |
| + |
| +var liID = regexp.MustCompile(`<li\b[^>]*\bdata-id="([^"]+)"`) |
| + |
| +// IDs lists the item ids shown on the page at path, in order. |
| +func (s *Session) IDs(path string) ([]string, error) { |
| + body, err := s.get(path) |
| + if err != nil { |
| + return nil, err |
| + } |
| + var ids []string |
| + for _, m := range liID.FindAllStringSubmatch(body, -1) { |
| + ids = append(ids, m[1]) |
| + } |
| + return ids, nil |
| +} |
| + |
| +// CookieHeader is the Cookie header this visitor sends, for tools like oha. |
| +func (s *Session) CookieHeader() string { |
| + var parts []string |
| + for _, c := range s.Client.Jar.Cookies(s.Base) { |
| + parts = append(parts, c.Name+"="+c.Value) |
| + } |
| + return strings.Join(parts, "; ") |
| +} |
| + |
| +// Seed adds "item 1".."item n" and completes every second one, returning |
| +// the ids in list order. |
| +func Seed(s *Session, n int) ([]string, error) { |
| + for i := 1; i <= n; i++ { |
| + if err := s.Add(fmt.Sprintf("item %d", i)); err != nil { |
| + return nil, err |
| + } |
| + } |
| + ids, err := s.IDs("/") |
| + if err != nil { |
| + return nil, err |
| + } |
| + if len(ids) != n { |
| + return nil, fmt.Errorf("seeded %d items, page shows %d", n, len(ids)) |
| + } |
| + for i := 1; i < n; i += 2 { |
| + if err := s.Toggle(ids[i]); err != nil { |
| + return nil, err |
| + } |
| + } |
| + return ids, nil |
| +} |
diff --git a/harness/testdata/buildtree/Makefile b/harness/testdata/buildtree/Makefile| new file mode 100644 |
| index 0000000..b07318e |
| --- /dev/null |
| +++ b/harness/testdata/buildtree/Makefile |
| @@ -0,0 +1,11 @@ |
| +clean: |
| + touch .clean |
| +clean-deps: |
| + touch .cold |
| +build-arm64: |
| + @if [ -f .cold ]; then sleep 0.4; elif [ -f .clean ]; then sleep 0.2; else sleep 0.05; fi |
| + @rm -f .cold .clean; mkdir -p dist; printf binary > dist/app-linux-arm64 |
| +bump: |
| + touch .bumped |
| +unbump: |
| + rm -f .bumped; touch .unbumped |
diff --git a/harness/testdata/claude.jsonl b/harness/testdata/claude.jsonl| new file mode 100644 |
| index 0000000..f9b5ea1 |
| --- /dev/null |
| +++ b/harness/testdata/claude.jsonl |
| @@ -0,0 +1,2 @@ |
| +{"type": "system", "subtype": "init", "session_id": "3f18d62d-a39b-491d-a21c-033c0a23cf8c", "model": "claude-opus-5-5"} |
| +{"duration_api_ms":1009,"stop_reason":"end_turn","session_id":"3f18d62d-a39b-491d-a21c-033c0a23cf8c","total_cost_usd":0.259256,"usage":{"input_tokens":2,"cache_creation_input_tokens":32396,"cache_read_input_tokens":0,"output_tokens":4,"output_tokens_details":{"thinking_tokens":0},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":32396,"ephemeral_5m_input_tokens":0},"inference_geo":"global","iterations":[{"input_tokens":2,"output_tokens":4,"cache_read_input_tokens":0,"cache_creation_input_tokens":32396,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":32396},"type":"message"}],"speed":"standard","fallback_credit":null},"modelUsage":{"claude-opus-5-5":{"inputTokens":2,"outputTokens":4,"cacheReadInputTokens":0,"cacheCreationInputTokens":32396,"webSearchRequests":0,"costUSD":0.259256,"contextWindow":1000000,"maxOutputTokens":128000,"thinkingTokens":0,"canonicalModel":"claude-opus-5-5","provider":"firstParty","costBasis":"list"}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","fast_mode_disabled_reason":"sdk_opt_in_required","subagent_stats":{"spawned":0,"requested":{"background":0,"foreground":0,"unset":0},"started_in_background":0,"max_depth":0,"spawned_by_subagents":0,"completed":0,"failed":0,"killed":{"parent":0,"user":0,"system":0},"refused":{"depth_limit":0,"concurrency_limit":0,"budget":0},"by_type":{}},"safety_stops":0,"is_error":false,"num_turns":1,"subtype":"success","api_error_status":null,"result":"ok","ttft_ms":1050,"type":"result","duration_ms":1081,"uuid":"877244f9-f3f5-4ea4-b949-56bb8e95d0ac","ttft_stream_ms":1046,"time_to_request_ms":141,"first_content_frame_ms":1046,"queued_turn_count":0,"result_index":0} |
diff --git a/harness/testdata/codetree/DEPS b/harness/testdata/codetree/DEPS| new file mode 100644 |
| index 0000000..8f3b64f |
| --- /dev/null |
| +++ b/harness/testdata/codetree/DEPS |
| @@ -0,0 +1,2 @@ |
| +modernc.org/sqlite |
| +github.com/x/y |
diff --git a/harness/testdata/codetree/README.md b/harness/testdata/codetree/README.md| new file mode 100644 |
| index 0000000..77ebc65 |
| --- /dev/null |
| +++ b/harness/testdata/codetree/README.md |
| @@ -0,0 +1 @@ |
| +# hi |
diff --git a/harness/testdata/codetree/go.sum b/harness/testdata/codetree/go.sum| new file mode 100644 |
| index 0000000..21c26a2 |
| --- /dev/null |
| +++ b/harness/testdata/codetree/go.sum |
| @@ -0,0 +1 @@ |
| +x v1 |
diff --git a/harness/testdata/codetree/main.go b/harness/testdata/codetree/main.go| new file mode 100644 |
| index 0000000..da29a2c |
| --- /dev/null |
| +++ b/harness/testdata/codetree/main.go |
| @@ -0,0 +1,4 @@ |
| +package main |
| + |
| +func main() { |
| +} |
diff --git a/harness/testdata/codetree/main_test.go b/harness/testdata/codetree/main_test.go| new file mode 100644 |
| index 0000000..02e0b75 |
| --- /dev/null |
| +++ b/harness/testdata/codetree/main_test.go |
| @@ -0,0 +1,3 @@ |
| +package main |
| + |
| +func TestX() {} |
diff --git a/harness/testdata/codetree/static/app.js b/harness/testdata/codetree/static/app.js| new file mode 100644 |
| index 0000000..ec1a904 |
| --- /dev/null |
| +++ b/harness/testdata/codetree/static/app.js |
| @@ -0,0 +1,5 @@ |
| +a() |
| +b() |
| + |
| +c() |
| +d() |
diff --git a/harness/testdata/codetree/static/index.css b/harness/testdata/codetree/static/index.css| new file mode 100644 |
| index 0000000..c0e1a88 |
| --- /dev/null |
| +++ b/harness/testdata/codetree/static/index.css |
| @@ -0,0 +1 @@ |
| +body{} |
diff --git a/harness/testdata/codetree/static/van-1.5.2.min.js b/harness/testdata/codetree/static/van-1.5.2.min.js| new file mode 100644 |
| index 0000000..0d583b2 |
| --- /dev/null |
| +++ b/harness/testdata/codetree/static/van-1.5.2.min.js |
| @@ -0,0 +1 @@ |
| +van() |
diff --git a/harness/testdata/codetree/vendor/x.go b/harness/testdata/codetree/vendor/x.go| new file mode 100644 |
| index 0000000..823aafd |
| --- /dev/null |
| +++ b/harness/testdata/codetree/vendor/x.go |
| @@ -0,0 +1 @@ |
| +package x |
diff --git a/harness/testdata/codetree/views/index.html b/harness/testdata/codetree/views/index.html| new file mode 100644 |
| index 0000000..14dc35f |
| --- /dev/null |
| +++ b/harness/testdata/codetree/views/index.html |
| @@ -0,0 +1,3 @@ |
| +<ul> |
| + |
| +</ul> |
diff --git a/harness/testdata/codex.jsonl b/harness/testdata/codex.jsonl| new file mode 100644 |
| index 0000000..0a303a8 |
| --- /dev/null |
| +++ b/harness/testdata/codex.jsonl |
| @@ -0,0 +1,4 @@ |
| +{"type":"thread.started","thread_id":"01a16a0a-5383-7e52-8256-b3587c7f3e34"} |
| +{"type":"turn.started"} |
| +{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"ok"}} |
| +{"type":"turn.completed","usage":{"input_tokens":16532,"cached_input_tokens":12032,"cache_write_input_tokens":0,"output_tokens":5,"reasoning_output_tokens":0}} |
diff --git a/harness/testdata/oha.json b/harness/testdata/oha.json| new file mode 100644 |
| index 0000000..2547003 |
| --- /dev/null |
| +++ b/harness/testdata/oha.json |
| @@ -0,0 +1,113 @@ |
| +{ |
| + "summary": { |
| + "successRate": 1.0, |
| + "total": 2.002044707, |
| + "slowest": 0.001128352, |
| + "fastest": 0.000024325, |
| + "average": 0.00004434557004230867, |
| + "requestsPerSec": 88072.95830282375, |
| + "totalData": 202770300, |
| + "sizePerRequest": 1150, |
| + "sizePerSec": 101281604.39725885 |
| + }, |
| + "metrics": { |
| + "success_rate": 1.0, |
| + "requests_per_sec": 88072.95830282375, |
| + "latency_ms": { |
| + "min": 0.024, |
| + "mean": 0.044, |
| + "p50": 0.041, |
| + "p95": 0.064, |
| + "p99": 0.122, |
| + "max": 1.128 |
| + } |
| + }, |
| + "responseTimeHistogram": { |
| + "0.000024325": 1, |
| + "0.0001347277": 174835, |
| + "0.0002451304": 1331, |
| + "0.0003555331": 86, |
| + "0.0004659358": 27, |
| + "0.0005763385": 8, |
| + "0.0006867411999999999": 9, |
| + "0.0007971438999999999": 11, |
| + "0.0009075466": 6, |
| + "0.0010179492999999999": 3, |
| + "0.001128352": 5 |
| + }, |
| + "latencyPercentiles": { |
| + "p10": 0.00003236, |
| + "p25": 0.000035586, |
| + "p50": 0.000041016, |
| + "p75": 0.000047889, |
| + "p90": 0.000056946, |
| + "p95": 0.00006431, |
| + "p99": 0.000121646, |
| + "p99.9": 0.000231642, |
| + "p99.99": 0.000770035 |
| + }, |
| + "firstByteHistogram": { |
| + "0.000024265": 1, |
| + "0.0001346637": 174835, |
| + "0.00024506239999999997": 1331, |
| + "0.0003554611": 86, |
| + "0.00046585979999999995": 27, |
| + "0.0005762584999999999": 8, |
| + "0.0006866572": 9, |
| + "0.0007970558999999999": 11, |
| + "0.0009074545999999999": 6, |
| + "0.0010178533": 3, |
| + "0.001128252": 5 |
| + }, |
| + "firstBytePercentiles": { |
| + "p10": 0.0000323, |
| + "p25": 0.000035516, |
| + "p50": 0.000040927, |
| + "p75": 0.000047789, |
| + "p90": 0.000056836, |
| + "p95": 0.000064199, |
| + "p99": 0.000121566, |
| + "p99.9": 0.000231512, |
| + "p99.99": 0.000769834 |
| + }, |
| + "rps": { |
| + "mean": 88106.24588366134, |
| + "stddev": 5088.620854504324, |
| + "max": 97430.19361700135, |
| + "min": 71435.4391213481, |
| + "percentiles": { |
| + "p10": 80919.3882854328, |
| + "p25": 84988.29155995119, |
| + "p50": 89145.47310583276, |
| + "p75": 91732.7394146948, |
| + "p90": 93820.32148163307, |
| + "p95": 94789.10426300664, |
| + "p99": 96751.88803755454, |
| + "p99.9": 97430.19361700135, |
| + "p99.99": 97430.19361700135 |
| + } |
| + }, |
| + "details": { |
| + "DNSDialup": { |
| + "average": 0.00011522725, |
| + "fastest": 0.000056355, |
| + "slowest": 0.000245667 |
| + }, |
| + "DNSLookup": { |
| + "average": 9.272250000000001e-6, |
| + "fastest": 3.637e-6, |
| + "slowest": 0.000018875 |
| + }, |
| + "firstByte": { |
| + "average": 0.00004425681693719456, |
| + "fastest": 0.000024265, |
| + "slowest": 0.001128252 |
| + } |
| + }, |
| + "statusCodeDistribution": { |
| + "200": 176322 |
| + }, |
| + "errorDistribution": { |
| + "aborted due to deadline": 4 |
| + } |
| +} |
| \ No newline at end of file |
diff --git a/harness/testdata/playwright.json b/harness/testdata/playwright.json| new file mode 100644 |
| index 0000000..5759fa3 |
| --- /dev/null |
| +++ b/harness/testdata/playwright.json |
| @@ -0,0 +1,4 @@ |
| +{"stats":{"expected":2,"skipped":1,"unexpected":1},"suites":[{"title":"todomvc.spec.ts","specs":[ |
| + {"title":"a","tests":[{"projectName":"js-on","status":"expected"},{"projectName":"js-off","status":"unexpected"}]}], |
| + "suites":[{"title":"Editing","specs":[ |
| + {"title":"b","tests":[{"projectName":"js-on","status":"flaky"},{"projectName":"js-off","status":"skipped"}]}]}]}]} |
diff --git a/harness/tokens.go b/harness/tokens.go| new file mode 100644 |
| index 0000000..6f583d2 |
| --- /dev/null |
| +++ b/harness/tokens.go |
| @@ -0,0 +1,111 @@ |
| +package bakeoff |
| + |
| +import ( |
| + "bufio" |
| + "encoding/json" |
| + "errors" |
| + "io" |
| +) |
| + |
| +// Tokens is what one agent session cost. |
| +type Tokens struct { |
| + Input, Output, CacheRead, CacheWrite int64 |
| + Turns int |
| + Seconds float64 |
| + CostUSD float64 |
| +} |
| + |
| +// Add sums two sessions. |
| +func (t Tokens) Add(o Tokens) Tokens { |
| + return Tokens{t.Input + o.Input, t.Output + o.Output, t.CacheRead + o.CacheRead, t.CacheWrite + o.CacheWrite, |
| + t.Turns + o.Turns, t.Seconds + o.Seconds, t.CostUSD + o.CostUSD} |
| +} |
| + |
| +func scanLines(r io.Reader, f func([]byte)) error { |
| + sc := bufio.NewScanner(r) |
| + sc.Buffer(make([]byte, 1<<20), 1<<26) |
| + for sc.Scan() { |
| + f(sc.Bytes()) |
| + } |
| + return sc.Err() |
| +} |
| + |
| +// ParseClaudeLog reads `claude -p --output-format stream-json` output and |
| +// returns the final result event's totals. Per-model usage (which includes |
| +// any subagents) is summed when present. |
| +func ParseClaudeLog(r io.Reader) (Tokens, error) { |
| + type usage struct { |
| + Input int64 `json:"input_tokens"` |
| + Output int64 `json:"output_tokens"` |
| + CacheRead int64 `json:"cache_read_input_tokens"` |
| + CacheWrite int64 `json:"cache_creation_input_tokens"` |
| + } |
| + type modelUsage struct { |
| + Input int64 `json:"inputTokens"` |
| + Output int64 `json:"outputTokens"` |
| + CacheRead int64 `json:"cacheReadInputTokens"` |
| + CacheWrite int64 `json:"cacheCreationInputTokens"` |
| + } |
| + var last *Tokens |
| + err := scanLines(r, func(b []byte) { |
| + var e struct { |
| + Type string `json:"type"` |
| + Usage usage `json:"usage"` |
| + ModelUsage map[string]modelUsage `json:"modelUsage"` |
| + Turns int `json:"num_turns"` |
| + DurationMs float64 `json:"duration_ms"` |
| + Cost float64 `json:"total_cost_usd"` |
| + } |
| + if json.Unmarshal(b, &e) != nil || e.Type != "result" { |
| + return |
| + } |
| + t := Tokens{Input: e.Usage.Input, Output: e.Usage.Output, CacheRead: e.Usage.CacheRead, CacheWrite: e.Usage.CacheWrite, |
| + Turns: e.Turns, Seconds: e.DurationMs / 1000, CostUSD: e.Cost} |
| + if len(e.ModelUsage) > 0 { |
| + t.Input, t.Output, t.CacheRead, t.CacheWrite = 0, 0, 0, 0 |
| + for _, m := range e.ModelUsage { |
| + t.Input += m.Input |
| + t.Output += m.Output |
| + t.CacheRead += m.CacheRead |
| + t.CacheWrite += m.CacheWrite |
| + } |
| + } |
| + last = &t |
| + }) |
| + if err != nil { |
| + return Tokens{}, err |
| + } |
| + if last == nil { |
| + return Tokens{}, errors.New("no result event in claude log") |
| + } |
| + return *last, nil |
| +} |
| + |
| +// ParseCodexLog sums the usage on every turn.completed event of |
| +// `codex exec --json` output. Codex's input count includes cached input. |
| +func ParseCodexLog(r io.Reader) (Tokens, error) { |
| + var t Tokens |
| + err := scanLines(r, func(b []byte) { |
| + var e struct { |
| + Type string `json:"type"` |
| + Usage struct { |
| + Input int64 `json:"input_tokens"` |
| + Cached int64 `json:"cached_input_tokens"` |
| + CacheWrite int64 `json:"cache_write_input_tokens"` |
| + Output int64 `json:"output_tokens"` |
| + } `json:"usage"` |
| + } |
| + if json.Unmarshal(b, &e) != nil || e.Type != "turn.completed" { |
| + return |
| + } |
| + t.Input += e.Usage.Input |
| + t.CacheRead += e.Usage.Cached |
| + t.CacheWrite += e.Usage.CacheWrite |
| + t.Output += e.Usage.Output |
| + t.Turns++ |
| + }) |
| + if err == nil && t.Turns == 0 { |
| + err = errors.New("no turn.completed event in codex log") |
| + } |
| + return t, err |
| +} |
diff --git a/harness/ttfb.go b/harness/ttfb.go| new file mode 100644 |
| index 0000000..a27d341 |
| --- /dev/null |
| +++ b/harness/ttfb.go |
| @@ -0,0 +1,62 @@ |
| +package bakeoff |
| + |
| +import ( |
| + "crypto/tls" |
| + "io" |
| + "net/http" |
| + "net/http/httptrace" |
| + "sort" |
| + "time" |
| +) |
| + |
| +// Stats summarises a set of samples in milliseconds. |
| +type Stats struct { |
| + N int |
| + P50, P95, P99, Min, Max float64 |
| +} |
| + |
| +func summarise(ms []float64) Stats { |
| + sort.Float64s(ms) |
| + pick := func(p float64) float64 { return ms[int(p*float64(len(ms)-1)+0.5)] } |
| + return Stats{N: len(ms), P50: pick(0.5), P95: pick(0.95), P99: pick(0.99), Min: ms[0], Max: ms[len(ms)-1]} |
| +} |
| + |
| +// timeToFirstByte makes one GET on a fresh connection (DNS, TCP and TLS |
| +// included, as a new visitor would pay them) and returns the milliseconds |
| +// until the first response byte. |
| +func (s *Session) timeToFirstByte(path string) (float64, error) { |
| + tr := &http.Transport{DisableKeepAlives: true, TLSClientConfig: &tls.Config{}} |
| + defer tr.CloseIdleConnections() |
| + c := &http.Client{Transport: tr, Jar: s.Client.Jar, CheckRedirect: s.Client.CheckRedirect, Timeout: s.Client.Timeout} |
| + var first time.Time |
| + req, _ := http.NewRequest("GET", s.Base.String()+path, nil) |
| + req = req.WithContext(httptrace.WithClientTrace(req.Context(), &httptrace.ClientTrace{ |
| + GotFirstResponseByte: func() { first = time.Now() }, |
| + })) |
| + start := time.Now() |
| + res, err := c.Do(req) |
| + if err != nil { |
| + return 0, err |
| + } |
| + io.Copy(io.Discard, res.Body) |
| + res.Body.Close() |
| + return float64(first.Sub(start).Microseconds()) / 1000, nil |
| +} |
| + |
| +// MeasureTTFB discards warmup requests, then times n. |
| +func MeasureTTFB(s *Session, path string, warmup, n int) (Stats, error) { |
| + for i := 0; i < warmup; i++ { |
| + if _, err := s.timeToFirstByte(path); err != nil { |
| + return Stats{}, err |
| + } |
| + } |
| + ms := make([]float64, 0, n) |
| + for i := 0; i < n; i++ { |
| + v, err := s.timeToFirstByte(path) |
| + if err != nil { |
| + return Stats{}, err |
| + } |
| + ms = append(ms, v) |
| + } |
| + return summarise(ms), nil |
| +} |
diff --git a/harness/wake.go b/harness/wake.go| new file mode 100644 |
| index 0000000..5727373 |
| --- /dev/null |
| +++ b/harness/wake.go |
| @@ -0,0 +1,20 @@ |
| +package bakeoff |
| + |
| +import ( |
| + "os/exec" |
| + "strings" |
| +) |
| + |
| +// IsAsleep asks the platform whether the instance at host is hibernating. |
| +func IsAsleep(app, host string) (bool, error) { |
| + out, err := exec.Command("carlos", "instances", "list", "--account", "bfj", "--app", app, "--health", "asleep").CombinedOutput() |
| + if err != nil { |
| + return false, err |
| + } |
| + return strings.Contains(string(out), host), nil |
| +} |
| + |
| +// MeasureWake times the first request to a sleeping instance. |
| +func MeasureWake(s *Session, path string) (float64, error) { |
| + return s.timeToFirstByte(path) |
| +} |