amadan
carlos / todomvc-bakeoff Public
feat: one cold build, three warm and one-line-change builds
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
4 files changed,
+24
−14
harness/buildstats.go+13 −7harness/cmd/bakeoff/main.go+5 −4harness/harness_test.go+4 −1scripts/measure-build.sh+2 −2
diff --git a/harness/buildstats.go b/harness/buildstats.go| index 96e3735..7f307ba 100644 |
| --- a/harness/buildstats.go |
| +++ b/harness/buildstats.go |
| @@ -15,6 +15,7 @@ type Build struct { |
| ColdSeconds, WarmSeconds, EditSeconds float64 |
| Bytes int64 |
| Host string |
| + ColdRuns, Runs int // how many cold builds, and warm and edit builds, each median is of |
| } |
| func mk(dir string, targets ...string) error { |
| @@ -37,15 +38,17 @@ func median(v []float64) float64 { |
| return v[len(v)/2] |
| } |
| -// BuildStats measures cold (clean-deps; COLD=1 for Go, which passes -a), |
| -// warm (clean) and one-line-change (bump) builds, runs times each. |
| -func BuildStats(dir string, goCold bool, runs int) (Build, error) { |
| +// BuildStats measures cold (clean-deps; COLD=1 for Go, which passes -a) |
| +// builds coldRuns times, and warm (clean) and one-line-change (bump) |
| +// builds runs times each. Cold builds under emulation take up to an hour, |
| +// hence the separate count. |
| +func BuildStats(dir string, goCold bool, coldRuns, runs int) (Build, error) { |
| coldTarget := []string{"build-arm64"} |
| if goCold { |
| coldTarget = append(coldTarget, "COLD=1") |
| } |
| var cold, warm, edit []float64 |
| - for i := 0; i < runs; i++ { |
| + for i := 0; i < coldRuns; i++ { |
| if err := mk(dir, "clean", "clean-deps"); err != nil { |
| return Build{}, err |
| } |
| @@ -54,11 +57,13 @@ func BuildStats(dir string, goCold bool, runs int) (Build, error) { |
| return Build{}, err |
| } |
| cold = append(cold, s) |
| - |
| + } |
| + for i := 0; i < runs; i++ { |
| if err := mk(dir, "clean"); err != nil { |
| return Build{}, err |
| } |
| - if s, err = timed(dir, "build-arm64"); err != nil { |
| + s, err := timed(dir, "build-arm64") |
| + if err != nil { |
| return Build{}, err |
| } |
| warm = append(warm, s) |
| @@ -83,5 +88,6 @@ func BuildStats(dir string, goCold bool, runs int) (Build, error) { |
| return Build{}, err |
| } |
| host, _ := os.Hostname() |
| - return Build{ColdSeconds: median(cold), WarmSeconds: median(warm), EditSeconds: median(edit), Bytes: fi.Size(), Host: host}, nil |
| + return Build{ColdSeconds: median(cold), WarmSeconds: median(warm), EditSeconds: median(edit), Bytes: fi.Size(), Host: host, |
| + ColdRuns: coldRuns, Runs: runs}, nil |
| } |
diff --git a/harness/cmd/bakeoff/main.go b/harness/cmd/bakeoff/main.go| index 6661615..e26a3fe 100644 |
| --- a/harness/cmd/bakeoff/main.go |
| +++ b/harness/cmd/bakeoff/main.go |
| @@ -31,7 +31,8 @@ func main() { |
| url := fs.String("url", "", "app base URL (default https://todo-<slug>.bfj.oncarlos.com)") |
| results := fs.String("results", "results", "results directory") |
| dir := fs.String("dir", "", "app directory (default apps/<slug>)") |
| - runs := fs.Int("runs", 3, "build: runs per mode") |
| + runs := fs.Int("runs", 3, "build: warm and one-line-change runs") |
| + coldRuns := fs.Int("cold-runs", 1, "build: cold runs") |
| dur := fs.Duration("duration", 10*time.Second, "load: length of each step") |
| rest := fs.Duration("rest", 10*time.Second, "load: pause between steps") |
| levels := fs.String("levels", "1,2,4,8,16,32,64,128,256,512,1024", "load: concurrency levels, doubling") |
| @@ -72,7 +73,7 @@ func main() { |
| case "code": |
| err = code(*slug, *dir) |
| case "build": |
| - err = build(*slug, *dir, *runs) |
| + err = build(*slug, *dir, *coldRuns, *runs) |
| case "tokens": |
| err = tokens(*slug, *logs, *phase, *since) |
| case "behaviour": |
| @@ -229,8 +230,8 @@ func code(slug, dir string) error { |
| return b.Update(slug, func(r *b.AppResult) { r.Code = c }) |
| } |
| -func build(slug, dir string, runs int) error { |
| - st, err := b.BuildStats(dir, strings.HasPrefix(slug, "go"), runs) |
| +func build(slug, dir string, coldRuns, runs int) error { |
| + st, err := b.BuildStats(dir, strings.HasPrefix(slug, "go"), coldRuns, runs) |
| if err != nil { |
| return err |
| } |
diff --git a/harness/harness_test.go b/harness/harness_test.go| index 3d93025..1c31c61 100644 |
| --- a/harness/harness_test.go |
| +++ b/harness/harness_test.go |
| @@ -145,7 +145,7 @@ func TestBuildStats(t *testing.T) { |
| dir := t.TempDir() |
| mk, _ := os.ReadFile("testdata/buildtree/Makefile") |
| os.WriteFile(filepath.Join(dir, "Makefile"), mk, 0o644) |
| - b, err := BuildStats(dir, false, 1) |
| + b, err := BuildStats(dir, false, 1, 2) |
| if err != nil { |
| t.Fatal(err) |
| } |
| @@ -158,6 +158,9 @@ func TestBuildStats(t *testing.T) { |
| if _, err := os.Stat(filepath.Join(dir, ".unbumped")); err != nil { |
| t.Fatal("unbump was not run") |
| } |
| + if b.ColdRuns != 1 || b.Runs != 2 { |
| + t.Fatalf("want 1 cold run and 2 warm/edit runs recorded, got %+v", b) |
| + } |
| } |
| func TestUpdateRoundTrip(t *testing.T) { |
diff --git a/scripts/measure-build.sh b/scripts/measure-build.sh| index ad8b6ca..883a0e3 100755 |
| --- a/scripts/measure-build.sh |
| +++ b/scripts/measure-build.sh |
| @@ -1,6 +1,6 @@ |
| #!/usr/bin/env bash |
| # Build measurements for one app, in its worktree: cold, warm and |
| -# one-line-change builds of the arm64 binary (median of 3), its size, and |
| +# one-line-change builds of the arm64 binary (cold once; warm and edit median of 3), its size, and |
| # the code counts. Run apps one at a time, with nothing else heavy running. |
| # scripts/measure-build.sh <slug> [runs] |
| set -euo pipefail |
| @@ -9,4 +9,4 @@ cd "$(dirname "$0")/.." |
| source scripts/env.sh |
| dir=.worktrees/$slug/apps/$slug |
| bakeoff code --slug "$slug" --dir "$dir" |
| -bakeoff build --slug "$slug" --dir "$dir" --runs "$runs" |
| +bakeoff build --slug "$slug" --dir "$dir" --cold-runs 1 --runs "$runs" |