amadan
carlos / todomvc-bakeoff Public
fix: one-line-change builds append a unique comment, so no cache can answer them
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
13 files changed,
+182
−52
harness/buildstats.go+91 −8harness/harness_test.go+12 −2harness/testdata/buildtree/Makefile+3 −3harness/testdata/buildtree/src.go+3 −0results/go-rastrillo.json+15 −11results/go.json+15 −11results/haskell.json+6 −2results/nim.json+6 −2results/ocaml.json+6 −2results/rust.json+10 −6results/spinel.json+3 −1results/swift.json+6 −2results/zig.json+6 −2
diff --git a/harness/buildstats.go b/harness/buildstats.go| index 7f307ba..81050f2 100644 |
| --- a/harness/buildstats.go |
| +++ b/harness/buildstats.go |
| @@ -1,11 +1,14 @@ |
| package bakeoff |
| import ( |
| + "crypto/sha256" |
| "fmt" |
| + "io/fs" |
| "os" |
| "os/exec" |
| "path/filepath" |
| "sort" |
| + "strings" |
| "time" |
| ) |
| @@ -39,7 +42,7 @@ func median(v []float64) float64 { |
| } |
| // BuildStats measures cold (clean-deps; COLD=1 for Go, which passes -a) |
| -// builds coldRuns times, and warm (clean) and one-line-change (bump) |
| +// builds coldRuns times, and warm (clean) and one-line-change (editBuild) |
| // builds runs times each. Cold builds under emulation take up to an hour, |
| // hence the separate count. |
| func BuildStats(dir string, goCold bool, coldRuns, runs int) (Build, error) { |
| @@ -47,6 +50,10 @@ func BuildStats(dir string, goCold bool, coldRuns, runs int) (Build, error) { |
| if goCold { |
| coldTarget = append(coldTarget, "COLD=1") |
| } |
| + editFile, err := bumpFile(dir) |
| + if err != nil { |
| + return Build{}, err |
| + } |
| var cold, warm, edit []float64 |
| for i := 0; i < coldRuns; i++ { |
| if err := mk(dir, "clean", "clean-deps"); err != nil { |
| @@ -68,13 +75,7 @@ func BuildStats(dir string, goCold bool, coldRuns, runs int) (Build, error) { |
| } |
| warm = append(warm, s) |
| - if err := mk(dir, "bump"); err != nil { |
| - return Build{}, err |
| - } |
| - s, err = timed(dir, "build-arm64") |
| - if uerr := mk(dir, "unbump"); err == nil { |
| - err = uerr |
| - } |
| + s, err = editBuild(dir, editFile, i) |
| if err != nil { |
| return Build{}, err |
| } |
| @@ -91,3 +92,85 @@ func BuildStats(dir string, goCold bool, coldRuns, runs int) (Build, error) { |
| return Build{ColdSeconds: median(cold), WarmSeconds: median(warm), EditSeconds: median(edit), Bytes: fi.Size(), Host: host, |
| ColdRuns: coldRuns, Runs: runs}, nil |
| } |
| + |
| +// bumpFile finds the source file `make bump` edits (the one holding the |
| +// GET / handler, per the brief) by comparing every file before and after, |
| +// then runs `make unbump`. |
| +func bumpFile(dir string) (string, error) { |
| + before, err := snapshot(dir) |
| + if err != nil { |
| + return "", err |
| + } |
| + if err := mk(dir, "bump"); err != nil { |
| + return "", err |
| + } |
| + after, err := snapshot(dir) |
| + if uerr := mk(dir, "unbump"); err == nil { |
| + err = uerr |
| + } |
| + if err != nil { |
| + return "", err |
| + } |
| + for f, h := range after { |
| + if before[f] != h { |
| + return f, nil |
| + } |
| + } |
| + return "", fmt.Errorf("make bump changed no file in %s", dir) |
| +} |
| + |
| +// snapshot hashes every file under dir outside build output and caches. |
| +func snapshot(dir string) (map[string][32]byte, error) { |
| + out := map[string][32]byte{} |
| + err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { |
| + if err != nil { |
| + return err |
| + } |
| + if d.IsDir() { |
| + if p != dir && (skipDirs[d.Name()] || strings.HasPrefix(d.Name(), ".")) { |
| + return filepath.SkipDir |
| + } |
| + return nil |
| + } |
| + b, err := os.ReadFile(p) |
| + if err != nil { |
| + return err |
| + } |
| + out[p] = sha256.Sum256(b) |
| + return nil |
| + }) |
| + return out, err |
| +} |
| + |
| +// commentPrefix is the line-comment syntax for a source file. |
| +func commentPrefix(path string) string { |
| + switch strings.ToLower(filepath.Ext(path)) { |
| + case ".hs": |
| + return "-- " |
| + case ".nim", ".rb", ".mojo", ".py": |
| + return "# " |
| + case ".ml", ".mli": |
| + return "(* " |
| + } |
| + return "// " |
| +} |
| + |
| +// editBuild is the one-line change: append a comment line no compiler |
| +// cache has seen to file, time the rebuild, and put the file back. A |
| +// comment changes the source's content (and mtime), so content-addressed |
| +// caches (Go, Zig, Nim, dune) recompile it as surely as mtime-based ones. |
| +func editBuild(dir, file string, run int) (float64, error) { |
| + orig, err := os.ReadFile(file) |
| + if err != nil { |
| + return 0, err |
| + } |
| + defer os.WriteFile(file, orig, 0o644) |
| + line := fmt.Sprintf("%sbakeoff edit %d %d", commentPrefix(file), time.Now().UnixNano(), run) |
| + if strings.HasPrefix(line, "(* ") { |
| + line += " *)" |
| + } |
| + if err := os.WriteFile(file, append(append(orig, '\n'), line+"\n"...), 0o644); err != nil { |
| + return 0, err |
| + } |
| + return timed(dir, "build-arm64") |
| +} |
diff --git a/harness/harness_test.go b/harness/harness_test.go| index 1c31c61..5ee90d4 100644 |
| --- a/harness/harness_test.go |
| +++ b/harness/harness_test.go |
| @@ -143,8 +143,10 @@ func TestMeasureTTFB(t *testing.T) { |
| func TestBuildStats(t *testing.T) { |
| dir := t.TempDir() |
| - mk, _ := os.ReadFile("testdata/buildtree/Makefile") |
| - os.WriteFile(filepath.Join(dir, "Makefile"), mk, 0o644) |
| + for _, f := range []string{"Makefile", "src.go"} { |
| + b, _ := os.ReadFile("testdata/buildtree/" + f) |
| + os.WriteFile(filepath.Join(dir, f), b, 0o644) |
| + } |
| b, err := BuildStats(dir, false, 1, 2) |
| if err != nil { |
| t.Fatal(err) |
| @@ -158,6 +160,14 @@ func TestBuildStats(t *testing.T) { |
| if _, err := os.Stat(filepath.Join(dir, ".unbumped")); err != nil { |
| t.Fatal("unbump was not run") |
| } |
| + last, _ := os.ReadFile(filepath.Join(dir, "dist", "history")) |
| + if !strings.Contains(string(last), "// bakeoff edit ") { |
| + t.Fatalf("the edit build must compile a unique comment line, built:\n%s", last) |
| + } |
| + src, _ := os.ReadFile(filepath.Join(dir, "src.go")) |
| + if string(src) != "package main\n\nconst marker = \"steady\"\n" { |
| + t.Fatalf("source not restored:\n%s", src) |
| + } |
| if b.ColdRuns != 1 || b.Runs != 2 { |
| t.Fatalf("want 1 cold run and 2 warm/edit runs recorded, got %+v", b) |
| } |
diff --git a/harness/testdata/buildtree/Makefile b/harness/testdata/buildtree/Makefile| index b07318e..e3e66b4 100644 |
| --- a/harness/testdata/buildtree/Makefile |
| +++ b/harness/testdata/buildtree/Makefile |
| @@ -4,8 +4,8 @@ clean-deps: |
| touch .cold |
| build-arm64: |
| @if [ -f .cold ]; then sleep 0.4; elif [ -f .clean ]; then sleep 0.2; else sleep 0.05; fi |
| - @rm -f .cold .clean; mkdir -p dist; printf binary > dist/app-linux-arm64 |
| + @rm -f .cold .clean; mkdir -p dist; printf binary > dist/app-linux-arm64; cat src.go >> dist/history |
| bump: |
| - touch .bumped |
| + sed -i 's/steady/bumped/' src.go |
| unbump: |
| - rm -f .bumped; touch .unbumped |
| + sed -i 's/bumped/steady/' src.go; touch .unbumped |
diff --git a/harness/testdata/buildtree/src.go b/harness/testdata/buildtree/src.go| new file mode 100644 |
| index 0000000..bf426b6 |
| --- /dev/null |
| +++ b/harness/testdata/buildtree/src.go |
| @@ -0,0 +1,3 @@ |
| +package main |
| + |
| +const marker = "steady" |
diff --git a/results/go-rastrillo.json b/results/go-rastrillo.json| index ece525b..fe2baa3 100644 |
| --- a/results/go-rastrillo.json |
| +++ b/results/go-rastrillo.json |
| @@ -32,7 +32,9 @@ |
| "Min": 48.03, |
| "Max": 64.063 |
| }, |
| - "Wake": null, |
| + "Wake": [ |
| + 1435.117 |
| + ], |
| "Load": [ |
| { |
| "Scenario": "read", |
| @@ -292,18 +294,20 @@ |
| "JSBytes": 3325 |
| }, |
| "Code": { |
| - "Server": 0, |
| - "Templates": 0, |
| - "JS": 0, |
| - "Tests": 0, |
| - "Deps": 0 |
| + "Server": 614, |
| + "Templates": 102, |
| + "JS": 86, |
| + "Tests": 503, |
| + "Deps": 3 |
| }, |
| "Build": { |
| - "ColdSeconds": 0, |
| - "WarmSeconds": 0, |
| - "EditSeconds": 0, |
| - "Bytes": 0, |
| - "Host": "" |
| + "ColdSeconds": 11.408177673, |
| + "WarmSeconds": 0.28805728, |
| + "EditSeconds": 0.285644601, |
| + "Bytes": 14090402, |
| + "Host": "DOCS-T1", |
| + "ColdRuns": 1, |
| + "Runs": 3 |
| }, |
| "TokensBuild": { |
| "Input": 124, |
diff --git a/results/go.json b/results/go.json| index 9ca4aee..d1b0c14 100644 |
| --- a/results/go.json |
| +++ b/results/go.json |
| @@ -32,7 +32,9 @@ |
| "Min": 86.404, |
| "Max": 108.968 |
| }, |
| - "Wake": null, |
| + "Wake": [ |
| + 84.556 |
| + ], |
| "Load": [ |
| { |
| "Scenario": "read", |
| @@ -281,18 +283,20 @@ |
| "JSBytes": 4689 |
| }, |
| "Code": { |
| - "Server": 0, |
| - "Templates": 0, |
| - "JS": 0, |
| - "Tests": 0, |
| - "Deps": 0 |
| + "Server": 665, |
| + "Templates": 83, |
| + "JS": 132, |
| + "Tests": 517, |
| + "Deps": 1 |
| }, |
| "Build": { |
| - "ColdSeconds": 0, |
| - "WarmSeconds": 0, |
| - "EditSeconds": 0, |
| - "Bytes": 0, |
| - "Host": "" |
| + "ColdSeconds": 10.280152685000001, |
| + "WarmSeconds": 0.221965481, |
| + "EditSeconds": 0.09772033, |
| + "Bytes": 12320930, |
| + "Host": "DOCS-T1", |
| + "ColdRuns": 1, |
| + "Runs": 3 |
| }, |
| "TokensBuild": { |
| "Input": 82, |
diff --git a/results/haskell.json b/results/haskell.json| index d148388..739efe7 100644 |
| --- a/results/haskell.json |
| +++ b/results/haskell.json |
| @@ -32,7 +32,9 @@ |
| "Min": 89.266, |
| "Max": 168.219 |
| }, |
| - "Wake": null, |
| + "Wake": [ |
| + 1373.548 |
| + ], |
| "Load": [ |
| { |
| "Scenario": "read", |
| @@ -304,7 +306,9 @@ |
| "WarmSeconds": 0, |
| "EditSeconds": 0, |
| "Bytes": 0, |
| - "Host": "" |
| + "Host": "", |
| + "ColdRuns": 0, |
| + "Runs": 0 |
| }, |
| "TokensBuild": { |
| "Input": 140, |
diff --git a/results/nim.json b/results/nim.json| index 798d11e..3a1c0d0 100644 |
| --- a/results/nim.json |
| +++ b/results/nim.json |
| @@ -32,7 +32,9 @@ |
| "Min": 42.03, |
| "Max": 55.361 |
| }, |
| - "Wake": null, |
| + "Wake": [ |
| + 1244.605 |
| + ], |
| "Load": [ |
| { |
| "Scenario": "read", |
| @@ -352,7 +354,9 @@ |
| "WarmSeconds": 0, |
| "EditSeconds": 0, |
| "Bytes": 0, |
| - "Host": "" |
| + "Host": "", |
| + "ColdRuns": 0, |
| + "Runs": 0 |
| }, |
| "TokensBuild": { |
| "Input": 140, |
diff --git a/results/ocaml.json b/results/ocaml.json| index f6e51d5..dd5915d 100644 |
| --- a/results/ocaml.json |
| +++ b/results/ocaml.json |
| @@ -32,7 +32,9 @@ |
| "Min": 41.961, |
| "Max": 125.31 |
| }, |
| - "Wake": null, |
| + "Wake": [ |
| + 1352.486 |
| + ], |
| "Load": [ |
| { |
| "Scenario": "read", |
| @@ -354,7 +356,9 @@ |
| "WarmSeconds": 0, |
| "EditSeconds": 0, |
| "Bytes": 0, |
| - "Host": "" |
| + "Host": "", |
| + "ColdRuns": 0, |
| + "Runs": 0 |
| }, |
| "TokensBuild": { |
| "Input": 202, |
diff --git a/results/rust.json b/results/rust.json| index e797fab..de783ef 100644 |
| --- a/results/rust.json |
| +++ b/results/rust.json |
| @@ -32,7 +32,9 @@ |
| "Min": 79.697, |
| "Max": 102.563 |
| }, |
| - "Wake": null, |
| + "Wake": [ |
| + 1156.261 |
| + ], |
| "Load": [ |
| { |
| "Scenario": "read", |
| @@ -354,18 +356,20 @@ |
| "JSBytes": 4596 |
| }, |
| "Code": { |
| - "Server": 0, |
| + "Server": 702, |
| "Templates": 0, |
| - "JS": 0, |
| - "Tests": 0, |
| - "Deps": 0 |
| + "JS": 113, |
| + "Tests": 203, |
| + "Deps": 7 |
| }, |
| "Build": { |
| "ColdSeconds": 0, |
| "WarmSeconds": 0, |
| "EditSeconds": 0, |
| "Bytes": 0, |
| - "Host": "" |
| + "Host": "", |
| + "ColdRuns": 0, |
| + "Runs": 0 |
| }, |
| "TokensBuild": { |
| "Input": 94, |
diff --git a/results/spinel.json b/results/spinel.json| index bc9ad00..b507d7b 100644 |
| --- a/results/spinel.json |
| +++ b/results/spinel.json |
| @@ -32,7 +32,9 @@ |
| "Min": 82.851, |
| "Max": 108.763 |
| }, |
| - "Wake": null, |
| + "Wake": [ |
| + 1300.982 |
| + ], |
| "Load": [ |
| { |
| "Scenario": "read", |
diff --git a/results/swift.json b/results/swift.json| index 9803d8c..d64613b 100644 |
| --- a/results/swift.json |
| +++ b/results/swift.json |
| @@ -32,7 +32,9 @@ |
| "Min": 47.876, |
| "Max": 63.425 |
| }, |
| - "Wake": null, |
| + "Wake": [ |
| + 1363.467 |
| + ], |
| "Load": [ |
| { |
| "Scenario": "read", |
| @@ -376,7 +378,9 @@ |
| "WarmSeconds": 0, |
| "EditSeconds": 0, |
| "Bytes": 0, |
| - "Host": "" |
| + "Host": "", |
| + "ColdRuns": 0, |
| + "Runs": 0 |
| }, |
| "TokensBuild": { |
| "Input": 120, |
diff --git a/results/zig.json b/results/zig.json| index e585344..7cc3819 100644 |
| --- a/results/zig.json |
| +++ b/results/zig.json |
| @@ -32,7 +32,9 @@ |
| "Min": 48.154, |
| "Max": 65.122 |
| }, |
| - "Wake": null, |
| + "Wake": [ |
| + 1239.057 |
| + ], |
| "Load": [ |
| { |
| "Scenario": "read", |
| @@ -351,7 +353,9 @@ |
| "WarmSeconds": 0, |
| "EditSeconds": 0, |
| "Bytes": 0, |
| - "Host": "" |
| + "Host": "", |
| + "ColdRuns": 0, |
| + "Runs": 0 |
| }, |
| "TokensBuild": { |
| "Input": 172, |