c2b9263 · carlos/todomvc-bakeoff

diff --git a/docs/superpowers/specs/2026-10-23-todomvc-bakeoff-design.md b/docs/superpowers/specs/2026-10-23-todomvc-bakeoff-design.md

index e9f0ab9..08bfdca 100644

--- a/docs/superpowers/specs/2026-10-23-todomvc-bakeoff-design.md

+++ b/docs/superpowers/specs/2026-10-23-todomvc-bakeoff-design.md

@@ -212,7 +212,7 @@ All of them are collected by scripts in this repo and can be re-run.

| Build | Three build times for the arm64 binary that ships: **cold** (the app and every third-party library compiled from already-downloaded sources; compiler and standard library as shipped), **warm** (libraries already compiled, app rebuilt from clean), and **one-line change** (rebuild after a trivial edit in the request-handling code). Plus binary size. Paul asked for cold and warm, 2026-10-23. Measured on docs-t1 where the app cross-compiles there; otherwise on the machine that built it, with the machine named beside the number and kept out of the ranking. |

| Response time | Time to the first byte for `GET /` with an empty list and with 50 items, from docs-t1, median of repeated runs. |

| Wake time | Time to the first byte on the first request after the instance has gone to sleep. |

-| Under load | Fixed scenarios at fixed concurrency levels (read-heavy: `GET /` on a 50-item list; write-heavy: add and toggle). Reports requests per second, p50/p95/p99 latency and error rate. |

+| Under load | A ramp for reads (`GET /` on a 50-item list) and for writes (toggling an item): 1, 2, 4 … 1,024 concurrent connections, 10 s per step. Reports peak requests per second, latency at every step, and **where it slows down**: the most connections at which doubling the load still raised throughput by 10% or more. A ramp stops early after two flat doublings or more than 1% errors. Paul asked for this, 2026-10-23, replacing fixed loads at 1, 16 and 64. |

| Memory | Peak and idle memory from the platform, if it exposes them. If it doesn't, we file a request against the platform rather than measure inside the app. |

| Page weight | Bytes of HTML and JS sent for `GET /`. |

diff --git a/harness/cmd/bakeoff/main.go b/harness/cmd/bakeoff/main.go

index 0b69614..4ebe6e7 100644

--- a/harness/cmd/bakeoff/main.go

+++ b/harness/cmd/bakeoff/main.go

@@ -30,9 +30,9 @@ func main() {

results := fs.String("results", "results", "results directory")

dir := fs.String("dir", "", "app directory (default apps/<slug>)")

runs := fs.Int("runs", 3, "build: runs per mode")

- dur := fs.Duration("duration", 30*time.Second, "load: length of each run")

- rest := fs.Duration("rest", 60*time.Second, "load: pause between runs")

- levels := fs.String("levels", "1,16,64", "load: concurrency levels")

+ dur := fs.Duration("duration", 10*time.Second, "load: length of each step")

+ rest := fs.Duration("rest", 10*time.Second, "load: pause between steps")

+ levels := fs.String("levels", "1,2,4,8,16,32,64,128,256,512,1024", "load: concurrency levels, doubling")

phase := fs.String("phase", "", "tokens: build|fix|security|review")

logs := fs.String("logs", "builders/logs", "tokens: log directory")

since := fs.String("since", "", "tokens: log whose cumulative totals this resumed phase starts from (default: the previous phase's)")

@@ -157,30 +157,37 @@ func wake(slug, url string) error {

return b.Update(slug, func(r *b.AppResult) { r.Wake = append(r.Wake, ms) })

}

+// load ramps each scenario through the levels, stopping early when the

+// app has stopped scaling or is failing (b.SweepShouldStop).

func load(slug, url, levels string, d, rest time.Duration) error {

s, err := seeded(url, 50)

if err != nil {

return err

}

var out []b.LoadResult

- first := true

for _, sc := range []string{"read", "toggle"} {

+ var sweep []b.LoadResult

for _, l := range strings.Split(levels, ",") {

c, err := strconv.Atoi(strings.TrimSpace(l))

if err != nil {

return err

}

- if !first {

+ if len(out)+len(sweep) > 0 {

time.Sleep(rest)

}

- first = false

r, err := b.RunLoad(s, sc, c, d)

if err != nil {

return err

}

fmt.Printf("%s %s c=%d: %.0f rps, p50 %.1fms, p99 %.1fms, errors %.2f%%\n", slug, sc, c, r.RPS, r.P50Ms, r.P99Ms, r.ErrorRate*100)

- out = append(out, r)

+ sweep = append(sweep, r)

+ if b.SweepShouldStop(sweep) {

+ break

+ }

}

+ k := b.Knee(sweep)

+ fmt.Printf("%s %s: peak %.0f rps; slows down after %d (found %v, tested to %d)\n", slug, sc, k.PeakRPS, k.Concurrency, k.Found, k.MaxTested)

+ out = append(out, sweep...)

}

return b.Update(slug, func(r *b.AppResult) { r.Load = out })

}

diff --git a/harness/harness_test.go b/harness/harness_test.go

index 9be9ca3..57304b8 100644

--- a/harness/harness_test.go

+++ b/harness/harness_test.go

@@ -274,3 +274,30 @@ func TestParseClaudeLogSumsTurnsAcrossResults(t *testing.T) {

t.Fatalf("got %+v (usage and cost are cumulative: take the last; turns and time are per result: sum)", tk)

}

+

+func lr(c int, rps, p99, errs float64) LoadResult {

+ return LoadResult{Scenario: "read", Concurrency: c, RPS: rps, P99Ms: p99, ErrorRate: errs}

+}

+

+func TestKneeIsFirstDoublingWithLittleGain(t *testing.T) {

+ sweep := []LoadResult{lr(1, 20, 60, 0), lr(2, 40, 60, 0), lr(4, 79, 61, 0), lr(8, 150, 70, 0), lr(16, 160, 130, 0), lr(32, 162, 260, 0)}

+ k := Knee(sweep)

+ if !k.Found || k.Concurrency != 8 || k.PeakRPS != 162 {

+ t.Fatalf("got %+v, want slowdown after 8 (16 added under 10%%), peak 162", k)

+ }

+ if k := Knee(sweep[:4]); k.Found || k.PeakRPS != 150 || k.MaxTested != 8 {

+ t.Fatalf("got %+v, want not found within 8", k)

+ }

+}

+

+func TestSweepStopsOnErrorsOrFlatThroughput(t *testing.T) {

+ if !SweepShouldStop([]LoadResult{lr(1, 20, 60, 0), lr(2, 40, 60, 0.02)}) {

+ t.Fatal("should stop on >1% errors")

+ }

+ if !SweepShouldStop([]LoadResult{lr(4, 100, 60, 0), lr(8, 105, 90, 0), lr(16, 108, 180, 0)}) {

+ t.Fatal("should stop after two flat doublings")

+ }

+ if SweepShouldStop([]LoadResult{lr(4, 100, 60, 0), lr(8, 105, 90, 0), lr(16, 190, 90, 0)}) {

+ t.Fatal("one flat step then growth: keep going")

+ }

+}

diff --git a/harness/report/report.go b/harness/report/report.go

index 7809619..cbfe986 100644

--- a/harness/report/report.go

+++ b/harness/report/report.go

@@ -16,20 +16,20 @@ import (

var pageSrc string

var page = template.Must(template.New("page").Funcs(template.FuncMap{

- "md": func(c Copy, key string) template.HTML { return markdown(c[key]) },

- "ms": ms,

- "mb": mb,

- "num": num,

- "usd": usd,

- "secs": secs,

- "live": live,

- "total": func(r b.AppResult) b.Tokens { return r.TokensBuild.Add(r.TokensFix).Add(r.TokensSecurity) },

- "wake": func(r b.AppResult) float64 { return medianOf(r.Wake) },

- "read64": func(r b.AppResult) b.LoadResult { return load(r, "read", 64) },

- "loc": func(c b.CodeStats) int { return c.Server + c.Templates + c.JS },

- "add3": func(x b.Behaviour) int { return x.Passed + x.Failed + x.Skipped },

- "float": func(n int64) float64 { return float64(n) },

- "inc": func(i int) int { return i + 1 },

+ "md": func(c Copy, key string) template.HTML { return markdown(c[key]) },

+ "ms": ms,

+ "mb": mb,

+ "num": num,

+ "usd": usd,

+ "secs": secs,

+ "live": live,

+ "total": func(r b.AppResult) b.Tokens { return r.TokensBuild.Add(r.TokensFix).Add(r.TokensSecurity) },

+ "wake": func(r b.AppResult) float64 { return medianOf(r.Wake) },

+ "peak": func(r b.AppResult, sc string) float64 { return knee(r, sc).PeakRPS },

+ "loc": func(c b.CodeStats) int { return c.Server + c.Templates + c.JS },

+ "add3": func(x b.Behaviour) int { return x.Passed + x.Failed + x.Skipped },

+ "float": func(n int64) float64 { return float64(n) },

+ "inc": func(i int) int { return i + 1 },

}).Parse(pageSrc))

func live(r b.AppResult) bool { return strings.HasPrefix(r.Status, "LIVE") }

@@ -60,13 +60,30 @@ func medianOf(v []float64) float64 {

return c[len(c)/2]

}

-func load(r b.AppResult, scenario string, c int) b.LoadResult {

+func knee(r b.AppResult, scenario string) b.KneeResult {

+ var sweep []b.LoadResult

for _, l := range r.Load {

- if l.Scenario == scenario && l.Concurrency == c {

- return l

+ if l.Scenario == scenario {

+ sweep = append(sweep, l)

}

- return b.LoadResult{}

+ return b.Knee(sweep)

+}

+

+// kneeChart shows, per app, the concurrency it still scaled at; an app

+// that never stopped scaling shows that instead of a bar.

+func kneeChart(rs []b.AppResult, scenario, title, note string) Chart {

+ c := chart(rs, title, note, func(v float64) string { return num(v) + " connections" },

+ func(r b.AppResult) (float64, bool) {

+ k := knee(r, scenario)

+ return float64(k.Concurrency), k.Found

+ })

+ for i, r := range rs {

+ if k := knee(r, scenario); live(r) && !k.Found && k.MaxTested > 0 {

+ c.Rows[i].Missing = "still scaling at " + num(float64(k.MaxTested))

+ }

+ return c

}

// chart builds one chart over all results; get returns ok=false when the

@@ -112,12 +129,12 @@ func charts(rs []b.AppResult) []Chart {

func(r b.AppResult) (float64, bool) { return pos(r.TTFB50.P50) }),

chart(rs, "Wake from sleep", "Time to first byte on the first request after the instance went to sleep. Median of 3.", ms,

func(r b.AppResult) (float64, bool) { return pos(medianOf(r.Wake)) }),

- chart(rs, "Reads per second, 64 at once", "GET / on a 50-item list for 30 s with 64 concurrent connections.", num,

- func(r b.AppResult) (float64, bool) { return pos(load(r, "read", 64).RPS) }),

- chart(rs, "Read latency (p99), 64 at once", "The slowest 1% of those reads took at least this long.", ms,

- func(r b.AppResult) (float64, bool) { return pos(load(r, "read", 64).P99Ms) }),

- chart(rs, "Toggles per second, 64 at once", "POST toggling one item for 30 s with 64 concurrent connections.", num,

- func(r b.AppResult) (float64, bool) { return pos(load(r, "toggle", 64).RPS) }),

+ chart(rs, "Peak reads per second", "GET / on a 50-item list, ramping from 1 to 1,024 concurrent connections, 10 s per step.", num,

+ func(r b.AppResult) (float64, bool) { return pos(knee(r, "read").PeakRPS) }),

+ kneeChart(rs, "read", "Reads: where it slows down", "The most concurrent connections at which doubling the load still raised throughput by 10% or more."),

+ chart(rs, "Peak toggles per second", "POST toggling one item, ramping the same way.", num,

+ func(r b.AppResult) (float64, bool) { return pos(knee(r, "toggle").PeakRPS) }),

+ kneeChart(rs, "toggle", "Toggles: where it slows down", "The most concurrent connections at which doubling the load still raised throughput by 10% or more."),

chart(rs, "Binary size", "The linux-arm64 binary that ships.", mb,

func(r b.AppResult) (float64, bool) { return pos(float64(r.Build.Bytes)) }),

chart(rs, "Cold build", "Every library compiled from already-downloaded sources.", secs,

diff --git a/harness/report/report.html.tmpl b/harness/report/report.html.tmpl

index e96c5ac..69dfb16 100644

--- a/harness/report/report.html.tmpl

+++ b/harness/report/report.html.tmpl

@@ -56,13 +56,13 @@ dt { color: var(--muted); } dd { margin: 0; color: var(--ink-2); }

<h2>At a glance</h2>

<table>

-<thead><tr><th>Setup</th><th>Tests (JS on / off)</th><th>First byte</th><th>Wake</th><th>Reads/s @64</th><th>Binary</th><th>Cold / warm build</th><th>Lines</th><th>Deps</th><th>Cost</th><th>Readability</th></tr></thead>

+<thead><tr><th>Setup</th><th>Tests (JS on / off)</th><th>First byte</th><th>Wake</th><th>Peak reads/s</th><th>Binary</th><th>Cold / warm build</th><th>Lines</th><th>Deps</th><th>Cost</th><th>Readability</th></tr></thead>

<tbody>

<tr><td>{{.Language}}</td>

<td>{{.BehaviourJSOn.Passed}}/{{add3 .BehaviourJSOn}} · {{.BehaviourJSOff.Passed}}/{{add3 .BehaviourJSOff}}</td>

-<td>{{ms .TTFBEmpty.P50}}</td><td>{{ms (wake .)}}</td><td>{{num (read64 .).RPS}}</td>

+<td>{{ms .TTFBEmpty.P50}}</td><td>{{ms (wake .)}}</td><td>{{num (peak . "read")}}</td>

<td>{{mb (float .Build.Bytes)}}</td><td>{{secs .Build.ColdSeconds}} / {{secs .Build.WarmSeconds}}</td>

<td>{{loc .Code}}</td><td>{{.Code.Deps}}</td><td>{{usd (total .).CostUSD}}</td><td>{{.Readability.Overall}}/10</td></tr>

diff --git a/harness/report/report_test.go b/harness/report/report_test.go

index 81a0b58..f72bfb9 100644

--- a/harness/report/report_test.go

+++ b/harness/report/report_test.go

@@ -14,8 +14,9 @@ func fixture() []b.AppResult {

Stack: "net/http, html/template, modernc.org/sqlite",

BehaviourJSOn: b.Behaviour{Passed: 35}, BehaviourJSOff: b.Behaviour{Passed: 33, Skipped: 2},

TTFBEmpty: b.Stats{N: 30, P50: 41.2}, TTFB50: b.Stats{N: 30, P50: 45.9},

- Wake: []float64{120, 140, 130},

- Load: []b.LoadResult{{Scenario: "read", Concurrency: 64, RPS: 2100, P50Ms: 30, P95Ms: 40, P99Ms: 55}},

+ Wake: []float64{120, 140, 130},

+ Load: []b.LoadResult{{Scenario: "read", Concurrency: 1, RPS: 20}, {Scenario: "read", Concurrency: 2, RPS: 40},

+ {Scenario: "read", Concurrency: 4, RPS: 41}, {Scenario: "toggle", Concurrency: 1, RPS: 15}},

Weight: b.Weight{HTMLBytes: 9000, JSBytes: 2400},

Code: b.CodeStats{Server: 300, Templates: 80, JS: 120, Tests: 200, Deps: 1},

Build: b.Build{ColdSeconds: 40, WarmSeconds: 2, EditSeconds: 1.1, Bytes: 9_400_000, Host: "docs-t1"},

@@ -35,7 +36,7 @@ func TestRenderAllApps(t *testing.T) {

out := buf.String()

for _, want := range []string{

"Go", "Mojo", "single-file binaries only", "https://todo-go.bfj.oncarlos.com", "<svg",

- "<p>Nine apps.</p>", `<a href="https://example.com">link</a>`, "Clear & tidy <ok>", "9.4 MB",

+ "<p>Nine apps.</p>", `<a href="https://example.com">link</a>`, "Clear & tidy <ok>", "9.4 MB", "2 connections", "still scaling at 1",

} {

if !strings.Contains(out, want) {

t.Errorf("output lacks %q", want)

diff --git a/harness/sweep.go b/harness/sweep.go

new file mode 100644

index 0000000..1075202

--- /dev/null

+++ b/harness/sweep.go

@@ -0,0 +1,49 @@

+package bakeoff

+

+// A sweep doubles the concurrent connections (1, 2, 4, …) and runs one

+// short load step at each, to find where an app starts to slow down.

+

+// flatGain is the throughput gain below which doubling the load counts

+// as no real gain: extra connections only add waiting.

+const flatGain = 1.10

+

+// maxErrorRate stops a sweep: past it, the numbers describe failures.

+const maxErrorRate = 0.01

+

+// KneeResult says where a sweep stopped scaling.

+type KneeResult struct {

+ Found bool // false: still scaling at MaxTested

+ Concurrency int // the last level that still scaled

+ PeakRPS float64 // the most requests per second any level reached

+ P50Ms float64 // median latency at Concurrency

+ MaxTested int

+}

+

+// Knee finds the first doubling that raised throughput by less than 10%:

+// the app slows down past the level before it.

+func Knee(sweep []LoadResult) KneeResult {

+ var k KneeResult

+ for i, r := range sweep {

+ if r.RPS > k.PeakRPS {

+ k.PeakRPS = r.RPS

+ }

+ k.MaxTested = r.Concurrency

+ if !k.Found && i > 0 && r.RPS < sweep[i-1].RPS*flatGain {

+ k.Found, k.Concurrency, k.P50Ms = true, sweep[i-1].Concurrency, sweep[i-1].P50Ms

+ }

+ return k

+}

+

+// SweepShouldStop is true once the last step had too many errors, or the

+// last two doublings were both flat.

+func SweepShouldStop(sweep []LoadResult) bool {

+ n := len(sweep)

+ if n == 0 {

+ return false

+ }

+ if sweep[n-1].ErrorRate > maxErrorRate {

+ return true

+ }

+ return n >= 3 && sweep[n-1].RPS < sweep[n-2].RPS*flatGain && sweep[n-2].RPS < sweep[n-3].RPS*flatGain

+}

diff --git a/scripts/measure-live.sh b/scripts/measure-live.sh

index a684c04..a5e9336 100755

--- a/scripts/measure-live.sh

+++ b/scripts/measure-live.sh

@@ -13,4 +13,5 @@ scripts/test-app.sh "$url" "$out/suite.json" >/dev/null 2>&1 || true

bakeoff behaviour --slug "$slug" --file "$out/suite.json"

bakeoff ttfb --slug "$slug"

bakeoff weight --slug "$slug"

+ulimit -n 8192 2>/dev/null || true

bakeoff load --slug "$slug"

carlos / todomvc-bakeoff Public

feat: load ramp to find where each app slows down