amadan
carlos / todomvc-bakeoff Public
feat: read ramp after the write ramp; two-step instance reset
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
5 files changed,
+32
−4
harness/cmd/bakeoff/main.go+1 −1harness/harness_test.go+1 −1harness/load.go+4 −2harness/report/report.go+2 −0scripts/fresh-instance.sh+24 −0
diff --git a/harness/cmd/bakeoff/main.go b/harness/cmd/bakeoff/main.go| index 0726f7d..f80a2b6 100644 |
| --- a/harness/cmd/bakeoff/main.go |
| +++ b/harness/cmd/bakeoff/main.go |
| @@ -169,7 +169,7 @@ func load(slug, url, levels string, d, rest time.Duration) error { |
| return err |
| } |
| var out []b.LoadResult |
| - for _, sc := range []string{"read", "toggle"} { |
| + for _, sc := range []string{"read", "toggle", "read-after"} { |
| var sweep []b.LoadResult |
| for _, l := range strings.Split(levels, ",") { |
| c, err := strconv.Atoi(strings.TrimSpace(l)) |
diff --git a/harness/harness_test.go b/harness/harness_test.go| index f336385..0609ac2 100644 |
| --- a/harness/harness_test.go |
| +++ b/harness/harness_test.go |
| @@ -186,7 +186,7 @@ func TestRunLoadAgainstRefapp(t *testing.T) { |
| if _, err := Seed(s, 10); err != nil { |
| t.Fatal(err) |
| } |
| - for _, sc := range []string{"read", "toggle"} { |
| + for _, sc := range []string{"read", "toggle", "read-after"} { |
| r, err := RunLoad(s, sc, 2, time.Second) |
| if err != nil { |
| t.Fatal(err) |
diff --git a/harness/load.go b/harness/load.go| index f48bf69..9611f50 100644 |
| --- a/harness/load.go |
| +++ b/harness/load.go |
| @@ -64,14 +64,16 @@ func ParseOha(b []byte, ok int) (LoadResult, error) { |
| } |
| // RunLoad drives the app with oha for d at the given concurrency. |
| -// "read" is GET / (200); "toggle" is POST /todos/<first id>/toggle (303). |
| +// "read" is GET / (200); "toggle" is POST /todos/<first id>/toggle (303); |
| +// "read-after" is "read" again, run after the toggle ramp, to show any |
| +// slow-down left behind by heavy writes. |
| func RunLoad(s *Session, scenario string, concurrency int, d time.Duration) (LoadResult, error) { |
| args := []string{"--no-tui", "--output-format", "json", "-z", d.String(), "-c", strconv.Itoa(concurrency), |
| "--redirect", "0", "-H", "Cookie: " + s.CookieHeader(), "-H", "Origin: " + s.origin()} |
| ok := 200 |
| target := s.Base.String() + "/" |
| switch scenario { |
| - case "read": |
| + case "read", "read-after": |
| case "toggle": |
| ids, err := s.IDs("/") |
| if err != nil || len(ids) == 0 { |
diff --git a/harness/report/report.go b/harness/report/report.go| index cc23029..f876402 100644 |
| --- a/harness/report/report.go |
| +++ b/harness/report/report.go |
| @@ -139,6 +139,8 @@ func charts(rs []b.AppResult) []Chart { |
| kneeChart(rs, "read", "Reads: where it slows down", "The fewest concurrent connections from which more load never raised throughput by 10% or more. Errors: the first step where more than 1% of requests failed."), |
| chart(rs, "Peak toggles per second", "POST toggling one item, ramping the same way.", num, |
| func(r b.AppResult) (float64, bool) { return pos(knee(r, "toggle").PeakRPS) }), |
| + chart(rs, "Peak reads per second, after the write ramp", "The same read ramp, run again straight after the toggle ramp on the same database.", num, |
| + func(r b.AppResult) (float64, bool) { return pos(knee(r, "read-after").PeakRPS) }), |
| kneeChart(rs, "toggle", "Toggles: where it slows down", "The fewest concurrent connections from which more load never raised throughput by 10% or more. Errors: the first step where more than 1% of requests failed."), |
| chart(rs, "Binary size", "The linux-arm64 binary that ships.", mb, |
| func(r b.AppResult) (float64, bool) { return pos(float64(r.Build.Bytes)) }), |
diff --git a/scripts/fresh-instance.sh b/scripts/fresh-instance.sh| new file mode 100755 |
| index 0000000..fcc10e2 |
| --- /dev/null |
| +++ b/scripts/fresh-instance.sh |
| @@ -0,0 +1,24 @@ |
| +#!/usr/bin/env bash |
| +# Gives one app an empty database before it is measured. Deleting an |
| +# instance parks its data, and the hostname can't be reused until an |
| +# operator purges it, so this runs in two steps: |
| +# scripts/fresh-instance.sh delete <slug> then an operator purges the host |
| +# scripts/fresh-instance.sh create <slug> recreate, wait for the recorded version |
| +# Bakeoff test data only. Paul's call, 2026-10-24. |
| +set -euo pipefail |
| +step=$1 slug=$2 |
| +cd "$(dirname "$0")/.." |
| +app=todo-$slug host=todo-$slug.bfj.oncarlos.com |
| +want=$(python3 -c "import json; print(json.load(open('results/$slug.json'))['Version'])") |
| +if [ "$step" = delete ]; then |
| + carlos instances delete --account bfj --app "$app" --host "$host" --message "Empty the database before measuring (bakeoff)" |
| + exit 0 |
| +fi |
| +carlos instances create --account bfj --app "$app" --host "$host" --channel edge --message "Recreate with an empty database (bakeoff)" |
| +for _ in $(seq 1 60); do |
| + got=$(curl -s --max-time 10 "https://$host/api/version" || true) |
| + [ "$got" = "$want" ] && { echo "$slug: fresh instance serving $got"; carlos instances list --account bfj --app "$app" | tail -1; exit 0; } |
| + sleep 5 |
| +done |
| +echo "$slug: fresh instance not serving $want after 5 minutes (last: $got)" >&2 |
| +exit 1 |