amadan

carlos / todomvc-bakeoff Public

feat: local memory measurement (idle, peak under load, after)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Paul Campbell pushed by claude 5f0b836d9cbb1cea3abffc542639735d327e0234 parent d4fc4ac
6 files changed, +206 −0
  • copy-review/strings.json +32 −0
  • harness/cmd/bakeoff/main.go +17 −0
  • harness/harness_test.go +29 −0
  • harness/memory.go +124 −0
  • harness/report/report.go +3 −0
  • harness/results.go +1 −0
diff --git a/copy-review/strings.json b/copy-review/strings.json
index 3b19264..2c96fee 100644
--- a/copy-review/strings.json
+++ b/copy-review/strings.json
@@ -253,6 +253,38 @@
"context": "The line under the chart title that says exactly what was measured.",
"notes": "One sentence or two."
},
+ {
+ "id": "chart.memory_idle.title",
+ "section": "Charts",
+ "label": "Memory at rest: title",
+ "text": "Memory at rest",
+ "context": "Title of the chart showing how much memory each app uses when it has just started and nobody is using it.",
+ "notes": "Short."
+ },
+ {
+ "id": "chart.memory_idle.note",
+ "section": "Charts",
+ "label": "Memory at rest: note",
+ "text": "Resident memory just after start, measured on our own machine because Carloku doesn't show members memory figures.",
+ "context": "The line under that chart saying exactly what was measured, and why it was measured locally.",
+ "notes": "One sentence or two."
+ },
+ {
+ "id": "chart.memory_peak.title",
+ "section": "Charts",
+ "label": "Memory under load: title",
+ "text": "Memory under load",
+ "context": "Title of the chart showing the most memory each app used while being load-tested.",
+ "notes": "Short."
+ },
+ {
+ "id": "chart.memory_peak.note",
+ "section": "Charts",
+ "label": "Memory under load: note",
+ "text": "The most resident memory each app used during 10 seconds of 64 simultaneous page loads and then 10 seconds of 64 simultaneous toggles, on our own machine.",
+ "context": "The line under that chart saying exactly what was measured.",
+ "notes": "One sentence or two."
+ },
{
"id": "chart.binary.title",
"section": "Charts",
diff --git a/harness/cmd/bakeoff/main.go b/harness/cmd/bakeoff/main.go
index 354cd70..f33b0c8 100644
--- a/harness/cmd/bakeoff/main.go
+++ b/harness/cmd/bakeoff/main.go
@@ -90,6 +90,8 @@ func main() {
err = findings(*slug, *file)
case "probe":
err = probe(*url, *levels, *dur)
+ case "memory":
+ err = memory(*slug, *dir, *dur)
case "record":
if rec.URL = *recURL; rec.URL == "" && strings.HasPrefix(rec.Status, "LIVE") {
rec.URL = *url
@@ -446,3 +448,18 @@ func probe(url, levels string, d time.Duration) error {
}
return nil
}
+
+// memory measures the app's own host build (dist/app, from make build)
+// locally: resident memory at rest, peak under load, and after.
+func memory(slug, dir string, d time.Duration) error {
+ bin, err := filepath.Abs(filepath.Join(dir, "dist", "app"))
+ if err != nil {
+ return err
+ }
+ m, err := b.MeasureMemory(bin, nil, d)
+ if err != nil {
+ return err
+ }
+ fmt.Printf("%s memory: idle %d KB, peak %d KB, after %d KB\n", slug, m.IdleKB, m.PeakKB, m.AfterKB)
+ return b.Update(slug, func(r *b.AppResult) { r.Memory = m })
+}
diff --git a/harness/harness_test.go b/harness/harness_test.go
index 08c74d2..a9f2f7a 100644
--- a/harness/harness_test.go
+++ b/harness/harness_test.go
@@ -360,3 +360,32 @@ func TestAsleepFromLogs(t *testing.T) {
t.Fatal("no events: unknown, so not asleep")
}
}
+
+func TestParseProcStatus(t *testing.T) {
+ status := "Name:\tapp\nVmHWM:\t 20480 kB\nVmRSS:\t 12288 kB\nThreads:\t4\n"
+ rss, hwm, err := ParseProcStatus(status)
+ if err != nil || rss != 12288 || hwm != 20480 {
+ t.Fatalf("got rss %d hwm %d err %v", rss, hwm, err)
+ }
+}
+
+func TestMeasureMemoryOfRefapp(t *testing.T) {
+ if testing.Short() {
+ t.Skip("builds and loads the reference app")
+ }
+ if _, err := exec.LookPath("oha"); err != nil {
+ t.Skip("oha not installed")
+ }
+ bin := filepath.Join(t.TempDir(), "refapp")
+ if out, err := exec.Command("go", "build", "-o", bin, "./refapp").CombinedOutput(); err != nil {
+ t.Fatalf("%v\n%s", err, out)
+ }
+ shared, _ := filepath.Abs("../shared")
+ m, err := MeasureMemory(bin, []string{"--shared", shared}, time.Second)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if m.IdleKB <= 0 || m.PeakKB < m.IdleKB || m.AfterKB <= 0 {
+ t.Fatalf("implausible %+v", m)
+ }
+}
diff --git a/harness/memory.go b/harness/memory.go
new file mode 100644
index 0000000..e360b19
--- /dev/null
+++ b/harness/memory.go
@@ -0,0 +1,124 @@
+package bakeoff
+
+import (
+ "fmt"
+ "net"
+ "net/http"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "syscall"
+ "time"
+)
+
+// Memory is an app's resident memory, from Linux's /proc, in kilobytes:
+// just after start, the peak over a load run, and a few seconds after.
+type Memory struct{ IdleKB, PeakKB, AfterKB int }
+
+// ParseProcStatus reads VmRSS (resident now) and VmHWM (peak resident)
+// from /proc/<pid>/status.
+func ParseProcStatus(status string) (rss, hwm int, err error) {
+ for _, line := range strings.Split(status, "\n") {
+ k, v, ok := strings.Cut(line, ":")
+ if !ok {
+ continue
+ }
+ f := strings.Fields(v)
+ if len(f) == 0 {
+ continue
+ }
+ n, _ := strconv.Atoi(f[0])
+ switch k {
+ case "VmRSS":
+ rss = n
+ case "VmHWM":
+ hwm = n
+ }
+ }
+ if rss == 0 {
+ return 0, 0, fmt.Errorf("no VmRSS in status")
+ }
+ return rss, hwm, nil
+}
+
+func procStatus(pid int) (int, int, error) {
+ b, err := os.ReadFile(fmt.Sprintf("/proc/%d/status", pid))
+ if err != nil {
+ return 0, 0, err
+ }
+ return ParseProcStatus(string(b))
+}
+
+// MeasureMemory runs the app binary locally (on --listen, with a fresh
+// database), reads its memory at rest, drives it with 64 concurrent reads
+// and then 64 concurrent toggles for d each, and reads it again. The
+// working directory is the app's own (the binary's parent's parent, since
+// builds land in dist/).
+func MeasureMemory(bin string, extra []string, d time.Duration) (Memory, error) {
+ ln, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ return Memory{}, err
+ }
+ addr := ln.Addr().String()
+ ln.Close()
+ tmp, err := os.MkdirTemp(os.Getenv("TMPDIR"), "bakeoff-mem-")
+ if err != nil {
+ return Memory{}, err
+ }
+ defer os.RemoveAll(tmp)
+ cmd := exec.Command(bin, append(extra, "--listen", addr, "--db", filepath.Join(tmp, "app.db"))...)
+ cmd.Dir = filepath.Dir(filepath.Dir(bin))
+ cmd.Stdout, cmd.Stderr = nil, nil
+ if err := cmd.Start(); err != nil {
+ return Memory{}, err
+ }
+ defer func() {
+ cmd.Process.Signal(syscall.SIGTERM)
+ done := make(chan error, 1)
+ go func() { done <- cmd.Wait() }()
+ select {
+ case <-done:
+ case <-time.After(10 * time.Second):
+ cmd.Process.Kill()
+ }
+ }()
+ base := "http://" + addr
+ up := false
+ for i := 0; i < 150 && !up; i++ {
+ if res, err := http.Get(base + "/healthz"); err == nil {
+ res.Body.Close()
+ up = res.StatusCode == 200
+ }
+ if !up {
+ time.Sleep(200 * time.Millisecond)
+ }
+ }
+ if !up {
+ return Memory{}, fmt.Errorf("%s did not answer /healthz on %s", bin, addr)
+ }
+ time.Sleep(2 * time.Second)
+ var m Memory
+ if m.IdleKB, _, err = procStatus(cmd.Process.Pid); err != nil {
+ return m, err
+ }
+ s, err := NewSession(base)
+ if err != nil {
+ return m, err
+ }
+ if _, err := Seed(s, 50); err != nil {
+ return m, err
+ }
+ for _, sc := range []string{"read", "toggle"} {
+ if _, err := RunLoad(s, sc, 64, d); err != nil {
+ return m, err
+ }
+ }
+ if _, m.PeakKB, err = procStatus(cmd.Process.Pid); err != nil {
+ return m, err
+ }
+ time.Sleep(5 * time.Second)
+ m.AfterKB, _, err = procStatus(cmd.Process.Pid)
+ return m, err
+}
diff --git a/harness/report/report.go b/harness/report/report.go
index a48a505..78f25fd 100644
--- a/harness/report/report.go
+++ b/harness/report/report.go
@@ -44,6 +44,7 @@ func ms(v float64) string {
}
func mb(v float64) string { return fmt.Sprintf("%.1f MB", v/1e6) }
func usd(v float64) string { return fmt.Sprintf("$%.2f", v) }
+func kb(v float64) string { return fmt.Sprintf("%.1f MB", v/1024) }
func secs(v float64) string { return fmt.Sprintf("%.1f s", v) }
func num(v float64) string {
s := fmt.Sprintf("%.0f", v)
@@ -139,6 +140,8 @@ func charts(rs []b.AppResult, c Copy) []Chart {
chart(rs, c, "toggles_peak", num, func(r b.AppResult) (float64, bool) { return pos(knee(r, "toggle").PeakRPS) }),
kneeChart(rs, c, "toggle", "toggles_knee"),
chart(rs, c, "reads_after", num, func(r b.AppResult) (float64, bool) { return pos(knee(r, "read-after").PeakRPS) }),
+ chart(rs, c, "memory_idle", kb, func(r b.AppResult) (float64, bool) { return pos(float64(r.Memory.IdleKB)) }),
+ chart(rs, c, "memory_peak", kb, func(r b.AppResult) (float64, bool) { return pos(float64(r.Memory.PeakKB)) }),
chart(rs, c, "binary", mb, func(r b.AppResult) (float64, bool) { return pos(float64(r.Build.Bytes)) }),
chart(rs, c, "build_cold", secs, func(r b.AppResult) (float64, bool) { return pos(r.Build.ColdSeconds) }),
chart(rs, c, "build_warm", secs, func(r b.AppResult) (float64, bool) { return pos(r.Build.WarmSeconds) }),
diff --git a/harness/results.go b/harness/results.go
index 0d12ee8..91a5aaf 100644
--- a/harness/results.go
+++ b/harness/results.go
@@ -30,6 +30,7 @@ type AppResult struct {
Wake []float64
Load []LoadResult
Weight Weight
+ Memory Memory
Code CodeStats
Build Build
TokensBuild, TokensFix, TokensSecurity Tokens