amadan

carlos / todomvc-bakeoff Public

fix: decide sleep from the platform's own events; recollect wake samples

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Paul Campbell pushed by claude ceb74f431fdb6eb46e0c9164a7ab30c6372e0860 parent eac65d6
12 files changed, +78 −55
  • harness/cmd/bakeoff/main.go +5 −0
  • harness/harness_test.go +16 −0
  • harness/wake.go +23 −3
  • results/go-rastrillo.json +4 −6
  • results/go.json +1 −3
  • results/haskell.json +1 −3
  • results/nim.json +1 −3
  • results/ocaml.json +1 −3
  • results/rust.json +8 −10
  • results/spinel.json +1 −3
  • results/swift.json +5 −7
  • results/zig.json +12 −14
diff --git a/harness/cmd/bakeoff/main.go b/harness/cmd/bakeoff/main.go
index 1e6ad00..354cd70 100644
--- a/harness/cmd/bakeoff/main.go
+++ b/harness/cmd/bakeoff/main.go
@@ -49,6 +49,7 @@ func main() {
fs.StringVar(&rec.Note, "note", "", "record: a note shown in the app's section (added once)")
recURL := fs.String("app-url", "", "record: live URL")
clearNotes := fs.Bool("clear-notes", false, "record: drop existing notes first")
+ clearWake := fs.Bool("clear", false, "wake: drop recorded samples")
replies := fs.String("replies", "", "readability: comma-separated files, one scorer reply each")
fs.Parse(os.Args[2:])
if *slug == "" && cmd != "report" {
@@ -66,6 +67,10 @@ func main() {
case "ttfb":
err = ttfb(*slug, *url)
case "wake":
+ if *clearWake {
+ err = b.Update(*slug, func(r *b.AppResult) { r.Wake = nil })
+ break
+ }
err = wake(*slug, *url)
case "load":
err = load(*slug, *url, *levels, *dur, *rest)
diff --git a/harness/harness_test.go b/harness/harness_test.go
index 5ee90d4..08c74d2 100644
--- a/harness/harness_test.go
+++ b/harness/harness_test.go
@@ -344,3 +344,19 @@ func TestApplyRecordAppendsNoteOnce(t *testing.T) {
t.Fatalf("got %q", r.Notes)
}
}
+
+func TestAsleepFromLogs(t *testing.T) {
+ logs := `2026-10-24 09:50:52 [platform] todo-zig-2.bfj.oncarlos.com instance stopping
+2026-10-24 09:50:53 [platform] todo-zig-2.bfj.oncarlos.com instance exited (exit 0)
+2026-10-24 10:01:00 [platform] todo-zig.bfj.oncarlos.com instance woke in 1.1s (spawn 1s) (version x)`
+ if !AsleepFromLogs(logs, "todo-zig-2.bfj.oncarlos.com") {
+ t.Fatal("last event for the host is an exit: asleep")
+ }
+ logs += "\n2026-10-24 10:36:41 [platform] todo-zig-2.bfj.oncarlos.com instance woke in 1.156s (spawn 1.037s) (version 73268e0)"
+ if AsleepFromLogs(logs, "todo-zig-2.bfj.oncarlos.com") {
+ t.Fatal("last event is a wake: awake")
+ }
+ if AsleepFromLogs("", "todo-zig-2.bfj.oncarlos.com") {
+ t.Fatal("no events: unknown, so not asleep")
+ }
+}
diff --git a/harness/wake.go b/harness/wake.go
index 5727373..72f70db 100644
--- a/harness/wake.go
+++ b/harness/wake.go
@@ -5,13 +5,33 @@ import (
"strings"
)
-// IsAsleep asks the platform whether the instance at host is hibernating.
+// AsleepFromLogs reads the platform's own events for host in `carlos logs`
+// output: the instance is asleep when its latest event is "stopping" or
+// "exited", and awake after "woke". (The health column in `carlos
+// instances list` lags by minutes, so it can't be trusted for this.)
+func AsleepFromLogs(logs, host string) bool {
+ asleep := false
+ for _, line := range strings.Split(logs, "\n") {
+ if !strings.Contains(line, "[platform] "+host+" ") {
+ continue
+ }
+ switch {
+ case strings.Contains(line, "instance woke"):
+ asleep = false
+ case strings.Contains(line, "instance stopping"), strings.Contains(line, "instance exited"):
+ asleep = true
+ }
+ }
+ return asleep
+}
+
+// IsAsleep asks the platform's log whether the instance at host is asleep.
func IsAsleep(app, host string) (bool, error) {
- out, err := exec.Command("carlos", "instances", "list", "--account", "bfj", "--app", app, "--health", "asleep").CombinedOutput()
+ out, err := exec.Command("carlos", "logs", "--account", "bfj", "--app", app).CombinedOutput()
if err != nil {
return false, err
}
- return strings.Contains(string(out), host), nil
+ return AsleepFromLogs(string(out), host), nil
}
// MeasureWake times the first request to a sleeping instance.
diff --git a/results/go-rastrillo.json b/results/go-rastrillo.json
index fe2baa3..ddb4176 100644
--- a/results/go-rastrillo.json
+++ b/results/go-rastrillo.json
@@ -32,9 +32,7 @@
"Min": 48.03,
"Max": 64.063
},
- "Wake": [
- 1435.117
- ],
+ "Wake": null,
"Load": [
{
"Scenario": "read",
@@ -301,9 +299,9 @@
"Deps": 3
},
"Build": {
- "ColdSeconds": 11.408177673,
- "WarmSeconds": 0.28805728,
- "EditSeconds": 0.285644601,
+ "ColdSeconds": 10.390341005,
+ "WarmSeconds": 0.272297767,
+ "EditSeconds": 0.139181192,
"Bytes": 14090402,
"Host": "DOCS-T1",
"ColdRuns": 1,
diff --git a/results/go.json b/results/go.json
index d1b0c14..496e736 100644
--- a/results/go.json
+++ b/results/go.json
@@ -32,9 +32,7 @@
"Min": 86.404,
"Max": 108.968
},
- "Wake": [
- 84.556
- ],
+ "Wake": null,
"Load": [
{
"Scenario": "read",
diff --git a/results/haskell.json b/results/haskell.json
index 739efe7..9e6b385 100644
--- a/results/haskell.json
+++ b/results/haskell.json
@@ -32,9 +32,7 @@
"Min": 89.266,
"Max": 168.219
},
- "Wake": [
- 1373.548
- ],
+ "Wake": null,
"Load": [
{
"Scenario": "read",
diff --git a/results/nim.json b/results/nim.json
index 3a1c0d0..eb3c94e 100644
--- a/results/nim.json
+++ b/results/nim.json
@@ -32,9 +32,7 @@
"Min": 42.03,
"Max": 55.361
},
- "Wake": [
- 1244.605
- ],
+ "Wake": null,
"Load": [
{
"Scenario": "read",
diff --git a/results/ocaml.json b/results/ocaml.json
index dd5915d..ab1d39e 100644
--- a/results/ocaml.json
+++ b/results/ocaml.json
@@ -32,9 +32,7 @@
"Min": 41.961,
"Max": 125.31
},
- "Wake": [
- 1352.486
- ],
+ "Wake": null,
"Load": [
{
"Scenario": "read",
diff --git a/results/rust.json b/results/rust.json
index de783ef..e5f8ce7 100644
--- a/results/rust.json
+++ b/results/rust.json
@@ -32,9 +32,7 @@
"Min": 79.697,
"Max": 102.563
},
- "Wake": [
- 1156.261
- ],
+ "Wake": null,
"Load": [
{
"Scenario": "read",
@@ -363,13 +361,13 @@
"Deps": 7
},
"Build": {
- "ColdSeconds": 0,
- "WarmSeconds": 0,
- "EditSeconds": 0,
- "Bytes": 0,
- "Host": "",
- "ColdRuns": 0,
- "Runs": 0
+ "ColdSeconds": 49.623494074,
+ "WarmSeconds": 8.477199217,
+ "EditSeconds": 5.752862848,
+ "Bytes": 3155304,
+ "Host": "DOCS-T1",
+ "ColdRuns": 1,
+ "Runs": 3
},
"TokensBuild": {
"Input": 94,
diff --git a/results/spinel.json b/results/spinel.json
index b507d7b..bc9ad00 100644
--- a/results/spinel.json
+++ b/results/spinel.json
@@ -32,9 +32,7 @@
"Min": 82.851,
"Max": 108.763
},
- "Wake": [
- 1300.982
- ],
+ "Wake": null,
"Load": [
{
"Scenario": "read",
diff --git a/results/swift.json b/results/swift.json
index d64613b..12063c7 100644
--- a/results/swift.json
+++ b/results/swift.json
@@ -32,9 +32,7 @@
"Min": 47.876,
"Max": 63.425
},
- "Wake": [
- 1363.467
- ],
+ "Wake": null,
"Load": [
{
"Scenario": "read",
@@ -367,11 +365,11 @@
"JSBytes": 3110
},
"Code": {
- "Server": 0,
+ "Server": 1251,
"Templates": 0,
- "JS": 0,
- "Tests": 0,
- "Deps": 0
+ "JS": 75,
+ "Tests": 309,
+ "Deps": 3
},
"Build": {
"ColdSeconds": 0,
diff --git a/results/zig.json b/results/zig.json
index 7cc3819..34fdc4a 100644
--- a/results/zig.json
+++ b/results/zig.json
@@ -32,9 +32,7 @@
"Min": 48.154,
"Max": 65.122
},
- "Wake": [
- 1239.057
- ],
+ "Wake": null,
"Load": [
{
"Scenario": "read",
@@ -342,20 +340,20 @@
"JSBytes": 3854
},
"Code": {
- "Server": 0,
+ "Server": 964,
"Templates": 0,
- "JS": 0,
- "Tests": 0,
- "Deps": 0
+ "JS": 94,
+ "Tests": 59,
+ "Deps": 1
},
"Build": {
- "ColdSeconds": 0,
- "WarmSeconds": 0,
- "EditSeconds": 0,
- "Bytes": 0,
- "Host": "",
- "ColdRuns": 0,
- "Runs": 0
+ "ColdSeconds": 151.72579616,
+ "WarmSeconds": 5.378776474,
+ "EditSeconds": 5.416322927,
+ "Bytes": 1918040,
+ "Host": "DOCS-T1",
+ "ColdRuns": 1,
+ "Runs": 3
},
"TokensBuild": {
"Input": 172,