run: a follow gives up on a run the record let go; DeleteRun keeps result.json until the output is gone (final review I1)
DeleteRun deletes result.json in its own tier after in/ and out/ and before request.json, so a delete that failed part way still answers a follower with how the run ended. followRun treated a page with no state and no result (the record no longer names the run, request.json still there) as nothing to act on and polled until the orphan sweep deleted the run, up to 24 hours. It now asks for runPageGiveUp, since a result may still arrive, then prints RUN-C86 and exits 125. RUN-C86 is added to the deck (draft text under copy review).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
6 files changed,
+124
−16
approved_run_copy_test.go+5 −5docs/superpowers/plans/2026-10-08-carlos-run-copy.md+1 −0internal/runs/runs_test.go+29 −0internal/runs/store.go+12 −8run.go+23 −3run_test.go+54 −0
diff --git a/approved_run_copy_test.go b/approved_run_copy_test.go| index fd238e849..b9921052f 100644 |
| --- a/approved_run_copy_test.go |
| +++ b/approved_run_copy_test.go |
| @@ -15,17 +15,17 @@ import ( |
| // parser and checked with its helpers, so "applied" means the same thing in |
| // both gates: |
| // |
| -// - TestRunCopyRecordIsComplete: the deck names exactly RUN-C1 to RUN-C85; |
| +// - TestRunCopyRecordIsComplete: the deck names exactly RUN-C1 to RUN-C86; |
| // - TestRunBuiltCopyIsApproved: an id in runCopyBuilt is an approved row |
| // (a string reaches code only after Paul has approved it); |
| // - TestRunBuiltCopyIsApplied: a built row's text is in the file it names; |
| // - TestRunCopyHasNoEmDash. |
| const runCopyRecordPath = "docs/superpowers/plans/2026-10-08-carlos-run-copy.md" |
| -// runCopyIDs is the deck's inventory: RUN-C1 to RUN-C85, pinned. |
| +// runCopyIDs is the deck's inventory: RUN-C1 to RUN-C86, pinned. |
| var runCopyIDs = func() []string { |
| - ids := make([]string, 0, 85) |
| - for i := 1; i <= 85; i++ { |
| + ids := make([]string, 0, 86) |
| + for i := 1; i <= 86; i++ { |
| ids = append(ids, "RUN-C"+strconv.Itoa(i)) |
| } |
| return ids |
| @@ -33,7 +33,7 @@ var runCopyIDs = func() []string { |
| // runCopyBuilt is every deck id whose text the code or docs now carry. The |
| // task that writes a row's text adds its id here in the same commit. |
| -var runCopyBuilt = []string{"RUN-C1", "RUN-C2", "RUN-C3", "RUN-C4", "RUN-C5", "RUN-C6", "RUN-C7", "RUN-C8", "RUN-C9", "RUN-C10", "RUN-C11", "RUN-C12", "RUN-C13", "RUN-C14", "RUN-C15", "RUN-C16", "RUN-C17", "RUN-C18", "RUN-C19", "RUN-C20", "RUN-C21", "RUN-C22", "RUN-C23", "RUN-C24", "RUN-C25", "RUN-C26", "RUN-C27", "RUN-C28", "RUN-C29", "RUN-C30", "RUN-C31", "RUN-C32", "RUN-C33", "RUN-C34", "RUN-C35", "RUN-C36", "RUN-C37", "RUN-C38", "RUN-C39", "RUN-C40", "RUN-C41", "RUN-C42", "RUN-C43", "RUN-C44", "RUN-C45", "RUN-C46", "RUN-C47", "RUN-C48", "RUN-C49", "RUN-C50", "RUN-C51", "RUN-C52", "RUN-C53", "RUN-C54", "RUN-C55", "RUN-C56", "RUN-C57", "RUN-C58", "RUN-C59", "RUN-C60", "RUN-C61", "RUN-C62", "RUN-C63", "RUN-C64", "RUN-C65", "RUN-C66", "RUN-C67", "RUN-C68", "RUN-C69", "RUN-C70", "RUN-C71", "RUN-C72", "RUN-C73", "RUN-C74", "RUN-C75", "RUN-C76", "RUN-C77", "RUN-C78", "RUN-C79", "RUN-C80", "RUN-C81", "RUN-C82", "RUN-C83", "RUN-C84", "RUN-C85"} |
| +var runCopyBuilt = []string{"RUN-C1", "RUN-C2", "RUN-C3", "RUN-C4", "RUN-C5", "RUN-C6", "RUN-C7", "RUN-C8", "RUN-C9", "RUN-C10", "RUN-C11", "RUN-C12", "RUN-C13", "RUN-C14", "RUN-C15", "RUN-C16", "RUN-C17", "RUN-C18", "RUN-C19", "RUN-C20", "RUN-C21", "RUN-C22", "RUN-C23", "RUN-C24", "RUN-C25", "RUN-C26", "RUN-C27", "RUN-C28", "RUN-C29", "RUN-C30", "RUN-C31", "RUN-C32", "RUN-C33", "RUN-C34", "RUN-C35", "RUN-C36", "RUN-C37", "RUN-C38", "RUN-C39", "RUN-C40", "RUN-C41", "RUN-C42", "RUN-C43", "RUN-C44", "RUN-C45", "RUN-C46", "RUN-C47", "RUN-C48", "RUN-C49", "RUN-C50", "RUN-C51", "RUN-C52", "RUN-C53", "RUN-C54", "RUN-C55", "RUN-C56", "RUN-C57", "RUN-C58", "RUN-C59", "RUN-C60", "RUN-C61", "RUN-C62", "RUN-C63", "RUN-C64", "RUN-C65", "RUN-C66", "RUN-C67", "RUN-C68", "RUN-C69", "RUN-C70", "RUN-C71", "RUN-C72", "RUN-C73", "RUN-C74", "RUN-C75", "RUN-C76", "RUN-C77", "RUN-C78", "RUN-C79", "RUN-C80", "RUN-C81", "RUN-C82", "RUN-C83", "RUN-C84", "RUN-C85", "RUN-C86"} |
| func mustParseRunRecord(t *testing.T) []copyRow { |
| t.Helper() |
diff --git a/docs/superpowers/plans/2026-10-08-carlos-run-copy.md b/docs/superpowers/plans/2026-10-08-carlos-run-copy.md| index ea3915559..3896f8d03 100644 |
| --- a/docs/superpowers/plans/2026-10-08-carlos-run-copy.md |
| +++ b/docs/superpowers/plans/2026-10-08-carlos-run-copy.md |
| @@ -131,3 +131,4 @@ error; `<url>` a console URL; `<role>` a quoted role word. |
| | RUN-C81 | docs/site/cli.md | Its arguments are exactly what followed --, after your binary. The platform adds nothing. It runs in the instance's data directory. On a cancel, a timeout or an instance stop it gets SIGTERM, then SIGKILL 20 seconds later. | approved | |
| | RUN-C84 | run.go | not a regular file | approved | |
| | RUN-C85 | run.go | the file changed while carlos was reading it | approved | |
| +| RUN-C86 | run.go | <host>: the instance no longer holds run <id>, and no result came back. it may have done part of its work | approved | |
diff --git a/internal/runs/runs_test.go b/internal/runs/runs_test.go| index 4b88a0c10..4f76a1a88 100644 |
| --- a/internal/runs/runs_test.go |
| +++ b/internal/runs/runs_test.go |
| @@ -285,6 +285,35 @@ func TestDeleteRunKeepsTheMarkerAndRequestUntilTheRestIsGone(t *testing.T) { |
| } |
| } |
| +// A chunk that will not delete stops DeleteRun before result.json, so a |
| +// follower of a half-deleted run still reads how it ended. Then the result |
| +// goes before request.json. |
| +func TestDeleteRunKeepsTheResultUntilTheOutputIsGone(t *testing.T) { |
| + ctx := context.Background() |
| + inner := release.NewDirStore(t.TempDir()) |
| + id, _ := MintID() |
| + chunk := ChunkKey("bab", "shop", id, 1, StreamOut) |
| + for _, k := range []string{AuditedKey("bab", "shop", id), RequestKey("bab", "shop", id), ResultKey("bab", "shop", id), chunk, InputKey("bab", "shop", id, "data.csv")} { |
| + if _, err := inner.Put(ctx, k, []byte("{}"), ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + s := &failDelete{Store: inner, fail: chunk} |
| + if err := DeleteRun(ctx, s, "bab", "shop", id); err == nil { |
| + t.Fatal("a failed delete must be returned") |
| + } |
| + if r, err := ReadResult(ctx, inner, "bab", "shop", id); r == nil || err != nil { |
| + t.Fatalf("result.json went before the output: %v, %v", r, err) |
| + } |
| + s.fail = RequestKey("bab", "shop", id) |
| + if err := DeleteRun(ctx, s, "bab", "shop", id); err == nil { |
| + t.Fatal("a failed delete must be returned") |
| + } |
| + if r, _ := ReadResult(ctx, inner, "bab", "shop", id); r != nil { |
| + t.Fatal("result.json must go before request.json") |
| + } |
| +} |
| + |
| type failDelete struct { |
| release.Store |
| fail string |
diff --git a/internal/runs/store.go b/internal/runs/store.go| index 2911bfa58..45429fdb9 100644 |
| --- a/internal/runs/store.go |
| +++ b/internal/runs/store.go |
| @@ -127,11 +127,13 @@ func LastChunkSeq(ctx context.Context, s release.Store, acct, app, id string) (i |
| // DeleteRun deletes every object of the run, in an order that keeps a |
| // half-deleted run findable and audited once. The input and output objects |
| -// and result.json go first; request.json, which the orphan sweep ages a run |
| -// by and which says whose run it was, goes only after they are all gone; the |
| -// audited marker goes last, so a run that still has any object still has its |
| -// marker, and nothing can write the run-end row a second time. A tier that |
| -// fails stops the later ones, and the call can be repeated. |
| +// go first; result.json goes only after they are all gone, so a run whose |
| +// delete failed part way still answers a follower with how it ended; |
| +// request.json, which the orphan sweep ages a run by and which says whose |
| +// run it was, goes after the result; the audited marker goes last, so a run |
| +// that still has any object still has its marker, and nothing can write the |
| +// run-end row a second time. A tier that fails stops the later ones, and the |
| +// call can be repeated. |
| func DeleteRun(ctx context.Context, s release.Store, acct, app, id string) error { |
| if err := checkRun(acct, app, id); err != nil { |
| return err |
| @@ -140,13 +142,15 @@ func DeleteRun(ctx context.Context, s release.Store, acct, app, id string) error |
| if err != nil { |
| return err |
| } |
| - request, audited := RequestKey(acct, app, id), AuditedKey(acct, app, id) |
| - tiers := [3][]string{} |
| + result, request, audited := ResultKey(acct, app, id), RequestKey(acct, app, id), AuditedKey(acct, app, id) |
| + tiers := [4][]string{} |
| for _, k := range keys { |
| switch k { |
| case audited: |
| - tiers[2] = append(tiers[2], k) |
| + tiers[3] = append(tiers[3], k) |
| case request: |
| + tiers[2] = append(tiers[2], k) |
| + case result: |
| tiers[1] = append(tiers[1], k) |
| default: |
| tiers[0] = append(tiers[0], k) |
diff --git a/run.go b/run.go| index 55ce87994..f8140c777 100644 |
| --- a/run.go |
| +++ b/run.go |
| @@ -338,6 +338,12 @@ func runNotPickedUp(host string) string { |
| return fmt.Sprintf("%s: not picked up within 60 seconds. the run did not start. try again", host) |
| } |
| +// runLetGo is RUN-C86: the record no longer names the run and no result |
| +// came back for it within runPageGiveUp. |
| +func runLetGo(host, id string) string { |
| + return fmt.Sprintf("%s: the instance no longer holds run %s, and no result came back. it may have done part of its work", host, id) |
| +} |
| + |
| // runConflict reports whether err is the console's 409 with exactly msg. |
| func runConflict(err error, msg string) bool { |
| var ce *runConsoleError |
| @@ -383,13 +389,16 @@ func ackRun(ctx context.Context, c runClient, t runTerm, id string, lastSeq int6 |
| // cut short is followed at once; the result is in hand only when a page |
| // carries it and is not cut short, because the console reads the result |
| // before it lists chunks. A run still queued at expireAt is expired (§5.2); |
| -// zero never expires. The first Ctrl-C cancels the run on the box and keeps |
| -// following; the second stops waiting (§5.7). |
| +// zero never expires. A page with no state and no result is a run the |
| +// record no longer names (the instance was deleted, or the run was dropped): |
| +// a result may still arrive, so it is asked for again for runPageGiveUp, and |
| +// then the follow ends with RUN-C86 and 125. The first Ctrl-C cancels the |
| +// run on the box and keeps following; the second stops waiting (§5.7). |
| func followRun(ctx context.Context, c runClient, t runTerm, host, id string, expireAt time.Time) int { |
| var after int64 |
| watch := &runWatch{ch: t.interrupts} |
| cancel := runCancel{w: watch} |
| - var failingSince time.Time |
| + var failingSince, letGoSince time.Time |
| for { |
| var p runs.Page |
| var err error |
| @@ -457,6 +466,17 @@ func followRun(ctx context.Context, c runClient, t runTerm, host, id string, exp |
| } |
| return code |
| } |
| + if p.State != "" { |
| + letGoSince = time.Time{} |
| + } else { |
| + if letGoSince.IsZero() { |
| + letGoSince = t.now() |
| + } |
| + if t.now().Sub(letGoSince) >= runPageGiveUp { |
| + t.say(runLetGo(host, id)) |
| + return 125 |
| + } |
| + } |
| switch p.State { |
| case provision.RunExpired: |
| t.say(runNotPickedUp(host)) |
diff --git a/run_test.go b/run_test.go| index e9976b795..be5532f5c 100644 |
| --- a/run_test.go |
| +++ b/run_test.go |
| @@ -587,6 +587,60 @@ func TestFollowRunStatesWithoutAResult(t *testing.T) { |
| } |
| } |
| +// A page with no state and no result is a run the record no longer names. |
| +// A result may still arrive, so the loop asks for a while, then gives up |
| +// with RUN-C86 instead of polling until the sweep deletes the run. |
| +func TestFollowRunGivesUpOnARunTheRecordLetGo(t *testing.T) { |
| + t.Run("no result comes", func(t *testing.T) { |
| + f := newTermFx() |
| + start := f.now |
| + s := &scriptedRun{pages: []runs.Page{{Chunks: []runs.Chunk{chunk(1, "o", "part\n")}}, {}}} |
| + if code := followRun(context.Background(), s, f.term(), "shop.example.com", "run_a", time.Time{}); code != 125 { |
| + t.Fatalf("exit %d stderr %q", code, f.err.String()) |
| + } |
| + if got := f.err.String(); got != "shop.example.com: the instance no longer holds run run_a, and no result came back. it may have done part of its work\n" { |
| + t.Fatalf("stderr %q", got) |
| + } |
| + if waited := f.now.Sub(start); waited < runPageGiveUp || waited > runPageGiveUp+2*time.Second { |
| + t.Fatalf("gave up after %s, want %s", waited, runPageGiveUp) |
| + } |
| + if f.out.String() != "part\n" || len(s.acked) != 0 { |
| + t.Fatalf("stdout %q acked %v", f.out.String(), s.acked) |
| + } |
| + }) |
| + t.Run("a result arrives in time", func(t *testing.T) { |
| + f := newTermFx() |
| + pages := make([]runs.Page, 0, 91) |
| + for range 90 { |
| + pages = append(pages, runs.Page{}) |
| + } |
| + pages = append(pages, runs.Page{Result: &runs.Result{Interrupted: runs.InterruptedLost}}) |
| + s := &scriptedRun{pages: pages} |
| + if code := followRun(context.Background(), s, f.term(), "shop.example.com", "run_a", time.Time{}); code != 125 || !strings.Contains(f.err.String(), "its box stopped reporting") { |
| + t.Fatalf("exit %d stderr %q", code, f.err.String()) |
| + } |
| + if len(s.acked) != 1 { |
| + t.Fatalf("acked %v", s.acked) |
| + } |
| + }) |
| + t.Run("a state in between starts the wait again", func(t *testing.T) { |
| + f := newTermFx() |
| + var pages []runs.Page |
| + for range 100 { |
| + pages = append(pages, runs.Page{}) |
| + } |
| + pages = append(pages, runs.Page{State: "picked"}) |
| + for range 100 { |
| + pages = append(pages, runs.Page{}) |
| + } |
| + pages = append(pages, runs.Page{State: "done", Result: &runs.Result{Exit: 4}}) |
| + s := &scriptedRun{pages: pages} |
| + if code := followRun(context.Background(), s, f.term(), "shop.example.com", "run_a", time.Time{}); code != 4 { |
| + t.Fatalf("exit %d stderr %q", code, f.err.String()) |
| + } |
| + }) |
| +} |
| + |
| func TestFollowRunPageErrors(t *testing.T) { |
| for name, c := range map[string]struct { |
| err error |