amadan
bot: member bots back off on repeated failures instead of apologising hourly (#809)
A charter run that errors, comes back empty, or reaches NONE of its sources (every fetch failed — the all-403 case, where the model writes an apology that posted as a normal edition every occurrence) now counts as a failed edition: the first failure of an outage posts the model's explanation once with a keep-trying note, further consecutive failures post nothing, and interval schedules stretch the next attempt (2x, 4x, 8x... the interval, capped at a day). A successful run clears the state and editions resume by themselves. The same all-sources-down signal hardens the builder's dry-run gate: a bot can no longer be born against sources that are all down, with its "first edition" an apology.
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
8 files changed,
+368
−38
bot_charterfail_test.go+144 −0changelog/2026-07-12-bot-failure-backoff.md+5 −0docs/member-bots.md+13 −2internal/bot/botbuild.go+16 −20internal/bot/chartersched.go+99 −5internal/bot/chartersched_test.go+56 −0internal/bot/runtime.go+8 −0internal/bot/task.go+27 −11
diff --git a/bot_charterfail_test.go b/bot_charterfail_test.go| new file mode 100644 |
| index 0000000..43ef486 |
| --- /dev/null |
| +++ b/bot_charterfail_test.go |
| @@ -0,0 +1,144 @@ |
| +package main |
| + |
| +// End-to-end test of charter failure backoff (internal/bot/chartersched.go): |
| +// a member bot whose every source is down posts ONE notice (the model's |
| +// explanation plus the keep-trying note), stays silent on further failed |
| +// attempts, and resumes normal editions the moment a source answers again. |
| +// The all-403 case is the important one — the run doesn't error, the model |
| +// writes an apology, and before backoff that apology posted every occurrence. |
| + |
| +import ( |
| + "bytes" |
| + "context" |
| + "encoding/json" |
| + "net/http" |
| + "net/http/httptest" |
| + "strings" |
| + "sync/atomic" |
| + "testing" |
| + "time" |
| + |
| + "lchat/internal/bot" |
| + "lchat/internal/e2ee" |
| +) |
| + |
| +func TestCharterFailingRunBacksOffAndRecovers(t *testing.T) { |
| + t.Setenv("LCHAT_BOT_ALLOW_PRIVATE_FETCH", "1") |
| + t.Setenv("LCHAT_BOT_MIN_EVERY", "1s") |
| + |
| + // The "publication": down (403) until the test flips it healthy. |
| + var healthy, hits atomic.Int64 |
| + site := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + hits.Add(1) |
| + if healthy.Load() == 0 { |
| + w.WriteHeader(http.StatusForbidden) |
| + return |
| + } |
| + w.Write([]byte(`<rss><item><title>Budget talks resume</title></item></rss>`)) |
| + })) |
| + defer site.Close() |
| + |
| + // Each run is two model calls: a fetch, then the final text. Runs 1 and 2 |
| + // end in apologies (their fetch 403s); run 3 is the recovery edition (the |
| + // last response repeats if further ticks fire). |
| + fetchCall := []map[string]any{{"type": "tool_use", "id": "tu_f", "name": "fetch", "input": map[string]any{"url": site.URL + "/feed"}}} |
| + fakeAI := scriptedAnthropic(t, |
| + fetchCall, |
| + []map[string]any{{"type": "text", "text": "I couldn't reach the news feed this time."}}, |
| + fetchCall, |
| + []map[string]any{{"type": "text", "text": "Still can't reach the feed, I'm afraid."}}, |
| + fetchCall, |
| + []map[string]any{{"type": "text", "text": "Back in business: budget talks resume."}}, |
| + ) |
| + t.Setenv("ANTHROPIC_BASE_URL", fakeAI.URL) |
| + |
| + ts, s := newTestServer(t) |
| + owner := provisionNamed(t, ts.URL, "Owner", false) |
| + eleven := provisionNamed(t, ts.URL, "Eleven", true) |
| + ownerUID, _ := s.db.deviceTokenUser(owner.Token) |
| + instKey := grantInstanceKey(t, ts.URL, owner, eleven, s) |
| + threadID, threadKey := createThreadKind(t, ts.URL, s, "group", owner, eleven) |
| + |
| + charter := map[string]any{ |
| + "v": 1, "name": "flaky-news", "title": "Flaky News", "glyph": "📰", |
| + "schedule": map[string]any{"every": "1s"}, |
| + "task": map[string]any{ |
| + "prompt": "Fetch the feed and summarise it.", |
| + "tools": []any{"fetch"}, |
| + "fetch": map[string]any{"allow": []any{site.URL}}, |
| + }, |
| + "thread": threadID, "owner": ownerUID, |
| + } |
| + blob, err := e2ee.EncryptJSON(instKey, charter) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + body, _ := json.Marshal(map[string]string{"token": eleven.Token, "blob": blob, "owner_id": ownerUID}) |
| + req, _ := http.NewRequest(http.MethodPut, ts.URL+"/api/member-bots/bot-flaky1", bytes.NewReader(body)) |
| + req.Header.Set("Content-Type", "application/json") |
| + if resp, err := http.DefaultClient.Do(req); err != nil || resp.StatusCode != http.StatusOK { |
| + t.Fatalf("seed charter: %v %d", err, resp.StatusCode) |
| + } |
| + |
| + dir := t.TempDir() |
| + sessionPath := dir + "/eleven.session.json" |
| + if err := eleven.Save(sessionPath); err != nil { |
| + t.Fatal(err) |
| + } |
| + ready := make(chan struct{}) |
| + ctx, cancel := context.WithCancel(context.Background()) |
| + defer cancel() |
| + eng := &bot.Engine{Base: ts.URL, Session: eleven, SessionPath: sessionPath, |
| + Recipe: bot.DefaultRecipe(), AIKey: "test-key", Charters: true, |
| + Ready: func() { close(ready) }, Logf: t.Logf} |
| + go eng.Run(ctx) |
| + awaitReady(t, ready) |
| + |
| + // editions returns what the persona has posted so far. |
| + editions := func() []string { |
| + var out []string |
| + for _, rp := range botReplies(t, s, threadID, threadKey, "Flaky News") { |
| + out = append(out, rp.Text) |
| + } |
| + return out |
| + } |
| + |
| + // First failed run: exactly one notice — the model's apology plus the |
| + // keep-trying note (the interval variant, since this schedule stretches). |
| + waitFor(t, "first failure notice", func() bool { |
| + s.sweepDueScheduled() |
| + return len(editions()) == 1 |
| + }) |
| + if got := editions()[0]; !strings.Contains(got, "couldn't reach") || !strings.Contains(got, "keep trying quietly") { |
| + t.Fatalf("first notice should carry the apology + note: %q", got) |
| + } |
| + |
| + // Second failed run: the site sees another attempt, the thread doesn't. |
| + // (Ticks only fire when this test sweeps, so the count below is stable.) |
| + first := hits.Load() |
| + waitFor(t, "a backed-off second attempt", func() bool { |
| + s.sweepDueScheduled() |
| + return hits.Load() > first |
| + }) |
| + time.Sleep(300 * time.Millisecond) // let the run settle past its (non-)post decision |
| + if got := editions(); len(got) != 1 { |
| + t.Fatalf("a repeat failure must post nothing, got %d editions: %q", len(got), got) |
| + } |
| + |
| + // The source recovers: the next attempt posts a normal edition and the |
| + // outage leaves exactly one notice behind. (Two stages, because the second |
| + // failure pushed this attempt out to ~4s — one 5s waitFor would be tight.) |
| + healthy.Store(1) |
| + second := hits.Load() |
| + waitFor(t, "a recovery attempt reaching the site", func() bool { |
| + s.sweepDueScheduled() |
| + return hits.Load() > second |
| + }) |
| + waitFor(t, "recovery edition", func() bool { |
| + eds := editions() |
| + return len(eds) == 2 && strings.Contains(eds[1], "budget talks resume") |
| + }) |
| + if got := editions()[1]; strings.Contains(got, "keep trying") { |
| + t.Fatalf("the recovery edition must post clean, got %q", got) |
| + } |
| +} |
diff --git a/changelog/2026-07-12-bot-failure-backoff.md b/changelog/2026-07-12-bot-failure-backoff.md| new file mode 100644 |
| index 0000000..d2e8e72 |
| --- /dev/null |
| +++ b/changelog/2026-07-12-bot-failure-backoff.md |
| @@ -0,0 +1,5 @@ |
| +🤖 Your bots handle outages gracefully now. When a bot can't reach any of its |
| +sources (a site goes down or blocks readers for a while), it tells you once — |
| +then retries quietly, a little less often while the outage lasts, and posts |
| +again by itself the moment a source answers. No more identical "I couldn't |
| +reach the news" messages every hour. |
diff --git a/docs/member-bots.md b/docs/member-bots.md| index e01c695..6fcdd90 100644 |
| --- a/docs/member-bots.md |
| +++ b/docs/member-bots.md |
| @@ -85,6 +85,16 @@ charter, a closed tool set, execute tool calls, post the final text. |
| render and the text (the mandatory plain fallback) shows instead. |
| - **Budgets**: `max_tool_calls` (default 6, cap 16), fetch bodies truncated |
| for context, one card per run. |
| +- **Failure backoff**: a run that errors, comes back empty, or reaches *none* |
| + of its sources (every fetch failed — the all-403 case, where the model |
| + writes an apology that would otherwise post as a normal edition every |
| + occurrence) is a **failed edition**. The first failure of an outage posts |
| + the model's explanation once, with a keep-trying note; further consecutive |
| + failures post nothing, and interval schedules stretch the next attempt |
| + (2×, 4×, 8×… the interval, capped at a day — daily bots keep their day). |
| + A successful run clears the state and editions resume by themselves. The |
| + counter is in-memory (`chartersched.go`); a sidecar restart forgives, |
| + costing at most one extra notice. |
| - **Injection story is containment, not detection**: fetched content arrives |
| fenced in tool_result blocks and is framed as data; the model can only call |
| the tools the charter names; output can only go where the charter's thread |
| @@ -110,8 +120,9 @@ the setup on its own goroutine, reporting progress on one action line. |
| (`<link rel="alternate" type="…rss/atom…">`). A site with no advertised |
| feed falls back to the page itself; the dry run judges. |
| 3. **The dry run is the gate**: the task is executed once, immediately. If it |
| - fails or comes back empty, *nothing is created* and the member sees why — |
| - never a bot that fails silently at 8am. |
| + fails, comes back empty, or can't read any of its sources, *nothing is |
| + created* and the member sees why — never a bot that fails silently at 8am, |
| + and never one born apologising. |
| 4. On success: charter stored (owner = requester), dedicated thread created, |
| first alarm armed, and the dry run's output delivered as the bot's **first |
| edition** — proof, not promise. |
diff --git a/internal/bot/botbuild.go b/internal/bot/botbuild.go| index 621247a..255e2a3 100644 |
| --- a/internal/bot/botbuild.go |
| +++ b/internal/bot/botbuild.go |
| @@ -349,15 +349,21 @@ func (r *Runtime) runBotBuild(tc *ThreadCtx, req botBuildReq, requester, request |
| // THE DRY RUN IS THE GATE: run the task once, now. No result → no bot. |
| phase <- "running a first edition as a test" |
| - sample, card, err := r.execTask(charter.Title, &charter.Task) |
| + sample, err := r.execTask(charter.Title, &charter.Task) |
| if err != nil { |
| fail(fmt.Sprintf("the test run failed (%v). Nothing was created.", err)) |
| return |
| } |
| - if strings.TrimSpace(sample) == "" { |
| + if strings.TrimSpace(sample.text) == "" { |
| fail("the test run came back empty — the sources may not be readable. Nothing was created.") |
| return |
| } |
| + if sample.sourcesDown() { |
| + // The model reached NONE of the sources: the "edition" is an apology, |
| + // not proof. Same gate as empty — never mint a bot that's born failing. |
| + fail("the test run couldn't read any of its sources — they may be down or blocking readers right now. Nothing was created.") |
| + return |
| + } |
| // The dedicated thread: the bot's own chat with its owner, titled after it. |
| if charter.Thread == "" { |
| @@ -388,15 +394,11 @@ func (r *Runtime) runBotBuild(tc *ThreadCtx, req botBuildReq, requester, request |
| } |
| // Re-arm cleanly: drop any pending alarm for this bot (an update may have |
| - // changed the schedule), then arm the next occurrence. |
| - if pending, err := r.listSchedules(selfID); err == nil { |
| - for _, p := range pending { |
| - var tp tickPayload |
| - if e2ee.DecryptJSON(selfKey, p.Blob, &tp) == nil && tp.Type == "bot.tick" && tp.Bot == id { |
| - _ = r.unschedule(p.ID) |
| - } |
| - } |
| - } |
| + // changed the schedule), then arm the next occurrence. A fresh charter also |
| + // starts with a clean slate — the dry run just proved it works, so any |
| + // backoff from its previous shape is stale. |
| + r.dropCharterAlarms(selfID, selfKey, id) |
| + r.clearCharterFails(id) |
| r.armCharter(selfID, selfKey, id, charter.Schedule.next(time.Now())) |
| // The store changed shape — refresh the instance's charter wake-time so a |
| // hibernating instance is woken for the new/updated schedule |
| @@ -407,7 +409,7 @@ func (r *Runtime) runBotBuild(tc *ThreadCtx, req botBuildReq, requester, request |
| // comes back from createCharterThread only on the create path; on updates |
| // (and any refetch) read it via the threads endpoint to stay off r.keys. |
| if key, err := r.wrappedKeyFor(charter.Thread); err == nil { |
| - r.postTaskResult(charter.Thread, key, charter.Title, &charter.Task, sample, card) |
| + r.postTaskResult(charter.Thread, key, charter.Title, &charter.Task, sample.text, sample.card) |
| } else { |
| r.logf("bot build %q: first-edition post: %v", req.Name, err) |
| } |
| @@ -459,15 +461,9 @@ func (r *Runtime) runBotRemove(tc *ThreadCtx, botID, requesterUID string) Reply |
| return Reply{Text: fmt.Sprintf("Removing it failed (%v) — try again in a moment.", err)} |
| } |
| if selfID, selfKey, ok := r.selfThread(); ok { |
| - if pending, err := r.listSchedules(selfID); err == nil { |
| - for _, p := range pending { |
| - var tp tickPayload |
| - if e2ee.DecryptJSON(selfKey, p.Blob, &tp) == nil && tp.Type == "bot.tick" && tp.Bot == botID { |
| - _ = r.unschedule(p.ID) |
| - } |
| - } |
| - } |
| + r.dropCharterAlarms(selfID, selfKey, botID) |
| } |
| + r.clearCharterFails(botID) |
| // The retired charter may have been the earliest — refresh the instance's |
| // charter wake-time so it doesn't wake a hibernating space for a bot that |
| // no longer exists. |
diff --git a/internal/bot/chartersched.go b/internal/bot/chartersched.go| index b4293be..6cc5208 100644 |
| --- a/internal/bot/chartersched.go |
| +++ b/internal/bot/chartersched.go |
| @@ -243,16 +243,110 @@ func (r *Runtime) charterTick(selfID string, selfKey []byte, tp *tickPayload) { |
| return |
| } |
| r.logf("charters: running %s (%s)", c.Name, c.Schedule.Summary()) |
| - text, card, err := r.execTask(c.Title, &c.Task) |
| - if err != nil { |
| + res, err := r.execTask(c.Title, &c.Task) |
| + switch { |
| + case err != nil: |
| r.logf("charters: %s: %v", c.Name, err) |
| + r.charterRunFailed(selfID, selfKey, tp.Bot, c, key, taskResult{}) |
| + case res.text == "": |
| + r.logf("charters: %s: empty result — nothing posted", c.Name) |
| + r.charterRunFailed(selfID, selfKey, tp.Bot, c, key, taskResult{}) |
| + case res.sourcesDown(): |
| + r.logf("charters: %s: reached none of its sources (%d fetches)", c.Name, res.fetches) |
| + r.charterRunFailed(selfID, selfKey, tp.Bot, c, key, res) |
| + default: |
| + r.clearCharterFails(tp.Bot) |
| + r.postTaskResult(c.Thread, key, c.Title, &c.Task, res.text, res.card) |
| + } |
| +} |
| + |
| +// How a failing charter behaves (the RTE-down case: a source 403s for a day, |
| +// an hourly bot): ONE notice per outage, not one per attempt, and the retries |
| +// space themselves out instead of hammering a dead feed on schedule. |
| +const charterBackoffCap = 24 * time.Hour |
| + |
| +// The one notice a failing bot posts, appended to the model's own explanation |
| +// of what went wrong. Two variants: interval schedules also stretch their |
| +// retries (backoffDelay), daily ones keep their day. |
| +const ( |
| + failNoteStretch = "I'll keep trying quietly — a little less often while my sources are unreachable — and post again as soon as I can read them." |
| + failNotePlain = "I'll keep trying quietly on my schedule and post again as soon as I can read my sources." |
| +) |
| + |
| +// backoffDelay is how long after `fails` consecutive failed runs the next |
| +// attempt should wait: interval schedules double per failure (2×, 4×, 8×… |
| +// the interval) capped at a day; daily (at) schedules return 0 — their |
| +// cadence is already gentle, and the one-notice rule alone stops the spam. |
| +// Pure; the schedule must have been validated (compiled `every`). |
| +func backoffDelay(s *ScheduleSpec, fails int) time.Duration { |
| + if s.Every == "" || fails < 1 { |
| + return 0 |
| + } |
| + d := s.every |
| + for i := 0; i < fails && d < charterBackoffCap; i++ { |
| + d *= 2 |
| + } |
| + return min(d, charterBackoffCap) |
| +} |
| + |
| +// charterRunFailed folds one failed run into the backoff state: bump the |
| +// consecutive count, push the pre-armed next alarm out by the backoff, and |
| +// post the model's explanation — but only on the FIRST failure of an outage |
| +// (err/empty runs have no text and post nothing, exactly as before). |
| +func (r *Runtime) charterRunFailed(selfID string, selfKey []byte, botID string, c *Charter, key []byte, res taskResult) { |
| + n := r.bumpCharterFails(botID) |
| + if d := backoffDelay(&c.Schedule, n); d > 0 { |
| + // charterTick armed the next ordinary occurrence before the run (the |
| + // crash-safety ordering); replace it — a source that's down now is |
| + // usually still down one interval later. |
| + r.dropCharterAlarms(selfID, selfKey, botID) |
| + r.armCharter(selfID, selfKey, botID, time.Now().Add(d)) |
| + r.logf("charters: %s: failure #%d — next attempt in %s", c.Name, n, d) |
| + } else { |
| + r.logf("charters: %s: failure #%d", c.Name, n) |
| + } |
| + if n > 1 || res.text == "" { |
| return |
| } |
| - if text == "" { |
| - r.logf("charters: %s: empty result — nothing posted", c.Name) |
| + note := failNotePlain |
| + if c.Schedule.Every != "" { |
| + note = failNoteStretch |
| + } |
| + // The explanation posts without any card the run half-composed — a failure |
| + // notice should read as one, not dress up as an edition. |
| + r.postTaskResult(c.Thread, key, c.Title, &c.Task, res.text+"\n\n"+note, nil) |
| +} |
| + |
| +// bumpCharterFails / clearCharterFails track consecutive failed runs per |
| +// charter — in memory only (a restart forgives; worst case one extra notice). |
| +// Own lock: ticks run on the frame goroutine, but builds and removals clear |
| +// from their own goroutines. |
| +func (r *Runtime) bumpCharterFails(botID string) int { |
| + r.failMu.Lock() |
| + defer r.failMu.Unlock() |
| + r.charterFails[botID]++ |
| + return r.charterFails[botID] |
| +} |
| + |
| +func (r *Runtime) clearCharterFails(botID string) { |
| + r.failMu.Lock() |
| + defer r.failMu.Unlock() |
| + delete(r.charterFails, botID) |
| +} |
| + |
| +// dropCharterAlarms cancels every pending alarm for one charter in the |
| +// self-thread — schedule edits, retirement, and backoff re-arms all need it. |
| +func (r *Runtime) dropCharterAlarms(selfID string, selfKey []byte, botID string) { |
| + pending, err := r.listSchedules(selfID) |
| + if err != nil { |
| return |
| } |
| - r.postTaskResult(c.Thread, key, c.Title, &c.Task, text, card) |
| + for _, p := range pending { |
| + var tp tickPayload |
| + if e2ee.DecryptJSON(selfKey, p.Blob, &tp) == nil && tp.Type == "bot.tick" && tp.Bot == botID { |
| + _ = r.unschedule(p.ID) |
| + } |
| + } |
| } |
| // threadKeyFor returns the unwrapped key for a thread the bot holds. Frame |
diff --git a/internal/bot/chartersched_test.go b/internal/bot/chartersched_test.go| new file mode 100644 |
| index 0000000..f65b848 |
| --- /dev/null |
| +++ b/internal/bot/chartersched_test.go |
| @@ -0,0 +1,56 @@ |
| +package bot |
| + |
| +import ( |
| + "testing" |
| + "time" |
| +) |
| + |
| +func TestBackoffDelay(t *testing.T) { |
| + hourly := &ScheduleSpec{Every: "1h"} |
| + if err := hourly.validate(); err != nil { |
| + t.Fatal(err) |
| + } |
| + cases := []struct { |
| + fails int |
| + want time.Duration |
| + }{ |
| + {0, 0}, // no failures — the schedule stands |
| + {1, 2 * time.Hour}, |
| + {2, 4 * time.Hour}, |
| + {3, 8 * time.Hour}, |
| + {5, 24 * time.Hour}, // 32h doubled — capped at a day |
| + {50, 24 * time.Hour}, // deep outage must not overflow past the cap |
| + } |
| + for _, c := range cases { |
| + if got := backoffDelay(hourly, c.fails); got != c.want { |
| + t.Errorf("backoffDelay(1h, %d) = %s, want %s", c.fails, got, c.want) |
| + } |
| + } |
| + |
| + // Daily (at) schedules never stretch — their cadence is already gentle. |
| + daily := &ScheduleSpec{At: "08:00", TZ: "Europe/Dublin"} |
| + if err := daily.validate(); err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, fails := range []int{0, 1, 7} { |
| + if got := backoffDelay(daily, fails); got != 0 { |
| + t.Errorf("backoffDelay(daily, %d) = %s, want 0", fails, got) |
| + } |
| + } |
| +} |
| + |
| +func TestTaskResultSourcesDown(t *testing.T) { |
| + cases := []struct { |
| + res taskResult |
| + want bool |
| + }{ |
| + {taskResult{fetches: 2, fetchOK: 0}, true}, // every source failed |
| + {taskResult{fetches: 2, fetchOK: 1}, false}, // partial reads still count as an edition |
| + {taskResult{fetches: 0, fetchOK: 0}, false}, // a no-fetch task (daily poem) can't be "down" |
| + } |
| + for _, c := range cases { |
| + if got := c.res.sourcesDown(); got != c.want { |
| + t.Errorf("sourcesDown(%+v) = %v, want %v", c.res, got, c.want) |
| + } |
| + } |
| +} |
diff --git a/internal/bot/runtime.go b/internal/bot/runtime.go| index f75440e..32900ee 100644 |
| --- a/internal/bot/runtime.go |
| +++ b/internal/bot/runtime.go |
| @@ -96,6 +96,13 @@ type Runtime struct { |
| // guard for the connect-time catch-up arm (see charterTick). Frame |
| // goroutine only, like muted. |
| charterRuns map[string]time.Time |
| + |
| + // charterFails counts consecutive failed runs per charter — the backoff |
| + // dial (charterRunFailed, chartersched.go). Unlike charterRuns it has its |
| + // own lock: ticks bump it on the frame goroutine, but a successful build |
| + // or a removal clears it from the builder goroutine. |
| + failMu sync.Mutex |
| + charterFails map[string]int |
| } |
| // ThreadCtx hands a Handler the thread's services: decrypted history, the |
| @@ -226,6 +233,7 @@ func (r *Runtime) Run(ctx context.Context) error { |
| r.agents = map[string]bool{} |
| r.muted = map[string]time.Time{} |
| r.charterRuns = map[string]time.Time{} |
| + r.charterFails = map[string]int{} |
| r.mention = regexp.MustCompile(`(?i)(^|[^a-z0-9])@` + regexp.QuoteMeta(r.Session.Name) + `\b`) |
| r.bare = regexp.MustCompile(`(?i)(^|[^a-z0-9])` + regexp.QuoteMeta(r.Session.Name) + `\b`) |
| if r.AIKey != "" { |
diff --git a/internal/bot/task.go b/internal/bot/task.go| index 7a03cd7..98bd56c 100644 |
| --- a/internal/bot/task.go |
| +++ b/internal/bot/task.go |
| @@ -135,14 +135,30 @@ func (r *Runtime) taskTools(t *TaskSpec) []anthTool { |
| return tools |
| } |
| +// taskResult is everything one run produced: the text, any composed card, and |
| +// the fetch tally the failure classification reads (chartersched.go). The |
| +// tally exists because an all-sources-down run doesn't error — the model gets |
| +// "fetch failed" tool_results and writes an apology, which would otherwise |
| +// post as a normal edition every single occurrence. |
| +type taskResult struct { |
| + text string |
| + card map[string]any |
| + fetches int // fetch tool calls the model attempted |
| + fetchOK int // ...that returned a body |
| +} |
| + |
| +// sourcesDown reports a run that tried its sources and reached none of them. |
| +func (res *taskResult) sourcesDown() bool { return res.fetches > 0 && res.fetchOK == 0 } |
| + |
| // execTask is the bounded model loop, shared by scheduled runs (the charter |
| // scheduler) and the builder's dry run (botbuild.go), which captures the |
| // output as the gate + first post instead of a separate delivery. |
| -func (r *Runtime) execTask(persona string, t *TaskSpec) (string, map[string]any, error) { |
| +func (r *Runtime) execTask(persona string, t *TaskSpec) (taskResult, error) { |
| + var res taskResult |
| if r.ai == nil { |
| // Fail closed but not silent: a scheduled task that can never run is a |
| // misconfiguration the owner should see once per firing, in the log. |
| - return "", nil, fmt.Errorf("no model available (ANTHROPIC_API_KEY unset)") |
| + return res, fmt.Errorf("no model available (ANTHROPIC_API_KEY unset)") |
| } |
| model := firstOf(t.Model, r.model, defaultModel) |
| maxCalls := t.MaxToolCalls |
| @@ -157,8 +173,6 @@ func (r *Runtime) execTask(persona string, t *TaskSpec) (string, map[string]any, |
| }} |
| calls := 0 |
| - var final string |
| - var card map[string]any |
| for { |
| resp, err := r.ai.create(anthRequest{ |
| Model: model, |
| @@ -169,16 +183,16 @@ func (r *Runtime) execTask(persona string, t *TaskSpec) (string, map[string]any, |
| Messages: messages, |
| }) |
| if err != nil { |
| - return "", nil, fmt.Errorf("model: %w", err) |
| + return res, fmt.Errorf("model: %w", err) |
| } |
| uses := toolUses(resp.Content) |
| if len(uses) == 0 { |
| - final = strings.TrimSpace(resp.text()) |
| + res.text = strings.TrimSpace(resp.text()) |
| break |
| } |
| if calls+len(uses) > maxCalls { |
| r.logf("task: tool budget exhausted (%d calls)", maxCalls) |
| - final = strings.TrimSpace(resp.text()) // whatever it had so far |
| + res.text = strings.TrimSpace(resp.text()) // whatever it had so far |
| break |
| } |
| calls += len(uses) |
| @@ -188,24 +202,26 @@ func (r *Runtime) execTask(persona string, t *TaskSpec) (string, map[string]any, |
| messages = append(messages, anthMessage{Role: "assistant", Content: resp.Content}) |
| var results []anthBlock |
| for _, u := range uses { |
| - results = append(results, anthBlock{Type: "tool_result", ToolUseID: u.ID, Content: r.runTaskTool(t, u, &card)}) |
| + results = append(results, anthBlock{Type: "tool_result", ToolUseID: u.ID, Content: r.runTaskTool(t, u, &res)}) |
| } |
| messages = append(messages, anthMessage{Role: "user", Content: results}) |
| } |
| - return final, card, nil |
| + return res, nil |
| } |
| // runTaskTool executes one tool_use block. Errors become the result text — |
| // the model should know a source failed (or a card was malformed) and fix or |
| // say so, not hallucinate around it. |
| -func (r *Runtime) runTaskTool(t *TaskSpec, u anthBlock, card *map[string]any) string { |
| +func (r *Runtime) runTaskTool(t *TaskSpec, u anthBlock, res *taskResult) string { |
| switch u.Name { |
| case "fetch": |
| + res.fetches++ |
| rawURL, _ := u.Input["url"].(string) |
| body, err := fetchURL(rawURL, t.Fetch.Allow) |
| if err != nil { |
| return "fetch failed: " + err.Error() |
| } |
| + res.fetchOK++ |
| if len(body) > maxToolResultChars { |
| body = body[:maxToolResultChars] + "\n[truncated]" |
| } |
| @@ -218,7 +234,7 @@ func (r *Runtime) runTaskTool(t *TaskSpec, u anthBlock, card *map[string]any) st |
| if err != nil { |
| return "card invalid: " + err.Error() // the model retries with a fix |
| } |
| - *card = normalized // at most one card; the latest valid one wins |
| + res.card = normalized // at most one card; the latest valid one wins |
| return "card attached" |
| default: |
| return "unknown tool" |