| 1 | package main |
| 2 | |
| 3 | // End-to-end test of charter failure backoff (internal/bot/chartersched.go): |
| 4 | // a member bot whose every source is down posts ONE notice (the model's |
| 5 | // explanation plus the keep-trying note), stays silent on further failed |
| 6 | // attempts, and resumes normal editions the moment a source answers again. |
| 7 | // The all-403 case is the important one — the run doesn't error, the model |
| 8 | // writes an apology, and before backoff that apology posted every occurrence. |
| 9 | |
| 10 | import ( |
| 11 | "bytes" |
| 12 | "context" |
| 13 | "encoding/json" |
| 14 | "net/http" |
| 15 | "net/http/httptest" |
| 16 | "strings" |
| 17 | "sync/atomic" |
| 18 | "testing" |
| 19 | "time" |
| 20 | |
| 21 | "lchat/internal/bot" |
| 22 | "lchat/internal/e2ee" |
| 23 | ) |
| 24 | |
| 25 | func TestCharterFailingRunBacksOffAndRecovers(t *testing.T) { |
| 26 | t.Setenv("LCHAT_BOT_ALLOW_PRIVATE_FETCH", "1") |
| 27 | t.Setenv("LCHAT_BOT_MIN_EVERY", "1s") |
| 28 | |
| 29 | // The "publication": down (403) until the test flips it healthy. |
| 30 | var healthy, hits atomic.Int64 |
| 31 | site := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 32 | hits.Add(1) |
| 33 | if healthy.Load() == 0 { |
| 34 | w.WriteHeader(http.StatusForbidden) |
| 35 | return |
| 36 | } |
| 37 | w.Write([]byte(`<rss><item><title>Budget talks resume</title></item></rss>`)) |
| 38 | })) |
| 39 | defer site.Close() |
| 40 | |
| 41 | // Each run is two model calls: a fetch, then the final text. Runs 1 and 2 |
| 42 | // end in apologies (their fetch 403s); run 3 is the recovery edition (the |
| 43 | // last response repeats if further ticks fire). |
| 44 | fetchCall := []map[string]any{{"type": "tool_use", "id": "tu_f", "name": "fetch", "input": map[string]any{"url": site.URL + "/feed"}}} |
| 45 | fakeAI := scriptedAnthropic(t, |
| 46 | fetchCall, |
| 47 | []map[string]any{{"type": "text", "text": "I couldn't reach the news feed this time."}}, |
| 48 | fetchCall, |
| 49 | []map[string]any{{"type": "text", "text": "Still can't reach the feed, I'm afraid."}}, |
| 50 | fetchCall, |
| 51 | []map[string]any{{"type": "text", "text": "Back in business: budget talks resume."}}, |
| 52 | ) |
| 53 | t.Setenv("ANTHROPIC_BASE_URL", fakeAI.URL) |
| 54 | |
| 55 | ts, s := newTestServer(t) |
| 56 | owner := provisionNamed(t, ts.URL, "Owner", false) |
| 57 | eleven := provisionNamed(t, ts.URL, "Eleven", true) |
| 58 | ownerUID, _ := s.db.deviceTokenUser(owner.Token) |
| 59 | instKey := grantInstanceKey(t, ts.URL, owner, eleven, s) |
| 60 | threadID, threadKey := createThreadKind(t, ts.URL, s, "group", owner, eleven) |
| 61 | |
| 62 | charter := map[string]any{ |
| 63 | "v": 1, "name": "flaky-news", "title": "Flaky News", "glyph": "📰", |
| 64 | "schedule": map[string]any{"every": "1s"}, |
| 65 | "task": map[string]any{ |
| 66 | "prompt": "Fetch the feed and summarise it.", |
| 67 | "tools": []any{"fetch"}, |
| 68 | "fetch": map[string]any{"allow": []any{site.URL}}, |
| 69 | }, |
| 70 | "thread": threadID, "owner": ownerUID, |
| 71 | } |
| 72 | blob, err := e2ee.EncryptJSON(instKey, charter) |
| 73 | if err != nil { |
| 74 | t.Fatal(err) |
| 75 | } |
| 76 | body, _ := json.Marshal(map[string]string{"token": eleven.Token, "blob": blob, "owner_id": ownerUID}) |
| 77 | req, _ := http.NewRequest(http.MethodPut, ts.URL+"/api/member-bots/bot-flaky1", bytes.NewReader(body)) |
| 78 | req.Header.Set("Content-Type", "application/json") |
| 79 | if resp, err := http.DefaultClient.Do(req); err != nil || resp.StatusCode != http.StatusOK { |
| 80 | t.Fatalf("seed charter: %v %d", err, resp.StatusCode) |
| 81 | } |
| 82 | |
| 83 | dir := t.TempDir() |
| 84 | sessionPath := dir + "/eleven.session.json" |
| 85 | if err := eleven.Save(sessionPath); err != nil { |
| 86 | t.Fatal(err) |
| 87 | } |
| 88 | ready := make(chan struct{}) |
| 89 | ctx, cancel := context.WithCancel(context.Background()) |
| 90 | defer cancel() |
| 91 | eng := &bot.Engine{Base: ts.URL, Session: eleven, SessionPath: sessionPath, |
| 92 | Recipe: bot.DefaultRecipe(), AIKey: "test-key", Charters: true, |
| 93 | Ready: func() { close(ready) }, Logf: t.Logf} |
| 94 | go eng.Run(ctx) |
| 95 | awaitReady(t, ready) |
| 96 | |
| 97 | // editions returns what the persona has posted so far. |
| 98 | editions := func() []string { |
| 99 | var out []string |
| 100 | for _, rp := range botReplies(t, s, threadID, threadKey, "Flaky News") { |
| 101 | out = append(out, rp.Text) |
| 102 | } |
| 103 | return out |
| 104 | } |
| 105 | |
| 106 | // First failed run: exactly one notice — the model's apology plus the |
| 107 | // keep-trying note (the interval variant, since this schedule stretches). |
| 108 | waitFor(t, "first failure notice", func() bool { |
| 109 | s.sweepDueScheduled() |
| 110 | return len(editions()) == 1 |
| 111 | }) |
| 112 | if got := editions()[0]; !strings.Contains(got, "couldn't reach") || !strings.Contains(got, "keep trying quietly") { |
| 113 | t.Fatalf("first notice should carry the apology + note: %q", got) |
| 114 | } |
| 115 | |
| 116 | // Second failed run: the site sees another attempt, the thread doesn't. |
| 117 | // (Ticks only fire when this test sweeps, so the count below is stable.) |
| 118 | first := hits.Load() |
| 119 | waitFor(t, "a backed-off second attempt", func() bool { |
| 120 | s.sweepDueScheduled() |
| 121 | return hits.Load() > first |
| 122 | }) |
| 123 | time.Sleep(300 * time.Millisecond) // let the run settle past its (non-)post decision |
| 124 | if got := editions(); len(got) != 1 { |
| 125 | t.Fatalf("a repeat failure must post nothing, got %d editions: %q", len(got), got) |
| 126 | } |
| 127 | |
| 128 | // The source recovers: the next attempt posts a normal edition and the |
| 129 | // outage leaves exactly one notice behind. (Two stages, because the second |
| 130 | // failure pushed this attempt out to ~4s — one 5s waitFor would be tight.) |
| 131 | healthy.Store(1) |
| 132 | second := hits.Load() |
| 133 | waitFor(t, "a recovery attempt reaching the site", func() bool { |
| 134 | s.sweepDueScheduled() |
| 135 | return hits.Load() > second |
| 136 | }) |
| 137 | waitFor(t, "recovery edition", func() bool { |
| 138 | eds := editions() |
| 139 | return len(eds) == 2 && strings.Contains(eds[1], "budget talks resume") |
| 140 | }) |
| 141 | if got := editions()[1]; strings.Contains(got, "keep trying") { |
| 142 | t.Fatalf("the recovery edition must post clean, got %q", got) |
| 143 | } |
| 144 | } |
| 145 | |
amadan