| 1 | package memoria |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "errors" |
| 7 | "fmt" |
| 8 | "log/slog" |
| 9 | "net/http/httptest" |
| 10 | "strings" |
| 11 | "testing" |
| 12 | "time" |
| 13 | |
| 14 | "amadan.net/oficina/suite" |
| 15 | |
| 16 | "amadan.net/oficina/memoria/internal/memoria/provider" |
| 17 | ) |
| 18 | |
| 19 | // recorder is gate 39's stub provider. It records exactly the passages |
| 20 | // each call SENT, and computes its reply from them — so a fixture can |
| 21 | // script "return an address the model was never shown", which a fixed |
| 22 | // reply cannot express. |
| 23 | type recorder struct { |
| 24 | calls [][]string |
| 25 | reply func(sent []string) ([]provider.Pointer, error) |
| 26 | } |
| 27 | |
| 28 | func (r *recorder) ExtractAddresses(_ context.Context, passages []string) ([]provider.Pointer, error) { |
| 29 | r.calls = append(r.calls, append([]string(nil), passages...)) |
| 30 | if r.reply == nil { |
| 31 | return []provider.Pointer{}, nil |
| 32 | } |
| 33 | return r.reply(passages) |
| 34 | } |
| 35 | |
| 36 | // point returns a reply that proposes text in passage n. |
| 37 | func point(n int, text string) func([]string) ([]provider.Pointer, error) { |
| 38 | return func([]string) ([]provider.Pointer, error) { |
| 39 | return []provider.Pointer{{Passage: n, Text: text}}, nil |
| 40 | } |
| 41 | } |
| 42 | |
| 43 | func upload(t *testing.T, s *Store, title, body string) int64 { |
| 44 | t.Helper() |
| 45 | id, err := s.Ingest(context.Background(), IngestSpec{Source: "upload", Key: title, Title: title, Body: body}) |
| 46 | if err != nil { |
| 47 | t.Fatalf("ingest %s: %v", title, err) |
| 48 | } |
| 49 | return id |
| 50 | } |
| 51 | |
| 52 | func modelRows(p Page) []Fact { |
| 53 | var out []Fact |
| 54 | for _, f := range p.Facts { |
| 55 | if f.Model != "" { |
| 56 | out = append(out, f) |
| 57 | } |
| 58 | } |
| 59 | return out |
| 60 | } |
| 61 | |
| 62 | func sentContains(calls [][]string, needle string) bool { |
| 63 | for _, c := range calls { |
| 64 | for _, p := range c { |
| 65 | if strings.Contains(p, needle) { |
| 66 | return true |
| 67 | } |
| 68 | } |
| 69 | } |
| 70 | return false |
| 71 | } |
| 72 | |
| 73 | // GATE 39 — the model half sends only what it may, accepts only what it |
| 74 | // can locate, and never touches a validator-backed kind (§8.9, §10). |
| 75 | // |
| 76 | // **Every case asserts on what was SENT as well as on what rendered.** |
| 77 | // Draft 1 of this gate asserted only the page, and could not see an |
| 78 | // egress that had already happened by the time the box rendered nothing. |
| 79 | func TestGate39TheModelHalf(t *testing.T) { |
| 80 | const addr = "12 Main Street, Dublin 2, D02 X285" |
| 81 | |
| 82 | // PRESENCE. Everything below that asserts an absence is an assertion |
| 83 | // about an empty world without this one (§10's second rule). |
| 84 | t.Run("presence: a located span renders as the document's, marked and named", func(t *testing.T) { |
| 85 | s := storeForTest(t) |
| 86 | id := upload(t, s, "Tito office", "## Contact\n\nOffice address: "+addr) |
| 87 | rec := &recorder{reply: point(0, addr)} |
| 88 | s.extractor = rec |
| 89 | |
| 90 | p := searchFacts(t, s, "office address") |
| 91 | rows := modelRows(p) |
| 92 | if len(rows) != 1 { |
| 93 | t.Fatalf("model rows = %+v, want one", rows) |
| 94 | } |
| 95 | got := rows[0] |
| 96 | want := Fact{Kind: "address", Title: "Tito office", Trail: "Contact", Label: "Office address", |
| 97 | Span: addr, DocumentID: id, Ordinal: 0, Provider: provider.Name, Model: provider.Model} |
| 98 | if got != want { |
| 99 | t.Errorf("row = %+v\nwant %+v", got, want) |
| 100 | } |
| 101 | if len(rec.calls) != 1 || !sentContains(rec.calls, addr) { |
| 102 | t.Errorf("sent = %q; want one call carrying the passage", rec.calls) |
| 103 | } |
| 104 | }) |
| 105 | |
| 106 | // §8.9: the top eight ruled results, and not one more. Ten documents |
| 107 | // match; the recorder must see eight, and the two it did not see must |
| 108 | // be the two ranked lowest — asserted by content, since every body is |
| 109 | // distinct. |
| 110 | t.Run("exactly the top eight ruled results are sent", func(t *testing.T) { |
| 111 | s := storeForTest(t) |
| 112 | for i := 0; i < 10; i++ { |
| 113 | upload(t, s, fmt.Sprintf("Office %d", i), fmt.Sprintf("## Contact\n\nOffice address: %d Main Street, Dublin 2", i+1)) |
| 114 | } |
| 115 | rec := &recorder{} |
| 116 | s.extractor = rec |
| 117 | p := searchFacts(t, s, "office address") |
| 118 | if len(p.Results) < 10 { |
| 119 | t.Fatalf("only %d results; this fixture needs ten to be wrong about", len(p.Results)) |
| 120 | } |
| 121 | if len(rec.calls) != 1 || len(rec.calls[0]) != ModelPassages { |
| 122 | t.Fatalf("sent %d calls of %v passages; want one call of %d", len(rec.calls), lens(rec.calls), ModelPassages) |
| 123 | } |
| 124 | // **Which eight, and in which order** — against the page's own |
| 125 | // ranking, which Search produced before modelFacts ran. A count |
| 126 | // alone passed a review's "send the LAST eight" mutation. |
| 127 | for i := 0; i < ModelPassages; i++ { |
| 128 | want := p.Results[i].Title // "Office N", whose body names "N+1 Main Street" |
| 129 | var n int |
| 130 | fmt.Sscanf(want, "Office %d", &n) |
| 131 | addr := fmt.Sprintf("%d Main Street", n+1) |
| 132 | if !strings.Contains(rec.calls[0][i], addr) { |
| 133 | t.Errorf("sent passage %d is not rank %d's (%s): %q", i, i, want, rec.calls[0][i]) |
| 134 | } |
| 135 | } |
| 136 | }) |
| 137 | |
| 138 | t.Run("a span in no passage renders nothing, and the refusal is ours", func(t *testing.T) { |
| 139 | s := storeForTest(t) |
| 140 | upload(t, s, "Tito office", "## Contact\n\nOffice address: "+addr) |
| 141 | rec := &recorder{reply: point(0, "99 Invented Road, Galway")} |
| 142 | s.extractor = rec |
| 143 | if rows := modelRows(searchFacts(t, s, "office address")); len(rows) != 0 { |
| 144 | t.Errorf("rendered an address that is in no document: %+v", rows) |
| 145 | } |
| 146 | // Delivered: so "nothing rendered" is memoria refusing, not a |
| 147 | // call that never happened. |
| 148 | if len(rec.calls) != 1 { |
| 149 | t.Errorf("calls = %d, want 1", len(rec.calls)) |
| 150 | } |
| 151 | }) |
| 152 | |
| 153 | // An authorised result ranked ninth was never sent. A check run |
| 154 | // against every authorised result instead of the SENT set passes every |
| 155 | // other fixture here, and this is the one that can see it. |
| 156 | t.Run("a span only in an unsent result renders nothing", func(t *testing.T) { |
| 157 | s := storeForTest(t) |
| 158 | for i := 0; i < 10; i++ { |
| 159 | upload(t, s, fmt.Sprintf("Office %d", i), fmt.Sprintf("## Contact\n\nOffice address: %d Main Street, Dublin 2", i+1)) |
| 160 | } |
| 161 | var unsent string |
| 162 | rec := &recorder{reply: func(sent []string) ([]provider.Pointer, error) { |
| 163 | for i := 1; i <= 10; i++ { |
| 164 | a := fmt.Sprintf("%d Main Street, Dublin 2", i) |
| 165 | if !sentContains([][]string{sent}, a) { |
| 166 | unsent = a |
| 167 | break |
| 168 | } |
| 169 | } |
| 170 | // Claimed in a real sent passage, and at an index past the |
| 171 | // end — both must drop. Plus a control that must render. |
| 172 | ctrl := sent[0][strings.Index(sent[0], ": ")+2:] |
| 173 | return []provider.Pointer{{Passage: 3, Text: unsent}, {Passage: 8, Text: unsent}, {Passage: 0, Text: ctrl}}, nil |
| 174 | }} |
| 175 | s.extractor = rec |
| 176 | rows := modelRows(searchFacts(t, s, "office address")) |
| 177 | if unsent == "" { |
| 178 | t.Fatal("every address was sent; this fixture proves nothing") |
| 179 | } |
| 180 | if len(rows) != 1 || rows[0].Span == unsent { |
| 181 | t.Errorf("rows = %+v; want only the control, never the unsent %q", rows, unsent) |
| 182 | } |
| 183 | }) |
| 184 | |
| 185 | // A withheld document is never SENT — asserted on the recorded |
| 186 | // request, because asserting only the page cannot tell "never sent" |
| 187 | // from "sent, then refused". With a visible control in the same |
| 188 | // search, and the same address in both, so the citation's every field |
| 189 | // has to come from the visible one. |
| 190 | t.Run("a withheld passage is never sent, and the citation is the visible one", func(t *testing.T) { |
| 191 | s := storeForTest(t) |
| 192 | s.custody = &perItemCustody{apps: docsRoster(), give: map[string]suite.Authorization{"hidden": suite.NOT_VISIBLE}} |
| 193 | ctx := context.Background() |
| 194 | var shown int64 |
| 195 | for _, d := range [][2]string{{"hidden", "Hidden office"}, {"shown", "Visible office"}} { |
| 196 | id, err := s.Ingest(ctx, IngestSpec{ |
| 197 | Source: "Docs", Key: d[0], Title: d[1], |
| 198 | Body: "## " + d[1] + "\n\nOffice address: " + addr + " — " + d[0] + " copy", |
| 199 | ItemID: d[0], SiblingRevision: 1, |
| 200 | }) |
| 201 | if err != nil { |
| 202 | t.Fatal(err) |
| 203 | } |
| 204 | if d[0] == "shown" { |
| 205 | shown = id |
| 206 | } |
| 207 | } |
| 208 | rec := &recorder{reply: point(0, addr)} |
| 209 | s.extractor = rec |
| 210 | p := searchFacts(t, s, "office address") |
| 211 | if sentContains(rec.calls, "hidden copy") { |
| 212 | t.Fatal("the withheld passage was SENT to the provider") |
| 213 | } |
| 214 | if !sentContains(rec.calls, "shown copy") { |
| 215 | t.Fatal("the visible control was not sent; this fixture proves nothing") |
| 216 | } |
| 217 | rows := modelRows(p) |
| 218 | if len(rows) != 1 || rows[0].DocumentID != shown || rows[0].Title != "Visible office" || rows[0].Trail != "Visible office" { |
| 219 | t.Errorf("rows = %+v; want one, cited to the visible document", rows) |
| 220 | } |
| 221 | }) |
| 222 | |
| 223 | // The same bytes twice in one passage, under different labels: the |
| 224 | // first by offset decides — the label AND the citation — and the |
| 225 | // provider's return order does not. |
| 226 | t.Run("two occurrences in a passage: the first by offset", func(t *testing.T) { |
| 227 | s := storeForTest(t) |
| 228 | upload(t, s, "Tito office", "## Contact\n\nPostal address: 5 High Street, Cork. Delivery address: 5 High Street, Cork.") |
| 229 | s.extractor = &recorder{reply: point(0, "5 High Street, Cork")} |
| 230 | rows := modelRows(searchFacts(t, s, "postal address")) |
| 231 | if len(rows) != 1 || rows[0].Label != "Postal address" { |
| 232 | t.Errorf("rows = %+v; want one, labelled by the first occurrence", rows) |
| 233 | } |
| 234 | }) |
| 235 | |
| 236 | // Byte-exact, deliberately: each of these is a refusal, and a recall |
| 237 | // limit §8.9 states rather than a mapping back to source bytes. |
| 238 | t.Run("no normalisation: each returned in the wrong form is refused", func(t *testing.T) { |
| 239 | for _, c := range []struct{ name, doc, model string }{ |
| 240 | {"nbsp versus space", "12 Main Street, Dublin 2", "12 Main Street, Dublin 2"}, |
| 241 | {"curly versus straight quotes", "“Rose Cottage”, 4 Main Street, Kinsale", "\"Rose Cottage\", 4 Main Street, Kinsale"}, |
| 242 | {"a flattened line break", "12 Main Street\nDublin 2", "12 Main Street Dublin 2"}, |
| 243 | {"a changed case", "12 Main Street, Dublin 2", "12 MAIN STREET, DUBLIN 2"}, |
| 244 | } { |
| 245 | t.Run(c.name, func(t *testing.T) { |
| 246 | s := storeForTest(t) |
| 247 | upload(t, s, "Tito office", "## Contact\n\nOffice address: "+c.doc) |
| 248 | rec := &recorder{reply: point(0, c.model)} |
| 249 | s.extractor = rec |
| 250 | if rows := modelRows(searchFacts(t, s, "office address")); len(rows) != 0 { |
| 251 | t.Errorf("accepted a normalised span: %+v", rows) |
| 252 | } |
| 253 | if len(rec.calls) != 1 { |
| 254 | t.Errorf("calls = %d; the refusal must be ours, after delivery", len(rec.calls)) |
| 255 | } |
| 256 | }) |
| 257 | } |
| 258 | }) |
| 259 | |
| 260 | t.Run("short, empty, mid-word and embedded spans are refused", func(t *testing.T) { |
| 261 | for _, c := range []struct{ name, model string }{ |
| 262 | {"a bare place name", "Dublin"}, |
| 263 | {"the empty string", ""}, |
| 264 | {"a digit", "2"}, |
| 265 | {"starts mid-word", "ain Street, Dublin 2"}, |
| 266 | {"inside a recognised email", "12-main-street"}, |
| 267 | } { |
| 268 | t.Run(c.name, func(t *testing.T) { |
| 269 | s := storeForTest(t) |
| 270 | upload(t, s, "Tito office", "## Contact\n\nOffice address: 12 Main Street, Dublin 2. Mail info.12-main-street@tito.io") |
| 271 | rec := &recorder{reply: point(0, c.model)} |
| 272 | s.extractor = rec |
| 273 | rows := modelRows(searchFacts(t, s, "office address")) |
| 274 | // **Delivered first.** A review switched extraction off |
| 275 | // entirely and all five of these stayed green: no call, no |
| 276 | // rows, "refused". The refusal has to be memoria's, after |
| 277 | // the model answered. |
| 278 | if len(rec.calls) != 1 || !sentContains(rec.calls, "12 Main Street") { |
| 279 | t.Fatalf("calls = %q: the model was never asked, so this proves nothing", rec.calls) |
| 280 | } |
| 281 | if len(rows) != 0 { |
| 282 | t.Errorf("accepted %q: %+v", c.model, rows) |
| 283 | } |
| 284 | }) |
| 285 | } |
| 286 | }) |
| 287 | |
| 288 | // The provider's order never picks: proposals are sorted by (rank, |
| 289 | // offset) before anything is kept. |
| 290 | t.Run("the provider's return order does not decide the rows", func(t *testing.T) { |
| 291 | s := storeForTest(t) |
| 292 | upload(t, s, "Tito office", "## Contact\n\nOffice address: 1 First Street, Cork. Second address: 2 Second Street, Cork.") |
| 293 | s.extractor = &recorder{reply: func([]string) ([]provider.Pointer, error) { |
| 294 | return []provider.Pointer{{Passage: 0, Text: "2 Second Street, Cork"}, {Passage: 0, Text: "1 First Street, Cork"}}, nil |
| 295 | }} |
| 296 | rows := modelRows(searchFacts(t, s, "office address")) |
| 297 | if len(rows) != 2 || rows[0].Span != "1 First Street, Cork" { |
| 298 | t.Errorf("rows = %+v; want document order", rows) |
| 299 | } |
| 300 | }) |
| 301 | |
| 302 | // §8.8's rules, applied to the ACCEPTED proposals. |
| 303 | t.Run("cap, collision and over-long labels apply to accepted proposals", func(t *testing.T) { |
| 304 | // **Every body names "address", so the search retrieves it.** The |
| 305 | // first version of these fixtures did not, returned no results, |
| 306 | // never called the model, rendered nothing — and passed, as |
| 307 | // assertions about an empty world. The call count below is the |
| 308 | // presence half that stops that recurring. |
| 309 | four := "## Our addresses\n\nA: 1 First Street, Cork. B: 2 Second Street, Cork. C: 3 Third Street, Cork. D: 4 Fourth Street, Cork." |
| 310 | for _, c := range []struct { |
| 311 | name, body string |
| 312 | spans []string |
| 313 | }{ |
| 314 | {"four accepted renders none", four, []string{"1 First Street, Cork", "2 Second Street, Cork", "3 Third Street, Cork", "4 Fourth Street, Cork"}}, |
| 315 | {"a colliding line renders none", "## Contact address\n\nWe are at 1 First Street, Cork and at 2 Second Street, Cork too.", []string{"1 First Street, Cork", "2 Second Street, Cork"}}, |
| 316 | {"an over-long label renders none", "## Contact\n\nThe address you must never give to any customer or supplier because it is private: 1 First Street, Cork", []string{"1 First Street, Cork"}}, |
| 317 | } { |
| 318 | t.Run(c.name, func(t *testing.T) { |
| 319 | s := storeForTest(t) |
| 320 | upload(t, s, "Tito office", c.body) |
| 321 | spans := c.spans |
| 322 | rec := &recorder{reply: func([]string) ([]provider.Pointer, error) { |
| 323 | var out []provider.Pointer |
| 324 | for _, sp := range spans { |
| 325 | out = append(out, provider.Pointer{Passage: 0, Text: sp}) |
| 326 | } |
| 327 | return out, nil |
| 328 | }} |
| 329 | s.extractor = rec |
| 330 | rows := modelRows(searchFacts(t, s, "address")) |
| 331 | if len(rec.calls) != 1 { |
| 332 | t.Fatalf("calls = %d: the model was never asked, so this proves nothing", len(rec.calls)) |
| 333 | } |
| 334 | if len(rows) != 0 { |
| 335 | t.Errorf("rendered %d rows", len(rows)) |
| 336 | } |
| 337 | }) |
| 338 | } |
| 339 | }) |
| 340 | |
| 341 | // **Eligibility is the kind, never the outcome.** Each corpus leaves |
| 342 | // §8.8's phone box empty for a different reason, and none of them is |
| 343 | // licence to ask a model for a phone number. Counted at the recorder, |
| 344 | // by call — a page assertion cannot tell "called and ignored" from |
| 345 | // "never called", and the first is an egress nobody authorised. |
| 346 | t.Run("a validator-backed kind never reaches the provider", func(t *testing.T) { |
| 347 | for _, c := range []struct{ name, body string }{ |
| 348 | {"zero valid candidates", "## Contact\n\nRing our phone any weekday."}, |
| 349 | {"suppressed by the cap", "## Contact\n\nA phone: 01 547 7854. B phone: 01 598 1234. C phone: 01 555 0101. D phone: 01 555 0202."}, |
| 350 | {"suppressed by a collision", "## Phone\n\n- 01 547 7854\n- 01 598 1234"}, |
| 351 | {"suppressed by an over-long label", "## Contact\n\nThe number you must never give to any customer because it rings the boss: 01 547 7854"}, |
| 352 | {"a validated candidate rendered", "## Contact\n\nOffice phone: 01 547 7854"}, |
| 353 | } { |
| 354 | t.Run(c.name, func(t *testing.T) { |
| 355 | s := storeForTest(t) |
| 356 | upload(t, s, "Tito office", c.body) |
| 357 | rec := &recorder{} |
| 358 | s.extractor = rec |
| 359 | // Retrieved, or "zero calls" is true of an empty search and |
| 360 | // the case proves nothing — addFacts returns before the model |
| 361 | // path when there are no results. |
| 362 | if p := searchFacts(t, s, "phone number"); len(p.Results) == 0 { |
| 363 | t.Fatal("the corpus was not retrieved; this case proves nothing") |
| 364 | } |
| 365 | if len(rec.calls) != 0 { |
| 366 | t.Errorf("a phone query sent %d requests to the model", len(rec.calls)) |
| 367 | } |
| 368 | }) |
| 369 | } |
| 370 | }) |
| 371 | |
| 372 | t.Run("a mixed query sends exactly one request, for the address", func(t *testing.T) { |
| 373 | s := storeForTest(t) |
| 374 | upload(t, s, "Tito office", "## Contact\n\nOffice phone: 01 547 7854\n\nOffice address: "+addr) |
| 375 | rec := &recorder{reply: point(0, addr)} |
| 376 | s.extractor = rec |
| 377 | p := searchFacts(t, s, "phone number and postal address") |
| 378 | if len(rec.calls) != 1 { |
| 379 | t.Errorf("calls = %d, want exactly 1", len(rec.calls)) |
| 380 | } |
| 381 | var phone, address int |
| 382 | for _, f := range p.Facts { |
| 383 | switch f.Kind { |
| 384 | case "phone": |
| 385 | phone++ |
| 386 | if f.Model != "" { |
| 387 | t.Errorf("a phone row is marked as model output: %+v", f) |
| 388 | } |
| 389 | case "address": |
| 390 | address++ |
| 391 | } |
| 392 | } |
| 393 | if phone != 1 || address != 1 { |
| 394 | t.Errorf("phone rows %d, address rows %d; want one each, independently", phone, address) |
| 395 | } |
| 396 | }) |
| 397 | |
| 398 | // Decision 5's floor, with its presence half: the SAME store and query |
| 399 | // call the model once when it is wired and never when it is not. |
| 400 | t.Run("AI off sends nothing; AI on sends one", func(t *testing.T) { |
| 401 | s := storeForTest(t) |
| 402 | upload(t, s, "Tito office", "## Contact\n\nOffice address: "+addr) |
| 403 | rec := &recorder{reply: point(0, addr)} |
| 404 | |
| 405 | s.extractor = nil |
| 406 | off := searchFacts(t, s, "office address") |
| 407 | if len(rec.calls) != 0 || len(modelRows(off)) != 0 { |
| 408 | t.Fatalf("AI off: %d calls, %d model rows", len(rec.calls), len(modelRows(off))) |
| 409 | } |
| 410 | s.extractor = rec |
| 411 | on := searchFacts(t, s, "office address") |
| 412 | if len(rec.calls) != 1 || len(modelRows(on)) != 1 { |
| 413 | t.Fatalf("AI on: %d calls, %d model rows; the absence above proves nothing without this", len(rec.calls), len(modelRows(on))) |
| 414 | } |
| 415 | }) |
| 416 | |
| 417 | // The undetectable omission, asserted as the documented behaviour so |
| 418 | // it is a known limit and not a surprise. §8.9: four addresses in the |
| 419 | // passages, the model returns one, and every check passes. The day |
| 420 | // something can detect this, this case should go red on purpose. |
| 421 | t.Run("an omitted address cannot be detected (a stated limit)", func(t *testing.T) { |
| 422 | s := storeForTest(t) |
| 423 | upload(t, s, "Tito office", "## Our addresses\n\nA: 1 First Street, Cork. B: 2 Second Street, Cork. C: 3 Third Street, Cork. D: 4 Fourth Street, Cork.") |
| 424 | rec := &recorder{reply: point(0, "1 First Street, Cork")} |
| 425 | s.extractor = rec |
| 426 | rows := modelRows(searchFacts(t, s, "address")) |
| 427 | if len(rec.calls) != 1 { |
| 428 | t.Fatalf("calls = %d: the model was never asked", len(rec.calls)) |
| 429 | } |
| 430 | if len(rows) != 1 { |
| 431 | t.Errorf("rows = %d; this documents the limit, and the limit has changed", len(rows)) |
| 432 | } |
| 433 | }) |
| 434 | } |
| 435 | |
| 436 | // A provider failure renders the page as AI-off would, and the log line |
| 437 | // carries no content — even when the error handed back DOES carry some. |
| 438 | // The provider package promises its errors are content-free; the store |
| 439 | // does not rely on it. |
| 440 | func TestAModelFailureLogsNoContent(t *testing.T) { |
| 441 | const canary = "CANARY-9e1b-a-passage-quoted-in-an-error" |
| 442 | for _, c := range []struct { |
| 443 | name string |
| 444 | err error |
| 445 | want string |
| 446 | }{ |
| 447 | {"an error quoting content", fmt.Errorf("upstream said: %s", canary), "why=other"}, |
| 448 | {"a truncation", fmt.Errorf("%w: %s", provider.ErrTruncated, canary), "why=truncated"}, |
| 449 | {"a timeout", fmt.Errorf("%w: %s", context.DeadlineExceeded, canary), "why=deadline"}, |
| 450 | } { |
| 451 | t.Run(c.name, func(t *testing.T) { |
| 452 | s := storeForTest(t) |
| 453 | var buf bytes.Buffer |
| 454 | s.logger = slog.New(slog.NewTextHandler(&buf, nil)) |
| 455 | upload(t, s, "Tito office", "## Contact\n\nOffice address: 12 Main Street, Dublin 2") |
| 456 | err := c.err |
| 457 | s.extractor = &recorder{reply: func([]string) ([]provider.Pointer, error) { return nil, err }} |
| 458 | p := searchFacts(t, s, "office address") |
| 459 | if len(p.Results) == 0 { |
| 460 | t.Fatal("the search itself failed; a model failure must not fail the page") |
| 461 | } |
| 462 | if len(modelRows(p)) != 0 { |
| 463 | t.Errorf("model rows rendered after a failure") |
| 464 | } |
| 465 | // Presence: the failure WAS logged, and classified. |
| 466 | if !strings.Contains(buf.String(), c.want) { |
| 467 | t.Errorf("log = %q; want it to record %s", buf.String(), c.want) |
| 468 | } |
| 469 | if strings.Contains(buf.String(), canary) { |
| 470 | t.Errorf("the log carries content: %q", buf.String()) |
| 471 | } |
| 472 | }) |
| 473 | } |
| 474 | |
| 475 | // An abandoned keystroke is ordinary and says nothing at all. |
| 476 | t.Run("a cancelled request logs nothing", func(t *testing.T) { |
| 477 | s := storeForTest(t) |
| 478 | var buf bytes.Buffer |
| 479 | s.logger = slog.New(slog.NewTextHandler(&buf, nil)) |
| 480 | upload(t, s, "Tito office", "## Contact\n\nOffice address: 12 Main Street, Dublin 2") |
| 481 | rec := &recorder{reply: func([]string) ([]provider.Pointer, error) { return nil, context.Canceled }} |
| 482 | s.extractor = rec |
| 483 | searchFacts(t, s, "office address") |
| 484 | if len(rec.calls) != 1 { |
| 485 | t.Fatal("the model was never asked; silence here proves nothing") |
| 486 | } |
| 487 | if buf.Len() != 0 { |
| 488 | t.Errorf("logged %q for an abandoned keystroke", buf.String()) |
| 489 | } |
| 490 | }) |
| 491 | } |
| 492 | |
| 493 | func lens(calls [][]string) []int { |
| 494 | var out []int |
| 495 | for _, c := range calls { |
| 496 | out = append(out, len(c)) |
| 497 | } |
| 498 | return out |
| 499 | } |
| 500 | |
| 501 | var _ = errors.Is // keep errors imported for future cases |
| 502 | |
| 503 | // GATE 39, continued — what the first draft could not see. |
| 504 | // |
| 505 | // Each subtest below exists because a review (Codex GPT-6-Astra, |
| 506 | // 2026-09-27) found a mutation of modelFacts that every case above let |
| 507 | // through. The mutation is named in each; each was re-run against this |
| 508 | // file and goes red. |
| 509 | func TestGate39WhatTheFirstDraftCouldNotSee(t *testing.T) { |
| 510 | two := func(t *testing.T) (*Store, map[int64]string) { |
| 511 | // Two single-passage documents whose ranks are fixed by how often |
| 512 | // they say the query's words. |
| 513 | s := storeForTest(t) |
| 514 | ids := map[int64]string{} |
| 515 | ids[upload(t, s, "First office", "## Contact\n\nOffice address: 1 First Street, Cork. Our office address is the office address.")] = "1 First Street, Cork" |
| 516 | ids[upload(t, s, "Second office", "## Contact\n\nOffice address: 2 Second Street, Cork")] = "2 Second Street, Cork" |
| 517 | return s, ids |
| 518 | } |
| 519 | |
| 520 | // Mutation: reverse passage priority in the sort. The earlier order |
| 521 | // fixture used one passage and could not see it. |
| 522 | t.Run("rows follow rank across passages, not the provider's order", func(t *testing.T) { |
| 523 | s, _ := two(t) |
| 524 | s.extractor = &recorder{reply: func(sent []string) ([]provider.Pointer, error) { |
| 525 | var out []provider.Pointer |
| 526 | for i := len(sent) - 1; i >= 0; i-- { // deliberately backwards |
| 527 | out = append(out, provider.Pointer{Passage: i, Text: sent[i][strings.Index(sent[i], ": ")+2 : strings.Index(sent[i], "Cork")+4]}) |
| 528 | } |
| 529 | return out, nil |
| 530 | }} |
| 531 | p := searchFacts(t, s, "office address") |
| 532 | rows := modelRows(p) |
| 533 | if len(rows) != 2 { |
| 534 | t.Fatalf("rows = %+v", rows) |
| 535 | } |
| 536 | if rows[0].DocumentID != p.Results[0].DocumentID || rows[1].DocumentID != p.Results[1].DocumentID { |
| 537 | t.Errorf("row order %d,%d does not follow result order %d,%d", |
| 538 | rows[0].DocumentID, rows[1].DocumentID, p.Results[0].DocumentID, p.Results[1].DocumentID) |
| 539 | } |
| 540 | }) |
| 541 | |
| 542 | // Mutation: disable span dedupe. Same bytes in two passages are ONE |
| 543 | // candidate, cited to the better-ranked. |
| 544 | t.Run("the same span in two passages is one row, cited to the better-ranked", func(t *testing.T) { |
| 545 | s := storeForTest(t) |
| 546 | upload(t, s, "First office", "## Contact\n\nOffice address: 9 Shared Street, Cork. The office address again: office address.") |
| 547 | upload(t, s, "Second office", "## Contact\n\nOffice address: 9 Shared Street, Cork") |
| 548 | s.extractor = &recorder{reply: func(sent []string) ([]provider.Pointer, error) { |
| 549 | return []provider.Pointer{{Passage: 1, Text: "9 Shared Street, Cork"}, {Passage: 0, Text: "9 Shared Street, Cork"}}, nil |
| 550 | }} |
| 551 | p := searchFacts(t, s, "office address") |
| 552 | rows := modelRows(p) |
| 553 | if len(rows) != 1 || rows[0].DocumentID != p.Results[0].DocumentID { |
| 554 | t.Errorf("rows = %+v; want one, cited to result 0 (doc %d)", rows, p.Results[0].DocumentID) |
| 555 | } |
| 556 | }) |
| 557 | |
| 558 | // Mutation: cap written >= rather than >. The cap fixtures showed only |
| 559 | // that four renders none. |
| 560 | t.Run("three accepted addresses render three", func(t *testing.T) { |
| 561 | s := storeForTest(t) |
| 562 | upload(t, s, "Tito office", "## Our addresses\n\nA: 1 First Street, Cork. B: 2 Second Street, Cork. C: 3 Third Street, Cork.") |
| 563 | s.extractor = &recorder{reply: func([]string) ([]provider.Pointer, error) { |
| 564 | return []provider.Pointer{{Passage: 0, Text: "1 First Street, Cork"}, {Passage: 0, Text: "2 Second Street, Cork"}, {Passage: 0, Text: "3 Third Street, Cork"}}, nil |
| 565 | }} |
| 566 | if rows := modelRows(searchFacts(t, s, "address")); len(rows) != 3 { |
| 567 | t.Errorf("rows = %d, want 3: three is under the cap", len(rows)) |
| 568 | } |
| 569 | }) |
| 570 | |
| 571 | // Mutation: apply the cap to RAW proposals, rejected ones included. |
| 572 | t.Run("rejected proposals do not count toward the cap", func(t *testing.T) { |
| 573 | s := storeForTest(t) |
| 574 | upload(t, s, "Tito office", "## Our addresses\n\nA: 1 First Street, Cork. B: 2 Second Street, Cork. C: 3 Third Street, Cork.") |
| 575 | s.extractor = &recorder{reply: func([]string) ([]provider.Pointer, error) { |
| 576 | return []provider.Pointer{ |
| 577 | {Passage: 0, Text: "1 First Street, Cork"}, {Passage: 0, Text: "2 Second Street, Cork"}, {Passage: 0, Text: "3 Third Street, Cork"}, |
| 578 | {Passage: 0, Text: "99 Invented Road, Galway"}, {Passage: 5, Text: "1 First Street, Cork"}, // both dropped |
| 579 | }, nil |
| 580 | }} |
| 581 | if rows := modelRows(searchFacts(t, s, "address")); len(rows) != 3 { |
| 582 | t.Errorf("rows = %d, want 3: two of five proposals were refused", len(rows)) |
| 583 | } |
| 584 | }) |
| 585 | |
| 586 | // Mutation: set every citation's ordinal to zero. Every earlier |
| 587 | // fixture was a single-passage document. |
| 588 | t.Run("the citation names the passage's own ordinal", func(t *testing.T) { |
| 589 | s := storeForTest(t) |
| 590 | intro := "## About\n\n" + strings.Repeat("We are a small office team that works on software. ", 12) |
| 591 | upload(t, s, "Tito office", intro+"\n\n## Visiting\n\nOffice address: 12 Main Street, Dublin 2") |
| 592 | s.extractor = &recorder{reply: func(sent []string) ([]provider.Pointer, error) { |
| 593 | for i, p := range sent { |
| 594 | if strings.Contains(p, "12 Main Street") { |
| 595 | return []provider.Pointer{{Passage: i, Text: "12 Main Street, Dublin 2"}}, nil |
| 596 | } |
| 597 | } |
| 598 | return nil, nil |
| 599 | }} |
| 600 | p := searchFacts(t, s, "office address") |
| 601 | var want int = -1 |
| 602 | for _, r := range p.Results { |
| 603 | if strings.Contains(r.Heading, "Visiting") { |
| 604 | want = r.Ordinal |
| 605 | } |
| 606 | } |
| 607 | if want < 1 { |
| 608 | t.Fatalf("the address passage is ordinal %d; this fixture needs it past 0", want) |
| 609 | } |
| 610 | if rows := modelRows(p); len(rows) != 1 || rows[0].Ordinal != want { |
| 611 | t.Errorf("rows = %+v; want ordinal %d", rows, want) |
| 612 | } |
| 613 | }) |
| 614 | |
| 615 | // Mutation: when text is absent from its claimed passage, look in the |
| 616 | // other sent passages and accept it there. |
| 617 | t.Run("a span in a different passage from the one claimed is refused", func(t *testing.T) { |
| 618 | s, _ := two(t) |
| 619 | rec := &recorder{reply: func(sent []string) ([]provider.Pointer, error) { |
| 620 | // Claim passage 0 for the text that is only in passage 1. |
| 621 | other := sent[1][strings.Index(sent[1], ": ")+2 : strings.Index(sent[1], "Cork")+4] |
| 622 | return []provider.Pointer{{Passage: 0, Text: other}}, nil |
| 623 | }} |
| 624 | s.extractor = rec |
| 625 | rows := modelRows(searchFacts(t, s, "office address")) |
| 626 | if len(rec.calls) != 1 || len(rec.calls[0]) < 2 { |
| 627 | t.Fatal("fewer than two passages were sent; this proves nothing") |
| 628 | } |
| 629 | if len(rows) != 0 { |
| 630 | t.Errorf("accepted text from a passage the model did not name: %+v", rows) |
| 631 | } |
| 632 | }) |
| 633 | |
| 634 | // Mutation: remove the negative-index guard (it panics without it). |
| 635 | t.Run("a negative passage index is refused", func(t *testing.T) { |
| 636 | s, _ := two(t) |
| 637 | s.extractor = &recorder{reply: point(-1, "1 First Street, Cork")} |
| 638 | if rows := modelRows(searchFacts(t, s, "office address")); len(rows) != 0 { |
| 639 | t.Errorf("rows = %+v", rows) |
| 640 | } |
| 641 | }) |
| 642 | |
| 643 | // Mutation: check over-long labels BEFORE dedupe. Only the occurrence |
| 644 | // actually cited decides whether the kind renders (§8.8's order). |
| 645 | t.Run("a later occurrence's over-long label does not silence the cited one", func(t *testing.T) { |
| 646 | s := storeForTest(t) |
| 647 | upload(t, s, "First office", "## Contact\n\nOffice address: 7 Quay Street, Galway. The office address again: office address.") |
| 648 | upload(t, s, "Second office", "## Contact\n\nThe office address you must never give to any customer or supplier at all: 7 Quay Street, Galway") |
| 649 | s.extractor = &recorder{reply: func([]string) ([]provider.Pointer, error) { |
| 650 | return []provider.Pointer{{Passage: 0, Text: "7 Quay Street, Galway"}, {Passage: 1, Text: "7 Quay Street, Galway"}}, nil |
| 651 | }} |
| 652 | p := searchFacts(t, s, "office address") |
| 653 | if len(p.Results) < 2 || !strings.Contains(p.Results[0].Title, "First") { |
| 654 | t.Fatal("the short-labelled document did not rank first; this fixture proves nothing") |
| 655 | } |
| 656 | if rows := modelRows(p); len(rows) != 1 || rows[0].Label != "Office address" { |
| 657 | t.Errorf("rows = %+v; the cited occurrence's label is fine", rows) |
| 658 | } |
| 659 | }) |
| 660 | |
| 661 | // Mutation: remove only the END word-boundary check. |
| 662 | t.Run("a span ending mid-word is refused", func(t *testing.T) { |
| 663 | s := storeForTest(t) |
| 664 | upload(t, s, "Tito office", "## Contact\n\nOffice address: 12 Main Streetside, Dublin 2") |
| 665 | rec := &recorder{reply: point(0, "12 Main Street")} |
| 666 | s.extractor = rec |
| 667 | rows := modelRows(searchFacts(t, s, "office address")) |
| 668 | if len(rec.calls) != 1 { |
| 669 | t.Fatal("never asked") |
| 670 | } |
| 671 | if len(rows) != 0 { |
| 672 | t.Errorf("accepted a span ending inside a word: %+v", rows) |
| 673 | } |
| 674 | }) |
| 675 | |
| 676 | // Mutation: collide on label alone. Different titles disambiguate. |
| 677 | t.Run("the same label under different titles renders both", func(t *testing.T) { |
| 678 | s, _ := two(t) |
| 679 | s.extractor = &recorder{reply: func(sent []string) ([]provider.Pointer, error) { |
| 680 | var out []provider.Pointer |
| 681 | for i, p := range sent { |
| 682 | out = append(out, provider.Pointer{Passage: i, Text: p[strings.Index(p, ": ")+2 : strings.Index(p, "Cork")+4]}) |
| 683 | } |
| 684 | return out, nil |
| 685 | }} |
| 686 | if rows := modelRows(searchFacts(t, s, "office address")); len(rows) != 2 { |
| 687 | t.Errorf("rows = %d, want 2: the titles tell them apart", len(rows)) |
| 688 | } |
| 689 | }) |
| 690 | |
| 691 | // Mutation: keep the 12-byte minimum, drop the digit/comma/newline |
| 692 | // requirement. |
| 693 | t.Run("a long span with no digit, comma or line break is refused", func(t *testing.T) { |
| 694 | s := storeForTest(t) |
| 695 | upload(t, s, "Tito office", "## Contact\n\nOffice address: Rose Cottage Lane near the harbour") |
| 696 | rec := &recorder{reply: point(0, "Rose Cottage Lane")} |
| 697 | s.extractor = rec |
| 698 | rows := modelRows(searchFacts(t, s, "office address")) |
| 699 | if len(rec.calls) != 1 { |
| 700 | t.Fatal("never asked") |
| 701 | } |
| 702 | if len(rows) != 0 { |
| 703 | t.Errorf("accepted %+v", rows) |
| 704 | } |
| 705 | }) |
| 706 | |
| 707 | // Mutation: reject only STRICTLY interior recognised spans, letting a |
| 708 | // span exactly equal to one through. |
| 709 | t.Run("a span exactly equal to a recognised fact is refused", func(t *testing.T) { |
| 710 | s := storeForTest(t) |
| 711 | upload(t, s, "Tito office", "## Contact\n\nOffice address: write to info.12-main@tito.io today") |
| 712 | rec := &recorder{reply: point(0, "info.12-main@tito.io")} |
| 713 | s.extractor = rec |
| 714 | rows := modelRows(searchFacts(t, s, "office address")) |
| 715 | if len(rec.calls) != 1 { |
| 716 | t.Fatal("never asked") |
| 717 | } |
| 718 | if len(rows) != 0 { |
| 719 | t.Errorf("accepted an email address as a postal one: %+v", rows) |
| 720 | } |
| 721 | }) |
| 722 | |
| 723 | // Mutation: pass context.Background() to the extractor. The request's |
| 724 | // context must reach it, or an abandoned keystroke is not cancelled. |
| 725 | t.Run("the request's context reaches the provider", func(t *testing.T) { |
| 726 | s := storeForTest(t) |
| 727 | upload(t, s, "Tito office", "## Contact\n\nOffice address: 12 Main Street, Dublin 2") |
| 728 | type key struct{} |
| 729 | var saw any |
| 730 | s.extractor = ctxRecorder(func(ctx context.Context) { saw = ctx.Value(key{}) }) |
| 731 | ctx := context.WithValue(context.Background(), key{}, "the request") |
| 732 | if _, err := s.Search(ctx, suite.Subject{Kind: suite.SubjectAddress, Value: "paul@tito.io"}, "office address", 25); err != nil { |
| 733 | t.Fatal(err) |
| 734 | } |
| 735 | if saw != "the request" { |
| 736 | t.Errorf("the provider saw a different context (value %v)", saw) |
| 737 | } |
| 738 | }) |
| 739 | |
| 740 | // Mutation: log the accepted proposal's text on success. Gate 4: no |
| 741 | // content in any log line, and success is a path too. |
| 742 | t.Run("a successful extraction logs no content", func(t *testing.T) { |
| 743 | s := storeForTest(t) |
| 744 | var buf bytes.Buffer |
| 745 | s.logger = slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) |
| 746 | upload(t, s, "Tito office", "## Contact\n\nOffice address: 12 Main Street, Dublin 2") |
| 747 | s.extractor = &recorder{reply: point(0, "12 Main Street, Dublin 2")} |
| 748 | if rows := modelRows(searchFacts(t, s, "office address")); len(rows) != 1 { |
| 749 | t.Fatal("no row; the success path did not run") |
| 750 | } |
| 751 | if strings.Contains(buf.String(), "Main Street") { |
| 752 | t.Errorf("the log carries content: %q", buf.String()) |
| 753 | } |
| 754 | }) |
| 755 | |
| 756 | // The sent set is bounded by RANK and then thinned, never refilled. |
| 757 | // The first build counted surviving chunks, so a chunk deleted between |
| 758 | // the ruling and the read let result nine in. Reproduced as the review |
| 759 | // did: rule a page, delete result one's document, then run addFacts on |
| 760 | // the already-ruled page. |
| 761 | t.Run("a chunk deleted after the ruling does not let result nine in", func(t *testing.T) { |
| 762 | s := storeForTest(t) |
| 763 | addrOf := map[int64]string{} |
| 764 | for i := 0; i < 10; i++ { |
| 765 | a := fmt.Sprintf("%d Main Street, Dublin 2", i+1) |
| 766 | addrOf[upload(t, s, fmt.Sprintf("Office %d", i), "## Contact\n\nOffice address: "+a)] = a |
| 767 | } |
| 768 | ruled := searchFacts(t, s, "office address") // extractor nil: no call |
| 769 | if len(ruled.Results) < 10 { |
| 770 | t.Fatalf("only %d results", len(ruled.Results)) |
| 771 | } |
| 772 | if err := s.Delete(context.Background(), ruled.Results[0].DocumentID); err != nil { |
| 773 | t.Fatal(err) |
| 774 | } |
| 775 | rec := &recorder{} |
| 776 | s.extractor = rec |
| 777 | page := Page{Results: ruled.Results} |
| 778 | s.addFacts(context.Background(), &page, "office address") |
| 779 | if len(rec.calls) != 1 { |
| 780 | t.Fatalf("calls = %d", len(rec.calls)) |
| 781 | } |
| 782 | if got := len(rec.calls[0]); got != ModelPassages-1 { |
| 783 | t.Errorf("sent %d passages; want %d — results 2 to 8", got, ModelPassages-1) |
| 784 | } |
| 785 | ninth := addrOf[ruled.Results[ModelPassages].DocumentID] |
| 786 | if sentContains(rec.calls, ninth) { |
| 787 | t.Errorf("result nine (%q) was sent after result one went away", ninth) |
| 788 | } |
| 789 | |
| 790 | // **After thinning, a pointer's index is into the THINNED list.** |
| 791 | // Passage 0 is now result two, and the citation must say so. A |
| 792 | // review looked the result up as results[c.passage] instead of |
| 793 | // sent[c.passage].result — citing the deleted result one — and |
| 794 | // every case stayed green, because none returned a pointer here. |
| 795 | second := ruled.Results[1] |
| 796 | rec2 := &recorder{reply: point(0, addrOf[second.DocumentID])} |
| 797 | s.extractor = rec2 |
| 798 | page2 := Page{Results: ruled.Results} |
| 799 | s.addFacts(context.Background(), &page2, "office address") |
| 800 | rows := modelRows(page2) |
| 801 | if len(rows) != 1 || rows[0].DocumentID != second.DocumentID || rows[0].Title != second.Title || rows[0].Ordinal != second.Ordinal { |
| 802 | t.Errorf("rows = %+v; want one, cited to result two (doc %d)", rows, second.DocumentID) |
| 803 | } |
| 804 | }) |
| 805 | } |
| 806 | |
| 807 | // ctxRecorder is an Extractor that reports the context it was given. |
| 808 | type ctxRecorder func(context.Context) |
| 809 | |
| 810 | func (f ctxRecorder) ExtractAddresses(ctx context.Context, _ []string) ([]provider.Pointer, error) { |
| 811 | f(ctx) |
| 812 | return []provider.Pointer{}, nil |
| 813 | } |
| 814 | |
| 815 | // The provider is NAMED on a model row, not only the model — rendered, |
| 816 | // because the review that found the omission found it in the page. |
| 817 | // Decision 4 names the provider: "deepseek/…" alone names DeepSeek's |
| 818 | // weights and never the party that received the passages. |
| 819 | func TestAModelRowNamesTheProviderAndTheModel(t *testing.T) { |
| 820 | rec := httptest.NewRecorder() |
| 821 | renderFragment(rec, "results", page{ |
| 822 | Searched: true, Query: "office address", |
| 823 | Results: []Result{{DocumentID: 1, Title: "Tito office", Snippet: "x"}}, |
| 824 | Facts: []Fact{ |
| 825 | {Kind: "address", Title: "Tito office", Label: "Office address", Span: "12 Main Street", |
| 826 | DocumentID: 1, Provider: provider.Name, Model: provider.Model}, |
| 827 | {Kind: "eircode", Title: "Tito office", Label: "Eircode", Span: "D02 X285", DocumentID: 1}, |
| 828 | }, |
| 829 | }) |
| 830 | body := rec.Body.String() |
| 831 | for _, want := range []string{"TensorX", provider.Model, "rst-fact-model"} { |
| 832 | if !strings.Contains(body, want) { |
| 833 | t.Errorf("the model row does not carry %q", want) |
| 834 | } |
| 835 | } |
| 836 | // And the validated row does not: exactly one row is marked. |
| 837 | if n := strings.Count(body, "rst-fact-model"); n != 1 { |
| 838 | t.Errorf("%d rows marked as model output, want 1", n) |
| 839 | } |
| 840 | } |
| 841 | |
| 842 | // **A model response is bounded in bytes, and its processing must be |
| 843 | // bounded in time too.** Each subtest guards one mechanism, and each was |
| 844 | // found unguarded by review: a thousand-identical-proposals test passed |
| 845 | // with caching removed, with dedupe removed, and with the per-proposal |
| 846 | // deadline check removed. The fixtures are sized from measurement so the |
| 847 | // version WITHOUT the mechanism lands past the 800 ms deadline and the |
| 848 | // version with it lands far inside: one recogniser pass over this 1.23 MB |
| 849 | // passage is ~200 ms, one over the 410 KB one ~70 ms, and a near-miss |
| 850 | // strings.Index over 1.23 MB ~1 ms (measured 2026-09-27). |
| 851 | // |
| 852 | // Every passage is ONE chunk — chunk.Split bounds by words, and the dense |
| 853 | // part has no spaces — asserted, because the first fixture here was split |
| 854 | // small by the chunker and passed against the unfixed code. |
| 855 | func TestAModelResponseIsCheapToProcess(t *testing.T) { |
| 856 | const unit = "D02X285,info@tito.io," // two recognised facts, no spaces |
| 857 | build := func(t *testing.T, prefix string, reps int) *Store { |
| 858 | t.Helper() |
| 859 | s := storeForTest(t) |
| 860 | upload(t, s, "Tito office", "Office address: "+prefix+strings.Repeat(unit, reps)) |
| 861 | return s |
| 862 | } |
| 863 | onePassage := func(t *testing.T, calls [][]string) { |
| 864 | t.Helper() |
| 865 | if len(calls) != 1 || len(calls[0]) != 1 { |
| 866 | t.Fatalf("sent %v passages; the fixture must be ONE chunk or it measures nothing", lens(calls)) |
| 867 | } |
| 868 | } |
| 869 | timed := func(t *testing.T, s *Store, q string) (time.Duration, []Fact) { |
| 870 | t.Helper() |
| 871 | st := time.Now() |
| 872 | p := searchFacts(t, s, q) |
| 873 | took := time.Since(st) |
| 874 | t.Logf("search took %v", took) |
| 875 | return took, modelRows(p) |
| 876 | } |
| 877 | |
| 878 | // Mutation: the only deadline check is on entry to each proposal. The |
| 879 | // proposal arrives with 10 ms left, and its recogniser pass takes ~200 |
| 880 | // ms: the row must not come back. The presence half, with no wait, is |
| 881 | // what shows the row exists to be dropped. |
| 882 | t.Run("a row whose processing crosses the deadline is dropped", func(t *testing.T) { |
| 883 | for _, late := range []bool{false, true} { |
| 884 | s := build(t, "12 Main Street, Dublin 2,", 60000) |
| 885 | rec := &recorder{} |
| 886 | s.extractor = ctxAwareRecorder(rec, func(ctx context.Context) { |
| 887 | if late { |
| 888 | if dl, ok := ctx.Deadline(); ok { |
| 889 | time.Sleep(time.Until(dl) - 10*time.Millisecond) |
| 890 | } |
| 891 | } |
| 892 | }, point(0, "12 Main Street, Dublin 2")) |
| 893 | _, rows := timed(t, s, "office address") |
| 894 | onePassage(t, rec.calls) |
| 895 | if !late && len(rows) != 1 { |
| 896 | t.Fatalf("no row with time to spare (%d); the late case proves nothing", len(rows)) |
| 897 | } |
| 898 | if late && len(rows) != 0 { |
| 899 | t.Errorf("a row came back after its deadline had passed: %+v", rows) |
| 900 | } |
| 901 | } |
| 902 | }) |
| 903 | |
| 904 | // **Counted, not timed.** The three guards below were wall-clock |
| 905 | // thresholds, and a review ran them with ten CPUs busy and watched all |
| 906 | // three fail on correct code — each blaming a missing mechanism that |
| 907 | // was present. The probe counts what modelFacts actually did, so each |
| 908 | // assertion is about the mechanism and nothing else can move it. |
| 909 | |
| 910 | // Mutation: recompute the recognisers per proposal. |
| 911 | t.Run("the recognisers run once per passage", func(t *testing.T) { |
| 912 | var prefix strings.Builder |
| 913 | var props []provider.Pointer |
| 914 | for i := 0; i < 1000; i++ { |
| 915 | a := fmt.Sprintf("Unit%d-Harbour-Cork", i) |
| 916 | prefix.WriteString(a + ",") |
| 917 | props = append(props, provider.Pointer{Passage: 0, Text: a}) |
| 918 | } |
| 919 | s := build(t, prefix.String(), 200) |
| 920 | s.probe = &modelProbe{} |
| 921 | rec := &recorder{reply: func([]string) ([]provider.Pointer, error) { return props, nil }} |
| 922 | s.extractor = rec |
| 923 | searchFacts(t, s, "office address") |
| 924 | onePassage(t, rec.calls) |
| 925 | if s.probe.examinations != 1000 { |
| 926 | t.Fatalf("examined %d proposals, want 1000: the fixture is not exercising the cache", s.probe.examinations) |
| 927 | } |
| 928 | if s.probe.recognitions != 1 { |
| 929 | t.Errorf("the recognisers ran %d times over one passage; want 1", s.probe.recognitions) |
| 930 | } |
| 931 | }) |
| 932 | |
| 933 | // Mutation: drop the dedupe. |
| 934 | t.Run("identical proposals are examined once", func(t *testing.T) { |
| 935 | s := build(t, "", 200) |
| 936 | s.probe = &modelProbe{} |
| 937 | props := make([]provider.Pointer, 1000) |
| 938 | for i := range props { |
| 939 | props[i] = provider.Pointer{Passage: 0, Text: "D02X285,info@tito.iq"} |
| 940 | } |
| 941 | rec := &recorder{reply: func([]string) ([]provider.Pointer, error) { return props, nil }} |
| 942 | s.extractor = rec |
| 943 | searchFacts(t, s, "office address") |
| 944 | onePassage(t, rec.calls) |
| 945 | if s.probe.examinations != 1 { |
| 946 | t.Errorf("examined %d identical proposals; want 1", s.probe.examinations) |
| 947 | } |
| 948 | }) |
| 949 | |
| 950 | // Mutation: drop the per-proposal deadline check. The request is |
| 951 | // cancelled after the tenth examination — a known point, not a |
| 952 | // measured one — and nothing after it may be examined. |
| 953 | t.Run("the deadline stops the scan, not the end of the list", func(t *testing.T) { |
| 954 | s := build(t, "", 200) |
| 955 | ctx, cancel := context.WithCancel(context.Background()) |
| 956 | defer cancel() |
| 957 | s.probe = &modelProbe{} |
| 958 | s.probe.afterExamine = func() { |
| 959 | if s.probe.examinations == 10 { |
| 960 | cancel() |
| 961 | } |
| 962 | } |
| 963 | props := make([]provider.Pointer, 2500) |
| 964 | for i := range props { |
| 965 | props[i] = provider.Pointer{Passage: 0, Text: fmt.Sprintf("D02X285,info@tito.%dq", i)} |
| 966 | } |
| 967 | rec := &recorder{reply: func([]string) ([]provider.Pointer, error) { return props, nil }} |
| 968 | s.extractor = rec |
| 969 | p, err := s.Search(ctx, suite.Subject{Kind: suite.SubjectAddress, Value: "paul@tito.io"}, "office address", 25) |
| 970 | if err != nil { |
| 971 | t.Fatal(err) |
| 972 | } |
| 973 | onePassage(t, rec.calls) |
| 974 | if s.probe.examinations != 10 { |
| 975 | t.Errorf("examined %d proposals after the deadline passed at 10", s.probe.examinations) |
| 976 | } |
| 977 | if rows := modelRows(p); len(rows) != 0 { |
| 978 | t.Errorf("rows = %+v", rows) |
| 979 | } |
| 980 | }) |
| 981 | |
| 982 | // **The deadline is a time, not a callback.** ctx.Err() reports it only |
| 983 | // once the runtime's timer has fired, and that lags: measured with one |
| 984 | // CPU and a busy wait just past the deadline, Err() was still nil in |
| 985 | // three or four runs of five — and in the others the timer had fired, |
| 986 | // so a single late run is a coin toss. A review got a late row five |
| 987 | // times out of five this way. |
| 988 | // |
| 989 | // So the lagging timer is modelled EXACTLY rather than raced for: a |
| 990 | // context whose deadline has passed and whose Err() is still nil — |
| 991 | // the state the race reaches by accident, reached on purpose. The |
| 992 | // presence half is the same search with the deadline ahead. |
| 993 | t.Run("a row is dropped by the clock, whether or not the timer has fired", func(t *testing.T) { |
| 994 | for _, late := range []bool{false, true} { |
| 995 | s := storeForTest(t) |
| 996 | upload(t, s, "Tito office", "## Contact\n\nOffice address: 12 Main Street, Dublin 2") |
| 997 | rec := &recorder{reply: point(0, "12 Main Street, Dublin 2")} |
| 998 | s.extractor = rec |
| 999 | dl := time.Now().Add(time.Hour) |
| 1000 | if late { |
| 1001 | dl = time.Now().Add(-time.Millisecond) |
| 1002 | } |
| 1003 | ctx := timerNotFired{Context: context.Background(), deadline: dl} |
| 1004 | p, err := s.Search(ctx, suite.Subject{Kind: suite.SubjectAddress, Value: "paul@tito.io"}, "office address", 25) |
| 1005 | if err != nil { |
| 1006 | t.Fatal(err) |
| 1007 | } |
| 1008 | if len(rec.calls) != 1 { |
| 1009 | t.Fatalf("calls = %d; the model path did not run", len(rec.calls)) |
| 1010 | } |
| 1011 | rows := modelRows(p) |
| 1012 | if !late && len(rows) != 1 { |
| 1013 | t.Fatalf("no row with the deadline ahead (%d); the late case proves nothing", len(rows)) |
| 1014 | } |
| 1015 | if late && len(rows) != 0 { |
| 1016 | t.Errorf("a row came back after the deadline, because the timer had not fired: %+v", rows) |
| 1017 | } |
| 1018 | } |
| 1019 | }) |
| 1020 | |
| 1021 | // **The deadline passing DURING processing**, with the timer still not |
| 1022 | // fired. The test above starts already expired, so it could not tell |
| 1023 | // which check caught it: a review swapped the post-processing checks |
| 1024 | // back to ctx.Err() — and separately the per-proposal entry check — |
| 1025 | // and both passed. These two fix the deadline's position. |
| 1026 | // |
| 1027 | // movingDeadline's Err() is always nil and its deadline is set by the |
| 1028 | // test mid-run. It starts at 700 ms so that modelFacts' own 800 ms |
| 1029 | // WithTimeout defers to it (a parent deadline that is sooner is kept), |
| 1030 | // and a deadline the test moves is then the one modelFacts reads. |
| 1031 | t.Run("a deadline passing mid-processing drops the row", func(t *testing.T) { |
| 1032 | for _, late := range []bool{false, true} { |
| 1033 | s := build(t, "12 Main Street, Dublin 2,", 20000) // ~70 ms recogniser pass |
| 1034 | dl := time.Now().Add(700 * time.Millisecond) |
| 1035 | ctx := movingDeadline{Context: context.Background(), dl: &dl} |
| 1036 | rec := &recorder{} |
| 1037 | s.extractor = ctxAwareRecorder(rec, func(context.Context) { |
| 1038 | if late { |
| 1039 | dl = time.Now().Add(20 * time.Millisecond) // passes during the pass |
| 1040 | } else { |
| 1041 | dl = time.Now().Add(time.Hour) |
| 1042 | } |
| 1043 | }, point(0, "12 Main Street, Dublin 2")) |
| 1044 | p, err := s.Search(ctx, suite.Subject{Kind: suite.SubjectAddress, Value: "paul@tito.io"}, "office address", 25) |
| 1045 | if err != nil { |
| 1046 | t.Fatal(err) |
| 1047 | } |
| 1048 | onePassage(t, rec.calls) |
| 1049 | rows := modelRows(p) |
| 1050 | if !late && len(rows) != 1 { |
| 1051 | t.Fatalf("no row with time to spare (%d); the late case proves nothing", len(rows)) |
| 1052 | } |
| 1053 | if late && len(rows) != 0 { |
| 1054 | t.Errorf("a row came back though the deadline passed during processing: %+v", rows) |
| 1055 | } |
| 1056 | } |
| 1057 | }) |
| 1058 | |
| 1059 | t.Run("a deadline passing between proposals stops the scan", func(t *testing.T) { |
| 1060 | s := build(t, "", 200) |
| 1061 | dl := time.Now().Add(700 * time.Millisecond) |
| 1062 | ctx := movingDeadline{Context: context.Background(), dl: &dl} |
| 1063 | s.probe = &modelProbe{afterExamine: func() {}} |
| 1064 | s.probe.afterExamine = func() { |
| 1065 | if s.probe.examinations == 10 { |
| 1066 | dl = time.Now().Add(-time.Millisecond) // passed; Err() stays nil |
| 1067 | } |
| 1068 | } |
| 1069 | props := make([]provider.Pointer, 2500) |
| 1070 | for i := range props { |
| 1071 | props[i] = provider.Pointer{Passage: 0, Text: fmt.Sprintf("D02X285,info@tito.%dq", i)} |
| 1072 | } |
| 1073 | rec := &recorder{reply: func([]string) ([]provider.Pointer, error) { return props, nil }} |
| 1074 | s.extractor = rec |
| 1075 | if _, err := s.Search(ctx, suite.Subject{Kind: suite.SubjectAddress, Value: "paul@tito.io"}, "office address", 25); err != nil { |
| 1076 | t.Fatal(err) |
| 1077 | } |
| 1078 | if s.probe.examinations != 10 { |
| 1079 | t.Errorf("examined %d proposals; the deadline passed by the clock at 10", s.probe.examinations) |
| 1080 | } |
| 1081 | }) |
| 1082 | |
| 1083 | // The whole model path runs under one deadline: the extractor is |
| 1084 | // handed a context that expires within the provider's budget even |
| 1085 | // when the request's own context never does. |
| 1086 | t.Run("the model path carries its own deadline", func(t *testing.T) { |
| 1087 | s := storeForTest(t) |
| 1088 | upload(t, s, "Tito office", "## Contact\n\nOffice address: 12 Main Street, Dublin 2") |
| 1089 | var left time.Duration |
| 1090 | var had bool |
| 1091 | s.extractor = ctxRecorder(func(ctx context.Context) { |
| 1092 | var dl time.Time |
| 1093 | dl, had = ctx.Deadline() |
| 1094 | left = time.Until(dl) |
| 1095 | }) |
| 1096 | searchFacts(t, s, "office address") // context.Background: no deadline of its own |
| 1097 | if !had || left <= 0 || left > provider.Deadline { |
| 1098 | t.Errorf("deadline present=%v, remaining %v; want one within %v", had, left, provider.Deadline) |
| 1099 | } |
| 1100 | }) |
| 1101 | } |
| 1102 | |
| 1103 | // ctxAwareRecorder records like recorder, runs wait with the context it |
| 1104 | // was given (to spend the budget), then replies. |
| 1105 | func ctxAwareRecorder(rec *recorder, wait func(context.Context), reply func([]string) ([]provider.Pointer, error)) Extractor { |
| 1106 | return ctxThen{rec: rec, wait: wait, reply: reply} |
| 1107 | } |
| 1108 | |
| 1109 | type ctxThen struct { |
| 1110 | rec *recorder |
| 1111 | wait func(context.Context) |
| 1112 | reply func([]string) ([]provider.Pointer, error) |
| 1113 | } |
| 1114 | |
| 1115 | func (c ctxThen) ExtractAddresses(ctx context.Context, passages []string) ([]provider.Pointer, error) { |
| 1116 | c.rec.calls = append(c.rec.calls, append([]string(nil), passages...)) |
| 1117 | c.wait(ctx) |
| 1118 | return c.reply(passages) |
| 1119 | } |
| 1120 | |
| 1121 | // timerNotFired is a context whose deadline may have passed while its |
| 1122 | // Err() is still nil — exactly what a context looks like between its |
| 1123 | // deadline and the runtime firing its timer. context.WithTimeout over it |
| 1124 | // keeps the earlier deadline and never cancels, so it reaches modelFacts |
| 1125 | // intact. |
| 1126 | type timerNotFired struct { |
| 1127 | context.Context |
| 1128 | deadline time.Time |
| 1129 | } |
| 1130 | |
| 1131 | func (c timerNotFired) Deadline() (time.Time, bool) { return c.deadline, true } |
| 1132 | |
| 1133 | // expired's own contract, directly: the clock decides, not the callback. |
| 1134 | func TestExpiredReadsTheClock(t *testing.T) { |
| 1135 | past := timerNotFired{Context: context.Background(), deadline: time.Now().Add(-time.Millisecond)} |
| 1136 | if past.Err() != nil { |
| 1137 | t.Fatal("the fixture's Err() is not nil; it does not model a timer that has not fired") |
| 1138 | } |
| 1139 | if !expired(past) { |
| 1140 | t.Error("expired() missed a passed deadline whose timer had not fired") |
| 1141 | } |
| 1142 | if expired(timerNotFired{Context: context.Background(), deadline: time.Now().Add(time.Hour)}) { |
| 1143 | t.Error("expired() reported a deadline an hour away") |
| 1144 | } |
| 1145 | cancelled, cancel := context.WithCancel(context.Background()) |
| 1146 | cancel() |
| 1147 | if !expired(cancelled) { |
| 1148 | t.Error("expired() missed a cancellation") |
| 1149 | } |
| 1150 | if expired(context.Background()) { |
| 1151 | t.Error("expired() reported a context with no deadline") |
| 1152 | } |
| 1153 | } |
| 1154 | |
| 1155 | // movingDeadline is a context whose deadline the test moves mid-run and |
| 1156 | // whose Err() never fires — the lagging-timer state, arriving at a point |
| 1157 | // the test chooses. |
| 1158 | type movingDeadline struct { |
| 1159 | context.Context |
| 1160 | dl *time.Time |
| 1161 | } |
| 1162 | |
| 1163 | func (c movingDeadline) Deadline() (time.Time, bool) { return *c.dl, true } |
| 1164 | |
amadan