| index b8112fd..98bb810 100644 |
| --- a/internal/memoria/modelfacts_test.go |
| +++ b/internal/memoria/modelfacts_test.go |
| @@ -6,6 +6,7 @@ import ( |
| "errors" |
| "fmt" |
| "log/slog" |
| + "net/http/httptest" |
| "strings" |
| "testing" |
| |
| @@ -92,7 +93,7 @@ func TestGate39TheModelHalf(t *testing.T) { |
| } |
| got := rows[0] |
| want := Fact{Kind: "address", Title: "Tito office", Trail: "Contact", Label: "Office address", |
| - Span: addr, DocumentID: id, Ordinal: 0, Model: provider.Model} |
| + Span: addr, DocumentID: id, Ordinal: 0, Provider: provider.Name, Model: provider.Model} |
| if got != want { |
| t.Errorf("row = %+v\nwant %+v", got, want) |
| } |
| @@ -119,6 +120,18 @@ func TestGate39TheModelHalf(t *testing.T) { |
| if len(rec.calls) != 1 || len(rec.calls[0]) != ModelPassages { |
| t.Fatalf("sent %d calls of %v passages; want one call of %d", len(rec.calls), lens(rec.calls), ModelPassages) |
| } |
| + // **Which eight, and in which order** — against the page's own |
| + // ranking, which Search produced before modelFacts ran. A count |
| + // alone passed a review's "send the LAST eight" mutation. |
| + for i := 0; i < ModelPassages; i++ { |
| + want := p.Results[i].Title // "Office N", whose body names "N+1 Main Street" |
| + var n int |
| + fmt.Sscanf(want, "Office %d", &n) |
| + addr := fmt.Sprintf("%d Main Street", n+1) |
| + if !strings.Contains(rec.calls[0][i], addr) { |
| + t.Errorf("sent passage %d is not rank %d's (%s): %q", i, i, want, rec.calls[0][i]) |
| + } |
| + } |
| }) |
| |
| t.Run("a span in no passage renders nothing, and the refusal is ours", func(t *testing.T) { |
| @@ -254,8 +267,17 @@ func TestGate39TheModelHalf(t *testing.T) { |
| t.Run(c.name, func(t *testing.T) { |
| s := storeForTest(t) |
| upload(t, s, "Tito office", "## Contact\n\nOffice address: 12 Main Street, Dublin 2. Mail info.12-main-street@tito.io") |
| - s.extractor = &recorder{reply: point(0, c.model)} |
| - if rows := modelRows(searchFacts(t, s, "office address")); len(rows) != 0 { |
| + rec := &recorder{reply: point(0, c.model)} |
| + s.extractor = rec |
| + rows := modelRows(searchFacts(t, s, "office address")) |
| + // **Delivered first.** A review switched extraction off |
| + // entirely and all five of these stayed green: no call, no |
| + // rows, "refused". The refusal has to be memoria's, after |
| + // the model answered. |
| + if len(rec.calls) != 1 || !sentContains(rec.calls, "12 Main Street") { |
| + t.Fatalf("calls = %q: the model was never asked, so this proves nothing", rec.calls) |
| + } |
| + if len(rows) != 0 { |
| t.Errorf("accepted %q: %+v", c.model, rows) |
| } |
| }) |
| @@ -455,8 +477,12 @@ func TestAModelFailureLogsNoContent(t *testing.T) { |
| var buf bytes.Buffer |
| s.logger = slog.New(slog.NewTextHandler(&buf, nil)) |
| upload(t, s, "Tito office", "## Contact\n\nOffice address: 12 Main Street, Dublin 2") |
| - s.extractor = &recorder{reply: func([]string) ([]provider.Pointer, error) { return nil, context.Canceled }} |
| + rec := &recorder{reply: func([]string) ([]provider.Pointer, error) { return nil, context.Canceled }} |
| + s.extractor = rec |
| searchFacts(t, s, "office address") |
| + if len(rec.calls) != 1 { |
| + t.Fatal("the model was never asked; silence here proves nothing") |
| + } |
| if buf.Len() != 0 { |
| t.Errorf("logged %q for an abandoned keystroke", buf.String()) |
| } |
| @@ -472,3 +498,327 @@ func lens(calls [][]string) []int { |
| } |
| |
| var _ = errors.Is // keep errors imported for future cases |
| + |
| +// GATE 39, continued — what the first draft could not see. |
| +// |
| +// Each subtest below exists because a review (Codex GPT-6-Astra, |
| +// 2026-09-27) found a mutation of modelFacts that every case above let |
| +// through. The mutation is named in each; each was re-run against this |
| +// file and goes red. |
| +func TestGate39WhatTheFirstDraftCouldNotSee(t *testing.T) { |
| + two := func(t *testing.T) (*Store, map[int64]string) { |
| + // Two single-passage documents whose ranks are fixed by how often |
| + // they say the query's words. |
| + s := storeForTest(t) |
| + ids := map[int64]string{} |
| + ids[upload(t, s, "First office", "## Contact\n\nOffice address: 1 First Street, Cork. Our office address is the office address.")] = "1 First Street, Cork" |
| + ids[upload(t, s, "Second office", "## Contact\n\nOffice address: 2 Second Street, Cork")] = "2 Second Street, Cork" |
| + return s, ids |
| + } |
| + |
| + // Mutation: reverse passage priority in the sort. The earlier order |
| + // fixture used one passage and could not see it. |
| + t.Run("rows follow rank across passages, not the provider's order", func(t *testing.T) { |
| + s, _ := two(t) |
| + s.extractor = &recorder{reply: func(sent []string) ([]provider.Pointer, error) { |
| + var out []provider.Pointer |
| + for i := len(sent) - 1; i >= 0; i-- { // deliberately backwards |
| + out = append(out, provider.Pointer{Passage: i, Text: sent[i][strings.Index(sent[i], ": ")+2 : strings.Index(sent[i], "Cork")+4]}) |
| + } |
| + return out, nil |
| + }} |
| + p := searchFacts(t, s, "office address") |
| + rows := modelRows(p) |
| + if len(rows) != 2 { |
| + t.Fatalf("rows = %+v", rows) |
| + } |
| + if rows[0].DocumentID != p.Results[0].DocumentID || rows[1].DocumentID != p.Results[1].DocumentID { |
| + t.Errorf("row order %d,%d does not follow result order %d,%d", |
| + rows[0].DocumentID, rows[1].DocumentID, p.Results[0].DocumentID, p.Results[1].DocumentID) |
| + } |
| + }) |
| + |
| + // Mutation: disable span dedupe. Same bytes in two passages are ONE |
| + // candidate, cited to the better-ranked. |
| + t.Run("the same span in two passages is one row, cited to the better-ranked", func(t *testing.T) { |
| + s := storeForTest(t) |
| + upload(t, s, "First office", "## Contact\n\nOffice address: 9 Shared Street, Cork. The office address again: office address.") |
| + upload(t, s, "Second office", "## Contact\n\nOffice address: 9 Shared Street, Cork") |
| + s.extractor = &recorder{reply: func(sent []string) ([]provider.Pointer, error) { |
| + return []provider.Pointer{{Passage: 1, Text: "9 Shared Street, Cork"}, {Passage: 0, Text: "9 Shared Street, Cork"}}, nil |
| + }} |
| + p := searchFacts(t, s, "office address") |
| + rows := modelRows(p) |
| + if len(rows) != 1 || rows[0].DocumentID != p.Results[0].DocumentID { |
| + t.Errorf("rows = %+v; want one, cited to result 0 (doc %d)", rows, p.Results[0].DocumentID) |
| + } |
| + }) |
| + |
| + // Mutation: cap written >= rather than >. The cap fixtures showed only |
| + // that four renders none. |
| + t.Run("three accepted addresses render three", func(t *testing.T) { |
| + s := storeForTest(t) |
| + upload(t, s, "Tito office", "## Our addresses\n\nA: 1 First Street, Cork. B: 2 Second Street, Cork. C: 3 Third Street, Cork.") |
| + s.extractor = &recorder{reply: func([]string) ([]provider.Pointer, error) { |
| + return []provider.Pointer{{Passage: 0, Text: "1 First Street, Cork"}, {Passage: 0, Text: "2 Second Street, Cork"}, {Passage: 0, Text: "3 Third Street, Cork"}}, nil |
| + }} |
| + if rows := modelRows(searchFacts(t, s, "address")); len(rows) != 3 { |
| + t.Errorf("rows = %d, want 3: three is under the cap", len(rows)) |
| + } |
| + }) |
| + |
| + // Mutation: apply the cap to RAW proposals, rejected ones included. |
| + t.Run("rejected proposals do not count toward the cap", func(t *testing.T) { |
| + s := storeForTest(t) |
| + upload(t, s, "Tito office", "## Our addresses\n\nA: 1 First Street, Cork. B: 2 Second Street, Cork. C: 3 Third Street, Cork.") |
| + s.extractor = &recorder{reply: func([]string) ([]provider.Pointer, error) { |
| + return []provider.Pointer{ |
| + {Passage: 0, Text: "1 First Street, Cork"}, {Passage: 0, Text: "2 Second Street, Cork"}, {Passage: 0, Text: "3 Third Street, Cork"}, |
| + {Passage: 0, Text: "99 Invented Road, Galway"}, {Passage: 5, Text: "1 First Street, Cork"}, // both dropped |
| + }, nil |
| + }} |
| + if rows := modelRows(searchFacts(t, s, "address")); len(rows) != 3 { |
| + t.Errorf("rows = %d, want 3: two of five proposals were refused", len(rows)) |
| + } |
| + }) |
| + |
| + // Mutation: set every citation's ordinal to zero. Every earlier |
| + // fixture was a single-passage document. |
| + t.Run("the citation names the passage's own ordinal", func(t *testing.T) { |
| + s := storeForTest(t) |
| + intro := "## About\n\n" + strings.Repeat("We are a small office team that works on software. ", 12) |
| + upload(t, s, "Tito office", intro+"\n\n## Visiting\n\nOffice address: 12 Main Street, Dublin 2") |
| + s.extractor = &recorder{reply: func(sent []string) ([]provider.Pointer, error) { |
| + for i, p := range sent { |
| + if strings.Contains(p, "12 Main Street") { |
| + return []provider.Pointer{{Passage: i, Text: "12 Main Street, Dublin 2"}}, nil |
| + } |
| + } |
| + return nil, nil |
| + }} |
| + p := searchFacts(t, s, "office address") |
| + var want int = -1 |
| + for _, r := range p.Results { |
| + if strings.Contains(r.Heading, "Visiting") { |
| + want = r.Ordinal |
| + } |
| + } |
| + if want < 1 { |
| + t.Fatalf("the address passage is ordinal %d; this fixture needs it past 0", want) |
| + } |
| + if rows := modelRows(p); len(rows) != 1 || rows[0].Ordinal != want { |
| + t.Errorf("rows = %+v; want ordinal %d", rows, want) |
| + } |
| + }) |
| + |
| + // Mutation: when text is absent from its claimed passage, look in the |
| + // other sent passages and accept it there. |
| + t.Run("a span in a different passage from the one claimed is refused", func(t *testing.T) { |
| + s, _ := two(t) |
| + rec := &recorder{reply: func(sent []string) ([]provider.Pointer, error) { |
| + // Claim passage 0 for the text that is only in passage 1. |
| + other := sent[1][strings.Index(sent[1], ": ")+2 : strings.Index(sent[1], "Cork")+4] |
| + return []provider.Pointer{{Passage: 0, Text: other}}, nil |
| + }} |
| + s.extractor = rec |
| + rows := modelRows(searchFacts(t, s, "office address")) |
| + if len(rec.calls) != 1 || len(rec.calls[0]) < 2 { |
| + t.Fatal("fewer than two passages were sent; this proves nothing") |
| + } |
| + if len(rows) != 0 { |
| + t.Errorf("accepted text from a passage the model did not name: %+v", rows) |
| + } |
| + }) |
| + |
| + // Mutation: remove the negative-index guard (it panics without it). |
| + t.Run("a negative passage index is refused", func(t *testing.T) { |
| + s, _ := two(t) |
| + s.extractor = &recorder{reply: point(-1, "1 First Street, Cork")} |
| + if rows := modelRows(searchFacts(t, s, "office address")); len(rows) != 0 { |
| + t.Errorf("rows = %+v", rows) |
| + } |
| + }) |
| + |
| + // Mutation: check over-long labels BEFORE dedupe. Only the occurrence |
| + // actually cited decides whether the kind renders (§8.8's order). |
| + t.Run("a later occurrence's over-long label does not silence the cited one", func(t *testing.T) { |
| + s := storeForTest(t) |
| + upload(t, s, "First office", "## Contact\n\nOffice address: 7 Quay Street, Galway. The office address again: office address.") |
| + upload(t, s, "Second office", "## Contact\n\nThe office address you must never give to any customer or supplier at all: 7 Quay Street, Galway") |
| + s.extractor = &recorder{reply: func([]string) ([]provider.Pointer, error) { |
| + return []provider.Pointer{{Passage: 0, Text: "7 Quay Street, Galway"}, {Passage: 1, Text: "7 Quay Street, Galway"}}, nil |
| + }} |
| + p := searchFacts(t, s, "office address") |
| + if len(p.Results) < 2 || !strings.Contains(p.Results[0].Title, "First") { |
| + t.Fatal("the short-labelled document did not rank first; this fixture proves nothing") |
| + } |
| + if rows := modelRows(p); len(rows) != 1 || rows[0].Label != "Office address" { |
| + t.Errorf("rows = %+v; the cited occurrence's label is fine", rows) |
| + } |
| + }) |
| + |
| + // Mutation: remove only the END word-boundary check. |
| + t.Run("a span ending mid-word is refused", func(t *testing.T) { |
| + s := storeForTest(t) |
| + upload(t, s, "Tito office", "## Contact\n\nOffice address: 12 Main Streetside, Dublin 2") |
| + rec := &recorder{reply: point(0, "12 Main Street")} |
| + s.extractor = rec |
| + rows := modelRows(searchFacts(t, s, "office address")) |
| + if len(rec.calls) != 1 { |
| + t.Fatal("never asked") |
| + } |
| + if len(rows) != 0 { |
| + t.Errorf("accepted a span ending inside a word: %+v", rows) |
| + } |
| + }) |
| + |
| + // Mutation: collide on label alone. Different titles disambiguate. |
| + t.Run("the same label under different titles renders both", func(t *testing.T) { |
| + s, _ := two(t) |
| + s.extractor = &recorder{reply: func(sent []string) ([]provider.Pointer, error) { |
| + var out []provider.Pointer |
| + for i, p := range sent { |
| + out = append(out, provider.Pointer{Passage: i, Text: p[strings.Index(p, ": ")+2 : strings.Index(p, "Cork")+4]}) |
| + } |
| + return out, nil |
| + }} |
| + if rows := modelRows(searchFacts(t, s, "office address")); len(rows) != 2 { |
| + t.Errorf("rows = %d, want 2: the titles tell them apart", len(rows)) |
| + } |
| + }) |
| + |
| + // Mutation: keep the 12-byte minimum, drop the digit/comma/newline |
| + // requirement. |
| + t.Run("a long span with no digit, comma or line break is refused", func(t *testing.T) { |
| + s := storeForTest(t) |
| + upload(t, s, "Tito office", "## Contact\n\nOffice address: Rose Cottage Lane near the harbour") |
| + rec := &recorder{reply: point(0, "Rose Cottage Lane")} |
| + s.extractor = rec |
| + rows := modelRows(searchFacts(t, s, "office address")) |
| + if len(rec.calls) != 1 { |
| + t.Fatal("never asked") |
| + } |
| + if len(rows) != 0 { |
| + t.Errorf("accepted %+v", rows) |
| + } |
| + }) |
| + |
| + // Mutation: reject only STRICTLY interior recognised spans, letting a |
| + // span exactly equal to one through. |
| + t.Run("a span exactly equal to a recognised fact is refused", func(t *testing.T) { |
| + s := storeForTest(t) |
| + upload(t, s, "Tito office", "## Contact\n\nOffice address: write to info.12-main@tito.io today") |
| + rec := &recorder{reply: point(0, "info.12-main@tito.io")} |
| + s.extractor = rec |
| + rows := modelRows(searchFacts(t, s, "office address")) |
| + if len(rec.calls) != 1 { |
| + t.Fatal("never asked") |
| + } |
| + if len(rows) != 0 { |
| + t.Errorf("accepted an email address as a postal one: %+v", rows) |
| + } |
| + }) |
| + |
| + // Mutation: pass context.Background() to the extractor. The request's |
| + // context must reach it, or an abandoned keystroke is not cancelled. |
| + t.Run("the request's context reaches the provider", func(t *testing.T) { |
| + s := storeForTest(t) |
| + upload(t, s, "Tito office", "## Contact\n\nOffice address: 12 Main Street, Dublin 2") |
| + type key struct{} |
| + var saw any |
| + s.extractor = ctxRecorder(func(ctx context.Context) { saw = ctx.Value(key{}) }) |
| + ctx := context.WithValue(context.Background(), key{}, "the request") |
| + if _, err := s.Search(ctx, suite.Subject{Kind: suite.SubjectAddress, Value: "paul@tito.io"}, "office address", 25); err != nil { |
| + t.Fatal(err) |
| + } |
| + if saw != "the request" { |
| + t.Errorf("the provider saw a different context (value %v)", saw) |
| + } |
| + }) |
| + |
| + // Mutation: log the accepted proposal's text on success. Gate 4: no |
| + // content in any log line, and success is a path too. |
| + t.Run("a successful extraction logs no content", func(t *testing.T) { |
| + s := storeForTest(t) |
| + var buf bytes.Buffer |
| + s.logger = slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) |
| + upload(t, s, "Tito office", "## Contact\n\nOffice address: 12 Main Street, Dublin 2") |
| + s.extractor = &recorder{reply: point(0, "12 Main Street, Dublin 2")} |
| + if rows := modelRows(searchFacts(t, s, "office address")); len(rows) != 1 { |
| + t.Fatal("no row; the success path did not run") |
| + } |
| + if strings.Contains(buf.String(), "Main Street") { |
| + t.Errorf("the log carries content: %q", buf.String()) |
| + } |
| + }) |
| + |
| + // The sent set is bounded by RANK and then thinned, never refilled. |
| + // The first build counted surviving chunks, so a chunk deleted between |
| + // the ruling and the read let result nine in. Reproduced as the review |
| + // did: rule a page, delete result one's document, then run addFacts on |
| + // the already-ruled page. |
| + t.Run("a chunk deleted after the ruling does not let result nine in", func(t *testing.T) { |
| + s := storeForTest(t) |
| + addrOf := map[int64]string{} |
| + for i := 0; i < 10; i++ { |
| + a := fmt.Sprintf("%d Main Street, Dublin 2", i+1) |
| + addrOf[upload(t, s, fmt.Sprintf("Office %d", i), "## Contact\n\nOffice address: "+a)] = a |
| + } |
| + ruled := searchFacts(t, s, "office address") // extractor nil: no call |
| + if len(ruled.Results) < 10 { |
| + t.Fatalf("only %d results", len(ruled.Results)) |
| + } |
| + if err := s.Delete(context.Background(), ruled.Results[0].DocumentID); err != nil { |
| + t.Fatal(err) |
| + } |
| + rec := &recorder{} |
| + s.extractor = rec |
| + page := Page{Results: ruled.Results} |
| + s.addFacts(context.Background(), &page, "office address") |
| + if len(rec.calls) != 1 { |
| + t.Fatalf("calls = %d", len(rec.calls)) |
| + } |
| + if got := len(rec.calls[0]); got != ModelPassages-1 { |
| + t.Errorf("sent %d passages; want %d — results 2 to 8", got, ModelPassages-1) |
| + } |
| + ninth := addrOf[ruled.Results[ModelPassages].DocumentID] |
| + if sentContains(rec.calls, ninth) { |
| + t.Errorf("result nine (%q) was sent after result one went away", ninth) |
| + } |
| + }) |
| +} |
| + |
| +// ctxRecorder is an Extractor that reports the context it was given. |
| +type ctxRecorder func(context.Context) |
| + |
| +func (f ctxRecorder) ExtractAddresses(ctx context.Context, _ []string) ([]provider.Pointer, error) { |
| + f(ctx) |
| + return []provider.Pointer{}, nil |
| +} |
| + |
| +// The provider is NAMED on a model row, not only the model — rendered, |
| +// because the review that found the omission found it in the page. |
| +// Decision 4 names the provider: "deepseek/…" alone names DeepSeek's |
| +// weights and never the party that received the passages. |
| +func TestAModelRowNamesTheProviderAndTheModel(t *testing.T) { |
| + rec := httptest.NewRecorder() |
| + renderFragment(rec, "results", page{ |
| + Searched: true, Query: "office address", |
| + Results: []Result{{DocumentID: 1, Title: "Tito office", Snippet: "x"}}, |
| + Facts: []Fact{ |
| + {Kind: "address", Title: "Tito office", Label: "Office address", Span: "12 Main Street", |
| + DocumentID: 1, Provider: provider.Name, Model: provider.Model}, |
| + {Kind: "eircode", Title: "Tito office", Label: "Eircode", Span: "D02 X285", DocumentID: 1}, |
| + }, |
| + }) |
| + body := rec.Body.String() |
| + for _, want := range []string{"TensorX", provider.Model, "rst-fact-model"} { |
| + if !strings.Contains(body, want) { |
| + t.Errorf("the model row does not carry %q", want) |
| + } |
| + } |
| + // And the validated row does not: exactly one row is marked. |
| + if n := strings.Count(body, "rst-fact-model"); n != 1 { |
| + t.Errorf("%d rows marked as model output, want 1", n) |
| + } |
| +} |
amadan