| new file mode 100644 |
| index 0000000..b8112fd |
| --- /dev/null |
| +++ b/internal/memoria/modelfacts_test.go |
| @@ -0,0 +1,474 @@ |
| +package memoria |
| + |
| +import ( |
| + "bytes" |
| + "context" |
| + "errors" |
| + "fmt" |
| + "log/slog" |
| + "strings" |
| + "testing" |
| + |
| + "amadan.net/oficina/suite" |
| + |
| + "amadan.net/oficina/memoria/internal/memoria/provider" |
| +) |
| + |
| +// recorder is gate 39's stub provider. It records exactly the passages |
| +// each call SENT, and computes its reply from them — so a fixture can |
| +// script "return an address the model was never shown", which a fixed |
| +// reply cannot express. |
| +type recorder struct { |
| + calls [][]string |
| + reply func(sent []string) ([]provider.Pointer, error) |
| +} |
| + |
| +func (r *recorder) ExtractAddresses(_ context.Context, passages []string) ([]provider.Pointer, error) { |
| + r.calls = append(r.calls, append([]string(nil), passages...)) |
| + if r.reply == nil { |
| + return []provider.Pointer{}, nil |
| + } |
| + return r.reply(passages) |
| +} |
| + |
| +// point returns a reply that proposes text in passage n. |
| +func point(n int, text string) func([]string) ([]provider.Pointer, error) { |
| + return func([]string) ([]provider.Pointer, error) { |
| + return []provider.Pointer{{Passage: n, Text: text}}, nil |
| + } |
| +} |
| + |
| +func upload(t *testing.T, s *Store, title, body string) int64 { |
| + t.Helper() |
| + id, err := s.Ingest(context.Background(), IngestSpec{Source: "upload", Key: title, Title: title, Body: body}) |
| + if err != nil { |
| + t.Fatalf("ingest %s: %v", title, err) |
| + } |
| + return id |
| +} |
| + |
| +func modelRows(p Page) []Fact { |
| + var out []Fact |
| + for _, f := range p.Facts { |
| + if f.Model != "" { |
| + out = append(out, f) |
| + } |
| + } |
| + return out |
| +} |
| + |
| +func sentContains(calls [][]string, needle string) bool { |
| + for _, c := range calls { |
| + for _, p := range c { |
| + if strings.Contains(p, needle) { |
| + return true |
| + } |
| + } |
| + } |
| + return false |
| +} |
| + |
| +// GATE 39 — the model half sends only what it may, accepts only what it |
| +// can locate, and never touches a validator-backed kind (§8.9, §10). |
| +// |
| +// **Every case asserts on what was SENT as well as on what rendered.** |
| +// Draft 1 of this gate asserted only the page, and could not see an |
| +// egress that had already happened by the time the box rendered nothing. |
| +func TestGate39TheModelHalf(t *testing.T) { |
| + const addr = "12 Main Street, Dublin 2, D02 X285" |
| + |
| + // PRESENCE. Everything below that asserts an absence is an assertion |
| + // about an empty world without this one (§10's second rule). |
| + t.Run("presence: a located span renders as the document's, marked and named", func(t *testing.T) { |
| + s := storeForTest(t) |
| + id := upload(t, s, "Tito office", "## Contact\n\nOffice address: "+addr) |
| + rec := &recorder{reply: point(0, addr)} |
| + s.extractor = rec |
| + |
| + p := searchFacts(t, s, "office address") |
| + rows := modelRows(p) |
| + if len(rows) != 1 { |
| + t.Fatalf("model rows = %+v, want one", rows) |
| + } |
| + got := rows[0] |
| + want := Fact{Kind: "address", Title: "Tito office", Trail: "Contact", Label: "Office address", |
| + Span: addr, DocumentID: id, Ordinal: 0, Model: provider.Model} |
| + if got != want { |
| + t.Errorf("row = %+v\nwant %+v", got, want) |
| + } |
| + if len(rec.calls) != 1 || !sentContains(rec.calls, addr) { |
| + t.Errorf("sent = %q; want one call carrying the passage", rec.calls) |
| + } |
| + }) |
| + |
| + // §8.9: the top eight ruled results, and not one more. Ten documents |
| + // match; the recorder must see eight, and the two it did not see must |
| + // be the two ranked lowest — asserted by content, since every body is |
| + // distinct. |
| + t.Run("exactly the top eight ruled results are sent", func(t *testing.T) { |
| + s := storeForTest(t) |
| + for i := 0; i < 10; i++ { |
| + upload(t, s, fmt.Sprintf("Office %d", i), fmt.Sprintf("## Contact\n\nOffice address: %d Main Street, Dublin 2", i+1)) |
| + } |
| + rec := &recorder{} |
| + s.extractor = rec |
| + p := searchFacts(t, s, "office address") |
| + if len(p.Results) < 10 { |
| + t.Fatalf("only %d results; this fixture needs ten to be wrong about", len(p.Results)) |
| + } |
| + if len(rec.calls) != 1 || len(rec.calls[0]) != ModelPassages { |
| + t.Fatalf("sent %d calls of %v passages; want one call of %d", len(rec.calls), lens(rec.calls), ModelPassages) |
| + } |
| + }) |
| + |
| + t.Run("a span in no passage renders nothing, and the refusal is ours", func(t *testing.T) { |
| + s := storeForTest(t) |
| + upload(t, s, "Tito office", "## Contact\n\nOffice address: "+addr) |
| + rec := &recorder{reply: point(0, "99 Invented Road, Galway")} |
| + s.extractor = rec |
| + if rows := modelRows(searchFacts(t, s, "office address")); len(rows) != 0 { |
| + t.Errorf("rendered an address that is in no document: %+v", rows) |
| + } |
| + // Delivered: so "nothing rendered" is memoria refusing, not a |
| + // call that never happened. |
| + if len(rec.calls) != 1 { |
| + t.Errorf("calls = %d, want 1", len(rec.calls)) |
| + } |
| + }) |
| + |
| + // An authorised result ranked ninth was never sent. A check run |
| + // against every authorised result instead of the SENT set passes every |
| + // other fixture here, and this is the one that can see it. |
| + t.Run("a span only in an unsent result renders nothing", func(t *testing.T) { |
| + s := storeForTest(t) |
| + for i := 0; i < 10; i++ { |
| + upload(t, s, fmt.Sprintf("Office %d", i), fmt.Sprintf("## Contact\n\nOffice address: %d Main Street, Dublin 2", i+1)) |
| + } |
| + var unsent string |
| + rec := &recorder{reply: func(sent []string) ([]provider.Pointer, error) { |
| + for i := 1; i <= 10; i++ { |
| + a := fmt.Sprintf("%d Main Street, Dublin 2", i) |
| + if !sentContains([][]string{sent}, a) { |
| + unsent = a |
| + break |
| + } |
| + } |
| + // Claimed in a real sent passage, and at an index past the |
| + // end — both must drop. Plus a control that must render. |
| + ctrl := sent[0][strings.Index(sent[0], ": ")+2:] |
| + return []provider.Pointer{{Passage: 3, Text: unsent}, {Passage: 8, Text: unsent}, {Passage: 0, Text: ctrl}}, nil |
| + }} |
| + s.extractor = rec |
| + rows := modelRows(searchFacts(t, s, "office address")) |
| + if unsent == "" { |
| + t.Fatal("every address was sent; this fixture proves nothing") |
| + } |
| + if len(rows) != 1 || rows[0].Span == unsent { |
| + t.Errorf("rows = %+v; want only the control, never the unsent %q", rows, unsent) |
| + } |
| + }) |
| + |
| + // A withheld document is never SENT — asserted on the recorded |
| + // request, because asserting only the page cannot tell "never sent" |
| + // from "sent, then refused". With a visible control in the same |
| + // search, and the same address in both, so the citation's every field |
| + // has to come from the visible one. |
| + t.Run("a withheld passage is never sent, and the citation is the visible one", func(t *testing.T) { |
| + s := storeForTest(t) |
| + s.custody = &perItemCustody{apps: docsRoster(), give: map[string]suite.Authorization{"hidden": suite.NOT_VISIBLE}} |
| + ctx := context.Background() |
| + var shown int64 |
| + for _, d := range [][2]string{{"hidden", "Hidden office"}, {"shown", "Visible office"}} { |
| + id, err := s.Ingest(ctx, IngestSpec{ |
| + Source: "Docs", Key: d[0], Title: d[1], |
| + Body: "## " + d[1] + "\n\nOffice address: " + addr + " — " + d[0] + " copy", |
| + ItemID: d[0], SiblingRevision: 1, |
| + }) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if d[0] == "shown" { |
| + shown = id |
| + } |
| + } |
| + rec := &recorder{reply: point(0, addr)} |
| + s.extractor = rec |
| + p := searchFacts(t, s, "office address") |
| + if sentContains(rec.calls, "hidden copy") { |
| + t.Fatal("the withheld passage was SENT to the provider") |
| + } |
| + if !sentContains(rec.calls, "shown copy") { |
| + t.Fatal("the visible control was not sent; this fixture proves nothing") |
| + } |
| + rows := modelRows(p) |
| + if len(rows) != 1 || rows[0].DocumentID != shown || rows[0].Title != "Visible office" || rows[0].Trail != "Visible office" { |
| + t.Errorf("rows = %+v; want one, cited to the visible document", rows) |
| + } |
| + }) |
| + |
| + // The same bytes twice in one passage, under different labels: the |
| + // first by offset decides — the label AND the citation — and the |
| + // provider's return order does not. |
| + t.Run("two occurrences in a passage: the first by offset", func(t *testing.T) { |
| + s := storeForTest(t) |
| + upload(t, s, "Tito office", "## Contact\n\nPostal address: 5 High Street, Cork. Delivery address: 5 High Street, Cork.") |
| + s.extractor = &recorder{reply: point(0, "5 High Street, Cork")} |
| + rows := modelRows(searchFacts(t, s, "postal address")) |
| + if len(rows) != 1 || rows[0].Label != "Postal address" { |
| + t.Errorf("rows = %+v; want one, labelled by the first occurrence", rows) |
| + } |
| + }) |
| + |
| + // Byte-exact, deliberately: each of these is a refusal, and a recall |
| + // limit §8.9 states rather than a mapping back to source bytes. |
| + t.Run("no normalisation: each returned in the wrong form is refused", func(t *testing.T) { |
| + for _, c := range []struct{ name, doc, model string }{ |
| + {"nbsp versus space", "12 Main Street, Dublin 2", "12 Main Street, Dublin 2"}, |
| + {"curly versus straight quotes", "“Rose Cottage”, 4 Main Street, Kinsale", "\"Rose Cottage\", 4 Main Street, Kinsale"}, |
| + {"a flattened line break", "12 Main Street\nDublin 2", "12 Main Street Dublin 2"}, |
| + {"a changed case", "12 Main Street, Dublin 2", "12 MAIN STREET, DUBLIN 2"}, |
| + } { |
| + t.Run(c.name, func(t *testing.T) { |
| + s := storeForTest(t) |
| + upload(t, s, "Tito office", "## Contact\n\nOffice address: "+c.doc) |
| + rec := &recorder{reply: point(0, c.model)} |
| + s.extractor = rec |
| + if rows := modelRows(searchFacts(t, s, "office address")); len(rows) != 0 { |
| + t.Errorf("accepted a normalised span: %+v", rows) |
| + } |
| + if len(rec.calls) != 1 { |
| + t.Errorf("calls = %d; the refusal must be ours, after delivery", len(rec.calls)) |
| + } |
| + }) |
| + } |
| + }) |
| + |
| + t.Run("short, empty, mid-word and embedded spans are refused", func(t *testing.T) { |
| + for _, c := range []struct{ name, model string }{ |
| + {"a bare place name", "Dublin"}, |
| + {"the empty string", ""}, |
| + {"a digit", "2"}, |
| + {"starts mid-word", "ain Street, Dublin 2"}, |
| + {"inside a recognised email", "12-main-street"}, |
| + } { |
| + t.Run(c.name, func(t *testing.T) { |
| + s := storeForTest(t) |
| + upload(t, s, "Tito office", "## Contact\n\nOffice address: 12 Main Street, Dublin 2. Mail info.12-main-street@tito.io") |
| + s.extractor = &recorder{reply: point(0, c.model)} |
| + if rows := modelRows(searchFacts(t, s, "office address")); len(rows) != 0 { |
| + t.Errorf("accepted %q: %+v", c.model, rows) |
| + } |
| + }) |
| + } |
| + }) |
| + |
| + // The provider's order never picks: proposals are sorted by (rank, |
| + // offset) before anything is kept. |
| + t.Run("the provider's return order does not decide the rows", func(t *testing.T) { |
| + s := storeForTest(t) |
| + upload(t, s, "Tito office", "## Contact\n\nOffice address: 1 First Street, Cork. Second address: 2 Second Street, Cork.") |
| + s.extractor = &recorder{reply: func([]string) ([]provider.Pointer, error) { |
| + return []provider.Pointer{{Passage: 0, Text: "2 Second Street, Cork"}, {Passage: 0, Text: "1 First Street, Cork"}}, nil |
| + }} |
| + rows := modelRows(searchFacts(t, s, "office address")) |
| + if len(rows) != 2 || rows[0].Span != "1 First Street, Cork" { |
| + t.Errorf("rows = %+v; want document order", rows) |
| + } |
| + }) |
| + |
| + // §8.8's rules, applied to the ACCEPTED proposals. |
| + t.Run("cap, collision and over-long labels apply to accepted proposals", func(t *testing.T) { |
| + // **Every body names "address", so the search retrieves it.** The |
| + // first version of these fixtures did not, returned no results, |
| + // never called the model, rendered nothing — and passed, as |
| + // assertions about an empty world. The call count below is the |
| + // presence half that stops that recurring. |
| + four := "## Our addresses\n\nA: 1 First Street, Cork. B: 2 Second Street, Cork. C: 3 Third Street, Cork. D: 4 Fourth Street, Cork." |
| + for _, c := range []struct { |
| + name, body string |
| + spans []string |
| + }{ |
| + {"four accepted renders none", four, []string{"1 First Street, Cork", "2 Second Street, Cork", "3 Third Street, Cork", "4 Fourth Street, Cork"}}, |
| + {"a colliding line renders none", "## Contact address\n\nWe are at 1 First Street, Cork and at 2 Second Street, Cork too.", []string{"1 First Street, Cork", "2 Second Street, Cork"}}, |
| + {"an over-long label renders none", "## Contact\n\nThe address you must never give to any customer or supplier because it is private: 1 First Street, Cork", []string{"1 First Street, Cork"}}, |
| + } { |
| + t.Run(c.name, func(t *testing.T) { |
| + s := storeForTest(t) |
| + upload(t, s, "Tito office", c.body) |
| + spans := c.spans |
| + rec := &recorder{reply: func([]string) ([]provider.Pointer, error) { |
| + var out []provider.Pointer |
| + for _, sp := range spans { |
| + out = append(out, provider.Pointer{Passage: 0, Text: sp}) |
| + } |
| + return out, nil |
| + }} |
| + s.extractor = rec |
| + rows := modelRows(searchFacts(t, s, "address")) |
| + if len(rec.calls) != 1 { |
| + t.Fatalf("calls = %d: the model was never asked, so this proves nothing", len(rec.calls)) |
| + } |
| + if len(rows) != 0 { |
| + t.Errorf("rendered %d rows", len(rows)) |
| + } |
| + }) |
| + } |
| + }) |
| + |
| + // **Eligibility is the kind, never the outcome.** Each corpus leaves |
| + // §8.8's phone box empty for a different reason, and none of them is |
| + // licence to ask a model for a phone number. Counted at the recorder, |
| + // by call — a page assertion cannot tell "called and ignored" from |
| + // "never called", and the first is an egress nobody authorised. |
| + t.Run("a validator-backed kind never reaches the provider", func(t *testing.T) { |
| + for _, c := range []struct{ name, body string }{ |
| + {"zero valid candidates", "## Contact\n\nRing our phone any weekday."}, |
| + {"suppressed by the cap", "## Contact\n\nA phone: 01 547 7854. B phone: 01 598 1234. C phone: 01 555 0101. D phone: 01 555 0202."}, |
| + {"suppressed by a collision", "## Phone\n\n- 01 547 7854\n- 01 598 1234"}, |
| + {"suppressed by an over-long label", "## Contact\n\nThe number you must never give to any customer because it rings the boss: 01 547 7854"}, |
| + {"a validated candidate rendered", "## Contact\n\nOffice phone: 01 547 7854"}, |
| + } { |
| + t.Run(c.name, func(t *testing.T) { |
| + s := storeForTest(t) |
| + upload(t, s, "Tito office", c.body) |
| + rec := &recorder{} |
| + s.extractor = rec |
| + // Retrieved, or "zero calls" is true of an empty search and |
| + // the case proves nothing — addFacts returns before the model |
| + // path when there are no results. |
| + if p := searchFacts(t, s, "phone number"); len(p.Results) == 0 { |
| + t.Fatal("the corpus was not retrieved; this case proves nothing") |
| + } |
| + if len(rec.calls) != 0 { |
| + t.Errorf("a phone query sent %d requests to the model", len(rec.calls)) |
| + } |
| + }) |
| + } |
| + }) |
| + |
| + t.Run("a mixed query sends exactly one request, for the address", func(t *testing.T) { |
| + s := storeForTest(t) |
| + upload(t, s, "Tito office", "## Contact\n\nOffice phone: 01 547 7854\n\nOffice address: "+addr) |
| + rec := &recorder{reply: point(0, addr)} |
| + s.extractor = rec |
| + p := searchFacts(t, s, "phone number and postal address") |
| + if len(rec.calls) != 1 { |
| + t.Errorf("calls = %d, want exactly 1", len(rec.calls)) |
| + } |
| + var phone, address int |
| + for _, f := range p.Facts { |
| + switch f.Kind { |
| + case "phone": |
| + phone++ |
| + if f.Model != "" { |
| + t.Errorf("a phone row is marked as model output: %+v", f) |
| + } |
| + case "address": |
| + address++ |
| + } |
| + } |
| + if phone != 1 || address != 1 { |
| + t.Errorf("phone rows %d, address rows %d; want one each, independently", phone, address) |
| + } |
| + }) |
| + |
| + // Decision 5's floor, with its presence half: the SAME store and query |
| + // call the model once when it is wired and never when it is not. |
| + t.Run("AI off sends nothing; AI on sends one", func(t *testing.T) { |
| + s := storeForTest(t) |
| + upload(t, s, "Tito office", "## Contact\n\nOffice address: "+addr) |
| + rec := &recorder{reply: point(0, addr)} |
| + |
| + s.extractor = nil |
| + off := searchFacts(t, s, "office address") |
| + if len(rec.calls) != 0 || len(modelRows(off)) != 0 { |
| + t.Fatalf("AI off: %d calls, %d model rows", len(rec.calls), len(modelRows(off))) |
| + } |
| + s.extractor = rec |
| + on := searchFacts(t, s, "office address") |
| + if len(rec.calls) != 1 || len(modelRows(on)) != 1 { |
| + t.Fatalf("AI on: %d calls, %d model rows; the absence above proves nothing without this", len(rec.calls), len(modelRows(on))) |
| + } |
| + }) |
| + |
| + // The undetectable omission, asserted as the documented behaviour so |
| + // it is a known limit and not a surprise. §8.9: four addresses in the |
| + // passages, the model returns one, and every check passes. The day |
| + // something can detect this, this case should go red on purpose. |
| + t.Run("an omitted address cannot be detected (a stated limit)", func(t *testing.T) { |
| + s := storeForTest(t) |
| + upload(t, s, "Tito office", "## Our addresses\n\nA: 1 First Street, Cork. B: 2 Second Street, Cork. C: 3 Third Street, Cork. D: 4 Fourth Street, Cork.") |
| + rec := &recorder{reply: point(0, "1 First Street, Cork")} |
| + s.extractor = rec |
| + rows := modelRows(searchFacts(t, s, "address")) |
| + if len(rec.calls) != 1 { |
| + t.Fatalf("calls = %d: the model was never asked", len(rec.calls)) |
| + } |
| + if len(rows) != 1 { |
| + t.Errorf("rows = %d; this documents the limit, and the limit has changed", len(rows)) |
| + } |
| + }) |
| +} |
| + |
| +// A provider failure renders the page as AI-off would, and the log line |
| +// carries no content — even when the error handed back DOES carry some. |
| +// The provider package promises its errors are content-free; the store |
| +// does not rely on it. |
| +func TestAModelFailureLogsNoContent(t *testing.T) { |
| + const canary = "CANARY-9e1b-a-passage-quoted-in-an-error" |
| + for _, c := range []struct { |
| + name string |
| + err error |
| + want string |
| + }{ |
| + {"an error quoting content", fmt.Errorf("upstream said: %s", canary), "why=other"}, |
| + {"a truncation", fmt.Errorf("%w: %s", provider.ErrTruncated, canary), "why=truncated"}, |
| + {"a timeout", fmt.Errorf("%w: %s", context.DeadlineExceeded, canary), "why=deadline"}, |
| + } { |
| + t.Run(c.name, func(t *testing.T) { |
| + s := storeForTest(t) |
| + var buf bytes.Buffer |
| + s.logger = slog.New(slog.NewTextHandler(&buf, nil)) |
| + upload(t, s, "Tito office", "## Contact\n\nOffice address: 12 Main Street, Dublin 2") |
| + err := c.err |
| + s.extractor = &recorder{reply: func([]string) ([]provider.Pointer, error) { return nil, err }} |
| + p := searchFacts(t, s, "office address") |
| + if len(p.Results) == 0 { |
| + t.Fatal("the search itself failed; a model failure must not fail the page") |
| + } |
| + if len(modelRows(p)) != 0 { |
| + t.Errorf("model rows rendered after a failure") |
| + } |
| + // Presence: the failure WAS logged, and classified. |
| + if !strings.Contains(buf.String(), c.want) { |
| + t.Errorf("log = %q; want it to record %s", buf.String(), c.want) |
| + } |
| + if strings.Contains(buf.String(), canary) { |
| + t.Errorf("the log carries content: %q", buf.String()) |
| + } |
| + }) |
| + } |
| + |
| + // An abandoned keystroke is ordinary and says nothing at all. |
| + t.Run("a cancelled request logs nothing", func(t *testing.T) { |
| + s := storeForTest(t) |
| + var buf bytes.Buffer |
| + s.logger = slog.New(slog.NewTextHandler(&buf, nil)) |
| + upload(t, s, "Tito office", "## Contact\n\nOffice address: 12 Main Street, Dublin 2") |
| + s.extractor = &recorder{reply: func([]string) ([]provider.Pointer, error) { return nil, context.Canceled }} |
| + searchFacts(t, s, "office address") |
| + if buf.Len() != 0 { |
| + t.Errorf("logged %q for an abandoned keystroke", buf.String()) |
| + } |
| + }) |
| +} |
| + |
| +func lens(calls [][]string) []int { |
| + var out []int |
| + for _, c := range calls { |
| + out = append(out, len(c)) |
| + } |
| + return out |
| +} |
| + |
| +var _ = errors.Is // keep errors imported for future cases |
amadan