| 1 | package index |
| 2 | |
| 3 | import ( |
| 4 | "strings" |
| 5 | "testing" |
| 6 | ) |
| 7 | |
| 8 | func build(t *testing.T, entries ...Entry) *Index { |
| 9 | t.Helper() |
| 10 | ix, err := Open() |
| 11 | if err != nil { |
| 12 | t.Fatal(err) |
| 13 | } |
| 14 | t.Cleanup(func() { ix.Close() }) |
| 15 | if err := ix.Add(entries...); err != nil { |
| 16 | t.Fatal(err) |
| 17 | } |
| 18 | return ix |
| 19 | } |
| 20 | |
| 21 | func TestSearchFindsAPassage(t *testing.T) { |
| 22 | ix := build(t, |
| 23 | Entry{ChunkID: 1, DocumentID: 10, Heading: "FY2024 › Revenue", Text: "June revenue was 1,350,000 across all regions."}, |
| 24 | Entry{ChunkID: 2, DocumentID: 10, Heading: "FY2024 › Costs", Text: "Hosting costs fell after the migration."}, |
| 25 | ) |
| 26 | hits, err := ix.Search("june revenue", 10) |
| 27 | if err != nil { |
| 28 | t.Fatal(err) |
| 29 | } |
| 30 | if len(hits) == 0 { |
| 31 | t.Fatal("no hits for a query that plainly matches") |
| 32 | } |
| 33 | if hits[0].ChunkID != 1 { |
| 34 | t.Errorf("best hit is chunk %d, want 1", hits[0].ChunkID) |
| 35 | } |
| 36 | if !strings.Contains(hits[0].Snippet, "mark>") { |
| 37 | t.Errorf("snippet carries no highlight: %q", hits[0].Snippet) |
| 38 | } |
| 39 | } |
| 40 | |
| 41 | // The heading is weighted above the body. A passage filed under Revenue |
| 42 | // should beat one that merely says the word. |
| 43 | func TestHeadingOutweighsBody(t *testing.T) { |
| 44 | ix := build(t, |
| 45 | Entry{ChunkID: 1, DocumentID: 1, Heading: "Revenue", Text: "Figures for the period."}, |
| 46 | Entry{ChunkID: 2, DocumentID: 2, Heading: "Minutes", Text: "We briefly mentioned revenue and moved on."}, |
| 47 | ) |
| 48 | hits, err := ix.Search("revenue", 10) |
| 49 | if err != nil { |
| 50 | t.Fatal(err) |
| 51 | } |
| 52 | if len(hits) != 2 { |
| 53 | t.Fatalf("want 2 hits, got %d", len(hits)) |
| 54 | } |
| 55 | if hits[0].ChunkID != 1 { |
| 56 | t.Errorf("heading weighting not applied: best hit is chunk %d", hits[0].ChunkID) |
| 57 | } |
| 58 | } |
| 59 | |
| 60 | // Ordinary English contains apostrophes and punctuation. Fed raw to |
| 61 | // MATCH these are syntax errors, and a syntax error surfaces as an empty |
| 62 | // page that reads exactly like "nothing matched" — the worst available |
| 63 | // failure shape. |
| 64 | func TestPunctuationDoesNotBreakSearch(t *testing.T) { |
| 65 | ix := build(t, Entry{ChunkID: 1, DocumentID: 1, Heading: "Notes", Text: "June revenue was strong."}) |
| 66 | for _, q := range []string{ |
| 67 | "what's june revenue?", |
| 68 | "revenue: june", |
| 69 | "NEAR revenue", |
| 70 | "OR", |
| 71 | `revenue "june`, |
| 72 | "(june)", |
| 73 | "revenue AND", |
| 74 | "-june", |
| 75 | "revenue*", |
| 76 | } { |
| 77 | hits, err := ix.Search(q, 10) |
| 78 | if err != nil { |
| 79 | t.Errorf("Search(%q) errored: %v", q, err) |
| 80 | continue |
| 81 | } |
| 82 | if strings.Contains(strings.ToLower(q), "revenue") && len(hits) == 0 { |
| 83 | t.Errorf("Search(%q) found nothing but mentions revenue", q) |
| 84 | } |
| 85 | } |
| 86 | } |
| 87 | |
| 88 | func TestEmptyQueryReturnsNothingWithoutError(t *testing.T) { |
| 89 | ix := build(t, Entry{ChunkID: 1, DocumentID: 1, Heading: "H", Text: "body"}) |
| 90 | for _, q := range []string{"", " ", "!!!", "???"} { |
| 91 | hits, err := ix.Search(q, 10) |
| 92 | if err != nil { |
| 93 | t.Errorf("Search(%q) errored: %v", q, err) |
| 94 | } |
| 95 | if len(hits) != 0 { |
| 96 | t.Errorf("Search(%q) returned %d hits", q, len(hits)) |
| 97 | } |
| 98 | } |
| 99 | } |
| 100 | |
| 101 | func TestRemoveDocumentDropsItsChunks(t *testing.T) { |
| 102 | ix := build(t, |
| 103 | Entry{ChunkID: 1, DocumentID: 10, Heading: "A", Text: "keepme text"}, |
| 104 | Entry{ChunkID: 2, DocumentID: 20, Heading: "B", Text: "keepme text"}, |
| 105 | ) |
| 106 | if err := ix.RemoveDocument(10); err != nil { |
| 107 | t.Fatal(err) |
| 108 | } |
| 109 | hits, err := ix.Search("keepme", 10) |
| 110 | if err != nil { |
| 111 | t.Fatal(err) |
| 112 | } |
| 113 | if len(hits) != 1 || hits[0].DocumentID != 20 { |
| 114 | t.Fatalf("after removing doc 10, got %d hits %+v", len(hits), hits) |
| 115 | } |
| 116 | n, err := ix.Count() |
| 117 | if err != nil { |
| 118 | t.Fatal(err) |
| 119 | } |
| 120 | if n != 1 { |
| 121 | t.Errorf("Count = %d, want 1", n) |
| 122 | } |
| 123 | } |
| 124 | |
| 125 | func TestMatchQueryQuotesAndOrs(t *testing.T) { |
| 126 | // The trailing operand requires the query's words to appear |
| 127 | // together. It is a RANKING signal: FTS5 sums per-phrase |
| 128 | // contributions, so a passage carrying the words together scores |
| 129 | // above one carrying them scattered, while both are still returned. |
| 130 | // See MatchQuery. |
| 131 | // |
| 132 | // NEAR and not a phrase, because a phrase is ordered and "revenue |
| 133 | // june" then ranked differently from "june revenue" — see |
| 134 | // TestWordOrderDoesNotChangeTheRanking. |
| 135 | // |
| 136 | // "what's up" is the interesting row. The apostrophe splits the query |
| 137 | // into three terms and they are required together as three, which |
| 138 | // still matches a document's "what's up": the tokenizer is |
| 139 | // `porter unicode61`, and unicode61 drops the apostrophe on both |
| 140 | // sides while porter stems what is left. The phrase logic depends on |
| 141 | // both halves of that tokenizer, so swapping either changes this. |
| 142 | for in, want := range map[string]string{ |
| 143 | "june revenue": `"june" OR "revenue" OR NEAR("june" "revenue", 2)`, |
| 144 | "what's up": `"what" OR "s" OR "up" OR NEAR("what" "s" "up", 2)`, |
| 145 | " spaced out ": `"spaced" OR "out" OR NEAR("spaced" "out", 2)`, |
| 146 | "!!!": ``, |
| 147 | "café": `"café"`, |
| 148 | } { |
| 149 | if got := MatchQuery(in); got != want { |
| 150 | t.Errorf("MatchQuery(%q) = %q, want %q", in, got, want) |
| 151 | } |
| 152 | } |
| 153 | } |
| 154 | |
| 155 | // OR keeps recall, so the discriminating has to be done by ranking. A |
| 156 | // passage matching both real terms must beat one matching only a common |
| 157 | // word, or the OR decision has traded a false empty for a false answer. |
| 158 | func TestOrRecallStillRanksBestMatchFirst(t *testing.T) { |
| 159 | ix := build(t, |
| 160 | Entry{ChunkID: 1, DocumentID: 1, Heading: "Notes", Text: "What we discussed today."}, |
| 161 | Entry{ChunkID: 2, DocumentID: 2, Heading: "Revenue", Text: "June revenue was strong."}, |
| 162 | ) |
| 163 | hits, err := ix.Search("what is june revenue", 10) |
| 164 | if err != nil { |
| 165 | t.Fatal(err) |
| 166 | } |
| 167 | if len(hits) == 0 { |
| 168 | t.Fatal("no hits; OR recall is not working") |
| 169 | } |
| 170 | if hits[0].ChunkID != 2 { |
| 171 | t.Errorf("best hit is chunk %d, want 2 — ranking is not compensating for OR", hits[0].ChunkID) |
| 172 | } |
| 173 | } |
| 174 | |
| 175 | // Two indexes in one process must be independent. They were not: the |
| 176 | // shared-cache DSN used a fixed name, so every Index was the same |
| 177 | // in-memory database. The visible symptom was "table chunks already |
| 178 | // exists"; the invisible one, had the schema matched, would have been two |
| 179 | // teams' passages merged into one searchable corpus. |
| 180 | func TestTwoIndexesAreIndependent(t *testing.T) { |
| 181 | a, err := Open() |
| 182 | if err != nil { |
| 183 | t.Fatal(err) |
| 184 | } |
| 185 | defer a.Close() |
| 186 | b, err := Open() |
| 187 | if err != nil { |
| 188 | t.Fatalf("opening a second index failed: %v", err) |
| 189 | } |
| 190 | defer b.Close() |
| 191 | |
| 192 | if err := a.Add(Entry{ChunkID: 1, DocumentID: 1, Heading: "A", Text: "alpha secret"}); err != nil { |
| 193 | t.Fatal(err) |
| 194 | } |
| 195 | hits, err := b.Search("alpha", 10) |
| 196 | if err != nil { |
| 197 | t.Fatal(err) |
| 198 | } |
| 199 | if len(hits) != 0 { |
| 200 | t.Fatalf("index B can see index A's passages: %+v — indexes are sharing state", hits) |
| 201 | } |
| 202 | na, err := a.Count() |
| 203 | if err != nil { |
| 204 | t.Fatal(err) |
| 205 | } |
| 206 | nb, err := b.Count() |
| 207 | if err != nil { |
| 208 | t.Fatal(err) |
| 209 | } |
| 210 | if na != 1 || nb != 0 { |
| 211 | t.Errorf("counts are A=%d B=%d, want 1 and 0", na, nb) |
| 212 | } |
| 213 | } |
| 214 | |
| 215 | // §3.1 rules that FTS5 postings are content, so the index must never be |
| 216 | // backed by a file. Nothing tested that: a review pointed the DSN at a |
| 217 | // path on disk and the entire suite stayed green with plaintext postings |
| 218 | // in a file, because the only plaintext gate scanned the app database's |
| 219 | // directory and nothing looked at the index at all. |
| 220 | // |
| 221 | // MUTATION-TESTED 2026-09-04: changing Open's DSN to |
| 222 | // "file:"+os.TempDir()+"/x.db" makes this fail naming the path. Reverted. |
| 223 | func TestIndexIsBackedByNoFile(t *testing.T) { |
| 224 | ix, err := Open() |
| 225 | if err != nil { |
| 226 | t.Fatal(err) |
| 227 | } |
| 228 | defer ix.Close() |
| 229 | if err := ix.Add(Entry{ChunkID: 1, DocumentID: 1, Heading: "H", Text: "a passage of text"}); err != nil { |
| 230 | t.Fatal(err) |
| 231 | } |
| 232 | file, err := ix.File() |
| 233 | if err != nil { |
| 234 | t.Fatal(err) |
| 235 | } |
| 236 | if file != "" { |
| 237 | t.Fatalf("the index is backed by %q; FTS5 postings are plaintext and §3.1 forbids them reaching a file", file) |
| 238 | } |
| 239 | } |
| 240 | |
amadan