| 1 | package index |
| 2 | |
| 3 | import ( |
| 4 | "fmt" |
| 5 | "strings" |
| 6 | "testing" |
| 7 | "time" |
| 8 | ) |
| 9 | |
| 10 | // The query that found this: "phone number" on the real corpus put a |
| 11 | // passage of Virgin Media ACCOUNT numbers above the passage that states |
| 12 | // the office phone number. Paul hit it on the live instance, and the |
| 13 | // keyboard prime shipped the same day sent Enter to the wrong document. |
| 14 | // |
| 15 | // The corpus below is what makes the test able to fail, and it took two |
| 16 | // attempts to build. A two-document version scored every hit at exactly |
| 17 | // 0.0000: FTS5's bm25 clamps IDF to ~1e-6 for a term that appears in |
| 18 | // every document, so the scores collapse and the order is rowid. "number" |
| 19 | // has to be COMMON here and "phone" RARE, as on the real corpus, or this |
| 20 | // measures nothing. |
| 21 | // |
| 22 | // Measured on that corpus before the fix: |
| 23 | // |
| 24 | // "phone" OR "number" → chunk 1 at 2.5607, chunk 2 at 1.8670 |
| 25 | // "number" alone → every score 0.0000 |
| 26 | // "phone" alone → identical to "phone" OR "number" |
| 27 | // |
| 28 | // The second line is the finding. A common term contributes nothing to |
| 29 | // ranking, so a two-word query ranked exactly as its rarest word alone, |
| 30 | // and "phone number" could not be told from "phone". |
| 31 | func rankCorpus(t *testing.T) *Index { |
| 32 | t.Helper() |
| 33 | ix, err := Open() |
| 34 | if err != nil { |
| 35 | t.Fatal(err) |
| 36 | } |
| 37 | t.Cleanup(func() { ix.Close() }) |
| 38 | |
| 39 | var entries []Entry |
| 40 | add := func(heading, title, text string) { |
| 41 | entries = append(entries, Entry{ |
| 42 | ChunkID: int64(len(entries) + 1), |
| 43 | DocumentID: int64(len(entries) + 1), |
| 44 | Ordinal: len(entries), |
| 45 | Title: title, |
| 46 | Path: "Team Tito Limited", |
| 47 | Heading: heading, |
| 48 | Text: text, |
| 49 | }) |
| 50 | } |
| 51 | |
| 52 | // Chunk 1: "Phone" in a short, 3×-weighted heading, and no phone |
| 53 | // number in the body. Chunk 2: the answer, in the body, twice. |
| 54 | add("Utilities › Internet & Phone", "Office", |
| 55 | "Virgin Media - login - Office 7: account number 80854641 - Office 2: account number 80753900") |
| 56 | add("Office Address", "Office", |
| 57 | "Turn the knob. The office phone number is: +353 (0)1 547 7854. We also have the following unused phone numbers: +353 (0)1 598 1234.") |
| 58 | |
| 59 | // "number" common across the corpus, as on the real one. |
| 60 | add("Business & VAT Numbers", "Business & VAT Numbers", |
| 61 | "Our company number is 566334. Our VAT number is IE3384527RH. Our DUNS/D-U-N-S number is 985632364.") |
| 62 | add("Bank", "Banking", "The account number is 12345678 and the sort code is 93-00-11.") |
| 63 | add("Payroll", "Payroll", "Each employee number is assigned on their first day.") |
| 64 | add("Insurance", "Insurance", "The policy number is renewed every January.") |
| 65 | add("Revenue", "Q3 board pack", "June revenue was 1,350,000 across all regions.") |
| 66 | add("Costs", "Q3 board pack", "Hosting costs fell after the migration.") |
| 67 | add("Employee Offboarding", "Processes", "Collect the laptop and the serial number of any device issued.") |
| 68 | add("Company Registration", "Registration", "The registration number appears on every invoice.") |
| 69 | add("Invoices", "Finance", "Every invoice carries a sequential number for the auditor.") |
| 70 | add("Passport numbers", "Travel", "Each passport number is stored in the safe, never in a document.") |
| 71 | add("Suppliers", "Finance", "Each supplier has a reference number in the ledger.") |
| 72 | add("Meeting, 12 August", "Minutes", "We will not build a suite index app.") |
| 73 | add("Sealing notes", "Sealing", "A sheet's contents are sealed at rest.") |
| 74 | |
| 75 | if err := ix.Add(entries...); err != nil { |
| 76 | t.Fatal(err) |
| 77 | } |
| 78 | return ix |
| 79 | } |
| 80 | |
| 81 | // Broken by removing the NEAR operand from MatchQuery — red, then |
| 82 | // reverted. Broken again by ANDing it rather than ORing it, which |
| 83 | // returns the one passage and drops the other fourteen — red on the |
| 84 | // recall assertion below, then reverted. 2026-09-05. |
| 85 | func TestAdjacentTermsOutrankAScatteredCommonWord(t *testing.T) { |
| 86 | ix := rankCorpus(t) |
| 87 | hits, err := ix.Search("phone number", 25) |
| 88 | if err != nil { |
| 89 | t.Fatal(err) |
| 90 | } |
| 91 | if len(hits) == 0 { |
| 92 | t.Fatal("no hits at all") |
| 93 | } |
| 94 | if hits[0].ChunkID != 2 { |
| 95 | t.Errorf("top hit is chunk %d (%q), want chunk 2 — the passage that states the number", |
| 96 | hits[0].ChunkID, hits[0].Heading) |
| 97 | } |
| 98 | // Recall is the other half, and the half a NEAR-only query loses. |
| 99 | // Every passage carrying either word must still come back: requiring |
| 100 | // the words together would turn a working search into a one-result |
| 101 | // page, which is the plausible-empty-list failure with a better score. |
| 102 | if len(hits) < 10 { |
| 103 | t.Errorf("only %d hits; the boost has narrowed recall instead of reordering it", len(hits)) |
| 104 | } |
| 105 | } |
| 106 | |
| 107 | // The boost must be earned by the words being TOGETHER, not by both |
| 108 | // being present — which is what AND already gave, and what did not help. |
| 109 | // |
| 110 | // The first version of this test could not fail. It used two documents |
| 111 | // and no filler, so both terms appeared in every document, FTS5 clamped |
| 112 | // both IDFs to ~1e-6, every score collapsed to 0 and the order fell back |
| 113 | // to rowid — which happened to be the order it asserted. Removing the fix |
| 114 | // entirely left it green. It needed the filler corpus for the same reason |
| 115 | // the test above does, and it needed the scattered passage to be WINNING |
| 116 | // beforehand: a test where the right answer is already first proves |
| 117 | // nothing about what put it there. |
| 118 | // |
| 119 | // So the scattered passage says both words three times each and the |
| 120 | // adjacent one says them once. On term frequency alone the scattered one |
| 121 | // wins; only togetherness turns it over. |
| 122 | // |
| 123 | // Broken by removing the NEAR operand — red, then reverted. 2026-09-05. |
| 124 | func TestBoostNeedsTheWordsTogether(t *testing.T) { |
| 125 | ix := rankCorpus(t) |
| 126 | if err := ix.Add( |
| 127 | Entry{ChunkID: 90, DocumentID: 90, Ordinal: 0, Heading: "Scattered", Title: "Desk", |
| 128 | Text: "The phone, the phone, the phone. Later, ask for a number, a number, another number."}, |
| 129 | Entry{ChunkID: 91, DocumentID: 91, Ordinal: 0, Heading: "Adjacent", Title: "Desk", |
| 130 | Text: "The phone number is on the desk."}, |
| 131 | ); err != nil { |
| 132 | t.Fatal(err) |
| 133 | } |
| 134 | hits, err := ix.Search("phone number", 25) |
| 135 | if err != nil { |
| 136 | t.Fatal(err) |
| 137 | } |
| 138 | var scattered, adjacent int |
| 139 | for i, h := range hits { |
| 140 | switch h.ChunkID { |
| 141 | case 90: |
| 142 | scattered = i + 1 |
| 143 | case 91: |
| 144 | adjacent = i + 1 |
| 145 | } |
| 146 | } |
| 147 | if scattered == 0 || adjacent == 0 { |
| 148 | t.Fatalf("both passages must be returned; scattered=%d adjacent=%d of %d hits", |
| 149 | scattered, adjacent, len(hits)) |
| 150 | } |
| 151 | if adjacent > scattered { |
| 152 | t.Errorf("the scattered passage ranked %d and the adjacent one %d — "+ |
| 153 | "adjacency is not being rewarded, only presence", scattered, adjacent) |
| 154 | } |
| 155 | } |
| 156 | |
| 157 | // A single-word query has nothing to be near, and must not grow a |
| 158 | // spurious operand. |
| 159 | func TestSingleTermQueryIsUnchanged(t *testing.T) { |
| 160 | if got, want := MatchQuery("revenue"), `"revenue"`; got != want { |
| 161 | t.Errorf("MatchQuery(%q) = %q, want %q", "revenue", got, want) |
| 162 | } |
| 163 | } |
| 164 | |
| 165 | // Typing the same two words the other way round must not change what |
| 166 | // comes back first. |
| 167 | // |
| 168 | // This is a regression the FIRST version of the togetherness boost |
| 169 | // introduced, found by review. It used an ordered phrase, so "revenue |
| 170 | // june" scored a coincidental adjacency across a full stop — "…helped |
| 171 | // revenue. June also saw…" — above the passage whose heading IS Revenue, |
| 172 | // and the keyboard prime then sent Enter there. Measured before the fix: |
| 173 | // "june revenue" put the right passage at 5.563 and the coincidence at |
| 174 | // 2.994; "revenue june" put the coincidence FIRST at 5.520. |
| 175 | // |
| 176 | // Before any boost existed the two spellings ranked identically, because |
| 177 | // order could not matter. NEAR restores that. |
| 178 | // |
| 179 | // Broken by replacing NEAR with the ordered phrase — red, then reverted. |
| 180 | // 2026-09-05. |
| 181 | func TestWordOrderDoesNotChangeTheRanking(t *testing.T) { |
| 182 | ix := rankCorpus(t) |
| 183 | if err := ix.Add( |
| 184 | Entry{ChunkID: 80, DocumentID: 80, Ordinal: 0, Heading: "Revenue", Title: "Q3 board pack", |
| 185 | Text: "June revenue was 1,350,000 across all regions, up from 1,180,000."}, |
| 186 | Entry{ChunkID: 81, DocumentID: 81, Ordinal: 0, Heading: "Costs", Title: "Q3 board pack", |
| 187 | Text: "Hosting fell, which helped revenue. June also saw the migration finish."}, |
| 188 | ); err != nil { |
| 189 | t.Fatal(err) |
| 190 | } |
| 191 | |
| 192 | rank := func(q string) []int64 { |
| 193 | hits, err := ix.Search(q, 25) |
| 194 | if err != nil { |
| 195 | t.Fatal(err) |
| 196 | } |
| 197 | var ids []int64 |
| 198 | for _, h := range hits { |
| 199 | if h.ChunkID == 80 || h.ChunkID == 81 { |
| 200 | ids = append(ids, h.ChunkID) |
| 201 | } |
| 202 | } |
| 203 | return ids |
| 204 | } |
| 205 | forward, reversed := rank("june revenue"), rank("revenue june") |
| 206 | if len(forward) != 2 || len(reversed) != 2 { |
| 207 | t.Fatalf("both passages must come back either way round: forward=%v reversed=%v", forward, reversed) |
| 208 | } |
| 209 | if forward[0] != 80 { |
| 210 | t.Errorf(`"june revenue" ranked chunk %d first, want 80`, forward[0]) |
| 211 | } |
| 212 | if reversed[0] != forward[0] { |
| 213 | t.Errorf(`"revenue june" ranked chunk %d first but "june revenue" ranked chunk %d — `+ |
| 214 | `word order is changing the answer`, reversed[0], forward[0]) |
| 215 | } |
| 216 | } |
| 217 | |
| 218 | // A search box is not a document. Without a cap the OR chain is |
| 219 | // quadratic: 2.8ms at 100 terms, 97ms at 1000, 2.25s at 5000 — all of it |
| 220 | // under the index's read lock, so one pasted essay stalls ingest. |
| 221 | // |
| 222 | // Broken by removing the cap — red on the timing assertion, then |
| 223 | // reverted. 2026-09-05. |
| 224 | func TestAbsurdlyLongQueryIsCapped(t *testing.T) { |
| 225 | var sb strings.Builder |
| 226 | for i := 0; i < 5000; i++ { |
| 227 | fmt.Fprintf(&sb, "term%d ", i) |
| 228 | } |
| 229 | q := MatchQuery(sb.String()) |
| 230 | if n := strings.Count(q, " OR ") + 1; n > 66 { |
| 231 | t.Errorf("a 5000-term query produced %d operands; the cap is not holding", n) |
| 232 | } |
| 233 | |
| 234 | // And it must still run quickly against a real index, which is the |
| 235 | // property the cap exists for rather than the operand count. |
| 236 | ix := rankCorpus(t) |
| 237 | start := time.Now() |
| 238 | if _, err := ix.Search(sb.String(), 25); err != nil { |
| 239 | t.Fatalf("a very long query errored: %v", err) |
| 240 | } |
| 241 | if d := time.Since(start); d > 250*time.Millisecond { |
| 242 | t.Errorf("a 5000-term search took %s; it holds the index read lock throughout", d) |
| 243 | } |
| 244 | } |
| 245 | |
amadan