| 1 | package index |
| 2 | |
| 3 | import ( |
| 4 | "fmt" |
| 5 | "testing" |
| 6 | ) |
| 7 | |
| 8 | // buildTrap fills an index with the fixture that reproduces the ancestry |
| 9 | // scoring trap, and returns it. |
| 10 | // |
| 11 | // Two things about this fixture are load-bearing, and the first version |
| 12 | // of it had neither. |
| 13 | // |
| 14 | // FIRST, the shape. bm25() SUMS per-column contributions, so a row |
| 15 | // matching in path AND body outranks a row matching only in path under |
| 16 | // any positive weights — a fixture where the "good" answer also carries |
| 17 | // the term in its ancestry proves nothing. The discriminating shape is |
| 18 | // many pages whose ONLY match is an inherited ancestor title, against |
| 19 | // one page, outside that subtree, whose BODY answers the query. |
| 20 | // |
| 21 | // SECOND, the filler, and this is the subtler one. Without the 200 |
| 22 | // unrelated chunks below, "handbook" appears in EVERY indexed row, BM25's |
| 23 | // IDF term collapses to about zero, every score comes back as -1e-06, and |
| 24 | // the resulting order is floating-point noise that happens to look like a |
| 25 | // ranking. Measured: with no filler every score was -0.000001; with it, |
| 26 | // the path-only rows score -3.196 at PathWeight and -7.218 with the path |
| 27 | // folded into the heading column, against -4.961 for the body match. Only |
| 28 | // the second set of numbers can distinguish a working weight from an |
| 29 | // inert one. |
| 30 | // |
| 31 | // pathAsHeading models the mutation: ancestry folded into the heading |
| 32 | // column at heading weight, which is what a naive implementation does. |
| 33 | func buildTrap(t *testing.T, pathAsHeading bool) *Index { |
| 34 | t.Helper() |
| 35 | ix, err := Open() |
| 36 | if err != nil { |
| 37 | t.Fatal(err) |
| 38 | } |
| 39 | t.Cleanup(func() { ix.Close() }) |
| 40 | |
| 41 | var entries []Entry |
| 42 | var id int64 |
| 43 | add := func(docID int64, heading, path, body string) { |
| 44 | id++ |
| 45 | // Title is the document's own name. Every document in the real |
| 46 | // index has one indexed at BodyWeight, so a fixture without one |
| 47 | // would be measuring a table this app no longer has — and a |
| 48 | // title column is exactly the ingredient that could reintroduce |
| 49 | // the trap. |
| 50 | e := Entry{ChunkID: id, DocumentID: docID, Ordinal: 0, Heading: heading, Text: body, Title: heading} |
| 51 | if pathAsHeading { |
| 52 | if path != "" { |
| 53 | e.Heading = path + " › " + heading |
| 54 | } |
| 55 | } else { |
| 56 | e.Path = path |
| 57 | } |
| 58 | entries = append(entries, e) |
| 59 | } |
| 60 | |
| 61 | // Filler, so the query term is rare and IDF means something. |
| 62 | for i := 0; i < 200; i++ { |
| 63 | add(int64(1000+i), fmt.Sprintf("Note %d", i), "Archive › Notes", |
| 64 | fmt.Sprintf("Meeting %d discussed procurement, latency budgets and the vendor renewal.", i)) |
| 65 | } |
| 66 | // 20 pages under "Handbook". No body says "handbook"; they match only |
| 67 | // through the ancestry they inherited. |
| 68 | for i := 0; i < 20; i++ { |
| 69 | add(int64(100+i), fmt.Sprintf("Section %d", i), "Handbook › Onboarding", |
| 70 | fmt.Sprintf("Week %d covers expenses, travel booking and the equipment request form.", i)) |
| 71 | } |
| 72 | // The Handbook page itself. It matches "handbook" on its own title, |
| 73 | // and SHOULD rank — a search for "handbook" returning the handbook is |
| 74 | // the right answer. What must not happen is its twenty descendants, |
| 75 | // which match only through inherited ancestry, burying real content. |
| 76 | add(50, "Handbook", "", "The starting point for everyone new.") |
| 77 | // One page elsewhere whose BODY is the actual answer. |
| 78 | add(999, "Tooling", "Engineering › Platform", |
| 79 | "The handbook is generated from this repository and published nightly.") |
| 80 | |
| 81 | if err := ix.Add(entries...); err != nil { |
| 82 | t.Fatal(err) |
| 83 | } |
| 84 | return ix |
| 85 | } |
| 86 | |
| 87 | // TestAncestryCannotOutrankContent is the gate for the trap described on |
| 88 | // PathWeight: a document's ancestry repeats into every chunk beneath it, |
| 89 | // so ancestry scored at heading weight buries real answers under the |
| 90 | // whole subtree that merely sits below a matching ancestor title. |
| 91 | // |
| 92 | // MUTATION-TESTED 2026-09-05: folding the path into the heading column |
| 93 | // (buildTrap's pathAsHeading branch, which is what a naive implementation |
| 94 | // does) makes this fail — the top three hits become Sections 0, 1 and 2, |
| 95 | // and the page that answers the query falls below twenty that do not. |
| 96 | // TestAncestryTrapFlipIsObserved asserts that flip so this gate cannot |
| 97 | // quietly stop discriminating. Reverted. |
| 98 | func TestAncestryCannotOutrankContent(t *testing.T) { |
| 99 | ix := buildTrap(t, false) |
| 100 | hits, err := ix.Search("handbook", 25) |
| 101 | if err != nil { |
| 102 | t.Fatal(err) |
| 103 | } |
| 104 | if len(hits) == 0 { |
| 105 | t.Fatal("no hits: the fixture matches nothing, so this gate proves nothing") |
| 106 | } |
| 107 | // The property is NOT "999 is first" — the Handbook page itself |
| 108 | // matching its own title and ranking highly is correct. It is that no |
| 109 | // page which matched ONLY through inherited ancestry outranks the |
| 110 | // page whose body answers the question. |
| 111 | var contentRank = -1 |
| 112 | for i, h := range hits { |
| 113 | if h.DocumentID == 999 { |
| 114 | contentRank = i |
| 115 | break |
| 116 | } |
| 117 | } |
| 118 | if contentRank < 0 { |
| 119 | t.Fatal("the page whose body answers the query did not match at all") |
| 120 | } |
| 121 | for i, h := range hits { |
| 122 | if i >= contentRank { |
| 123 | break |
| 124 | } |
| 125 | if h.DocumentID >= 100 && h.DocumentID < 200 { |
| 126 | t.Errorf("ancestry outranked content: document %d matched only through its "+ |
| 127 | "inherited ancestry and ranks above the page whose body answers the "+ |
| 128 | "query (position %d vs %d)", h.DocumentID, i, contentRank) |
| 129 | } |
| 130 | } |
| 131 | } |
| 132 | |
| 133 | // TestAncestryTrapFlipIsObserved fails if the mutation does NOT change the |
| 134 | // ranking. A gate for a ranking property is worthless unless the ranking |
| 135 | // has been seen to move, and this codebase has shipped three gates that |
| 136 | // could not fail for the thing they named. |
| 137 | func TestAncestryTrapFlipIsObserved(t *testing.T) { |
| 138 | // The measurement is the RELATIVE position of two kinds of hit, not |
| 139 | // which document is first. Once titles are indexed the Handbook page |
| 140 | // itself is top under either weighting — legitimately, it is called |
| 141 | // Handbook — so a top-hit comparison stopped discriminating and this |
| 142 | // test said so rather than passing. |
| 143 | probe := func(pathAsHeading bool) (content int, ancestry int) { |
| 144 | ix := buildTrap(t, pathAsHeading) |
| 145 | hits, err := ix.Search("handbook", 30) |
| 146 | if err != nil { |
| 147 | t.Fatal(err) |
| 148 | } |
| 149 | content, ancestry = -1, -1 |
| 150 | for i, h := range hits { |
| 151 | if h.DocumentID == 999 && content < 0 { |
| 152 | content = i |
| 153 | } |
| 154 | if h.DocumentID >= 100 && h.DocumentID < 200 && ancestry < 0 { |
| 155 | ancestry = i |
| 156 | } |
| 157 | } |
| 158 | if content < 0 || ancestry < 0 { |
| 159 | t.Fatalf("fixture did not produce both kinds of hit (content=%d ancestry=%d)", content, ancestry) |
| 160 | } |
| 161 | return content, ancestry |
| 162 | } |
| 163 | |
| 164 | goodContent, goodAncestry := probe(false) |
| 165 | badContent, badAncestry := probe(true) |
| 166 | t.Logf("path column at %.1f : body-match at %d, best ancestry-only at %d", PathWeight, goodContent, goodAncestry) |
| 167 | t.Logf("path in heading : body-match at %d, best ancestry-only at %d", badContent, badAncestry) |
| 168 | |
| 169 | if goodContent > goodAncestry { |
| 170 | t.Errorf("with the path column, an ancestry-only page already outranks the body match "+ |
| 171 | "(%d vs %d)", goodAncestry, goodContent) |
| 172 | } |
| 173 | if badAncestry > badContent { |
| 174 | t.Errorf("the mutation did NOT change the ranking: with ancestry at heading weight the "+ |
| 175 | "body match is still above every ancestry-only page (%d vs %d). This fixture cannot "+ |
| 176 | "detect the trap, so TestAncestryCannotOutrankContent is a gate written to pass. "+ |
| 177 | "Strengthen the fixture until the order flips.", badAncestry, badContent) |
| 178 | } |
| 179 | } |
| 180 | |
| 181 | // TestTrapFixtureCanDiscriminate guards the FIXTURE, not production code, |
| 182 | // and it exists because this fixture silently stopped being able to prove |
| 183 | // anything once already. |
| 184 | // |
| 185 | // The first measurement of the trap was taken on a corpus where the query |
| 186 | // term appeared in every indexed row. BM25's IDF term collapsed to about |
| 187 | // zero, every score came back as -0.000001, and moving the path weight |
| 188 | // from 0.5 to 99 changed nothing — yet the rows still came back in an |
| 189 | // order, and that order looked exactly like a working ranking. A weight |
| 190 | // that is silently ignored ranks the same as a weight that is wrong, so |
| 191 | // TestAncestryCannotOutrankContent was passing for a reason unrelated to |
| 192 | // the property it names. |
| 193 | // |
| 194 | // There is no production mutation for this one: it asserts that the |
| 195 | // corpus and the engine can tell two weightings apart at all. Its |
| 196 | // mutation is therefore to the fixture. |
| 197 | // |
| 198 | // MUTATION-TESTED 2026-09-05: deleting the filler loop from buildTrap |
| 199 | // makes this fail on the degenerate-IDF assertion. Reverted. |
| 200 | func TestTrapFixtureCanDiscriminate(t *testing.T) { |
| 201 | ix := buildTrap(t, false) |
| 202 | score := func(pw float64) float64 { |
| 203 | var v float64 |
| 204 | // Document 100 is one of the ancestry-only matches, so its score |
| 205 | // depends on the path weight and nothing else. |
| 206 | // FOUR weights, one per column. This probe passed three while the |
| 207 | // table had four, which SQLite accepts and which silently made |
| 208 | // the path weight inert — caught by this test's own assertion |
| 209 | // when the title column landed in a merge. |
| 210 | err := ix.db.QueryRow( |
| 211 | `SELECT bm25(chunks, ?, ?, ?, ?) FROM chunks WHERE chunks MATCH ? AND document_id = 100`, |
| 212 | HeadingWeight, BodyWeight, TitleWeight, pw, MatchQuery("handbook")).Scan(&v) |
| 213 | if err != nil { |
| 214 | t.Fatal(err) |
| 215 | } |
| 216 | return v |
| 217 | } |
| 218 | low, high := score(PathWeight), score(99) |
| 219 | t.Logf("ancestry-only row: path weight %.1f → %.4f, path weight 99 → %.4f", PathWeight, low, high) |
| 220 | if low == high { |
| 221 | t.Fatalf("path weight is inert: %.9f at both %.1f and 99", low, PathWeight) |
| 222 | } |
| 223 | if low > -1e-4 { |
| 224 | t.Fatalf("score %.9f is ~0, so IDF is degenerate and this corpus cannot "+ |
| 225 | "discriminate a working weight from an inert one", low) |
| 226 | } |
| 227 | } |
| 228 | |
amadan