amadan
fix(index): make the togetherness boost order-free, and cap the terms
Both from the second review of this fix. The ordered phrase I added reintroduced the bug it fixed, one spelling over. Measured: "june revenue" ranked the Revenue passage at 5.563 and a coincidental adjacency across a full stop — "…helped revenue. June also saw…" — at 2.994; "revenue june" ranked the coincidence FIRST at 5.520. Before the boost existed the two spellings ranked identically, because order could not matter. So the fix made a plausible query worse, in exactly the class it was written to close, and the prime then sends Enter to the coincidence. NEAR is order-free and restores that. The distance is measured, not picked. On "office phone number": NEAR/0 ranked the WRONG passage first, worse than the phrase; NEAR/2 got it right; NEAR/5 promoted a loose match over a tight one. Two tolerates the small words English puts between the words someone searched for. What NEAR still cannot see is punctuation, because FTS5 tokenises it away. That limit is now written where the next person will look, so they do not go back to the weights for it. Second: no cap on terms. 2.8ms at 100, 97ms at 1000, 2.25s at 5000 — all under the index read lock, so one pasted essay stalls ingest for everyone, and the per-keystroke /search route makes it cheap to do by accident. Capped at 64, which is longer than any query a person types. §9.1's 1 KiB cap on q would bound it at the request instead and is still not implemented; this does not wait for it. Seven mutations, all red then reverted: the operand removed, the ordered phrase restored, the distance changed, the operand ANDed, the cap removed, and the operand added to single-term queries. Also: the spec said gate 17's list "is read from" §9.1, which reads as a live guard. It is not — nothing walks the router in memoriatest, so the route list is guarded by nobody today, and the spec now says so. Claiming a guard the code does not have is the exact shape §1.4 is about. And the tokenizer is `porter unicode61`, not unicode61: stemming is part of why "phone numbers" matches, so the comment names both halves. Reviewed by Fable 5.1 (claude-fable-5-1), third pass on this branch. 🤖 Written by Claude Opus 5, reviewed by Fable 5.1 and Paul.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
4 files changed,
+169
−37
docs/superpowers/specs/2026-09-03-memoria-design.md+5 −2internal/memoria/index/index.go+46 −10internal/memoria/index/index_test.go+17 −10internal/memoria/index/rank_test.go+101 −15
diff --git a/docs/superpowers/specs/2026-09-03-memoria-design.md b/docs/superpowers/specs/2026-09-03-memoria-design.md| index 3b5db14..469168c 100644 |
| --- a/docs/superpowers/specs/2026-09-03-memoria-design.md |
| +++ b/docs/superpowers/specs/2026-09-03-memoria-design.md |
| @@ -2011,8 +2011,11 @@ memoria exposes exactly two surfaces: |
| path**, and a sibling presenting a connector token is refused: the two |
| credential sets are disjoint and separately scoped. |
| -**What the app actually serves, as of 2026-09-05**, because gate 17's |
| -list is literal and this is where it is read from: |
| +**What the app actually serves, as of 2026-09-05.** Gate 17 is specified |
| +and **not implemented on this branch** — nothing walks the router in |
| +`internal/memoriatest`, so this list is guarded by nobody today. It is |
| +written down so that the gate, when it lands, is read from a list that |
| +matches the app rather than from memory: |
| `GET /`, `GET /search`, `GET /documents/{id}`, `POST /documents`, |
| `POST /documents/{id}/delete`, the connector routes under |
diff --git a/internal/memoria/index/index.go b/internal/memoria/index/index.go| index 3db1e36..179eb9d 100644 |
| --- a/internal/memoria/index/index.go |
| +++ b/internal/memoria/index/index.go |
| @@ -299,12 +299,27 @@ func MatchQuery(raw string) string { |
| if len(terms) == 0 { |
| return "" |
| } |
| + // A search box is not a document. Beyond this the OR chain costs |
| + // more than the answer is worth: measured at 2.8ms for 100 terms, |
| + // 97ms for 1000 and 2.25s for 5000 — and Search holds the index's |
| + // read lock throughout, so one pasted essay stalls ingest for |
| + // everyone. 64 is longer than any query a person types and cheap |
| + // (~3ms). Terms past it are dropped, which narrows a query nobody |
| + // typed rather than refusing one somebody did. |
| + // |
| + // §9.1 also specifies q capped at 1 KiB, which would bound this at |
| + // the request instead and is not implemented; this cap does not |
| + // depend on that one arriving. |
| + const maxTerms = 64 |
| + if len(terms) > maxTerms { |
| + terms = terms[:maxTerms] |
| + } |
| quoted := make([]string, 0, len(terms)+1) |
| for _, t := range terms { |
| quoted = append(quoted, `"`+t+`"`) |
| } |
| - // The whole query as one adjacent phrase, added as a further OR |
| - // operand rather than replacing anything. |
| + // The query's words required to appear TOGETHER, added as a further |
| + // OR operand rather than replacing anything. |
| // |
| // This is the ranking half of the OR decision, and it was missing. |
| // OR keeps recall, and the comment on TestOrRecallStillRanksBestMatchFirst |
| @@ -316,17 +331,38 @@ func MatchQuery(raw string) string { |
| // passage that actually stated the office phone number. |
| // |
| // FTS5 scores each phrase in the expression and sums them, so a |
| - // passage carrying the words ADJACENTLY earns a third contribution |
| + // passage carrying the words together earns a further contribution |
| // and rises; one carrying them scattered is untouched. Measured on |
| - // the fixture in rank_test.go: the right passage 1.8670 → 4.3786, |
| - // the wrong one unchanged at 2.5607, all fifteen still returned. |
| + // the fixture in rank_test.go: the right passage 1.8670 → 4.0322, |
| + // the wrong one unchanged at 2.7670, all fifteen still returned. |
| + // |
| + // An OR operand and not AND, and not on its own: both of those |
| + // return only passages carrying the words together — one result |
| + // where there were twenty-five. Narrowing recall to fix an ordering |
| + // is the plausible-empty-list failure with a better score. |
| + // |
| + // NEAR rather than a plain phrase, and that was a review finding |
| + // rather than a first instinct. A phrase is ORDERED, so "revenue |
| + // june" scored a coincidental adjacency across a full stop above the |
| + // passage whose heading IS Revenue — the same failure this whole |
| + // function exists to fix, reintroduced by the fix. Before the phrase |
| + // existed the two spellings ranked identically, because order could |
| + // not matter; NEAR restores that and is what a person means by |
| + // typing two words. |
| + // |
| + // Distance 2, measured rather than picked. On "office phone number": |
| + // NEAR/0 ranked the WRONG passage first (worse than the phrase), |
| + // NEAR/2 got it right, NEAR/5 promoted a loose match over a tight |
| + // one. Two tolerates the small words English puts between the words |
| + // someone searched for. |
| // |
| - // An OR operand and not AND, and not the phrase alone: both of those |
| - // return only passages carrying the phrase — one result where there |
| - // were twenty-five. Narrowing recall to fix an ordering is the |
| - // plausible-empty-list failure with a better score. |
| + // What it cannot see is punctuation. FTS5 tokenises it away, so |
| + // "…helped revenue. June also…" is adjacent here and there is no |
| + // expression that says otherwise — the limit is the tokenizer's, not |
| + // this line's, and the next person should not go looking at the |
| + // weights for it. |
| if len(terms) > 1 { |
| - quoted = append(quoted, `"`+strings.Join(terms, " ")+`"`) |
| + quoted = append(quoted, `NEAR(`+strings.Join(quoted, " ")+`, 2)`) |
| } |
| return strings.Join(quoted, " OR ") |
| } |
diff --git a/internal/memoria/index/index_test.go b/internal/memoria/index/index_test.go| index 20a81b6..4c4bc06 100644 |
| --- a/internal/memoria/index/index_test.go |
| +++ b/internal/memoria/index/index_test.go |
| @@ -123,19 +123,26 @@ func TestRemoveDocumentDropsItsChunks(t *testing.T) { |
| } |
| func TestMatchQueryQuotesAndOrs(t *testing.T) { |
| - // The trailing operand is the whole query as one adjacent phrase. It |
| - // is a RANKING signal: FTS5 sums per-phrase contributions, so a |
| - // passage carrying the words together scores above one carrying them |
| - // scattered, while both are still returned. See MatchQuery. |
| + // The trailing operand requires the query's words to appear |
| + // together. It is a RANKING signal: FTS5 sums per-phrase |
| + // contributions, so a passage carrying the words together scores |
| + // above one carrying them scattered, while both are still returned. |
| + // See MatchQuery. |
| + // |
| + // NEAR and not a phrase, because a phrase is ordered and "revenue |
| + // june" then ranked differently from "june revenue" — see |
| + // TestWordOrderDoesNotChangeTheRanking. |
| // |
| // "what's up" is the interesting row. The apostrophe splits the query |
| - // into three terms, and the phrase reassembles them as "what s up" — |
| - // which matches, because unicode61 tokenises the apostrophe out of a |
| - // document's "what's up" in exactly the same way. |
| + // into three terms and they are required together as three, which |
| + // still matches a document's "what's up": the tokenizer is |
| + // `porter unicode61`, and unicode61 drops the apostrophe on both |
| + // sides while porter stems what is left. The phrase logic depends on |
| + // both halves of that tokenizer, so swapping either changes this. |
| for in, want := range map[string]string{ |
| - "june revenue": `"june" OR "revenue" OR "june revenue"`, |
| - "what's up": `"what" OR "s" OR "up" OR "what s up"`, |
| - " spaced out ": `"spaced" OR "out" OR "spaced out"`, |
| + "june revenue": `"june" OR "revenue" OR NEAR("june" "revenue", 2)`, |
| + "what's up": `"what" OR "s" OR "up" OR NEAR("what" "s" "up", 2)`, |
| + " spaced out ": `"spaced" OR "out" OR NEAR("spaced" "out", 2)`, |
| "!!!": ``, |
| "café": `"café"`, |
| } { |
diff --git a/internal/memoria/index/rank_test.go b/internal/memoria/index/rank_test.go| index 9f7d24f..34eab93 100644 |
| --- a/internal/memoria/index/rank_test.go |
| +++ b/internal/memoria/index/rank_test.go |
| @@ -1,6 +1,11 @@ |
| package index |
| -import "testing" |
| +import ( |
| + "fmt" |
| + "strings" |
| + "testing" |
| + "time" |
| +) |
| // The query that found this: "phone number" on the real corpus put a |
| // passage of Virgin Media ACCOUNT numbers above the passage that states |
| @@ -73,8 +78,8 @@ func rankCorpus(t *testing.T) *Index { |
| return ix |
| } |
| -// Broken by removing the phrase operand from MatchQuery — red, then |
| -// reverted. Broken again by adding the phrase but ANDing it, which |
| +// Broken by removing the NEAR operand from MatchQuery — red, then |
| +// reverted. Broken again by ANDing it rather than ORing it, which |
| // returns the one passage and drops the other fourteen — red on the |
| // recall assertion below, then reverted. 2026-09-05. |
| func TestAdjacentTermsOutrankAScatteredCommonWord(t *testing.T) { |
| @@ -90,17 +95,17 @@ func TestAdjacentTermsOutrankAScatteredCommonWord(t *testing.T) { |
| t.Errorf("top hit is chunk %d (%q), want chunk 2 — the passage that states the number", |
| hits[0].ChunkID, hits[0].Heading) |
| } |
| - // Recall is the other half, and the half a phrase-only query loses. |
| - // Every passage carrying either word must still come back: narrowing |
| - // to the phrase would turn a working search into a one-result page, |
| - // which is the plausible-empty-list failure wearing a better score. |
| + // Recall is the other half, and the half a NEAR-only query loses. |
| + // Every passage carrying either word must still come back: requiring |
| + // the words together would turn a working search into a one-result |
| + // page, which is the plausible-empty-list failure with a better score. |
| if len(hits) < 10 { |
| - t.Errorf("only %d hits; the phrase boost has narrowed recall instead of reordering it", len(hits)) |
| + t.Errorf("only %d hits; the boost has narrowed recall instead of reordering it", len(hits)) |
| } |
| } |
| -// The boost must be earned by ADJACENCY, not by both words merely being |
| -// present — which is what AND already gave, and what did not help. |
| +// The boost must be earned by the words being TOGETHER, not by both |
| +// being present — which is what AND already gave, and what did not help. |
| // |
| // The first version of this test could not fail. It used two documents |
| // and no filler, so both terms appeared in every document, FTS5 clamped |
| @@ -113,11 +118,10 @@ func TestAdjacentTermsOutrankAScatteredCommonWord(t *testing.T) { |
| // |
| // So the scattered passage says both words three times each and the |
| // adjacent one says them once. On term frequency alone the scattered one |
| -// wins; only adjacency turns it over. |
| +// wins; only togetherness turns it over. |
| // |
| -// Broken by removing the phrase operand — red, then reverted. |
| -// 2026-09-05. |
| -func TestPhraseBoostNeedsAdjacency(t *testing.T) { |
| +// Broken by removing the NEAR operand — red, then reverted. 2026-09-05. |
| +func TestBoostNeedsTheWordsTogether(t *testing.T) { |
| ix := rankCorpus(t) |
| if err := ix.Add( |
| Entry{ChunkID: 90, DocumentID: 90, Ordinal: 0, Heading: "Scattered", Title: "Desk", |
| @@ -150,9 +154,91 @@ func TestPhraseBoostNeedsAdjacency(t *testing.T) { |
| } |
| } |
| -// A single-word query has no phrase, and must not grow a spurious one. |
| +// A single-word query has nothing to be near, and must not grow a |
| +// spurious operand. |
| func TestSingleTermQueryIsUnchanged(t *testing.T) { |
| if got, want := MatchQuery("revenue"), `"revenue"`; got != want { |
| t.Errorf("MatchQuery(%q) = %q, want %q", "revenue", got, want) |
| } |
| } |
| + |
| +// Typing the same two words the other way round must not change what |
| +// comes back first. |
| +// |
| +// This is a regression the FIRST version of the togetherness boost |
| +// introduced, found by review. It used an ordered phrase, so "revenue |
| +// june" scored a coincidental adjacency across a full stop — "…helped |
| +// revenue. June also saw…" — above the passage whose heading IS Revenue, |
| +// and the keyboard prime then sent Enter there. Measured before the fix: |
| +// "june revenue" put the right passage at 5.563 and the coincidence at |
| +// 2.994; "revenue june" put the coincidence FIRST at 5.520. |
| +// |
| +// Before any boost existed the two spellings ranked identically, because |
| +// order could not matter. NEAR restores that. |
| +// |
| +// Broken by replacing NEAR with the ordered phrase — red, then reverted. |
| +// 2026-09-05. |
| +func TestWordOrderDoesNotChangeTheRanking(t *testing.T) { |
| + ix := rankCorpus(t) |
| + if err := ix.Add( |
| + Entry{ChunkID: 80, DocumentID: 80, Ordinal: 0, Heading: "Revenue", Title: "Q3 board pack", |
| + Text: "June revenue was 1,350,000 across all regions, up from 1,180,000."}, |
| + Entry{ChunkID: 81, DocumentID: 81, Ordinal: 0, Heading: "Costs", Title: "Q3 board pack", |
| + Text: "Hosting fell, which helped revenue. June also saw the migration finish."}, |
| + ); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + rank := func(q string) []int64 { |
| + hits, err := ix.Search(q, 25) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var ids []int64 |
| + for _, h := range hits { |
| + if h.ChunkID == 80 || h.ChunkID == 81 { |
| + ids = append(ids, h.ChunkID) |
| + } |
| + } |
| + return ids |
| + } |
| + forward, reversed := rank("june revenue"), rank("revenue june") |
| + if len(forward) != 2 || len(reversed) != 2 { |
| + t.Fatalf("both passages must come back either way round: forward=%v reversed=%v", forward, reversed) |
| + } |
| + if forward[0] != 80 { |
| + t.Errorf(`"june revenue" ranked chunk %d first, want 80`, forward[0]) |
| + } |
| + if reversed[0] != forward[0] { |
| + t.Errorf(`"revenue june" ranked chunk %d first but "june revenue" ranked chunk %d — `+ |
| + `word order is changing the answer`, reversed[0], forward[0]) |
| + } |
| +} |
| + |
| +// A search box is not a document. Without a cap the OR chain is |
| +// quadratic: 2.8ms at 100 terms, 97ms at 1000, 2.25s at 5000 — all of it |
| +// under the index's read lock, so one pasted essay stalls ingest. |
| +// |
| +// Broken by removing the cap — red on the timing assertion, then |
| +// reverted. 2026-09-05. |
| +func TestAbsurdlyLongQueryIsCapped(t *testing.T) { |
| + var sb strings.Builder |
| + for i := 0; i < 5000; i++ { |
| + fmt.Fprintf(&sb, "term%d ", i) |
| + } |
| + q := MatchQuery(sb.String()) |
| + if n := strings.Count(q, " OR ") + 1; n > 66 { |
| + t.Errorf("a 5000-term query produced %d operands; the cap is not holding", n) |
| + } |
| + |
| + // And it must still run quickly against a real index, which is the |
| + // property the cap exists for rather than the operand count. |
| + ix := rankCorpus(t) |
| + start := time.Now() |
| + if _, err := ix.Search(sb.String(), 25); err != nil { |
| + t.Fatalf("a very long query errored: %v", err) |
| + } |
| + if d := time.Since(start); d > 250*time.Millisecond { |
| + t.Errorf("a 5000-term search took %s; it holds the index read lock throughout", d) |
| + } |
| +} |