oficina / memoria Public

fix: index the document title, which was searchable nowhere

Reported from the live instance: a pasted document titled "Passport
numbers" could not be found by "passport". The page then said "There is 1
passage indexed, so this is a real absence rather than an empty index" —
asserting a real absence over the codebase's own worst failure shape,
on the line written to prevent it. It was not an absence. The word was
in the corpus; nothing looked at the column holding it.

The FTS5 table had two indexed columns, heading and body, and the title
is neither. A chunk's heading trail comes from Markdown headings *inside*
the body, so a plain paste has an empty trail and the only searchable
text is the body. The title was sealed, opened for display on every
result row, and never handed to the index at all.

Adds a title column, populated in indexDocument so that Ingest and
Rebuild both fill it — a title indexed on only one of those paths would
make a document findable until the next restart, which is a worse bug
than this one. It sits after body because snippet() and bm25() address
columns by declared position; putting it first would move body from 1 to
2 and silently change which column is highlighted.

Weighted 1.0, the same as body rather than the heading's 3.0. The bug is
that a title term found nothing, not that it ranked low, and a title
match outranking a document that genuinely discusses the term would be a
different wrong answer. The known consequence is the one the repeated
heading trail already has: a title match hits every chunk of that
document.

On the first test: it passed against the unfixed code. The search page
lists every document below the results, so asserting the title appears
somewhere in the HTML is satisfied by a 0-result page. It now counts
rst-result-title rows, and was watched failing 0-want-1 before the fix
and passing after.

No migration: the index is in-memory and rebuilt at boot, so an existing
corpus becomes searchable by title on restart with nothing re-ingested.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01YJRwhRTr2kBeBjQNuH4HWm

Paul Campbell pushed by paul@keymail.dev 1 month ago cf77e8ee91bcb0274871d5979ec00c92bbae4692 parent 87f2f54

3 files changed, +72 −4

internal/memoria/index/index.go +26 −4
internal/memoria/store.go +13 −0
internal/memoriatest/app_test.go +33 −0

diff --git a/internal/memoria/index/index.go b/internal/memoria/index/index.go

index 30d3795..d5aa976 100644

--- a/internal/memoria/index/index.go

+++ b/internal/memoria/index/index.go

@@ -76,6 +76,17 @@ func Open() (*Index, error) {

CREATE VIRTUAL TABLE chunks USING fts5(

heading,

body,

+ -- title is the document's own title, repeated into every one

+ -- of its chunks. Without it a document is unfindable by the

+ -- words in its name: a pasted "Passport numbers" whose body

+ -- is numbers matched nothing for "passport", and the page

+ -- reported that as a real absence.

+ --

+ -- It goes AFTER body deliberately. snippet() and bm25()

+ -- address columns by declared position, so putting it first

+ -- would move body from 1 to 2 and silently change which

+ -- column gets highlighted.

+ title,

chunk_id UNINDEXED,

document_id UNINDEXED,

ordinal UNINDEXED,

@@ -102,6 +113,9 @@ type Entry struct {

Ordinal int

Heading string

Text string

+ // Title is the document's title. The same value for every chunk of

+ // a document, as the heading trail is for every chunk of a section.

+ Title string

}

// Add indexes entries in one transaction.

@@ -113,13 +127,13 @@ func (i *Index) Add(entries ...Entry) error {

return err

}

defer tx.Rollback()

- st, err := tx.Prepare(`INSERT INTO chunks (heading, body, chunk_id, document_id, ordinal) VALUES (?, ?, ?, ?, ?)`)

+ st, err := tx.Prepare(`INSERT INTO chunks (heading, body, title, chunk_id, document_id, ordinal) VALUES (?, ?, ?, ?, ?, ?)`)

if err != nil {

return err

}

defer st.Close()

for _, e := range entries {

- if _, err := st.Exec(e.Heading, e.Text, e.ChunkID, e.DocumentID, e.Ordinal); err != nil {

+ if _, err := st.Exec(e.Heading, e.Text, e.Title, e.ChunkID, e.DocumentID, e.Ordinal); err != nil {

return err

}

@@ -149,6 +163,14 @@ func (i *Index) Count() (int, error) {

// Search runs a BM25 query. The heading column is weighted above the

// body: a passage under "Revenue" is a better answer to "revenue" than

// one that mentions the word in passing.

+//

+// Title carries the same weight as body, not more. The bug it fixes is

+// that a title term found nothing at all; ranking a title match above a

+// document that genuinely discusses the term would be a different and

+// worse answer. Note the consequence, which is the same one the repeated

+// heading trail already has: a title match hits every chunk of that

+// document, so a one-word title query can fill the page with one

+// document. Acceptable while documents are small; revisit with §12.

func (i *Index) Search(query string, limit int) ([]Hit, error) {

q := MatchQuery(query)

if q == "" {

@@ -162,10 +184,10 @@ func (i *Index) Search(query string, limit int) ([]Hit, error) {

rows, err := i.db.Query(`

SELECT chunk_id, document_id, ordinal, heading,

snippet(chunks, 1, '<mark>', '</mark>', '…', 24),

- bm25(chunks, 3.0, 1.0)

+ bm25(chunks, 3.0, 1.0, 1.0)

FROM chunks

WHERE chunks MATCH ?

- ORDER BY bm25(chunks, 3.0, 1.0)

+ ORDER BY bm25(chunks, 3.0, 1.0, 1.0)

LIMIT ?`, q, limit)

if err != nil {

return nil, err

diff --git a/internal/memoria/store.go b/internal/memoria/store.go

index fefde48..98525a6 100644

--- a/internal/memoria/store.go

+++ b/internal/memoria/store.go

@@ -195,6 +195,18 @@ func (s *Store) Ingest(ctx context.Context, source, key, title, body string) (in

// indexDocument opens a document's chunks and adds them to the index.

func (s *Store) indexDocument(ctx context.Context, docID int64) error {

+ // The title is indexed alongside every chunk, so it is read here

+ // rather than left to the caller: Ingest and Rebuild both land in

+ // this function, and a title indexed on only one of those paths

+ // would make a document findable until the next restart.

+ var doc Document

+ if err := s.db.G.WithContext(ctx).Where("id = ?", docID).First(&doc).Error; err != nil {

+ return err

+ }

+ title, err := s.sealer.Open(seal.PurposeTitle, doc.SealedTitle)

+ if err != nil {

+ return fmt.Errorf("open document title %d: %w", docID, err)

+ }

var rows []Chunk

if err := s.db.G.WithContext(ctx).Where("document_id = ?", docID).Order("ordinal").Find(&rows).Error; err != nil {

return err

@@ -215,6 +227,7 @@ func (s *Store) indexDocument(ctx context.Context, docID int64) error {

Ordinal: r.Ordinal,

Heading: string(heading),

Text: string(text),

+ Title: string(title),

})

}

if len(entries) == 0 {

diff --git a/internal/memoriatest/app_test.go b/internal/memoriatest/app_test.go

index 3b64fb2..283606d 100644

--- a/internal/memoriatest/app_test.go

+++ b/internal/memoriatest/app_test.go

@@ -211,3 +211,36 @@ func TestEditingUnderTheSameKeyUpdatesOneDocument(t *testing.T) {

t.Error("the new passage was not indexed")

}

+// A term that appears only in the document's title must find the

+// document. Reported from the live instance: a paste titled "Passport

+// numbers" whose body was numbers could not be found by "passport", and

+// the page said so confidently — "1 passage indexed, so this is a real

+// absence". It was not an absence. The index held two columns, heading

+// and body, and neither is the title: a chunk's heading trail comes from

+// Markdown headings *inside* the body, so a plain paste has none at all.

+// The title was sealed, shown on every result, and never indexed.

+func TestATermOnlyInTheTitleIsFound(t *testing.T) {

+ h := newApp(t)

+ store(t, h, "Passport numbers", "L898902C3\n\n7727833B1\n\n3041005D9")

+ body := get(t, h, "/?q=passport").Body.String()

+ // Count the result rows, not the title text. The same page lists

+ // every document below the results, so asserting on the title alone

+ // passes on a 0-result page — which is how the first draft of this

+ // test passed against the very bug it was written for.

+ if n := strings.Count(body, "rst-result-title"); n != 1 {

+ t.Errorf("a term that appears only in the title returns %d results, want 1:\n%s", n, body)

+ }

+// The title must not drown the body. Indexing the title into every chunk

+// makes it a term in each one; this checks a body match still ranks and

+// that the title has not become the only thing that matches.

+func TestBodyStillMatchesWhenTitleIsIndexed(t *testing.T) {

+ h := newApp(t)

+ store(t, h, "Passport numbers", "The renewal fee rose in June.")

+ body := get(t, h, "/?q=renewal").Body.String()

+ if n := strings.Count(body, "rst-result-title"); n != 1 {

+ t.Errorf("body term returns %d results, want 1:\n%s", n, body)

+ }