amadan
Search finds a title whichever Unicode form it was typed in
Branch task 24 on discussion #14, the half of it that is a defect rather than a judgement. "café" has two encodings — U+00E9, and "e" followed by combining acute U+0301 — and normaliseForSearch compared them as bytes, so they did not match. A title typed on a machine that produces decomposed text (a macOS filesystem, several IMEs) was unfindable from a machine that produces precomposed text: the person searching gets an empty result for a title they are looking straight at, with nothing on screen to suggest why. normaliseForSearch now applies NFC before it does anything else. It is applied inside that one function rather than at either call site, so the title and the query are normalised by the same code — normalising only one side fixes exactly one direction and looks fixed from whichever side you happen to test. This is NOT the accent-folding decision. That one — whether "cafe" should match "café", two different strings — is Paul's, filed as branch task 21, and is untouched: TestSearchDoesNotFoldAccents still passes, and the new test asserts the non-folding directly so a later reader cannot mistake this change for having settled it. The new test checks both directions, and fails first if the two literals are byte-identical — a test whose two inputs are the same string proves nothing while passing, which is this repo's recurring shape. golang.org/x/text moves from indirect to direct in go.mod. No new module: it was already in the build. Verified: red first in both directions (0 matches, want 1), green after; all 12 search tests pass.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WxucgadpWLE9XNWvfHBsnb
3 files changed,
+67
−2
go.mod+1 −1internal/docs/search.go+19 −1internal/docs/search_test.go+47 −0
diff --git a/go.mod b/go.mod| index 5c58304..4f54fbd 100644 |
| --- a/go.mod |
| +++ b/go.mod |
| @@ -7,6 +7,7 @@ require ( |
| github.com/go-chi/chi/v5 v5.3.2 |
| github.com/google/uuid v1.6.0 |
| github.com/tetratelabs/wazero v1.12.0 |
| + golang.org/x/text v0.20.0 |
| gorm.io/gorm v1.31.2 |
| ) |
| @@ -19,7 +20,6 @@ require ( |
| github.com/ncruces/go-strftime v1.0.0 // indirect |
| github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect |
| golang.org/x/sys v0.46.0 // indirect |
| - golang.org/x/text v0.20.0 // indirect |
| gorm.io/plugin/dbresolver v1.6.2 // indirect |
| modernc.org/libc v1.74.1 // indirect |
| modernc.org/mathutil v1.7.1 // indirect |
diff --git a/internal/docs/search.go b/internal/docs/search.go| index b835b54..32fe3f5 100644 |
| --- a/internal/docs/search.go |
| +++ b/internal/docs/search.go |
| @@ -15,6 +15,7 @@ import ( |
| "unicode" |
| "github.com/carlosframework/rastrillo" |
| + "golang.org/x/text/unicode/norm" |
| ) |
| // maxSearchResults caps what one query returns. |
| @@ -69,11 +70,28 @@ func rankTitle(title, query string) searchRank { |
| // straight one. Deliberately NOT a full Unicode fold: "café" and "cafe" |
| // stay different, because guessing at accent equivalence across twelve |
| // locales (render.go's Locales) is a design and not a detail. |
| +// |
| +// It DOES normalise to NFC first, and that is a different question from |
| +// the folding above rather than a softening of it. Folding is about two |
| +// different strings a person might want treated as one. NFC is about |
| +// one string with two encodings — "café" as U+00E9, and "café" as "e" |
| +// followed by U+0301 — which nobody has ever wanted treated as two. |
| +// Without it a title typed on a machine that produces decomposed text |
| +// (macOS's filesystem, several IMEs) is unfindable from a machine that |
| +// produces precomposed text, and the person searching sees an empty |
| +// result for a title they are looking straight at. |
| +// |
| +// Applied here rather than at either call site so the title and the |
| +// query go through the same function — normalising only one side fixes |
| +// exactly one direction and looks fixed from whichever side you tested. |
| +// NFC and not NFD because it is the shorter form and the web's own |
| +// default; the choice does not matter as long as it is the same on both |
| +// sides, which sharing this function is what guarantees. |
| func normaliseForSearch(s string) string { |
| var b strings.Builder |
| b.Grow(len(s)) |
| lastSpace := true |
| - for _, r := range strings.ToLower(s) { |
| + for _, r := range strings.ToLower(norm.NFC.String(s)) { |
| switch { |
| case unicode.IsSpace(r) || r == '—' || r == '–' || r == '-' || r == '_': |
| if !lastSpace { |
diff --git a/internal/docs/search_test.go b/internal/docs/search_test.go| index 533baba..8eca7f2 100644 |
| --- a/internal/docs/search_test.go |
| +++ b/internal/docs/search_test.go |
| @@ -110,6 +110,53 @@ func TestSearchDoesNotFoldAccents(t *testing.T) { |
| } |
| } |
| +// The same text in its two Unicode encodings is the same text, and this |
| +// is a different question from the accent folding above — which is why |
| +// it gets its own test rather than a case in that one. |
| +// |
| +// TestSearchDoesNotFoldAccents is about "café" and "cafe": two |
| +// DIFFERENT strings, which a person may or may not want treated as one, |
| +// and Paul's call (branch task 21). This is about "café" and "café": |
| +// the SAME string, once as U+00E9 and once as "e" + U+0301, which |
| +// nobody has ever wanted treated as two. A macOS filesystem hands over |
| +// the decomposed form, several IMEs produce it, and a title typed on |
| +// one machine and searched for on another can differ by nothing else. |
| +// |
| +// Both directions, because normalising only the query or only the title |
| +// fixes exactly one of them and looks fixed from whichever side the |
| +// author happened to test. |
| +func TestSearchNormalisesUnicodeForm(t *testing.T) { |
| + app, err := NewForTest(t) |
| + if err != nil { |
| + t.Fatalf("NewForTest: %v", err) |
| + } |
| + ctx := context.Background() |
| + |
| + const precomposed = "Café notes" // café, one rune |
| + const decomposed = "Café notes" // café, e + combining acute |
| + if precomposed == decomposed { |
| + t.Fatal("the two forms are byte-identical — this test proves nothing") |
| + } |
| + |
| + owner := seedTitles(t, app, "writer@example.com", precomposed) |
| + if got, _ := app.Store().Search(ctx, owner, decomposed); len(got) != 1 { |
| + t.Errorf("decomposed query against a precomposed title matched %d, want 1", len(got)) |
| + } |
| + |
| + // And the other way round: a title stored decomposed, queried |
| + // precomposed. |
| + other := seedTitles(t, app, "second@example.com", decomposed) |
| + if got, _ := app.Store().Search(ctx, other, precomposed); len(got) != 1 { |
| + t.Errorf("precomposed query against a decomposed title matched %d, want 1", len(got)) |
| + } |
| + |
| + // Normalising must not have quietly folded the accent away as a |
| + // side effect — that is task 21's decision, not this change's. |
| + if got, _ := app.Store().Search(ctx, owner, "cafe notes"); len(got) != 0 { |
| + t.Errorf(`"cafe notes" matched %d, want 0 — NFC must not fold accents`, len(got)) |
| + } |
| +} |
| + |
| // "No query" is not "match nothing", and it is not "match everything" |
| // either. The caller decides what an empty query shows; Search says |
| // nothing. |