| 1 | // Command bench measures the index, so §7's numbers are recorded |
| 2 | // measurements rather than assertions. |
| 3 | // |
| 4 | // A review of draft 2 pointed out that every figure in §7 — 3.2 GFLOP/s, |
| 5 | // 104 MB, 19 s — had no benchmark, command or machine behind it, and that |
| 6 | // the FTS size could not be reconciled with the stated schema, because a |
| 7 | // default FTS5 table keeps its own copy of the inserted text. Both are |
| 8 | // fair, and both are answerable by measuring instead of arguing. |
| 9 | package main |
| 10 | |
| 11 | import ( |
| 12 | "crypto/rand" |
| 13 | "database/sql" |
| 14 | "flag" |
| 15 | "fmt" |
| 16 | "math" |
| 17 | "os" |
| 18 | "path/filepath" |
| 19 | "runtime" |
| 20 | "strings" |
| 21 | "time" |
| 22 | |
| 23 | _ "modernc.org/sqlite" |
| 24 | |
| 25 | "amadan.net/oficina/memoria/internal/memoria/chunk" |
| 26 | ) |
| 27 | |
| 28 | func main() { |
| 29 | corpus := flag.String("corpus", "", "directory of real text to build the corpus from") |
| 30 | target := flag.Int("chunks", 90000, "how many chunks to grow the corpus to") |
| 31 | flag.Parse() |
| 32 | |
| 33 | fmt.Printf("machine: %s/%s, %d cpu, go %s\n\n", runtime.GOOS, runtime.GOARCH, runtime.NumCPU(), runtime.Version()) |
| 34 | |
| 35 | passages := realPassages(*corpus) |
| 36 | if len(passages) == 0 { |
| 37 | fmt.Fprintln(os.Stderr, "no corpus found; pass --corpus") |
| 38 | os.Exit(1) |
| 39 | } |
| 40 | avgWords := 0 |
| 41 | avgBytes := 0 |
| 42 | for _, p := range passages { |
| 43 | avgWords += len(strings.Fields(p)) |
| 44 | avgBytes += len(p) |
| 45 | } |
| 46 | fmt.Printf("real corpus: %d passages, mean %d words / %d bytes per passage\n", |
| 47 | len(passages), avgWords/len(passages), avgBytes/len(passages)) |
| 48 | |
| 49 | // Grow to the target by cycling the real passages. Cycling repeats |
| 50 | // vocabulary, which makes the postings list SMALLER than genuinely |
| 51 | // varied English would — so the FTS figure below is optimistic, and |
| 52 | // says so. |
| 53 | grown := make([]string, 0, *target) |
| 54 | for i := 0; i < *target; i++ { |
| 55 | grown = append(grown, passages[i%len(passages)]) |
| 56 | } |
| 57 | totalBytes := 0 |
| 58 | for _, p := range grown { |
| 59 | totalBytes += len(p) |
| 60 | } |
| 61 | fmt.Printf("grown corpus: %d chunks, %.1f MB of text\n\n", len(grown), float64(totalBytes)/(1<<20)) |
| 62 | |
| 63 | for _, contentless := range []bool{false, true} { |
| 64 | measureFTS(grown, contentless) |
| 65 | } |
| 66 | measureCosine() |
| 67 | } |
| 68 | |
| 69 | func measureFTS(chunks []string, contentless bool) { |
| 70 | label := "default (keeps its own copy of the text)" |
| 71 | schema := `CREATE VIRTUAL TABLE c USING fts5(body, tokenize='porter unicode61');` |
| 72 | if contentless { |
| 73 | label = "contentless (content='')" |
| 74 | schema = `CREATE VIRTUAL TABLE c USING fts5(body, content='', tokenize='porter unicode61');` |
| 75 | } |
| 76 | dir, err := os.MkdirTemp("/home/paulca/memoriadrive", "bench") |
| 77 | if err != nil { |
| 78 | panic(err) |
| 79 | } |
| 80 | defer os.RemoveAll(dir) |
| 81 | path := filepath.Join(dir, "fts.db") |
| 82 | db, err := sql.Open("sqlite", "file:"+path) |
| 83 | if err != nil { |
| 84 | panic(err) |
| 85 | } |
| 86 | defer db.Close() |
| 87 | if _, err := db.Exec(schema); err != nil { |
| 88 | panic(err) |
| 89 | } |
| 90 | |
| 91 | start := time.Now() |
| 92 | tx, _ := db.Begin() |
| 93 | st, err := tx.Prepare(`INSERT INTO c (body) VALUES (?)`) |
| 94 | if err != nil { |
| 95 | panic(err) |
| 96 | } |
| 97 | for _, ch := range chunks { |
| 98 | if _, err := st.Exec(ch); err != nil { |
| 99 | panic(err) |
| 100 | } |
| 101 | } |
| 102 | st.Close() |
| 103 | if err := tx.Commit(); err != nil { |
| 104 | panic(err) |
| 105 | } |
| 106 | if _, err := db.Exec(`INSERT INTO c(c) VALUES('optimize');`); err != nil { |
| 107 | panic(err) |
| 108 | } |
| 109 | build := time.Since(start) |
| 110 | db.Exec(`PRAGMA wal_checkpoint(TRUNCATE);`) |
| 111 | |
| 112 | var size int64 |
| 113 | for _, suffix := range []string{"", "-wal"} { |
| 114 | if fi, err := os.Stat(path + suffix); err == nil { |
| 115 | size += fi.Size() |
| 116 | } |
| 117 | } |
| 118 | |
| 119 | // A query, warm. |
| 120 | qstart := time.Now() |
| 121 | var n int |
| 122 | row := db.QueryRow(`SELECT count(*) FROM c WHERE c MATCH '"revenue" OR "sealed"'`) |
| 123 | if err := row.Scan(&n); err != nil { |
| 124 | fmt.Printf(" query failed: %v\n", err) |
| 125 | } |
| 126 | q := time.Since(qstart) |
| 127 | |
| 128 | fmt.Printf("FTS5 %s\n", label) |
| 129 | fmt.Printf(" index on disk : %.1f MB (%d bytes/chunk)\n", float64(size)/(1<<20), size/int64(len(chunks))) |
| 130 | fmt.Printf(" build time : %s\n", build.Round(time.Millisecond)) |
| 131 | fmt.Printf(" one BM25 query: %s (%d rows matched)\n\n", q.Round(time.Microsecond), n) |
| 132 | } |
| 133 | |
| 134 | // measureCosine times a brute-force pass, the number §7's latency table |
| 135 | // is derived from. |
| 136 | func measureCosine() { |
| 137 | fmt.Println("brute-force cosine, one goroutine, pure Go, no SIMD") |
| 138 | for _, c := range []struct{ n, d int }{{10000, 512}, {90000, 512}, {100000, 768}, {250000, 512}} { |
| 139 | vecs := make([]float32, c.n*c.d) |
| 140 | buf := make([]byte, len(vecs)*4) |
| 141 | rand.Read(buf) |
| 142 | for i := range vecs { |
| 143 | vecs[i] = float32(int8(buf[i*4])) / 128 |
| 144 | } |
| 145 | q := make([]float32, c.d) |
| 146 | for i := range q { |
| 147 | q[i] = float32(i%17) / 17 |
| 148 | } |
| 149 | // Three runs, report the best — a slow run measures the |
| 150 | // scheduler, not the arithmetic. |
| 151 | best := time.Duration(math.MaxInt64) |
| 152 | for run := 0; run < 3; run++ { |
| 153 | start := time.Now() |
| 154 | var bestScore float32 |
| 155 | for i := 0; i+c.d <= len(vecs); i += c.d { |
| 156 | var dot float32 |
| 157 | row := vecs[i : i+c.d] |
| 158 | for j, v := range row { |
| 159 | dot += v * q[j] |
| 160 | } |
| 161 | if dot > bestScore { |
| 162 | bestScore = dot |
| 163 | } |
| 164 | } |
| 165 | if e := time.Since(start); e < best { |
| 166 | best = e |
| 167 | } |
| 168 | } |
| 169 | flops := float64(2 * c.n * c.d) |
| 170 | fmt.Printf(" %7d chunks x %d dims: %8s (%.1f GFLOP/s) vectors %.0f MB f32 / %.0f MB int8\n", |
| 171 | c.n, c.d, best.Round(time.Microsecond), flops/best.Seconds()/1e9, |
| 172 | float64(c.n*c.d*4)/(1<<20), float64(c.n*c.d)/(1<<20)) |
| 173 | } |
| 174 | } |
| 175 | |
| 176 | // realPassages reads .md files and chunks them with MEMORIA'S OWN |
| 177 | // chunker, so the measurement describes the real product rather than an |
| 178 | // approximation of it. An earlier version split on blank lines and |
| 179 | // reported a mean of 81 words per passage, which is a paragraph, not a |
| 180 | // memoria chunk. |
| 181 | func realPassages(dir string) []string { |
| 182 | var out []string |
| 183 | if dir == "" { |
| 184 | return nil |
| 185 | } |
| 186 | filepath.Walk(dir, func(p string, info os.FileInfo, err error) error { |
| 187 | if err != nil || info.IsDir() { |
| 188 | return nil |
| 189 | } |
| 190 | if !strings.HasSuffix(p, ".md") || strings.Contains(p, "/.") || strings.Contains(p, "node_modules") { |
| 191 | return nil |
| 192 | } |
| 193 | raw, err := os.ReadFile(p) |
| 194 | if err != nil { |
| 195 | return nil |
| 196 | } |
| 197 | for _, c := range chunk.Split(string(raw)) { |
| 198 | out = append(out, c.Indexable()) |
| 199 | } |
| 200 | return nil |
| 201 | }) |
| 202 | return out |
| 203 | } |
| 204 | |
amadan