| 1 | package memoria |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "fmt" |
| 7 | "log/slog" |
| 8 | "sort" |
| 9 | "strings" |
| 10 | "time" |
| 11 | |
| 12 | "amadan.net/rastrillo/rastrillo/db" |
| 13 | |
| 14 | "amadan.net/oficina/suite" |
| 15 | |
| 16 | "amadan.net/oficina/memoria/internal/memoria/chunk" |
| 17 | "amadan.net/oficina/memoria/internal/memoria/fact" |
| 18 | "amadan.net/oficina/memoria/internal/memoria/index" |
| 19 | "amadan.net/oficina/memoria/internal/memoria/provider" |
| 20 | "amadan.net/oficina/memoria/internal/memoria/seal" |
| 21 | "amadan.net/oficina/memoria/internal/memoria/sibling" |
| 22 | ) |
| 23 | |
| 24 | // Store is the only thing that opens sealed rows. Everything above it |
| 25 | // works in plaintext that never reaches disk; everything below it works |
| 26 | // in ciphertext. |
| 27 | type Store struct { |
| 28 | db *db.DB |
| 29 | sealer *seal.Sealer |
| 30 | ix *index.Index |
| 31 | // logger is optional. A dropped breaker counter is not allowed to |
| 32 | // fail a search, but it is not allowed to be silent either: a |
| 33 | // breaker that quietly stops counting reads as "no anomaly" forever, |
| 34 | // which is the failure §8.1 spends a paragraph on. |
| 35 | logger *slog.Logger |
| 36 | // probe observes §8.9's processing for tests, and is nil in |
| 37 | // production. It exists so caching, dedupe and the deadline can be |
| 38 | // asserted by COUNTING what was done rather than by timing it: three |
| 39 | // wall-clock guards failed on correct code with ten CPUs busy. |
| 40 | probe *modelProbe |
| 41 | // extractor is §8.9's model. **nil means the model layer is off**, which |
| 42 | // is decision 5's floor and the default: nothing is sent anywhere. Set |
| 43 | // only by WireModel, only when the owner said MEMORIA_AI=on. |
| 44 | extractor Extractor |
| 45 | // custody is who memoria asks about a sibling's passages. |
| 46 | // |
| 47 | // **nil is a configuration, not a bug**, and it is slice 0's: no |
| 48 | // siblings, every document memoria-owned under §4.2. It is also the |
| 49 | // state a misconfigured slice-1 instance is in, so nil withholds |
| 50 | // every sibling passage rather than rendering it. See rule(). |
| 51 | custody Custody |
| 52 | } |
| 53 | |
| 54 | // NewStore wires the store. The index starts empty — call Rebuild. |
| 55 | // |
| 56 | // custody may be nil, and what that means is in the field's comment: no |
| 57 | // app of record can be asked, so nothing owned by one is spoken. |
| 58 | func NewStore(d *db.DB, sealer *seal.Sealer, ix *index.Index, custody Custody) *Store { |
| 59 | return &Store{db: d, sealer: sealer, ix: ix, custody: custody} |
| 60 | } |
| 61 | |
| 62 | // UseLogger supplies the log the store reports dropped counters to. |
| 63 | func (s *Store) UseLogger(l *slog.Logger) { s.logger = l } |
| 64 | |
| 65 | // logBreakerWriteFailed says a counter was lost, naming the app and |
| 66 | // nothing else — no asker, no item, no query text (AGENTS.md). |
| 67 | func (s *Store) logBreakerWriteFailed(app string) { |
| 68 | if s.logger == nil { |
| 69 | return |
| 70 | } |
| 71 | s.logger.Warn("breaker counter not written", "app", app) |
| 72 | } |
| 73 | |
| 74 | // Result is one search hit, with its citation resolved to plaintext. |
| 75 | type Result struct { |
| 76 | DocumentID int64 |
| 77 | ChunkID int64 |
| 78 | Ordinal int |
| 79 | Source string |
| 80 | Title string |
| 81 | // Path is the document's ancestry in its source. SHOWN, not merely |
| 82 | // indexed: two pages called "Week one" under different handbooks are |
| 83 | // the same result to a reader who cannot see where each came from, |
| 84 | // and carrying ancestry for findability while hiding it from the |
| 85 | // person reading the answer is half a feature. Found by looking at |
| 86 | // the rendered page, which is the only thing that would have. |
| 87 | Path string |
| 88 | Heading string |
| 89 | Snippet string |
| 90 | Score float64 |
| 91 | // item is the app of record's coordinate for this passage's document, |
| 92 | // zero for memoria-owned content (§4.2). |
| 93 | // |
| 94 | // Unexported: it goes to a sibling on the wire and never to a |
| 95 | // template. A reader has no use for another app's internal id, and a |
| 96 | // field a template can reach is a field a template eventually prints. |
| 97 | item sibling.Candidate |
| 98 | } |
| 99 | |
| 100 | // Page is one search, ruled on. |
| 101 | // |
| 102 | // **Results are the passages this person may read** — every one of them |
| 103 | // either memoria's own (§4.2) or carrying a VISIBLE from its app of |
| 104 | // record, asked for this person, for this revision, just now. |
| 105 | // |
| 106 | // **Notices are what could not be shown and why**, per §6.5, naming the |
| 107 | // app and never the document. A short page with no notice is the failure |
| 108 | // this whole design is arranged against; a short page WITH one is an |
| 109 | // honest partial answer, and §6.5 says partial pages are the normal case |
| 110 | // rather than an error path. |
| 111 | type Page struct { |
| 112 | Results []Result |
| 113 | Notices []string |
| 114 | // Facts is §8.8's box: validated facts pulled deterministically out |
| 115 | // of the passages above, with no model anywhere near them. |
| 116 | // |
| 117 | // **Derived from Results and from nothing else**, which is what makes |
| 118 | // it safe by construction rather than by care. See addFacts. |
| 119 | Facts []Fact |
| 120 | } |
| 121 | |
| 122 | // FactNote is the line under a narrowed kind's row: the box chose it on |
| 123 | // the reader's own words, and this many others were left out (§8.8, Paul |
| 124 | // 2026-09-29). Counted over Results, like everything else in the box. |
| 125 | type FactNote struct { |
| 126 | Kind string |
| 127 | // Matched are the qualifiers every shown row's label contains. |
| 128 | Matched []string |
| 129 | // Others is how many of this kind's candidates narrowing left out, |
| 130 | // counted after the §4.1 ruling. At least MaxFacts, because |
| 131 | // narrowing only runs once the cap has fired. |
| 132 | Others int |
| 133 | } |
| 134 | |
| 135 | // factNouns names a kind in the note. Plural only: the note is written |
| 136 | // when the cap has fired, so at least MaxFacts others are counted. |
| 137 | var factNouns = map[string]string{ |
| 138 | string(fact.Phone): "phone numbers", |
| 139 | string(fact.Email): "email addresses", |
| 140 | string(fact.Eircode): "eircodes", |
| 141 | string(fact.IBAN): "IBANs", |
| 142 | string(fact.VAT): "VAT numbers", |
| 143 | } |
| 144 | |
| 145 | // Text is the line as the reader sees it: |
| 146 | // "Matched “office”. 3 other phone numbers are in these results." |
| 147 | func (n FactNote) Text() string { |
| 148 | quoted := make([]string, len(n.Matched)) |
| 149 | for i, m := range n.Matched { |
| 150 | quoted[i] = "“" + m + "”" |
| 151 | } |
| 152 | words := strings.Join(quoted, ", ") |
| 153 | if len(quoted) > 1 { |
| 154 | words = strings.Join(quoted[:len(quoted)-1], ", ") + " and " + quoted[len(quoted)-1] |
| 155 | } |
| 156 | return fmt.Sprintf("Matched %s. %d other %s are in these results.", words, n.Others, factNouns[n.Kind]) |
| 157 | } |
| 158 | |
| 159 | // Summary is a document as the list shows it. |
| 160 | type Summary struct { |
| 161 | ID int64 |
| 162 | Source string |
| 163 | Title string |
| 164 | Path string |
| 165 | Bytes int |
| 166 | Chunks int |
| 167 | Updated string |
| 168 | } |
| 169 | |
| 170 | // externalID keys a document by its stable source identifier. |
| 171 | // |
| 172 | // Keyed, not a bare hash: an unkeyed digest of anything plaintext-derived |
| 173 | // lets someone with the bucket confirm what the team holds. See |
| 174 | // seal.Fingerprint. |
| 175 | func (s *Store) externalID(key string) (string, error) { |
| 176 | return s.sealer.Fingerprint(seal.PurposeExternal, []byte("key\x00"+key)) |
| 177 | } |
| 178 | |
| 179 | // IngestSpec is one document to store. |
| 180 | // |
| 181 | // A struct rather than a parameter list because the connectors added two |
| 182 | // more fields and a five-string call site is a bug waiting for someone |
| 183 | // to transpose two of them. |
| 184 | type IngestSpec struct { |
| 185 | // Source names where this came from — "upload", "paste", "notion". |
| 186 | Source string |
| 187 | // Key is the STABLE identifier in the source: a file path, a Notion |
| 188 | // page id, a pasted document's title. Not the body — see |
| 189 | // Document.ExternalID. |
| 190 | Key string |
| 191 | // Title and Body are content. |
| 192 | Title string |
| 193 | Body string |
| 194 | // Path is the document's ancestry in its source, outermost first. |
| 195 | // Empty for upload and paste, which have none. |
| 196 | Path []string |
| 197 | // AllowTitleOnly permits a document whose body is empty, storing the |
| 198 | // title as the body. |
| 199 | // |
| 200 | // Off for upload and paste, where an empty submission is a person's |
| 201 | // mistake and refusing it is the useful answer. On for connectors, |
| 202 | // where an empty page is a FACT ABOUT THE SOURCE — real workspaces |
| 203 | // are full of placeholders and title-only rows — and both |
| 204 | // alternatives are wrong: failing the run lets one empty page block |
| 205 | // an entire import, and skipping makes the page an absence with |
| 206 | // nothing to explain it. |
| 207 | AllowTitleOnly bool |
| 208 | // ItemID and SiblingRevision are the app of record's coordinate, set |
| 209 | // only for content ingested from a sibling (§4.1) and left empty for |
| 210 | // everything memoria owns outright (§4.2). |
| 211 | // |
| 212 | // **Both or neither.** A document with an item id and no revision |
| 213 | // would ask S2 about revision 0, which is a revision no sibling |
| 214 | // issued — and a sibling asked about a revision it never issued |
| 215 | // answers about a body memoria is not holding. Ingest refuses the |
| 216 | // half-set case rather than storing it, because the failure it |
| 217 | // produces is a wrong authorization rather than a missing one. |
| 218 | ItemID string |
| 219 | SiblingRevision int64 |
| 220 | } |
| 221 | |
| 222 | // Ingest stores a document and indexes it. |
| 223 | // |
| 224 | // Re-ingesting the same Key replaces the document's chunks and bumps its |
| 225 | // revision. The old chunks are deleted from the index *and* the database |
| 226 | // before the new ones land, so a passage that the new revision removed |
| 227 | // cannot survive as a citation to text that is no longer there — the |
| 228 | // stale-passage failure the spec's §4.1 refuses to allow. |
| 229 | // |
| 230 | // Two failures, and they are not the same failure. Anything before the |
| 231 | // commit means nothing was stored and the id comes back zero. Anything |
| 232 | // after it means the document IS stored and only the index is behind: the |
| 233 | // id comes back real and the error wraps ErrIndexLag. |
| 234 | func (s *Store) Ingest(ctx context.Context, spec IngestSpec) (int64, error) { |
| 235 | title := strings.TrimSpace(spec.Title) |
| 236 | body := strings.TrimSpace(spec.Body) |
| 237 | if body == "" { |
| 238 | if !spec.AllowTitleOnly { |
| 239 | return 0, errors.New("nothing to store: the document is empty") |
| 240 | } |
| 241 | if title == "" { |
| 242 | return 0, errors.New("nothing to store: the document has neither a body nor a title") |
| 243 | } |
| 244 | // A title-only document IS its title. Storing an empty body |
| 245 | // instead would index nothing and make the page unfindable by |
| 246 | // the only words it has. |
| 247 | body = title |
| 248 | } |
| 249 | if title == "" { |
| 250 | title = firstLine(body) |
| 251 | } |
| 252 | key := spec.Key |
| 253 | if strings.TrimSpace(key) == "" { |
| 254 | key = title |
| 255 | } |
| 256 | pathStr := chunk.Chunk{Path: spec.Path}.PathString() |
| 257 | |
| 258 | // Both or neither: see IngestSpec.ItemID. Refused here rather than |
| 259 | // defaulted, because either default is a lie — an item id with |
| 260 | // revision 0 asks about a body no sibling has, and a revision with no |
| 261 | // item id is a document nothing can be asked about that would |
| 262 | // otherwise render as memoria's own. |
| 263 | if (spec.ItemID == "") != (spec.SiblingRevision == 0) { |
| 264 | return 0, errors.New("ingest: a sibling document needs both an item id and a revision") |
| 265 | } |
| 266 | var sealedItemID []byte |
| 267 | if spec.ItemID != "" { |
| 268 | sealed, err := s.sealer.Seal(seal.PurposeItemID, []byte(spec.ItemID)) |
| 269 | if err != nil { |
| 270 | return 0, err |
| 271 | } |
| 272 | sealedItemID = sealed |
| 273 | } |
| 274 | |
| 275 | external, err := s.externalID(key) |
| 276 | if err != nil { |
| 277 | return 0, err |
| 278 | } |
| 279 | bodyPrint, err := s.sealer.Fingerprint(seal.PurposeExternal, []byte("body\x00"+body)) |
| 280 | if err != nil { |
| 281 | return 0, err |
| 282 | } |
| 283 | |
| 284 | sealedTitle, err := s.sealer.Seal(seal.PurposeTitle, []byte(title)) |
| 285 | if err != nil { |
| 286 | return 0, err |
| 287 | } |
| 288 | var sealedPath []byte |
| 289 | if pathStr != "" { |
| 290 | if sealedPath, err = s.sealer.Seal(seal.PurposePath, []byte(pathStr)); err != nil { |
| 291 | return 0, err |
| 292 | } |
| 293 | } |
| 294 | sealedBody, err := s.sealer.Seal(seal.PurposeBody, []byte(body)) |
| 295 | if err != nil { |
| 296 | return 0, err |
| 297 | } |
| 298 | |
| 299 | chunks := chunk.SplitUnder(spec.Path, body) |
| 300 | if len(chunks) == 0 { |
| 301 | return 0, errors.New("nothing to store: the document produced no passages") |
| 302 | } |
| 303 | |
| 304 | var docID int64 |
| 305 | var unchanged, rechunk bool |
| 306 | err = s.db.G.WithContext(ctx).Transaction(func(tx *gormTx) error { |
| 307 | var existing Document |
| 308 | res := tx.Where("source = ? AND external_id = ?", spec.Source, external).First(&existing) |
| 309 | switch { |
| 310 | case res.Error == nil: |
| 311 | docID = existing.ID |
| 312 | if existing.BodyPrint == bodyPrint { |
| 313 | // Same key, same body — but the page may have been |
| 314 | // RENAMED or MOVED, and neither touches the body. |
| 315 | // |
| 316 | // An earlier version returned here unconditionally, so a |
| 317 | // renamed page kept its old title forever and a moved one |
| 318 | // kept its old ancestry. Worse for a connector: an |
| 319 | // ancestor rename does not change a descendant's own edit |
| 320 | // time at all, so nothing upstream ever signalled it. |
| 321 | // Found by a design review. |
| 322 | moved, err := s.titleOrPathMoved(existing, title, pathStr) |
| 323 | if err != nil { |
| 324 | return err |
| 325 | } |
| 326 | if !moved { |
| 327 | // Genuinely unchanged. Do NOT bump memoria's own |
| 328 | // revision: a revision that moves without the content |
| 329 | // moving would make every cached authorization stale |
| 330 | // for no reason. |
| 331 | // |
| 332 | // **But the SIBLING's revision must still be |
| 333 | // recorded, and this returned without it.** The two |
| 334 | // numbers answer different questions. memoria's |
| 335 | // counts its own edits; the sibling's is the token S2 |
| 336 | // is asked about, and the sibling issues it for |
| 337 | // reasons memoria cannot see — a permissions change, |
| 338 | // a metadata edit, a re-save that touched no bytes. |
| 339 | // |
| 340 | // Left stale, the failure is permanent and invisible: |
| 341 | // the sibling announces revision 8 of a body |
| 342 | // identical to 7, memoria keeps 7, and every |
| 343 | // subsequent query asks S2 about `(item, 7)`. The |
| 344 | // sibling answers STALE — correctly, it has moved on |
| 345 | // — so the passage is withheld loudly on every page, |
| 346 | // forever, and re-polling cannot fix it because the |
| 347 | // body never changes. "Docs has moved on from 1 |
| 348 | // result; refetching" on every single query, for a |
| 349 | // document the reader is entitled to. |
| 350 | // |
| 351 | // Found by a review, 2026-09-07. |
| 352 | if spec.ItemID != "" && existing.SiblingRevision != spec.SiblingRevision { |
| 353 | unchanged = true |
| 354 | return tx.Model(&Document{}).Where("id = ?", docID).Updates(map[string]any{ |
| 355 | "sibling_revision": spec.SiblingRevision, |
| 356 | "sibling_item_id": sealedItemID, |
| 357 | }).Error |
| 358 | } |
| 359 | unchanged = true |
| 360 | return nil |
| 361 | } |
| 362 | // Title or path only. Still no revision bump — the |
| 363 | // passages a citation points at have not moved — and the |
| 364 | // chunks are left alone, because the path lives on the |
| 365 | // document. The index is refreshed below, since it is |
| 366 | // the index that carries the path. |
| 367 | // Title or path moved. The sibling's revision travels |
| 368 | // with it for the same reason as above: whatever made the |
| 369 | // sibling issue a new one, S2 will be asked about the new |
| 370 | // one or about nothing. |
| 371 | return tx.Model(&Document{}).Where("id = ?", docID).Updates(map[string]any{ |
| 372 | "sealed_title": sealedTitle, |
| 373 | "sealed_path": sealedPath, |
| 374 | "sibling_revision": spec.SiblingRevision, |
| 375 | "sibling_item_id": sealedItemID, |
| 376 | }).Error |
| 377 | } |
| 378 | rechunk = true |
| 379 | if err := tx.Where("document_id = ?", docID).Delete(&Chunk{}).Error; err != nil { |
| 380 | return err |
| 381 | } |
| 382 | if err := tx.Model(&Document{}).Where("id = ?", docID).Updates(map[string]any{ |
| 383 | "revision": existing.Revision + 1, |
| 384 | "sealed_title": sealedTitle, |
| 385 | "sealed_body": sealedBody, |
| 386 | "sealed_path": sealedPath, |
| 387 | "body_print": bodyPrint, |
| 388 | "bytes": len(body), |
| 389 | // The sibling's own revision moves with the body, and it |
| 390 | // is the sibling's number rather than an increment of |
| 391 | // ours: §4.1 forbids memoria inventing or ordering one. |
| 392 | "sibling_item_id": sealedItemID, |
| 393 | "sibling_revision": spec.SiblingRevision, |
| 394 | }).Error; err != nil { |
| 395 | return err |
| 396 | } |
| 397 | default: |
| 398 | rechunk = true |
| 399 | d := Document{ |
| 400 | Source: spec.Source, |
| 401 | ExternalID: external, |
| 402 | BodyPrint: bodyPrint, |
| 403 | Revision: 1, |
| 404 | SealedTitle: sealedTitle, |
| 405 | SealedBody: sealedBody, |
| 406 | SealedPath: sealedPath, |
| 407 | SiblingItemID: sealedItemID, |
| 408 | SiblingRevision: spec.SiblingRevision, |
| 409 | Bytes: len(body), |
| 410 | } |
| 411 | if err := tx.Create(&d).Error; err != nil { |
| 412 | return err |
| 413 | } |
| 414 | docID = d.ID |
| 415 | } |
| 416 | |
| 417 | if !rechunk { |
| 418 | return nil |
| 419 | } |
| 420 | rows := make([]Chunk, 0, len(chunks)) |
| 421 | for _, c := range chunks { |
| 422 | // Seal the body ALONE, not Indexable(): the heading has its |
| 423 | // own indexed column, and sealing heading+body made the |
| 424 | // heading count twice in BM25 and appear at the front of |
| 425 | // every snippet. Found by reading a rendered page. |
| 426 | st, err := s.sealer.Seal(seal.PurposeChunk, []byte(c.Text)) |
| 427 | if err != nil { |
| 428 | return err |
| 429 | } |
| 430 | sh, err := s.sealer.Seal(seal.PurposeTitle, []byte(c.Heading())) |
| 431 | if err != nil { |
| 432 | return err |
| 433 | } |
| 434 | rows = append(rows, Chunk{ |
| 435 | DocumentID: docID, |
| 436 | Ordinal: c.Ordinal, |
| 437 | SealedHeading: sh, |
| 438 | SealedText: st, |
| 439 | }) |
| 440 | } |
| 441 | return tx.Create(&rows).Error |
| 442 | }) |
| 443 | if err != nil { |
| 444 | return 0, err |
| 445 | } |
| 446 | if unchanged { |
| 447 | return docID, nil |
| 448 | } |
| 449 | |
| 450 | // The index is derived state, so it is refreshed after the |
| 451 | // transaction commits rather than inside it: a rolled-back |
| 452 | // transaction that had already mutated the index would leave the |
| 453 | // index claiming passages the database does not have. |
| 454 | // |
| 455 | // context.WithoutCancel because this runs AFTER the commit. With the |
| 456 | // request's context, a client that closed the tab between commit and |
| 457 | // index left the document stored and unsearchable, the footer count |
| 458 | // disagreeing with the table, and the person told "that did not |
| 459 | // work" about a document that exists. |
| 460 | // |
| 461 | // Everything from here on has ALREADY COMMITTED, so a failure here is |
| 462 | // not a failure to store and must not be returned as one. It is |
| 463 | // wrapped in ErrIndexLag and the document id comes back beside it: the |
| 464 | // row exists, it will appear in the list, and the only thing wrong |
| 465 | // with it is that this process cannot find it. The caller has to be |
| 466 | // able to tell those apart, because the sentence for one of them is a |
| 467 | // lie about the other — createDocument used to say "could not be |
| 468 | // stored" about a document sitting in the database. Found by a code |
| 469 | // review of this branch. |
| 470 | post := context.WithoutCancel(ctx) |
| 471 | if err := s.ix.RemoveDocument(docID); err != nil { |
| 472 | return docID, fmt.Errorf("%w: %v", ErrIndexLag, err) |
| 473 | } |
| 474 | if err := s.indexDocument(post, docID); err != nil { |
| 475 | return docID, fmt.Errorf("%w: %v", ErrIndexLag, err) |
| 476 | } |
| 477 | return docID, nil |
| 478 | } |
| 479 | |
| 480 | // ErrIndexLag is the committed-but-unsearchable outcome: the document is |
| 481 | // in the database and the in-memory index does not have it. |
| 482 | // |
| 483 | // It is a distinct error rather than a plain one because the two states |
| 484 | // need two different sentences. A document that was refused is not there; |
| 485 | // a document that lagged is there, is in the list, is in the byte count, |
| 486 | // and comes back the next time the index is rebuilt from the sealed rows, |
| 487 | // which is every boot. Telling someone the first about the second is the |
| 488 | // refusal this app never verified. |
| 489 | // |
| 490 | // The wrapped cause is a database or index error and carries no document |
| 491 | // content — see the AGENTS.md rule this error is allowed to be logged |
| 492 | // under. |
| 493 | var ErrIndexLag = errors.New("the document is stored, but refreshing the index failed") |
| 494 | |
| 495 | // titleOrPathMoved reports whether a document's title or ancestry has |
| 496 | // changed, by opening what is stored and comparing plaintext. |
| 497 | // |
| 498 | // Comparing the sealed blobs would be cheaper and wrong: AES-GCM uses a |
| 499 | // fresh nonce per seal, so identical plaintext seals to different bytes |
| 500 | // every time and every document would look moved on every ingest. |
| 501 | func (s *Store) titleOrPathMoved(d Document, title, pathStr string) (bool, error) { |
| 502 | cur, err := s.sealer.Open(seal.PurposeTitle, d.SealedTitle) |
| 503 | if err != nil { |
| 504 | return false, fmt.Errorf("open title %d: %w", d.ID, err) |
| 505 | } |
| 506 | if string(cur) != title { |
| 507 | return true, nil |
| 508 | } |
| 509 | var curPath string |
| 510 | if len(d.SealedPath) > 0 { |
| 511 | p, err := s.sealer.Open(seal.PurposePath, d.SealedPath) |
| 512 | if err != nil { |
| 513 | return false, fmt.Errorf("open path %d: %w", d.ID, err) |
| 514 | } |
| 515 | curPath = string(p) |
| 516 | } |
| 517 | return curPath != pathStr, nil |
| 518 | } |
| 519 | |
| 520 | // indexDocument opens a document's chunks and adds them to the index. |
| 521 | func (s *Store) indexDocument(ctx context.Context, docID int64) error { |
| 522 | // The title and the ancestry are both indexed alongside every chunk, |
| 523 | // so both are read here rather than left to the caller: Ingest and |
| 524 | // Rebuild both land in this function, and a value indexed on only one |
| 525 | // of those paths would make a document findable until the next |
| 526 | // restart. |
| 527 | var doc Document |
| 528 | if err := s.db.G.WithContext(ctx).Where("id = ?", docID).First(&doc).Error; err != nil { |
| 529 | return err |
| 530 | } |
| 531 | title, err := s.sealer.Open(seal.PurposeTitle, doc.SealedTitle) |
| 532 | if err != nil { |
| 533 | return fmt.Errorf("open document title %d: %w", docID, err) |
| 534 | } |
| 535 | // The path lives on the DOCUMENT and is stamped onto every one of its |
| 536 | // entries, because every passage of a nested page is equally nested. |
| 537 | var path string |
| 538 | if len(doc.SealedPath) > 0 { |
| 539 | p, err := s.sealer.Open(seal.PurposePath, doc.SealedPath) |
| 540 | if err != nil { |
| 541 | return fmt.Errorf("open document path %d: %w", docID, err) |
| 542 | } |
| 543 | path = string(p) |
| 544 | } |
| 545 | var rows []Chunk |
| 546 | if err := s.db.G.WithContext(ctx).Where("document_id = ?", docID).Order("ordinal").Find(&rows).Error; err != nil { |
| 547 | return err |
| 548 | } |
| 549 | entries := make([]index.Entry, 0, len(rows)) |
| 550 | for _, r := range rows { |
| 551 | text, err := s.sealer.Open(seal.PurposeChunk, r.SealedText) |
| 552 | if err != nil { |
| 553 | return fmt.Errorf("open chunk %d: %w", r.ID, err) |
| 554 | } |
| 555 | heading, err := s.sealer.Open(seal.PurposeTitle, r.SealedHeading) |
| 556 | if err != nil { |
| 557 | return fmt.Errorf("open chunk heading %d: %w", r.ID, err) |
| 558 | } |
| 559 | entries = append(entries, index.Entry{ |
| 560 | ChunkID: r.ID, |
| 561 | DocumentID: r.DocumentID, |
| 562 | Ordinal: r.Ordinal, |
| 563 | Heading: string(heading), |
| 564 | Text: string(text), |
| 565 | Title: string(title), |
| 566 | Path: path, |
| 567 | }) |
| 568 | } |
| 569 | if len(entries) == 0 { |
| 570 | return nil |
| 571 | } |
| 572 | return s.ix.Add(entries...) |
| 573 | } |
| 574 | |
| 575 | // Rebuild fills an empty index from the sealed rows. Called at boot, |
| 576 | // because the index lives in memory and nowhere else (see the index |
| 577 | // package for why). |
| 578 | func (s *Store) Rebuild(ctx context.Context) (int, error) { |
| 579 | var ids []int64 |
| 580 | if err := s.db.G.WithContext(ctx).Model(&Document{}).Order("id").Pluck("id", &ids).Error; err != nil { |
| 581 | return 0, err |
| 582 | } |
| 583 | for _, id := range ids { |
| 584 | if err := s.indexDocument(ctx, id); err != nil { |
| 585 | return 0, err |
| 586 | } |
| 587 | } |
| 588 | return s.ix.Count() |
| 589 | } |
| 590 | |
| 591 | // Search runs the query and resolves each hit's citation. |
| 592 | // Search finds passages this person may read. |
| 593 | // |
| 594 | // **The ruling happens in here, and that placement is the design.** §4.1 |
| 595 | // binds every path that would speak a sibling's passage, and the way to |
| 596 | // make that true of every path is to leave no way to obtain a passage |
| 597 | // without it: there is one method that returns renderable text, it takes |
| 598 | // the asker, and it does not return anything it has not asked about. A |
| 599 | // version that returned raw hits and trusted its callers to ask would be |
| 600 | // correct exactly as long as nobody added a second caller. |
| 601 | // |
| 602 | // An empty asker refuses outright. There is nobody to ask about, and a |
| 603 | // question S2 cannot answer for a person is not a question worth sending |
| 604 | // — §4.1's typed subject travels on the wire precisely so a receiver that |
| 605 | // cannot make sense of it says UNKNOWN rather than guessing. |
| 606 | func (s *Store) Search(ctx context.Context, asker suite.Subject, query string, limit int) (Page, error) { |
| 607 | return s.search(ctx, asker, query, limit, true) |
| 608 | } |
| 609 | |
| 610 | // SearchWithoutFacts is Search for a surface that cannot show the fact |
| 611 | // box: identical ruling, no box, and therefore no model call. |
| 612 | // |
| 613 | // **It exists because native search sent passages and threw the answer |
| 614 | // away.** POST /api/native/search went through Search, which builds the |
| 615 | // box — including §8.9's model call — and the native response has no field |
| 616 | // for facts. So the native client paid the latency and caused the egress |
| 617 | // without ever receiving the address it was for. Found by review. When the |
| 618 | // native client can show attributed facts, it moves back to Search. |
| 619 | func (s *Store) SearchWithoutFacts(ctx context.Context, asker suite.Subject, query string, limit int) (Page, error) { |
| 620 | return s.search(ctx, asker, query, limit, false) |
| 621 | } |
| 622 | |
| 623 | func (s *Store) search(ctx context.Context, asker suite.Subject, query string, limit int, facts bool) (Page, error) { |
| 624 | if asker.Value == "" { |
| 625 | return Page{}, errors.New("search: no asker; nothing can be authorized") |
| 626 | } |
| 627 | hits, err := s.ix.Search(query, limit) |
| 628 | if err != nil { |
| 629 | return Page{}, err |
| 630 | } |
| 631 | if len(hits) == 0 { |
| 632 | return Page{}, nil |
| 633 | } |
| 634 | docIDs := make([]int64, 0, len(hits)) |
| 635 | seen := map[int64]bool{} |
| 636 | for _, h := range hits { |
| 637 | if !seen[h.DocumentID] { |
| 638 | seen[h.DocumentID] = true |
| 639 | docIDs = append(docIDs, h.DocumentID) |
| 640 | } |
| 641 | } |
| 642 | var docs []Document |
| 643 | if err := s.db.G.WithContext(ctx).Where("id IN ?", docIDs).Find(&docs).Error; err != nil { |
| 644 | return Page{}, err |
| 645 | } |
| 646 | titles := map[int64]string{} |
| 647 | sources := map[int64]string{} |
| 648 | paths := map[int64]string{} |
| 649 | // items carries each document's app-of-record coordinate, absent for |
| 650 | // memoria-owned rows. |
| 651 | items := map[int64]sibling.Candidate{} |
| 652 | for _, d := range docs { |
| 653 | t, err := s.sealer.Open(seal.PurposeTitle, d.SealedTitle) |
| 654 | if err != nil { |
| 655 | return Page{}, fmt.Errorf("open title %d: %w", d.ID, err) |
| 656 | } |
| 657 | titles[d.ID] = string(t) |
| 658 | sources[d.ID] = d.Source |
| 659 | if len(d.SealedPath) > 0 { |
| 660 | p, err := s.sealer.Open(seal.PurposePath, d.SealedPath) |
| 661 | if err != nil { |
| 662 | return Page{}, fmt.Errorf("open path %d: %w", d.ID, err) |
| 663 | } |
| 664 | paths[d.ID] = string(p) |
| 665 | } |
| 666 | if len(d.SiblingItemID) == 0 { |
| 667 | continue |
| 668 | } |
| 669 | // A sibling document whose coordinate will not open is a document |
| 670 | // memoria cannot ask about. It is left with a zero Candidate, |
| 671 | // which rule() treats as unresolvable and withholds loudly — the |
| 672 | // error is not swallowed into "no restriction applies". |
| 673 | id, err := s.sealer.Open(seal.PurposeItemID, d.SiblingItemID) |
| 674 | if err != nil { |
| 675 | return Page{}, fmt.Errorf("open item id %d: %w", d.ID, err) |
| 676 | } |
| 677 | items[d.ID] = sibling.Candidate{ |
| 678 | Item: suite.ItemRevision{ItemID: string(id), Revision: d.SiblingRevision}, |
| 679 | } |
| 680 | } |
| 681 | // livePassage is the database's answer to "is there still such a |
| 682 | // passage", and the database is the only thing allowed to answer it. |
| 683 | // See the loop below for why the index is not. |
| 684 | chunkIDs := make([]int64, 0, len(hits)) |
| 685 | for _, h := range hits { |
| 686 | chunkIDs = append(chunkIDs, h.ChunkID) |
| 687 | } |
| 688 | var present []int64 |
| 689 | if err := s.db.G.WithContext(ctx).Model(&Chunk{}).Where("id IN ?", chunkIDs).Pluck("id", &present).Error; err != nil { |
| 690 | return Page{}, err |
| 691 | } |
| 692 | livePassage := map[int64]bool{} |
| 693 | for _, id := range present { |
| 694 | livePassage[id] = true |
| 695 | } |
| 696 | out := make([]Result, 0, len(hits)) |
| 697 | for _, h := range hits { |
| 698 | // A hit whose chunk row is no longer in the database is DROPPED, |
| 699 | // never rendered. The index proposes a passage; the database |
| 700 | // decides whether it may be spoken. |
| 701 | // |
| 702 | // The index is derived state and it can outlive the rows it |
| 703 | // describes. Both ways of getting there have happened in this |
| 704 | // file, and both end as a citation to text that is not there — |
| 705 | // the failure §4.1 exists to prevent: |
| 706 | // |
| 707 | // - Delete removed the rows and then the entries. A failure in |
| 708 | // between left the passages of a deleted document indexed, |
| 709 | // and the reader got a result with an empty title linking to |
| 710 | // a page that 404s. Delete now removes the entries first, so |
| 711 | // it cannot produce that state; an ingest racing a delete on |
| 712 | // the same document still can. |
| 713 | // - Ingest replaces a document's chunks in its transaction and |
| 714 | // drops the old entries AFTER the commit. A failure there |
| 715 | // leaves the PREVIOUS revision's passages indexed under a |
| 716 | // document that still exists — a snippet of text the author |
| 717 | // has since removed, quoted under a live title. |
| 718 | // |
| 719 | // It is the CHUNK that is checked, not the document, and the |
| 720 | // second case is why: there the document is still there and only |
| 721 | // the passage is gone. The document check that was written first |
| 722 | // caught nothing the chunk check does not — a chunk row cannot |
| 723 | // outlive its document row, they go in one transaction and the |
| 724 | // schema cascades — so it was removed rather than left as a guard |
| 725 | // no test can see fail. |
| 726 | // |
| 727 | // It also makes ErrIndexLag's sentence true. "Stored but not |
| 728 | // findable until this instance restarts" was a claim about a |
| 729 | // document whose previous revision could still be answering |
| 730 | // searches; with the stale passages dropped, the document really |
| 731 | // is unfindable and the sentence really is the whole of what is |
| 732 | // wrong with it. |
| 733 | // |
| 734 | // Dropped rather than reported: the passage is genuinely gone, so |
| 735 | // its absence from the results is the true answer and not a |
| 736 | // truncation of one. Nothing is logged, because the only thing |
| 737 | // worth logging here would sit beside the query. |
| 738 | if !livePassage[h.ChunkID] { |
| 739 | continue |
| 740 | } |
| 741 | out = append(out, Result{ |
| 742 | DocumentID: h.DocumentID, |
| 743 | ChunkID: h.ChunkID, |
| 744 | Ordinal: h.Ordinal, |
| 745 | Source: sources[h.DocumentID], |
| 746 | Title: titles[h.DocumentID], |
| 747 | Path: paths[h.DocumentID], |
| 748 | Heading: h.Heading, |
| 749 | Snippet: h.Snippet, |
| 750 | Score: h.Score, |
| 751 | item: items[h.DocumentID], |
| 752 | }) |
| 753 | } |
| 754 | page := s.rule(ctx, asker, out) |
| 755 | // **After the ruling, never before it.** §8.8's box is built from the |
| 756 | // passages rule() kept, so a withheld one cannot contribute a |
| 757 | // candidate and cannot be counted towards the cap. |
| 758 | if facts { |
| 759 | s.addFacts(ctx, &page, query) |
| 760 | } |
| 761 | return page, nil |
| 762 | } |
| 763 | |
| 764 | // rule asks every app of record about its own passages and keeps the ones |
| 765 | // it may speak. |
| 766 | // |
| 767 | // **Withheld passages are dropped, not refilled.** The slot a withheld |
| 768 | // passage occupied in top-k stays lost, and the page is shorter by one. |
| 769 | // §12 lists over-fetching and refilling as deliberately not built, and |
| 770 | // §6.5 says partial pages are the normal case — so the notice, and not a |
| 771 | // backfill, is what makes the page honest. (§4.1's post-restore rule 1 is |
| 772 | // the one exception and it acts earlier, by taking a reconciling source |
| 773 | // out of the ranking before anything is scored.) |
| 774 | func (s *Store) rule(ctx context.Context, asker suite.Subject, all []Result) Page { |
| 775 | // One disposition per result, and **the zero value is Withhold**, so |
| 776 | // a branch below that forgets to assign one withholds loudly rather |
| 777 | // than speaking. See sibling.Disposition's const block for why that |
| 778 | // is the first constant. |
| 779 | disp := make([]sibling.Disposition, len(all)) |
| 780 | var cands []sibling.Candidate |
| 781 | var at []int |
| 782 | for i, r := range all { |
| 783 | if r.item.Item.ItemID == "" { |
| 784 | // memoria-owned: §4.2 makes memoria the app of record, and |
| 785 | // there is nobody else to ask. |
| 786 | disp[i] = sibling.Render |
| 787 | continue |
| 788 | } |
| 789 | if s.custody == nil { |
| 790 | continue |
| 791 | } |
| 792 | sib, ok := s.custody.Sibling(r.Source) |
| 793 | if !ok { |
| 794 | // The document is here and its app of record is not |
| 795 | // configured. Nobody can be asked, so nothing is spoken — |
| 796 | // never "no restriction applies", which is how a sibling |
| 797 | // removed from the roster would become a disclosure. |
| 798 | continue |
| 799 | } |
| 800 | c := r.item |
| 801 | c.Sibling = sib |
| 802 | cands = append(cands, c) |
| 803 | at = append(at, i) |
| 804 | } |
| 805 | |
| 806 | if len(cands) > 0 { |
| 807 | got := s.custody.Rule(ctx, asker, cands) |
| 808 | for j, d := range got { |
| 809 | disp[at[j]] = d |
| 810 | } |
| 811 | // §8.1's breaker counters, written on the path that already made |
| 812 | // the call. |
| 813 | // |
| 814 | // **`WithoutCancel`, because this is an addition to derived |
| 815 | // state** and AGENTS.md's rule covers it: a reader closing the |
| 816 | // tab must not decide whether the counter lands. Without it the |
| 817 | // rows most likely to be lost are the ones from the slowest |
| 818 | // pages, which is not a random sample. |
| 819 | // |
| 820 | // **Failing to write one does not fail the search.** The caller |
| 821 | // is answering somebody's query, and trading a working page for a |
| 822 | // metrics row is the wrong direction. |
| 823 | // |
| 824 | // An earlier comment here claimed a gap "makes the breaker report |
| 825 | // cannot determine". It does not: nothing records the gap, so the |
| 826 | // period simply reads quieter than it was — the false-negative |
| 827 | // direction §8.1 forbids. The honest statement is that a dropped |
| 828 | // counter is a small, logged loss of fidelity in a signal that |
| 829 | // needs three askers to fire, and that losing one is better than |
| 830 | // failing the page. If that ever stops being true the fix is a |
| 831 | // gap marker, not a louder comment. |
| 832 | s.recordForBreaker(context.WithoutCancel(ctx), asker, cands, at, disp) |
| 833 | } |
| 834 | |
| 835 | // **Notices are aggregated once, here, in result order.** The first |
| 836 | // version of this built two Outcomes — one from the client, one for |
| 837 | // the sources that could not be resolved — and filled the second by |
| 838 | // ranging a `map[int]string`, so a page with two unreachable apps |
| 839 | // listed them in a different order on every reload. Go randomises map |
| 840 | // iteration precisely so that this shows up, and it showed up. |
| 841 | // |
| 842 | // Ranging `all` also means the lines arrive in the order the reader |
| 843 | // meets the results they are about, which is the order that reads |
| 844 | // like an explanation rather than a list. |
| 845 | notices := &sibling.Outcome{} |
| 846 | page := Page{Results: make([]Result, 0, len(all))} |
| 847 | for i, r := range all { |
| 848 | // Record before the keep check: a passage kept is a Render, which |
| 849 | // Record ignores, and a passage dropped is the one with something |
| 850 | // to say. Doing both from one loop is what keeps them in step. |
| 851 | notices.Record(r.Source, disp[i]) |
| 852 | if disp[i].Renders() { |
| 853 | page.Results = append(page.Results, r) |
| 854 | } |
| 855 | } |
| 856 | page.Notices = notices.Lines() |
| 857 | return page |
| 858 | } |
| 859 | |
| 860 | // Summaries lists every document, titles opened. |
| 861 | func (s *Store) Summaries(ctx context.Context) ([]Summary, error) { |
| 862 | var docs []Document |
| 863 | if err := s.db.G.WithContext(ctx).Order("updated_at desc").Find(&docs).Error; err != nil { |
| 864 | return nil, err |
| 865 | } |
| 866 | out := make([]Summary, 0, len(docs)) |
| 867 | for _, d := range docs { |
| 868 | t, err := s.sealer.Open(seal.PurposeTitle, d.SealedTitle) |
| 869 | if err != nil { |
| 870 | return nil, fmt.Errorf("open title %d: %w", d.ID, err) |
| 871 | } |
| 872 | var n int64 |
| 873 | if err := s.db.G.WithContext(ctx).Model(&Chunk{}).Where("document_id = ?", d.ID).Count(&n).Error; err != nil { |
| 874 | return nil, err |
| 875 | } |
| 876 | var path string |
| 877 | if len(d.SealedPath) > 0 { |
| 878 | p, err := s.sealer.Open(seal.PurposePath, d.SealedPath) |
| 879 | if err != nil { |
| 880 | return nil, fmt.Errorf("open path %d: %w", d.ID, err) |
| 881 | } |
| 882 | path = string(p) |
| 883 | } |
| 884 | out = append(out, Summary{ |
| 885 | ID: d.ID, |
| 886 | Source: d.Source, |
| 887 | Title: string(t), |
| 888 | Path: path, |
| 889 | Bytes: d.Bytes, |
| 890 | Chunks: int(n), |
| 891 | Updated: stamp(d.UpdatedAt), |
| 892 | }) |
| 893 | } |
| 894 | return out, nil |
| 895 | } |
| 896 | |
| 897 | // Body opens one document for display. |
| 898 | func (s *Store) Body(ctx context.Context, id int64) (title, body string, err error) { |
| 899 | var d Document |
| 900 | if err := s.db.G.WithContext(ctx).Where("id = ?", id).First(&d).Error; err != nil { |
| 901 | return "", "", err |
| 902 | } |
| 903 | t, err := s.sealer.Open(seal.PurposeTitle, d.SealedTitle) |
| 904 | if err != nil { |
| 905 | return "", "", err |
| 906 | } |
| 907 | b, err := s.sealer.Open(seal.PurposeBody, d.SealedBody) |
| 908 | if err != nil { |
| 909 | return "", "", err |
| 910 | } |
| 911 | return string(t), string(b), nil |
| 912 | } |
| 913 | |
| 914 | // Passages returns a document's stored chunks, opened. |
| 915 | // |
| 916 | // The document page used to re-run the chunker at view time, which meant |
| 917 | // every "#p{n}" anchor a search result linked to was derived from the |
| 918 | // CURRENT chunker while the ordinal in the citation came from the one |
| 919 | // that ran at ingest. Any change to chunking silently misaligned every |
| 920 | // citation on every existing document — and this branch changed the |
| 921 | // chunker twice. |
| 922 | func (s *Store) Passages(ctx context.Context, id int64) ([]Passage, error) { |
| 923 | var rows []Chunk |
| 924 | if err := s.db.G.WithContext(ctx).Where("document_id = ?", id).Order("ordinal").Find(&rows).Error; err != nil { |
| 925 | return nil, err |
| 926 | } |
| 927 | out := make([]Passage, 0, len(rows)) |
| 928 | for _, r := range rows { |
| 929 | text, err := s.sealer.Open(seal.PurposeChunk, r.SealedText) |
| 930 | if err != nil { |
| 931 | return nil, fmt.Errorf("open chunk %d: %w", r.ID, err) |
| 932 | } |
| 933 | heading, err := s.sealer.Open(seal.PurposeTitle, r.SealedHeading) |
| 934 | if err != nil { |
| 935 | return nil, fmt.Errorf("open chunk heading %d: %w", r.ID, err) |
| 936 | } |
| 937 | out = append(out, Passage{Ordinal: r.Ordinal, Heading: string(heading), Text: string(text)}) |
| 938 | } |
| 939 | return out, nil |
| 940 | } |
| 941 | |
| 942 | // Passage is one stored chunk, opened for display. |
| 943 | type Passage struct { |
| 944 | Ordinal int |
| 945 | Heading string |
| 946 | Text string |
| 947 | } |
| 948 | |
| 949 | // Meta returns a document's plaintext metadata. |
| 950 | func (s *Store) Meta(ctx context.Context, id int64) (source string, revision, bytes int, err error) { |
| 951 | var d Document |
| 952 | if err := s.db.G.WithContext(ctx).Where("id = ?", id).First(&d).Error; err != nil { |
| 953 | return "", 0, 0, err |
| 954 | } |
| 955 | return d.Source, d.Revision, d.Bytes, nil |
| 956 | } |
| 957 | |
| 958 | // Delete removes a document, its chunks, and its index entries. |
| 959 | // |
| 960 | // The index entries go FIRST, and that ordering is the whole of this |
| 961 | // function. It is the opposite of Ingest's, deliberately, because the two |
| 962 | // have opposite failure states and only one of them is survivable. |
| 963 | // |
| 964 | // AGENTS.md's rule — derived state is refreshed after the commit — is a |
| 965 | // rule about ADDING to the index, and its reason says so: a rolled-back |
| 966 | // transaction that had already mutated the index would leave the index |
| 967 | // claiming passages the database does not have. Removing entries cannot |
| 968 | // produce that. It can only make the index claim fewer passages than the |
| 969 | // database holds, which is the direction this codebase can survive: a |
| 970 | // document that is stored and not findable until the next boot, which is |
| 971 | // exactly what Ingest's ErrIndexLag already says out loud. |
| 972 | // |
| 973 | // Run the other way round — the way this did, and the way the rule read |
| 974 | // literally would have it — a failed removal leaves the rows gone and the |
| 975 | // passages indexed. Search then answers with a passage from a document |
| 976 | // that no longer exists, and the passage someone deleted may be precisely |
| 977 | // the passage that had to stop being findable. So a delete whose index |
| 978 | // removal fails is not reported, it is REFUSED: nothing is deleted, the |
| 979 | // document stays whole and findable, the person is told (see |
| 980 | // deleteDocument), and they can try again. An annoyance beats a citation |
| 981 | // to text that is not there. |
| 982 | // |
| 983 | // The residual case is a delete that removes the entries and then fails |
| 984 | // to commit: a stored document, unfindable until the next boot, and an |
| 985 | // error the person is shown. That is the survivable direction, and it is |
| 986 | // self-healing — the index is rebuilt from the sealed rows at every boot. |
| 987 | func (s *Store) Delete(ctx context.Context, id int64) error { |
| 988 | if err := s.ix.RemoveDocument(id); err != nil { |
| 989 | return err |
| 990 | } |
| 991 | return s.db.G.WithContext(ctx).Transaction(func(tx *gormTx) error { |
| 992 | if err := tx.Where("document_id = ?", id).Delete(&Chunk{}).Error; err != nil { |
| 993 | return err |
| 994 | } |
| 995 | return tx.Where("id = ?", id).Delete(&Document{}).Error |
| 996 | }) |
| 997 | } |
| 998 | |
| 999 | // IndexFile reports the file backing the index, empty when in memory. |
| 1000 | func (s *Store) IndexFile() (string, error) { return s.ix.File() } |
| 1001 | |
| 1002 | // Count returns the indexed passage count, for the UI to state what the |
| 1003 | // corpus holds rather than implying it is complete. |
| 1004 | func (s *Store) Count() (int, error) { return s.ix.Count() } |
| 1005 | |
| 1006 | func firstLine(body string) string { |
| 1007 | for _, line := range strings.Split(body, "\n") { |
| 1008 | line = strings.TrimSpace(strings.TrimLeft(line, "# ")) |
| 1009 | if line != "" { |
| 1010 | if len(line) > 120 { |
| 1011 | return line[:120] |
| 1012 | } |
| 1013 | return line |
| 1014 | } |
| 1015 | } |
| 1016 | return "Untitled" |
| 1017 | } |
| 1018 | |
| 1019 | // MaxFacts is §8.8's cap: three candidates render, four render nothing. |
| 1020 | // |
| 1021 | // Inclusive, and the boundary is the whole of it. A cap written `>= 3` |
| 1022 | // passes any suite that tests one, two and four, which is why gate 35 |
| 1023 | // tests three as well. |
| 1024 | const MaxFacts = 3 |
| 1025 | |
| 1026 | // Fact is one row of §8.8's box. |
| 1027 | // |
| 1028 | // It carries Title and Trail as well as Label for the reason Result does: |
| 1029 | // two pages called "Contact" under different documents are the same row |
| 1030 | // to a reader who cannot see which is which. It carries no ChunkID and no |
| 1031 | // chunk text — a field a template can reach is a field a template |
| 1032 | // eventually prints, which is why Result.item is unexported. |
| 1033 | type Fact struct { |
| 1034 | Kind string |
| 1035 | Title string |
| 1036 | Trail string |
| 1037 | // Label is what the DOCUMENT called this value, never composed. |
| 1038 | Label string |
| 1039 | // Span is the document's own bytes. The canonical form that keyed |
| 1040 | // this candidate's identity does not leave the store. |
| 1041 | Span string |
| 1042 | DocumentID int64 |
| 1043 | Ordinal int |
| 1044 | // Provider names the party the passages were SENT to (decision 4), |
| 1045 | // and Model names the weights that answered. Both, because a row |
| 1046 | // naming only "deepseek/…" names DeepSeek's model and never the |
| 1047 | // company that received the team's passages. Empty on a row the |
| 1048 | // deterministic layer validated. |
| 1049 | Provider string |
| 1050 | // Model names the model that pointed at this span, and is empty for a |
| 1051 | // row the deterministic layer validated. **The one field that tells a |
| 1052 | // reader which kind of claim they are looking at**: §8.9 is explicit |
| 1053 | // that a model row does not inherit §8.8's guarantees — the model can |
| 1054 | // omit, and its classification is a judgement — so the page must |
| 1055 | // distinguish the two at a glance. Decision 4's naming rule, reaching |
| 1056 | // a box that has no answer paragraph. |
| 1057 | Model string |
| 1058 | // Note is set on the one row narrowing left for its kind, and nil on |
| 1059 | // every other row. It is carried by the row rather than the page so |
| 1060 | // that it renders under the row it describes, not under whichever |
| 1061 | // kind happens to render last. |
| 1062 | Note *FactNote |
| 1063 | } |
| 1064 | |
| 1065 | // addFacts fills p.Facts for the kinds this query names. |
| 1066 | // |
| 1067 | // **It runs over p.Results — the RULED slice — and never over the hits |
| 1068 | // that produced it.** That is §8.8's ordering ruling and the whole of why |
| 1069 | // the box cannot leak: a candidate that is not in p.Results does not |
| 1070 | // exist, so §8.4's invariant ("every passage in a response passed §4.1 in |
| 1071 | // the request that produced it") is structural here rather than |
| 1072 | // something a later caller has to remember. |
| 1073 | // |
| 1074 | // The counts below — dedupe, collision, cap — therefore also happen after |
| 1075 | // the ruling. Counting before it is the leak rather than a tidiness |
| 1076 | // question: a box that vanishes for passing the cap because of documents |
| 1077 | // the reader may not see reports the existence of those documents, and so |
| 1078 | // does an ambiguity of two where the reader may read one. |
| 1079 | func (s *Store) addFacts(ctx context.Context, p *Page, query string) { |
| 1080 | want := fact.Wanted(query) |
| 1081 | if len(want) == 0 || len(p.Results) == 0 { |
| 1082 | // **No database work for a query that names no kind**, which is |
| 1083 | // almost every query. The box must not cost anything on the |
| 1084 | // searches it will never appear on. |
| 1085 | return |
| 1086 | } |
| 1087 | |
| 1088 | ids := make([]int64, 0, len(p.Results)) |
| 1089 | for _, r := range p.Results { |
| 1090 | ids = append(ids, r.ChunkID) |
| 1091 | } |
| 1092 | var rows []Chunk |
| 1093 | if err := s.db.G.WithContext(ctx).Where("id IN ?", ids).Find(&rows).Error; err != nil { |
| 1094 | // The search already succeeded and the page is already honest |
| 1095 | // without a box. Failing it now would trade a working answer for |
| 1096 | // a garnish, which is the wrong direction — and there is no |
| 1097 | // query text and no content in this line, per gate 4. |
| 1098 | if s.logger != nil { |
| 1099 | s.logger.Error("fact box: read passages", "err", err) |
| 1100 | } |
| 1101 | return |
| 1102 | } |
| 1103 | texts := make(map[int64]string, len(rows)) |
| 1104 | for _, r := range rows { |
| 1105 | t, err := s.sealer.Open(seal.PurposeChunk, r.SealedText) |
| 1106 | if err != nil { |
| 1107 | if s.logger != nil { |
| 1108 | s.logger.Error("fact box: open passage", "err", err) |
| 1109 | } |
| 1110 | continue |
| 1111 | } |
| 1112 | texts[r.ID] = string(t) |
| 1113 | } |
| 1114 | |
| 1115 | // One bucket per kind, because every rule below is per kind. A cap |
| 1116 | // counted across kinds lets two phone numbers and two addresses |
| 1117 | // suppress each other, and a collision check across kinds suppresses |
| 1118 | // a phone and an email that happen to share a label. |
| 1119 | found := map[fact.Kind][]factCandidate{} |
| 1120 | seen := map[fact.Kind]map[string]bool{} |
| 1121 | |
| 1122 | for _, r := range p.Results { |
| 1123 | text, ok := texts[r.ChunkID] |
| 1124 | if !ok { |
| 1125 | // The chunk row is gone between the ruling and this read. |
| 1126 | // Skipped silently, exactly as Search's livePassage check |
| 1127 | // drops a stale hit and for the reason recorded there: the |
| 1128 | // passage is genuinely gone, so its absence is the true |
| 1129 | // answer rather than a truncation of one. |
| 1130 | continue |
| 1131 | } |
| 1132 | for _, f := range fact.Scan(text, r.Heading, r.Title, want) { |
| 1133 | if seen[f.Kind] == nil { |
| 1134 | seen[f.Kind] = map[string]bool{} |
| 1135 | } |
| 1136 | // **Identity is checked BEFORE rule 4, so only the occurrence |
| 1137 | // that is actually cited decides whether its kind renders.** |
| 1138 | // The other order made one document give two different |
| 1139 | // answers depending on where chunk.Split happened to cut: the |
| 1140 | // same value written twice, once with a short label and once |
| 1141 | // with an over-long one, silenced the kind when the two |
| 1142 | // landed in different passages and did not when they landed |
| 1143 | // in the same one. A boundary the reader cannot see is not |
| 1144 | // allowed to decide what the page says. |
| 1145 | if seen[f.Kind][f.Canonical] { |
| 1146 | continue |
| 1147 | } |
| 1148 | // **Identity is the canonical value**, so one number written |
| 1149 | // two ways is one candidate and the same number in five |
| 1150 | // forwarded signatures is one too. First occurrence wins the |
| 1151 | // citation, and because p.Results is in rank order that is |
| 1152 | // the best-ranked passage carrying it. A NoBox occurrence is |
| 1153 | // a candidate like any other: it silences the kind when the |
| 1154 | // kind is ruled with it in the set (see narrow). |
| 1155 | seen[f.Kind][f.Canonical] = true |
| 1156 | found[f.Kind] = append(found[f.Kind], factCandidate{ |
| 1157 | fact: Fact{ |
| 1158 | Kind: string(f.Kind), |
| 1159 | Title: r.Title, |
| 1160 | Trail: r.Heading, |
| 1161 | Label: f.Label, |
| 1162 | Span: f.Span, |
| 1163 | DocumentID: r.DocumentID, |
| 1164 | Ordinal: r.Ordinal, |
| 1165 | }, |
| 1166 | own: f.Own, |
| 1167 | noBox: f.NoBox, |
| 1168 | }) |
| 1169 | } |
| 1170 | } |
| 1171 | |
| 1172 | // A slice ranged in a fixed order, never a map: Go randomises map |
| 1173 | // iteration precisely so that an order nobody chose shows up, and it |
| 1174 | // showed up once already in this file's notice aggregation. |
| 1175 | quals := fact.Qualifiers(query) |
| 1176 | for _, k := range fact.Kinds() { |
| 1177 | cs := found[k] |
| 1178 | var note *FactNote |
| 1179 | if len(cs) > MaxFacts { |
| 1180 | // The cap has fired. §8.8's narrowing is the only way this |
| 1181 | // kind renders now, and it renders exactly one row or none. |
| 1182 | one, ok := narrow(cs, quals) |
| 1183 | if !ok { |
| 1184 | continue |
| 1185 | } |
| 1186 | note = &FactNote{Kind: string(k), Matched: quals, Others: len(cs) - 1} |
| 1187 | cs = []factCandidate{one} |
| 1188 | } |
| 1189 | if len(cs) == 0 { |
| 1190 | continue |
| 1191 | } |
| 1192 | rows := make([]Fact, 0, len(cs)) |
| 1193 | silenced := false |
| 1194 | for _, c := range cs { |
| 1195 | silenced = silenced || c.noBox |
| 1196 | rows = append(rows, c.fact) |
| 1197 | } |
| 1198 | if silenced || collides(rows) { |
| 1199 | continue |
| 1200 | } |
| 1201 | if note != nil { |
| 1202 | rows[0].Note = note |
| 1203 | } |
| 1204 | p.Facts = append(p.Facts, rows...) |
| 1205 | } |
| 1206 | |
| 1207 | // §8.9, strictly after everything above. **Eligibility is the kind, |
| 1208 | // never the outcome**: a validator-backed kind is never sent, whether |
| 1209 | // it produced candidates, none, or some that a rule suppressed. And |
| 1210 | // with the model layer off, s.extractor is nil and nothing below runs. |
| 1211 | if s.extractor == nil { |
| 1212 | return |
| 1213 | } |
| 1214 | for _, k := range want { |
| 1215 | if fact.ModelKind(k) { |
| 1216 | p.Facts = append(p.Facts, s.modelFacts(ctx, p.Results, texts, k)...) |
| 1217 | } |
| 1218 | } |
| 1219 | } |
| 1220 | |
| 1221 | // factCandidate is one canonical value of one kind, before §8.8's rules |
| 1222 | // decide whether its kind renders. |
| 1223 | type factCandidate struct { |
| 1224 | fact Fact |
| 1225 | // own is §8.8 rule 2: the document wrote this label for this value. |
| 1226 | own bool |
| 1227 | // noBox is rule 4: that label is too long to show, and fact.Label |
| 1228 | // holds it for matching only. It is never rendered. |
| 1229 | noBox bool |
| 1230 | } |
| 1231 | |
| 1232 | // narrow is §8.8's narrowing by the reader's own words (Paul, |
| 1233 | // 2026-09-29), for a kind whose candidates are over the cap. It returns |
| 1234 | // the one candidate whose OWN label contains every qualifier, and false |
| 1235 | // when there are no qualifiers, no such candidate, or more than one. |
| 1236 | // Never picks between two: "if narrowing doesn't leave exactly one |
| 1237 | // candidate, still show nothing rather than guess." |
| 1238 | // |
| 1239 | // A fallback label — the trail or the title — never matches. On the page |
| 1240 | // that motivated this, the unused numbers fall to the heading "Office |
| 1241 | // Address", and matching it would leave three candidates, not one. |
| 1242 | func narrow(cs []factCandidate, quals []string) (factCandidate, bool) { |
| 1243 | var one factCandidate |
| 1244 | n := 0 |
| 1245 | for _, c := range cs { |
| 1246 | if c.own && fact.LabelMatches(c.fact.Label, quals) { |
| 1247 | one = c |
| 1248 | n++ |
| 1249 | } |
| 1250 | } |
| 1251 | return one, n == 1 |
| 1252 | } |
| 1253 | |
| 1254 | // ModelPassages is how many ruled results §8.9 sends: eight, matching |
| 1255 | // §8.6's answer-time budget, so the payload is bounded the same way the |
| 1256 | // answer path's is. An address in result eleven is a stated recall limit. |
| 1257 | const ModelPassages = 8 |
| 1258 | |
| 1259 | // sentPassage is one passage as it was sent, with the result it came from. |
| 1260 | type sentPassage struct { |
| 1261 | result Result |
| 1262 | text string |
| 1263 | } |
| 1264 | |
| 1265 | // modelFacts runs §8.9 for one model kind. Every rule in its numbered |
| 1266 | // list is here, in order, and each is a place a proposal is dropped. |
| 1267 | // |
| 1268 | // Nothing a model returns is ever rendered. A proposal either becomes an |
| 1269 | // immutable (passage, start, end) coordinate in a document the reader may |
| 1270 | // see, or it is dropped — and from the coordinate on, the span, the label |
| 1271 | // and the citation all come from the DOCUMENT. |
| 1272 | func (s *Store) modelFacts(ctx context.Context, results []Result, texts map[int64]string, k fact.Kind) []Fact { |
| 1273 | // **The deadline covers the whole model path, not only the call.** |
| 1274 | // Processing a response happens after the provider's own 800 ms has |
| 1275 | // run out, and a review made that processing take 6.8 seconds with a |
| 1276 | // valid response under the size cap. One budget for asking and |
| 1277 | // answering; past it, the page renders as AI-off would. |
| 1278 | ctx, cancel := context.WithTimeout(ctx, provider.Deadline) |
| 1279 | defer cancel() |
| 1280 | |
| 1281 | // What is sent: of the top eight ruled results, those still present, |
| 1282 | // in rank order. |
| 1283 | // |
| 1284 | // **Bounded by RANK first, and then thinned — never refilled.** The |
| 1285 | // first version counted surviving chunks instead, so a chunk deleted |
| 1286 | // between the ruling and this read let result nine in: a review |
| 1287 | // deleted result one and watched 2-9 go out, not 2-8. §8.9 excludes |
| 1288 | // the ninth result by name, and it could then supply a row, fill the |
| 1289 | // cap, collide or silence the kind. A gone passage makes the payload |
| 1290 | // smaller, as it makes §8.8's page shorter; it does not reach further. |
| 1291 | top := results |
| 1292 | if len(top) > ModelPassages { |
| 1293 | top = top[:ModelPassages] |
| 1294 | } |
| 1295 | var sent []sentPassage |
| 1296 | for _, r := range top { |
| 1297 | if t, ok := texts[r.ChunkID]; ok { |
| 1298 | sent = append(sent, sentPassage{result: r, text: t}) |
| 1299 | } |
| 1300 | } |
| 1301 | if len(sent) == 0 { |
| 1302 | return nil |
| 1303 | } |
| 1304 | passages := make([]string, len(sent)) |
| 1305 | for i, sp := range sent { |
| 1306 | passages[i] = sp.text |
| 1307 | } |
| 1308 | |
| 1309 | ptrs, err := s.extractor.ExtractAddresses(ctx, passages) |
| 1310 | if err != nil { |
| 1311 | // The page is already honest without model rows; a failure here |
| 1312 | // renders exactly what AI-off would. An abandoned keystroke is the |
| 1313 | // ordinary case and says nothing. Anything else is logged as the |
| 1314 | // provider package's fixed string — which carries no content and |
| 1315 | // no key by construction (gate 31) — and nothing else. |
| 1316 | if !errors.Is(err, context.Canceled) && s.logger != nil { |
| 1317 | s.logger.Warn("fact box: model extraction failed", "why", modelFailure(err)) |
| 1318 | } |
| 1319 | return nil |
| 1320 | } |
| 1321 | |
| 1322 | type coord struct { |
| 1323 | passage, start, end int |
| 1324 | } |
| 1325 | // **Work is per DISTINCT proposal, and per passage where it can be.** |
| 1326 | // A response may repeat one proposal a thousand times, and the |
| 1327 | // recognisers run once per passage rather than once per proposal. Both |
| 1328 | // preserve every rule below — dedupe by identical (passage, text) cannot |
| 1329 | // change which occurrence is found. What bounds the rest is the |
| 1330 | // deadline, checked on every proposal. |
| 1331 | // |
| 1332 | // (An early return once more distinct spans were accepted than the cap |
| 1333 | // allows was here too. It was removed rather than kept: with the |
| 1334 | // recognisers cached, its absence could not be made measurably slow, so |
| 1335 | // no test could ever see it go — AGENTS.md's unreachable-defence rule.) |
| 1336 | type proposal struct { |
| 1337 | passage int |
| 1338 | text string |
| 1339 | } |
| 1340 | tried := map[proposal]bool{} |
| 1341 | recognised := map[int][][2]int{} |
| 1342 | spansOf := func(i int) [][2]int { |
| 1343 | sp, ok := recognised[i] |
| 1344 | if !ok { |
| 1345 | sp = fact.RecognisedSpans(passages[i]) |
| 1346 | recognised[i] = sp |
| 1347 | if s.probe != nil { |
| 1348 | s.probe.recognitions++ |
| 1349 | } |
| 1350 | } |
| 1351 | return sp |
| 1352 | } |
| 1353 | var accepted []coord |
| 1354 | for _, ptr := range ptrs { |
| 1355 | if expired(ctx) { |
| 1356 | return nil |
| 1357 | } |
| 1358 | if tried[proposal{ptr.Passage, ptr.Text}] { |
| 1359 | continue |
| 1360 | } |
| 1361 | tried[proposal{ptr.Passage, ptr.Text}] = true |
| 1362 | if s.probe != nil { |
| 1363 | s.probe.examinations++ |
| 1364 | if s.probe.afterExamine != nil { |
| 1365 | s.probe.afterExamine() |
| 1366 | } |
| 1367 | } |
| 1368 | // 1. It must name a passage that was SENT. An authorised result |
| 1369 | // ranked ninth was never in the request, and a proposal pointing |
| 1370 | // at it is pointing at something the model was not shown. |
| 1371 | if ptr.Passage < 0 || ptr.Passage >= len(sent) { |
| 1372 | continue |
| 1373 | } |
| 1374 | // 2. Structural, not a validator: non-empty, twelve bytes, and a |
| 1375 | // digit, comma or line break. There to stop "Dublin" or "1" |
| 1376 | // passing because they occur somewhere — the empty string occurs |
| 1377 | // everywhere. |
| 1378 | if !admissibleSpan(ptr.Text) { |
| 1379 | continue |
| 1380 | } |
| 1381 | // 3 and 4. Exact bytes, in THAT passage, first occurrence by |
| 1382 | // offset. No normalisation of any kind: NBSP versus space, curly |
| 1383 | // versus straight quotes, a flattened line break are all refusals, |
| 1384 | // because a mapping back to source bytes would be a second parser. |
| 1385 | text := passages[ptr.Passage] |
| 1386 | start := strings.Index(text, ptr.Text) |
| 1387 | if start < 0 { |
| 1388 | continue |
| 1389 | } |
| 1390 | end := start + len(ptr.Text) |
| 1391 | // 5. On word boundaries, and not wholly inside a fact the |
| 1392 | // recognisers found — a street name inside an email address is |
| 1393 | // not an address. (An address that CONTAINS an eircode is fine; |
| 1394 | // most Irish ones do.) |
| 1395 | if !fact.WordBounded(text, start, end) || insideRecognised(spansOf(ptr.Passage), start, end) { |
| 1396 | continue |
| 1397 | } |
| 1398 | accepted = append(accepted, coord{ptr.Passage, start, end}) |
| 1399 | } |
| 1400 | // **Checked AFTER the work, not only before each proposal.** The loop |
| 1401 | // above checks on entry, so the last proposal's recogniser pass could |
| 1402 | // run past the deadline and its row still be returned: a review had |
| 1403 | // one proposal arrive with 10 ms left against a 1.26 MB passage and got |
| 1404 | // a model row back at 1.01 s. A row the deadline has passed is dropped |
| 1405 | // here, again in the labelling loop, and once more before returning. |
| 1406 | // |
| 1407 | // What this does NOT bound: a single recogniser pass is not |
| 1408 | // interruptible, so the path can overrun by one passage's scan. What |
| 1409 | // it does bound: no row is returned once the deadline has passed by |
| 1410 | // the CLOCK — see expired, and why ctx.Err() alone was not enough. |
| 1411 | if expired(ctx) { |
| 1412 | return nil |
| 1413 | } |
| 1414 | |
| 1415 | // **Sorted by (rank, offset) before anything is chosen**, so the |
| 1416 | // provider's return order never decides which occurrence is cited. |
| 1417 | // The same bytes in two passages are one candidate, and the sort makes |
| 1418 | // the kept one the better-ranked. |
| 1419 | sort.Slice(accepted, func(i, j int) bool { |
| 1420 | if accepted[i].passage != accepted[j].passage { |
| 1421 | return accepted[i].passage < accepted[j].passage |
| 1422 | } |
| 1423 | return accepted[i].start < accepted[j].start |
| 1424 | }) |
| 1425 | |
| 1426 | var out []Fact |
| 1427 | seen := map[string]bool{} |
| 1428 | for _, c := range accepted { |
| 1429 | if expired(ctx) { |
| 1430 | return nil |
| 1431 | } |
| 1432 | text := passages[c.passage] |
| 1433 | // 6. The span is sliced from the document. The model's own string |
| 1434 | // is not used again. |
| 1435 | span := text[c.start:c.end] |
| 1436 | // Identity before rule 4, as §8.8 orders it: only the occurrence |
| 1437 | // actually cited decides whether the kind renders. |
| 1438 | if seen[span] { |
| 1439 | continue |
| 1440 | } |
| 1441 | seen[span] = true |
| 1442 | r := sent[c.passage].result |
| 1443 | label, noBox := fact.LabelAtSpans(text, c.start, spansOf(c.passage), r.Heading, r.Title) |
| 1444 | if noBox { |
| 1445 | return nil |
| 1446 | } |
| 1447 | out = append(out, Fact{ |
| 1448 | Kind: string(k), |
| 1449 | Title: r.Title, |
| 1450 | Trail: r.Heading, |
| 1451 | Label: label, |
| 1452 | Span: span, |
| 1453 | DocumentID: r.DocumentID, |
| 1454 | Ordinal: r.Ordinal, |
| 1455 | Provider: provider.Name, |
| 1456 | Model: provider.Model, |
| 1457 | }) |
| 1458 | } |
| 1459 | |
| 1460 | // §8.8's cap and collision, applied to the ACCEPTED proposals — which |
| 1461 | // is all this layer can apply them to. §8.9 says what that costs: an |
| 1462 | // address the model did not return cannot trip the cap, and nothing |
| 1463 | // here can know it was there. |
| 1464 | if len(out) > MaxFacts || collides(out) || expired(ctx) { |
| 1465 | return nil |
| 1466 | } |
| 1467 | return out |
| 1468 | } |
| 1469 | |
| 1470 | // modelFailure names a provider failure with a FIXED string, never the |
| 1471 | // error's own text. |
| 1472 | // |
| 1473 | // The provider package promises its errors carry no content, and they |
| 1474 | // don't. This does not rely on it. A log line built from err.Error() is |
| 1475 | // one careless %w away from quoting a response body — and a provider's |
| 1476 | // response body is content (gate 31) in storage nobody can grep |
| 1477 | // (`carlos logs`). So the store names the class and nothing else, and |
| 1478 | // TestAModelFailureLogsNoContent feeds it an error that DOES carry a |
| 1479 | // canary to prove the line is content-free regardless. |
| 1480 | func modelFailure(err error) string { |
| 1481 | switch { |
| 1482 | case errors.Is(err, context.DeadlineExceeded): |
| 1483 | return "deadline" |
| 1484 | case errors.Is(err, provider.ErrTruncated): |
| 1485 | return "truncated" |
| 1486 | case errors.Is(err, provider.ErrMalformed): |
| 1487 | return "malformed" |
| 1488 | case errors.Is(err, provider.ErrTooLarge): |
| 1489 | return "too_large" |
| 1490 | case errors.Is(err, provider.ErrForbiddenHost): |
| 1491 | return "forbidden_host" |
| 1492 | case errors.Is(err, provider.ErrStatus): |
| 1493 | return "status" |
| 1494 | default: |
| 1495 | return "other" |
| 1496 | } |
| 1497 | } |
| 1498 | |
| 1499 | // expired reports whether ctx is done, reading the CLOCK as well as the |
| 1500 | // cancellation. |
| 1501 | // |
| 1502 | // ctx.Err() reports a deadline only once the runtime's timer has fired, |
| 1503 | // and that can lag the deadline itself. A review pinned GOMAXPROCS to 1, |
| 1504 | // had the model busy-wait half a millisecond past the deadline, and got |
| 1505 | // every row back — 5 of 5, twice — because the timer had not yet run when |
| 1506 | // the checks did. The deadline is a time; this compares against it. |
| 1507 | func expired(ctx context.Context) bool { |
| 1508 | if ctx.Err() != nil { |
| 1509 | return true |
| 1510 | } |
| 1511 | dl, ok := ctx.Deadline() |
| 1512 | return ok && !time.Now().Before(dl) |
| 1513 | } |
| 1514 | |
| 1515 | // modelProbe counts what modelFacts did, for tests. nil in production. |
| 1516 | type modelProbe struct { |
| 1517 | // recognitions is recogniser passes — one per passage if caching works. |
| 1518 | recognitions int |
| 1519 | // examinations is proposals examined after dedupe. |
| 1520 | examinations int |
| 1521 | // afterExamine runs after each examination, so a test can end the |
| 1522 | // deadline at a known point rather than by timing. |
| 1523 | afterExamine func() |
| 1524 | } |
| 1525 | |
| 1526 | // admissibleSpan is §8.9's structural bound on a model's proposal. |
| 1527 | func admissibleSpan(t string) bool { |
| 1528 | return len(t) >= 12 && strings.ContainsAny(t, "0123456789,\n") |
| 1529 | } |
| 1530 | |
| 1531 | // insideRecognised reports whether [start,end) lies wholly inside one of |
| 1532 | // spans — the passage's recognised facts, computed once by the caller. |
| 1533 | func insideRecognised(spans [][2]int, start, end int) bool { |
| 1534 | for _, sp := range spans { |
| 1535 | if sp[0] <= start && end <= sp[1] { |
| 1536 | return true |
| 1537 | } |
| 1538 | } |
| 1539 | return false |
| 1540 | } |
| 1541 | |
| 1542 | // collides reports whether two candidates would render an identical |
| 1543 | // citation line. |
| 1544 | // |
| 1545 | // **Keyed on the whole line and not on the label alone.** A supplier page |
| 1546 | // and the office page both falling through to a trail of "Contact" are |
| 1547 | // perfectly disambiguated by their titles, and suppressing that box would |
| 1548 | // lose a real answer to a rule aimed at a different problem. What the |
| 1549 | // rule is actually for is the case where the page would print one |
| 1550 | // identical row twice against two different values — not an enumeration |
| 1551 | // but a contradiction, with nothing for the reader to resolve it by. |
| 1552 | func collides(cs []Fact) bool { |
| 1553 | type line struct{ title, trail, label string } |
| 1554 | seen := map[line]bool{} |
| 1555 | for _, c := range cs { |
| 1556 | l := line{c.Title, c.Trail, c.Label} |
| 1557 | if seen[l] { |
| 1558 | return true |
| 1559 | } |
| 1560 | seen[l] = true |
| 1561 | } |
| 1562 | return false |
| 1563 | } |
| 1564 | |
amadan