| 1 | // Package fact recognises the facts §8.8's box is allowed to show. |
| 2 | // |
| 3 | // One table drives all of it: a kind names the words that ask for it AND |
| 4 | // the validator that admits it. Those two must not drift apart — a |
| 5 | // lexicon that primes `iban` next to a validator that no longer accepts |
| 6 | // one is a box that never appears and nothing that says why — so they are |
| 7 | // two columns of one declaration and live in one file. That is also why |
| 8 | // the lexicon is here rather than in query, which has its own single |
| 9 | // decision to make and says so. |
| 10 | // |
| 11 | // Nothing here touches a database, a template or an app. The spec's §8.8 |
| 12 | // is a long argument about authorization and ordering, and none of it is |
| 13 | // in this package: by the time Scan is called the passage has already |
| 14 | // passed §4.1. What is here is the part that can be wrong on its own — |
| 15 | // whether a string is really an IBAN, and what the document called it. |
| 16 | // |
| 17 | // **Three of the five kinds are checksums and two are not**, and the |
| 18 | // difference is stated rather than smoothed over. An eircode's routing |
| 19 | // key is a closed set, an IBAN has a mod-97 check, an Irish VAT number |
| 20 | // has a check letter derived from its own digits: those can be wrong, and |
| 21 | // a wrong one is refused. A phone number and an email address have no |
| 22 | // such property, and their bounds are structural. See phoneValid for the |
| 23 | // rule that keeps the weakest of them honest. |
| 24 | package fact |
| 25 | |
| 26 | import ( |
| 27 | "regexp" |
| 28 | "sort" |
| 29 | "strings" |
| 30 | "unicode" |
| 31 | "unicode/utf8" |
| 32 | ) |
| 33 | |
| 34 | // Kind is one recognisable fact type. |
| 35 | type Kind string |
| 36 | |
| 37 | const ( |
| 38 | Eircode Kind = "eircode" |
| 39 | Phone Kind = "phone" |
| 40 | IBAN Kind = "iban" |
| 41 | VAT Kind = "vat" |
| 42 | Email Kind = "email" |
| 43 | // Address is a postal address, and it is the first MODEL kind: no |
| 44 | // recogniser produces one, because an address has no checksum, no |
| 45 | // grammar and no length to validate against. §8.9 sends it to a |
| 46 | // model; this package only lets a query ask for it. |
| 47 | Address Kind = "address" |
| 48 | ) |
| 49 | |
| 50 | // modelKinds is §8.9's allowlist: the kinds that may be sent to a model. |
| 51 | // |
| 52 | // **An allowlist, and never "whatever has no recogniser".** Draft 1 of |
| 53 | // §8.9 phrased eligibility as "where the deterministic layer produced |
| 54 | // nothing", which invites asking a model for a phone number whenever the |
| 55 | // box happens to be empty — and §8.8's box is empty for five different |
| 56 | // reasons, from zero candidates to a collision. Eligibility is a |
| 57 | // property of the kind. TestNoModelKindHasARecogniser holds the two sets |
| 58 | // apart, so adding a recogniser for Address later forces a decision |
| 59 | // rather than quietly sending a validated kind to a model. |
| 60 | var modelKinds = map[Kind]bool{Address: true} |
| 61 | |
| 62 | // ModelKind reports whether k may be sent to a model under §8.9. |
| 63 | func ModelKind(k Kind) bool { return modelKinds[k] } |
| 64 | |
| 65 | // order is the order kinds are returned and rendered in, so that a |
| 66 | // two-kind query produces the same two boxes in the same places every |
| 67 | // time. A map's range order would not, and §8.8's gate asserts order. |
| 68 | var order = []Kind{Eircode, Phone, IBAN, VAT, Email, Address} |
| 69 | |
| 70 | // Found is one validated fact, located in a passage. |
| 71 | type Found struct { |
| 72 | Kind Kind |
| 73 | // Span is the document's own bytes, and the only thing rendered. |
| 74 | // Taken as a slice of the input rather than rebuilt from parts, so |
| 75 | // that "byte-identical to the document" is a property of this type |
| 76 | // and not a discipline somebody has to keep. §8.7's rule one noun |
| 77 | // over: a second formatter is the same bug class as a second |
| 78 | // calculator. |
| 79 | Span string |
| 80 | // Canonical is match-only. It keys candidate identity so that |
| 81 | // "01 547 7854" and "+353 (0)1 547 7854" are one number rather than |
| 82 | // two, and it is what the validator ran on. It is NEVER rendered and |
| 83 | // is never an operand — see the package doc and §8.8 for why that |
| 84 | // distinction is what keeps this inside §8.7 and decision 10. |
| 85 | Canonical string |
| 86 | // Label is what the document called this value. Never composed. |
| 87 | // When NoBox is set it holds the over-long label, which is never |
| 88 | // rendered: it is kept so §8.8's narrowing can tell whether the |
| 89 | // reader's words are in it. |
| 90 | Label string |
| 91 | // Own reports that Label is the document's own words for this value — |
| 92 | // the sentence-anchored run ending in a separator (§8.8 rule 2) — |
| 93 | // rather than the heading trail or title it falls back to. Only an |
| 94 | // own label can match the reader's words (§8.8, narrowing). |
| 95 | Own bool |
| 96 | // NoBox means this kind renders nothing at all, because a value |
| 97 | // carried a document-authored label too long to show. Substituting |
| 98 | // the heading trail would replace the document's words with vaguer |
| 99 | // ones — "the number you must never give to customers…" becoming |
| 100 | // "Contacts" — and dropping just this candidate would turn a |
| 101 | // two-candidate ambiguity into a confident single answer. Both are |
| 102 | // worse than silence. §8.8. |
| 103 | NoBox bool |
| 104 | } |
| 105 | |
| 106 | // --------------------------------------------------------------------- |
| 107 | // The lexicon: which words ask for which kind. |
| 108 | // --------------------------------------------------------------------- |
| 109 | |
| 110 | // phrase is one entry of the lexicon, as a token sequence. |
| 111 | type phrase struct { |
| 112 | toks []string |
| 113 | kind Kind |
| 114 | } |
| 115 | |
| 116 | // lexicon maps what a person types to a kind. Longest phrases are tried |
| 117 | // first, which is the whole reason "email address" can mean Email while |
| 118 | // "address" alone means nothing: postal and email addresses are |
| 119 | // ambiguous and only one of them has anything resembling a validator. |
| 120 | // |
| 121 | // **This is query.openers' trade made in the opposite direction.** That |
| 122 | // list is deliberately stingy, because a wrong Enter-prime navigates away |
| 123 | // from a page somebody was reading. This one can afford to be generous, |
| 124 | // and the reason is the validator rather than a shared instinct: "mobile |
| 125 | // app" primes Phone, finds no phone number, and renders nothing. A |
| 126 | // generous lexicon costs a box that does not appear. |
| 127 | var lexicon = []phrase{ |
| 128 | // Three tokens, and it must come first: "e-mail address" splits to |
| 129 | // e / mail / address, and without this the two-token "e mail" took |
| 130 | // the first two and left "address" to prime the POSTAL kind — an |
| 131 | // email query sending passages to a model. Found by review. |
| 132 | {[]string{"e", "mail", "address"}, Email}, |
| 133 | {[]string{"email", "address"}, Email}, |
| 134 | {[]string{"postal", "address"}, Address}, |
| 135 | {[]string{"phone", "number"}, Phone}, |
| 136 | {[]string{"bank", "details"}, IBAN}, |
| 137 | {[]string{"post", "code"}, Eircode}, |
| 138 | {[]string{"vat", "number"}, VAT}, |
| 139 | {[]string{"e", "mail"}, Email}, |
| 140 | {[]string{"eircode"}, Eircode}, |
| 141 | {[]string{"postcode"}, Eircode}, |
| 142 | {[]string{"telephone"}, Phone}, |
| 143 | {[]string{"phone"}, Phone}, |
| 144 | {[]string{"mobile"}, Phone}, |
| 145 | {[]string{"tel"}, Phone}, |
| 146 | {[]string{"iban"}, IBAN}, |
| 147 | {[]string{"vat"}, VAT}, |
| 148 | {[]string{"email"}, Email}, |
| 149 | // Bare "address" is POSTAL. It used to prime nothing, because it is |
| 150 | // ambiguous with email and only email had a validator; decision 15's |
| 151 | // amendment makes postal a model kind, and "email address" is matched |
| 152 | // as a phrase before this single word is ever tried. |
| 153 | {[]string{"address"}, Address}, |
| 154 | } |
| 155 | |
| 156 | // Wanted returns the kinds a query names, in `order`. Empty means no box. |
| 157 | // |
| 158 | // Deliberately absent: "number", which is generic and counts only in |
| 159 | // combination, and would prime on ordinary queries that are not asking |
| 160 | // for a fact. ("address" used to be absent too; it is now the postal |
| 161 | // model kind — see the lexicon.) |
| 162 | func Wanted(q string) []Kind { |
| 163 | toks := tokens(q) |
| 164 | want := map[Kind]bool{} |
| 165 | for i := 0; i < len(toks); i++ { |
| 166 | for _, p := range lexicon { |
| 167 | if len(p.toks) > len(toks)-i { |
| 168 | continue |
| 169 | } |
| 170 | if matchAt(toks, i, p.toks) { |
| 171 | want[p.kind] = true |
| 172 | i += len(p.toks) - 1 |
| 173 | break |
| 174 | } |
| 175 | } |
| 176 | } |
| 177 | var out []Kind |
| 178 | for _, k := range order { |
| 179 | if want[k] { |
| 180 | out = append(out, k) |
| 181 | } |
| 182 | } |
| 183 | return out |
| 184 | } |
| 185 | |
| 186 | // generic are words that name no kind and qualify nothing: "number" is |
| 187 | // how every kind is asked for. |
| 188 | var generic = map[string]bool{"number": true, "numbers": true, "no": true, "nr": true} |
| 189 | |
| 190 | // stopwords are the question and function words a query carries around |
| 191 | // its qualifiers. **Stated as incomplete**, like the abbreviation list: |
| 192 | // a word missing from here becomes a qualifier that no label contains, |
| 193 | // narrowing matches nothing, and the kind is ruled exactly as it would |
| 194 | // have been without narrowing. That is the safe direction. This must not |
| 195 | // grow into a parser. |
| 196 | var stopwords = map[string]bool{} |
| 197 | |
| 198 | func init() { |
| 199 | for _, w := range strings.Fields(` |
| 200 | what whats which who whose where when why how |
| 201 | is are was were be been am do does did has have had can could |
| 202 | will would should shall may might must |
| 203 | a an the this that these those there here |
| 204 | i me my mine we us our ours you your yours he him his she her hers |
| 205 | it its they them their theirs |
| 206 | of for to at in on by with from about as into onto and or but not |
| 207 | please tell give find show get need want know let see look |
| 208 | any some one all`) { |
| 209 | stopwords[w] = true |
| 210 | } |
| 211 | } |
| 212 | |
| 213 | // Qualifiers returns the words in a query that are neither how a kind is |
| 214 | // asked for nor how a question is asked — "office" in "what is the office |
| 215 | // phone number?". §8.8 narrows a kind to the candidates whose own label |
| 216 | // contains every one of them. In query order, without repeats. |
| 217 | func Qualifiers(q string) []string { |
| 218 | lex := map[string]bool{} |
| 219 | for _, p := range lexicon { |
| 220 | for _, t := range p.toks { |
| 221 | lex[t] = true |
| 222 | } |
| 223 | } |
| 224 | var out []string |
| 225 | seen := map[string]bool{} |
| 226 | for _, t := range tokens(q) { |
| 227 | if utf8.RuneCountInString(t) < 2 || lex[t] || generic[t] || stopwords[t] || seen[t] { |
| 228 | continue |
| 229 | } |
| 230 | seen[t] = true |
| 231 | out = append(out, t) |
| 232 | } |
| 233 | return out |
| 234 | } |
| 235 | |
| 236 | // LabelMatches reports whether label contains every qualifier as a whole |
| 237 | // token. Tokens, not substrings, so "office" does not match "offices" or |
| 238 | // "backoffice". No stemming: a missed match leaves the kind ruled as it |
| 239 | // would have been anyway. |
| 240 | func LabelMatches(label string, qualifiers []string) bool { |
| 241 | if len(qualifiers) == 0 { |
| 242 | return false |
| 243 | } |
| 244 | have := map[string]bool{} |
| 245 | for _, t := range tokens(label) { |
| 246 | have[t] = true |
| 247 | } |
| 248 | for _, q := range qualifiers { |
| 249 | if !have[q] { |
| 250 | return false |
| 251 | } |
| 252 | } |
| 253 | return true |
| 254 | } |
| 255 | |
| 256 | func matchAt(toks []string, i int, want []string) bool { |
| 257 | for j, w := range want { |
| 258 | if toks[i+j] != w { |
| 259 | return false |
| 260 | } |
| 261 | } |
| 262 | return true |
| 263 | } |
| 264 | |
| 265 | // tokens lowercases and splits on anything that is not a letter or |
| 266 | // digit, so "e-mail" becomes ["e","mail"] and punctuation is inert. The |
| 267 | // same shape as index.MatchQuery's term split, and for the same reason: a |
| 268 | // search box is not a query language. |
| 269 | func tokens(s string) []string { |
| 270 | return strings.FieldsFunc(strings.ToLower(s), func(r rune) bool { |
| 271 | return !unicode.IsLetter(r) && !unicode.IsDigit(r) |
| 272 | }) |
| 273 | } |
| 274 | |
| 275 | // --------------------------------------------------------------------- |
| 276 | // The recognisers. |
| 277 | // --------------------------------------------------------------------- |
| 278 | |
| 279 | // sep is every character that may separate the groups of a fact, and it |
| 280 | // includes U+00A0. |
| 281 | // |
| 282 | // Go's \s does NOT match a non-breaking space, and Word and Notion both |
| 283 | // paste them. chunk.Split does not normalise interior whitespace, so the |
| 284 | // bytes reach the sealed row exactly as pasted — which means a recogniser |
| 285 | // written with \s finds nothing at all in text pasted from either, and |
| 286 | // the failure is a box that silently never appears. |
| 287 | const sep = ` \x{00A0}` |
| 288 | |
| 289 | // phoneGroups builds the digit-group half of the phone pattern for one |
| 290 | // separator style. |
| 291 | // |
| 292 | // **Loose, and the group count is the validator's job — not this |
| 293 | // pattern's.** Bounding it here truncated real numbers instead of |
| 294 | // refusing them, which is far worse than either: capped at five digits a |
| 295 | // group, "Mobile: +353 87 1234567" was boxed as "+353 87 12345" — the |
| 296 | // commonest international spelling of an Irish mobile, rendered as a |
| 297 | // different number, cited, above the results. Capped at three groups, |
| 298 | // "+49 30 12 34 56 78" was boxed as "+49 30 12 34 56". Both pass every |
| 299 | // other check because what is left is a well-formed number; they are |
| 300 | // §8.7's "cites correctly while being wrong", produced by the pattern |
| 301 | // that was supposed to prevent it. |
| 302 | // |
| 303 | // So the groups are generous and phoneValid refuses by count. A refusal |
| 304 | // shows nothing; a truncation shows a wrong number. |
| 305 | // |
| 306 | // **And at least one, which is what makes the three styles alternate.** |
| 307 | // Written {0,3} the first branch matched zero groups and won, because |
| 308 | // Go's regexp prefers the earliest alternative that matches AT ALL — |
| 309 | // so "01-547-7854" came back as the two-digit span "01" and was refused, |
| 310 | // with the dash branch never tried. A branch that cannot match zero |
| 311 | // groups fails, and the next one gets its turn. |
| 312 | func phoneGroups(sepClass string) string { |
| 313 | return `(?:` + sepClass + `[0-9]{2,7}){1,6}` |
| 314 | } |
| 315 | |
| 316 | type recogniser struct { |
| 317 | kind Kind |
| 318 | re *regexp.Regexp |
| 319 | canon func(string) string |
| 320 | valid func(span, canonical string) bool |
| 321 | } |
| 322 | |
| 323 | // The regexes are deliberately LOOSE and the validators do the work. |
| 324 | // That is §8.8's "validation is not pattern-spotting" made structural: it |
| 325 | // is what stops the next person tightening a regex until it is a |
| 326 | // validator nobody can see, and it is why every reject below fails on an |
| 327 | // arithmetic property rather than on a character class. |
| 328 | var recognisers = []recogniser{ |
| 329 | { |
| 330 | kind: Eircode, |
| 331 | // The routing key's first letter is a closed set and the unique |
| 332 | // identifier's alphabet excludes the letters that misread. A |
| 333 | // pattern written [A-Z][0-9][0-9] gets Dublin 6W wrong and lets |
| 334 | // B12 3456 through; both are fixtures. |
| 335 | // |
| 336 | // **Case-SENSITIVE, and it was written (?i) first.** The review |
| 337 | // found what that costs: the identifier alphabet is most of the |
| 338 | // consonants, so lowercase English after a road number or a form |
| 339 | // code matches it. "Take the N11 then exit at Bray" boxed |
| 340 | // "N11 then"; "Form P45 then P60" boxed "P45 then". A query for |
| 341 | // an eircode then gets a cited fact that is an English word — |
| 342 | // §8.7's "cites correctly while being wrong", from a character |
| 343 | // class. Eircodes are written uppercase, and a lowercase one |
| 344 | // going unrecognised is the safe direction. |
| 345 | re: regexp.MustCompile(`\b[ACDEFHKNPRTVWXY][0-9][0-9W][` + sep + `]?[0-9ACDEFHKNPRTVWXY]{4}\b`), |
| 346 | canon: func(s string) string { return strings.ToUpper(stripSep(s)) }, |
| 347 | valid: func(_, _ string) bool { return true }, // the pattern IS the closed set |
| 348 | }, |
| 349 | { |
| 350 | kind: Phone, |
| 351 | // Built out of digit GROUPS rather than one character class, and |
| 352 | // that is the difference between working and not. |
| 353 | // |
| 354 | // A class — [0-9 ().-]{6,24} — is greedy across a LIST: given |
| 355 | // "Phone: 01 547 7854 - 01 598 1234" it swallows both numbers as |
| 356 | // one span of eighteen digits, which then fails validation, so |
| 357 | // the page shows nothing where it should show two. Requiring a |
| 358 | // SINGLE separator between groups stops the match at the " - ", |
| 359 | // and the second number then matches on its own. Found by the |
| 360 | // fixture, not by reading the pattern. |
| 361 | // |
| 362 | // The prefix is "+CC", optionally followed by the "(0)" that |
| 363 | // means "the trunk digit you omit from abroad", or a bare |
| 364 | // leading 0. Anchoring there is what keeps account numbers, |
| 365 | // company numbers and DUNS numbers out before any validation |
| 366 | // runs at all. |
| 367 | // |
| 368 | // **One separator style throughout, and phoneValid's 9-or-10 |
| 369 | // bound is why it has to be.** That bound was written to keep |
| 370 | // dates out, and a mixed-separator pattern walked straight past |
| 371 | // it: "The deadline is 01-02-2024 10:30" matched |
| 372 | // "01-02-2024 10" — ten digits, valid, boxed. A person writing a |
| 373 | // phone number uses one separator for all of it. RE2 has no |
| 374 | // backreferences, so the three styles are three alternatives. |
| 375 | re: regexp.MustCompile(`(?:\+[0-9]{1,3}[` + sep + `]*(?:\(0\))?|\b0)[` + sep + `]*` + |
| 376 | `[0-9]{1,9}(?:` + phoneGroups(`[`+sep+`]`) + `|` + phoneGroups(`-`) + `|` + phoneGroups(`\.`) + `)?` + |
| 377 | // An extension is part of the span, because §8.8 says the span |
| 378 | // is what the document wrote, and a number whose extension has |
| 379 | // been dropped is a number that does not reach the person. |
| 380 | // |
| 381 | // **An extension OR a word boundary, never "optionally |
| 382 | // neither".** Written `(?:…)?` the match could stop in the |
| 383 | // middle of a run of digits, which is the other half of how |
| 384 | // "+353 87 1234567" became "+353 87 12345". Requiring one or |
| 385 | // the other means a partial number is not a match at all. |
| 386 | `(?:(?i:[` + sep + `]*(?:ext|extension|x)\.?[` + sep + `]*[0-9]{1,6})|\b)`), |
| 387 | canon: phoneCanon, |
| 388 | valid: phoneValid, |
| 389 | }, |
| 390 | { |
| 391 | kind: IBAN, |
| 392 | // Two spellings, because a document writes an IBAN either solid |
| 393 | // or in groups of four. |
| 394 | // |
| 395 | // **The grouped alternative was {2,4}-sized groups and that made |
| 396 | // the kind inert on its commonest shape.** A trailing 2-4 letter |
| 397 | // word was swallowed into the last group, the canonical form came |
| 398 | // out the wrong length, and validation refused the whole thing: |
| 399 | // "Our IBAN is IE29 AIBK 9311 5212 3456 78 and our BIC is …" |
| 400 | // found nothing at all. Only a full stop immediately after the |
| 401 | // number worked. Groups of exactly four with one short tail stop |
| 402 | // at the space before "and", because "and" is not four |
| 403 | // characters — and case-sensitivity means it would not match |
| 404 | // even if it were. |
| 405 | // |
| 406 | // **The tail is digits only**, because an uppercase 1-3 character |
| 407 | // token is exactly what follows an IBAN in practice: "BE68 5390 |
| 408 | // 0754 7034 BIC: GEBABEBB" found nothing while the tail admitted |
| 409 | // letters, and every country whose length is a multiple of four |
| 410 | // (BE, ES, AT, PL, SE, LU) has no real tail for it to sit in. |
| 411 | re: regexp.MustCompile(`\b[A-Z]{2}[0-9]{2}(?:[A-Z0-9]{11,30}|(?:[` + sep + `][A-Z0-9]{4}){2,7}(?:[` + sep + `][0-9]{1,3})?)\b`), |
| 412 | canon: func(s string) string { return strings.ToUpper(stripSep(s)) }, |
| 413 | valid: func(_, c string) bool { return ibanValid(c) }, |
| 414 | }, |
| 415 | { |
| 416 | kind: VAT, |
| 417 | re: regexp.MustCompile(`(?i)\bIE[0-9]{7}[A-Z]{1,2}\b`), |
| 418 | canon: func(s string) string { return strings.ToUpper(stripSep(s)) }, |
| 419 | valid: func(_, c string) bool { return vatValid(c) }, |
| 420 | }, |
| 421 | { |
| 422 | kind: Email, |
| 423 | // The weakest kind here, and §8.8 says so rather than hiding it: |
| 424 | // local@domain is closer to pattern-spotting than to validation. |
| 425 | // On a corpus carrying forwarded mail every signature qualifies, |
| 426 | // the cap is passed, and the box is inert for this kind — which |
| 427 | // is the cap working, and fails towards silence rather than |
| 428 | // towards a wrong address. |
| 429 | re: regexp.MustCompile(`\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b`), |
| 430 | canon: strings.ToLower, |
| 431 | valid: func(_, _ string) bool { return true }, |
| 432 | }, |
| 433 | } |
| 434 | |
| 435 | // stripSep removes every separator a document might group a fact with, |
| 436 | // including U+00A0. Used only to build a Canonical; a Span is never |
| 437 | // passed through it, which is gate 36's whole subject. |
| 438 | func stripSep(s string) string { |
| 439 | return strings.Map(func(r rune) rune { |
| 440 | switch r { |
| 441 | case ' ', ' ', '-', '.', '(', ')', '\t': |
| 442 | return -1 |
| 443 | } |
| 444 | return r |
| 445 | }, s) |
| 446 | } |
| 447 | |
| 448 | // phoneCanon folds a written number onto E.164 so that "01 547 7854" and |
| 449 | // "+353 (0)1 547 7854" are ONE candidate rather than two. |
| 450 | // |
| 451 | // The "(0)" is dropped only after an international prefix, which is what |
| 452 | // that notation means: it is the trunk digit you dial domestically and |
| 453 | // omit from abroad. Keeping it would make the two spellings above |
| 454 | // different canonical values and put the same number in the box twice. |
| 455 | // |
| 456 | // Never rendered. §8.8 rules explicitly that a canonical form which is |
| 457 | // neither displayed nor an operand is not §8.7's forbidden second parser. |
| 458 | func phoneCanon(s string) string { |
| 459 | if base, ext := splitExt(s); ext != "" { |
| 460 | // An extension is part of the number somebody has to ring, so two |
| 461 | // entries differing only by extension are two candidates and not a |
| 462 | // duplicate. Appended AFTER the base is folded, so the base still |
| 463 | // dedupes across spellings. |
| 464 | return phoneCanon(base) + "x" + onlyDigits(ext) |
| 465 | } |
| 466 | intl := strings.HasPrefix(s, "+") |
| 467 | if intl { |
| 468 | s = strings.Replace(s, "(0)", "", 1) |
| 469 | } |
| 470 | var d strings.Builder |
| 471 | for _, r := range s { |
| 472 | if r >= '0' && r <= '9' { |
| 473 | d.WriteRune(r) |
| 474 | } |
| 475 | } |
| 476 | digits := d.String() |
| 477 | switch { |
| 478 | case intl: |
| 479 | return "+" + digits |
| 480 | case strings.HasPrefix(digits, "0"): |
| 481 | // A national number, which on this corpus means Ireland. Folded |
| 482 | // onto +353 so it dedupes against the international spelling. |
| 483 | return "+353" + digits[1:] |
| 484 | default: |
| 485 | return digits |
| 486 | } |
| 487 | } |
| 488 | |
| 489 | // phoneValid is the structural rule that keeps the one checksum-less |
| 490 | // numeric kind honest: **a bare run of digits is never a phone number.** |
| 491 | // |
| 492 | // A phone number has no check digit, so unlike an IBAN or a VAT number |
| 493 | // there is nothing arithmetic to refuse a wrong one with. What is left is |
| 494 | // how documents actually write phone numbers: with a country code, or in |
| 495 | // groups. Without this rule index/rank_test.go's own corpus produces a |
| 496 | // false positive on its first line — "DUNS/D-U-N-S number is 985632364" |
| 497 | // is nine digits and a plausible Irish mobile. |
| 498 | // |
| 499 | // **The cost is stated rather than discovered:** a document that writes |
| 500 | // 0871234567 with no separators at all is not boxed. That is the |
| 501 | // direction a refusal should fail in. |
| 502 | func phoneValid(span, _ string) bool { |
| 503 | // Validated on the BASE number. An extension carries its own |
| 504 | // digits and would otherwise push an ordinary number past E.164's |
| 505 | // fifteen — refusing it for carrying more information, not less. |
| 506 | base, _ := splitExt(span) |
| 507 | if !strings.HasPrefix(base, "+") && !strings.ContainsAny(base, " ().-") { |
| 508 | return false |
| 509 | } |
| 510 | // **Refused by group count, never truncated.** A phone number is at |
| 511 | // most four separator-delimited runs, or five once a country code is |
| 512 | // counted: "01 547 7854" is three, "+353 87 1234567" is three. |
| 513 | // "Numbers drawn: 01 05 12 23 34" is five, and "+49 30 12 34 56 78" |
| 514 | // is six — both refused here rather than cut down to something that |
| 515 | // would validate. Germany and France genuinely do write more groups |
| 516 | // than this; refusing their numbers is the cost, and it is the right |
| 517 | // way round, because the alternative is showing most of one. |
| 518 | groups := digitRuns(base) |
| 519 | max := 4 |
| 520 | if strings.HasPrefix(base, "+") { |
| 521 | max = 5 |
| 522 | } |
| 523 | if groups > max { |
| 524 | return false |
| 525 | } |
| 526 | digits := strings.TrimPrefix(phoneCanon(base), "+") |
| 527 | if strings.HasPrefix(base, "+") { |
| 528 | // E.164 caps at fifteen. The floor here is TEN rather than the |
| 529 | // eight E.164 technically permits, and that is a deliberate |
| 530 | // narrowing: a country code plus a national number reaches ten |
| 531 | // for every country anybody in this corpus dials, while eight |
| 532 | // admits "+10 000 000" out of the sentence "Growth of |
| 533 | // +10 000 000 users". Refusing a genuinely short international |
| 534 | // number is the safe direction; boxing a growth figure is not. |
| 535 | return len(digits) >= 10 && len(digits) <= 15 |
| 536 | } |
| 537 | // A national number, canonicalised to +353 above, so the national |
| 538 | // form was 0 plus what follows. Irish numbers are 9 or 10 digits |
| 539 | // including the trunk 0 — 01 xxx xxxx, 021 xxx xxxx, 08x xxx xxxx. |
| 540 | // |
| 541 | // Nine-or-ten rather than "eight or more" is load-bearing: a date |
| 542 | // written 01-02-2024 leads with a 0, carries separators, and is eight |
| 543 | // digits. Widening this bound puts dates in the box. |
| 544 | n := len(digits) - len("353") + 1 |
| 545 | return n == 9 || n == 10 |
| 546 | } |
| 547 | |
| 548 | // digitRuns counts the separator-delimited runs of digits in a span. |
| 549 | func digitRuns(s string) int { |
| 550 | n, in := 0, false |
| 551 | for i := 0; i < len(s); i++ { |
| 552 | if s[i] >= '0' && s[i] <= '9' { |
| 553 | if !in { |
| 554 | n++ |
| 555 | in = true |
| 556 | } |
| 557 | continue |
| 558 | } |
| 559 | in = false |
| 560 | } |
| 561 | return n |
| 562 | } |
| 563 | |
| 564 | // extRe finds a trailing extension, so canon and valid can treat the base |
| 565 | // number and the extension separately while the SPAN keeps both. |
| 566 | var extRe = regexp.MustCompile(`(?i)[` + sep + `]*(?:ext|extension|x)\.?[` + sep + `]*[0-9]{1,6}$`) |
| 567 | |
| 568 | func splitExt(s string) (base, ext string) { |
| 569 | if m := extRe.FindStringIndex(s); m != nil { |
| 570 | return s[:m[0]], s[m[0]:] |
| 571 | } |
| 572 | return s, "" |
| 573 | } |
| 574 | |
| 575 | func onlyDigits(s string) string { |
| 576 | var b strings.Builder |
| 577 | for _, r := range s { |
| 578 | if r >= '0' && r <= '9' { |
| 579 | b.WriteRune(r) |
| 580 | } |
| 581 | } |
| 582 | return b.String() |
| 583 | } |
| 584 | |
| 585 | // ibanAlphabet holds the lengths IBANs have, by country. An unknown |
| 586 | // country is refused rather than guessed: mod-97 alone admits roughly one |
| 587 | // random string in ninety-seven, and the length is what makes the check |
| 588 | // worth having. |
| 589 | var ibanLength = map[string]int{ |
| 590 | "IE": 22, "GB": 22, "DE": 22, "FR": 27, "ES": 24, "NL": 18, |
| 591 | "IT": 27, "BE": 16, "PT": 25, "AT": 20, "PL": 28, "SE": 24, |
| 592 | "DK": 18, "FI": 18, "NO": 15, "CH": 21, "LU": 20, "GR": 27, |
| 593 | } |
| 594 | |
| 595 | // ibanValid is ISO 13616's mod-97 check: move the first four characters |
| 596 | // to the end, map letters to two-digit numbers, and the whole thing |
| 597 | // modulo 97 must be 1. |
| 598 | // |
| 599 | // Verified before it was written, against IE29AIBK93115212345678, |
| 600 | // GB82WEST12345698765432 and DE89370400440532013000, and against the |
| 601 | // first of those with one digit changed. A checksum is the one thing in |
| 602 | // this package that can be checked against the world rather than against |
| 603 | // a fixture somebody here wrote, so it was. |
| 604 | func ibanValid(s string) bool { |
| 605 | if len(s) < 15 || len(s) > 34 { |
| 606 | return false |
| 607 | } |
| 608 | want, known := ibanLength[s[:2]] |
| 609 | if !known || len(s) != want { |
| 610 | return false |
| 611 | } |
| 612 | rem := 0 |
| 613 | for _, c := range s[4:] + s[:4] { |
| 614 | switch { |
| 615 | case c >= '0' && c <= '9': |
| 616 | rem = (rem*10 + int(c-'0')) % 97 |
| 617 | case c >= 'A' && c <= 'Z': |
| 618 | rem = (rem*100 + int(c-'A') + 10) % 97 |
| 619 | default: |
| 620 | return false |
| 621 | } |
| 622 | } |
| 623 | return rem == 1 |
| 624 | } |
| 625 | |
| 626 | // vatCheckAlphabet is Revenue's check-letter table, indexed by the |
| 627 | // weighted sum modulo 23. W is zero. |
| 628 | const vatCheckAlphabet = "WABCDEFGHIJKLMNOPQRSTUV" |
| 629 | |
| 630 | // vatValid checks an Irish VAT number's check letter. |
| 631 | // |
| 632 | // Modern format only — seven digits, a check letter, optionally a second |
| 633 | // letter. **The old style, with a letter or + or * in second position, is |
| 634 | // refused deliberately.** A refused VAT number is a box that does not |
| 635 | // appear; an admitted wrong one is a fact on the page, and between those |
| 636 | // two the choice is not close. |
| 637 | // |
| 638 | // Verified against IE6388047V and IE3628739L, and against the invented |
| 639 | // IE1234567X, which it refuses. index/rank_test.go's own IE3384527RH |
| 640 | // passes, which is what makes that fixture usable as a corpus here. |
| 641 | func vatValid(s string) bool { |
| 642 | s = strings.TrimPrefix(s, "IE") |
| 643 | if len(s) != 8 && len(s) != 9 { |
| 644 | return false |
| 645 | } |
| 646 | sum := 0 |
| 647 | for i := 0; i < 7; i++ { |
| 648 | if s[i] < '0' || s[i] > '9' { |
| 649 | return false |
| 650 | } |
| 651 | sum += int(s[i]-'0') * (8 - i) |
| 652 | } |
| 653 | if len(s) == 9 { |
| 654 | j := strings.IndexByte(vatCheckAlphabet, s[8]) |
| 655 | if j < 0 { |
| 656 | return false |
| 657 | } |
| 658 | sum += j * 9 |
| 659 | } |
| 660 | return s[7] == vatCheckAlphabet[sum%23] |
| 661 | } |
| 662 | |
| 663 | // --------------------------------------------------------------------- |
| 664 | // The label. |
| 665 | // --------------------------------------------------------------------- |
| 666 | |
| 667 | // MaxLabel is how long a document-authored label may be, in characters. |
| 668 | // |
| 669 | // **A guess.** What is not a guess is that it has to clear the motivating |
| 670 | // fixture in index/rank_test.go, whose longer label — "We also have the |
| 671 | // following unused phone numbers" — is 47. |
| 672 | const MaxLabel = 60 |
| 673 | |
| 674 | // abbreviations are the full stops that do not end a sentence. |
| 675 | // |
| 676 | // **Stated as incomplete, because that is the honest form.** Draft 2 of |
| 677 | // §8.8 claimed the sentence split "fails in the safe direction"; it does |
| 678 | // not. A boundary placed too LATE yields a shorter run that passes every |
| 679 | // other test and produces a label the document never wrote — "Dr. Smith's |
| 680 | // direct line:" becoming "Smith's direct line". That is the composing |
| 681 | // failure this whole function exists to prevent, arriving through the |
| 682 | // function itself, so the list is what holds it and the list is short. |
| 683 | // |
| 684 | // It must not grow into a sentence tokeniser. Where it is wrong it is |
| 685 | // wrong in the composing direction, and knowing that is the difference |
| 686 | // between a limit and a surprise. |
| 687 | var abbreviations = map[string]bool{ |
| 688 | "dr.": true, "mr.": true, "mrs.": true, "ms.": true, "st.": true, |
| 689 | "no.": true, "ext.": true, "ltd.": true, "co.": true, |
| 690 | } |
| 691 | |
| 692 | // separators are what a document puts between a label and its value. |
| 693 | // |
| 694 | // The en dash is here because Word and Notion autocorrect a hyphen |
| 695 | // between spaces into one, so "Office – 01 547 7854" is what a document |
| 696 | // actually contains rather than what its author typed. |
| 697 | func isSeparator(r rune) bool { |
| 698 | switch r { |
| 699 | case ':', '—', '–', '-', '=', '\t': |
| 700 | return true |
| 701 | } |
| 702 | return false |
| 703 | } |
| 704 | |
| 705 | // maxLookback bounds how far sentenceStart walks. |
| 706 | // |
| 707 | // Without it this is linear in the LINE, and chunk.Split bounds a chunk |
| 708 | // by words rather than bytes — so a 2 MiB upload with no whitespace is |
| 709 | // one word, one paragraph, one chunk, and one line. Measured before this |
| 710 | // bound: 0.49s at 100 KB, 1.94s at 200 KB, 48.5s at 1 MB, which at the |
| 711 | // upload cap is minutes of search latency bought by one person's file. |
| 712 | // |
| 713 | // **It changes an outcome in one case, and draft 2's claim that it |
| 714 | // changes none was wrong.** Four hundred bytes is always more than |
| 715 | // MaxLabel runes UNLESS most of them are whitespace, which TrimSpace |
| 716 | // then removes: five hundred characters, four hundred and fifty spaces, |
| 717 | // then "Phone: 01 547 7854" yields the document's own label where the |
| 718 | // uncapped walk yielded none. That is the better answer and it is the |
| 719 | // direction this file prefers everywhere else, so the bound stays — but |
| 720 | // stated, because "changes no outcome" is the kind of claim that stops |
| 721 | // anybody checking. |
| 722 | const maxLookback = 400 |
| 723 | |
| 724 | // sentenceStart walks back from at to the start of the enclosing sentence |
| 725 | // or line, bounded by maxLookback. |
| 726 | func sentenceStart(text string, at int) int { |
| 727 | floor := at - maxLookback |
| 728 | if floor < 0 { |
| 729 | floor = 0 |
| 730 | } |
| 731 | for i := at - 1; i > floor; i-- { |
| 732 | if text[i] == '\n' { |
| 733 | return i + 1 |
| 734 | } |
| 735 | if text[i] != ' ' || (text[i-1] != '.' && text[i-1] != '!' && text[i-1] != '?') { |
| 736 | continue |
| 737 | } |
| 738 | // A full stop followed by a space. Whether it ends a sentence |
| 739 | // depends on the word it is attached to. |
| 740 | w := i |
| 741 | for w > floor { |
| 742 | r, size := utf8.DecodeLastRuneInString(text[floor:w]) |
| 743 | if size == 0 || isWordBreak(r) { |
| 744 | break |
| 745 | } |
| 746 | w -= size |
| 747 | } |
| 748 | if isAbbreviation(text[w:i]) { |
| 749 | continue |
| 750 | } |
| 751 | return i + 1 |
| 752 | } |
| 753 | return floor |
| 754 | } |
| 755 | |
| 756 | // isWordBreak ends the backward word walk. **NBSP counts**, and it did |
| 757 | // not at first: Word and Notion glue "Contact\u00a0Dr." into one token, |
| 758 | // which then matched no abbreviation, so the boundary was taken and the |
| 759 | // label came back as "Smith's direct line" — the composing failure the |
| 760 | // abbreviation list exists to prevent, arriving through a space nobody |
| 761 | // can see. |
| 762 | func isWordBreak(r rune) bool { |
| 763 | return r == ' ' || r == '\n' || r == '\t' || r == '\u00a0' |
| 764 | } |
| 765 | |
| 766 | // isAbbreviation reports whether a token ending in "." is one that does |
| 767 | // not end a sentence. |
| 768 | // |
| 769 | // The fixed list, plus **any single letter**, which covers the initials |
| 770 | // the list cannot enumerate: "J. Smith's line:" yielded "Smith's line" |
| 771 | // before this, and there is no list of every initial. |
| 772 | func isAbbreviation(tok string) bool { |
| 773 | if abbreviations[strings.ToLower(tok)] { |
| 774 | return true |
| 775 | } |
| 776 | // Decoded as RUNES, not bytes. `len(tok) == 2 && rune(tok[0])` read a |
| 777 | // raw byte as a code point, so "Planta 2\u00ba. Tel\u00e9fono:" saw |
| 778 | // 0xBA as the letter "\u00ba" and swallowed the sentence boundary, |
| 779 | // while "\u00c9. Dupont's line:" was still cut to "Dupont's line" — |
| 780 | // wrong in both directions at once, and the second is the composing |
| 781 | // direction §8.8 calls the unsafe one. |
| 782 | r, size := utf8.DecodeRuneInString(tok) |
| 783 | return size > 0 && len(tok) == size+1 && tok[size] == '.' && unicode.IsLetter(r) |
| 784 | } |
| 785 | |
| 786 | // listMarker is one leading list marker: a bullet, a number, a to-do box |
| 787 | // after either, or a quote. memoria's own Notion renderer writes these |
| 788 | // (notion/markdown.go), and rule 1 anchors the run at the start of the |
| 789 | // line, so without this the label carried memoria's syntax: "- Office". |
| 790 | var listMarker = regexp.MustCompile(`^[ \t\x{00A0}]*(?:(?:[-*+•]|[0-9]{1,3}[.)])[ \x{00A0}]+(?:\[[ xX]\][ \x{00A0}]+)?|>[ \x{00A0}]+)`) |
| 791 | |
| 792 | // labelOf applies §8.8's label rules. It returns the label, whether the |
| 793 | // document wrote it (rule 2) rather than it being the fallback (rule 5), |
| 794 | // and whether the kind renders no box because that label is too long |
| 795 | // (rule 4) — in which case the label is returned too, for matching only. |
| 796 | // |
| 797 | // The order of the rules is the whole of it, and each one is here because |
| 798 | // a fixture broke without it — see gate 37, which carries one per case. |
| 799 | func labelOf(text string, start int, spans [][2]int, heading, title string) (string, bool, bool) { |
| 800 | b := sentenceStart(text, start) |
| 801 | // **An earlier recognised span inside the run is not part of the |
| 802 | // label.** Without this, "Phone: 01 547 7854 - 01 598 1234" labels |
| 803 | // the SECOND number with the first one's value — a box whose left |
| 804 | // column is another phone number. |
| 805 | // |
| 806 | // A binary search over spans sorted by start, not a scan of all of |
| 807 | // them: scanning made this quadratic in the facts a chunk holds, and |
| 808 | // a chunk can hold tens of thousands of them. |
| 809 | if e := lastSpanEndBefore(spans, start); e > b { |
| 810 | b = e |
| 811 | } |
| 812 | run := strings.TrimRight(text[b:start], " ") |
| 813 | // A leading list marker is markup. The pattern needs a space after |
| 814 | // the marker and run is already trimmed on the right, so whatever it |
| 815 | // matches has something after it: a bare bullet — "- +353 1 547 |
| 816 | // 7854" — trims to "-", does not match, and rule 3 still sends it to |
| 817 | // the trail. (A guard for "nothing follows" was written and removed: |
| 818 | // no input could reach it.) |
| 819 | if m := listMarker.FindStringIndex(run); m != nil { |
| 820 | run = run[m[1]:] |
| 821 | } |
| 822 | if r, size := utf8.DecodeLastRuneInString(run); size > 0 && isSeparator(r) { |
| 823 | lbl := strings.TrimSpace(run[:len(run)-size]) |
| 824 | switch { |
| 825 | case lbl == "": |
| 826 | // A markdown bullet: chunk.Split keeps lines verbatim, so |
| 827 | // "- +353 1 547 7854" gives a run of "-", which ends in a |
| 828 | // separator and names nothing. Falling through keeps a plain |
| 829 | // contact list boxable; returning "" would make two bullets |
| 830 | // collide and suppress it. |
| 831 | case utf8.RuneCountInString(lbl) > MaxLabel: |
| 832 | return lbl, true, true |
| 833 | default: |
| 834 | return lbl, true, false |
| 835 | } |
| 836 | } |
| 837 | if heading != "" { |
| 838 | return heading, false, false |
| 839 | } |
| 840 | return title, false, false |
| 841 | } |
| 842 | |
| 843 | // lastSpanEndBefore returns the end of the latest span finishing at or |
| 844 | // before at. spans must be sorted by END — see the sort in Scan. |
| 845 | func lastSpanEndBefore(spans [][2]int, at int) int { |
| 846 | lo, hi, out := 0, len(spans)-1, 0 |
| 847 | for lo <= hi { |
| 848 | mid := (lo + hi) / 2 |
| 849 | if spans[mid][1] <= at { |
| 850 | if spans[mid][1] > out { |
| 851 | out = spans[mid][1] |
| 852 | } |
| 853 | lo = mid + 1 |
| 854 | } else { |
| 855 | hi = mid - 1 |
| 856 | } |
| 857 | } |
| 858 | return out |
| 859 | } |
| 860 | |
| 861 | // --------------------------------------------------------------------- |
| 862 | // Scan. |
| 863 | // --------------------------------------------------------------------- |
| 864 | |
| 865 | // Scan returns every validated fact of the wanted kinds in text. |
| 866 | // |
| 867 | // heading and title are the label fallbacks, in that order, and title is |
| 868 | // the last one on purpose: an upload with no headings at all is the |
| 869 | // common pasted-file shape, and falling through to an empty label there |
| 870 | // is the case every fixture carrying a heading would hide. |
| 871 | // |
| 872 | // **This runs over a passage that has already passed §4.1.** Nothing in |
| 873 | // this package knows about authorization, and that is the point: by |
| 874 | // construction there is no way to reach it with a passage the reader may |
| 875 | // not see, because the caller has only the ruled results to hand it. |
| 876 | func Scan(text, heading, title string, want []Kind) []Found { |
| 877 | wanted := map[Kind]bool{} |
| 878 | for _, k := range want { |
| 879 | wanted[k] = true |
| 880 | } |
| 881 | |
| 882 | // Every span of EVERY kind is located first, and every label is |
| 883 | // derived afterwards. Two separate reasons, both found by review: |
| 884 | // |
| 885 | // - labelOf needs to know about the spans to its left, and cannot |
| 886 | // if it is called as they are found. |
| 887 | // - **the spans it needs are of every kind, not just the wanted |
| 888 | // ones.** Locating only what was asked for made a fact's label |
| 889 | // depend on which kinds the query named: "Email paul@tito.io or |
| 890 | // phone: 01 547 7854" asked for `phone` alone came back labelled |
| 891 | // "Email paul@tito.io or phone", one fact's value inside |
| 892 | // another's label — which is exactly what the earlier-span rule |
| 893 | // exists to stop, reappearing because the rule could not see the |
| 894 | // span. |
| 895 | type hit struct { |
| 896 | kind Kind |
| 897 | at [2]int |
| 898 | span string |
| 899 | canonical string |
| 900 | } |
| 901 | var hits []hit |
| 902 | var spans [][2]int |
| 903 | for _, r := range recognisers { |
| 904 | for _, m := range r.re.FindAllStringIndex(text, -1) { |
| 905 | span := text[m[0]:m[1]] |
| 906 | c := r.canon(span) |
| 907 | if !r.valid(span, c) { |
| 908 | continue |
| 909 | } |
| 910 | spans = append(spans, [2]int{m[0], m[1]}) |
| 911 | if wanted[r.kind] { |
| 912 | hits = append(hits, hit{r.kind, [2]int{m[0], m[1]}, span, c}) |
| 913 | } |
| 914 | } |
| 915 | } |
| 916 | // Sorted by END, not by start, because lastSpanEndBefore searches on |
| 917 | // the end and spans NEST: a phone number inside an email address, or |
| 918 | // inside an IBAN, is a real pair. Sorted by start the predicate is |
| 919 | // non-monotonic and the binary search can miss the enclosing span, |
| 920 | // which puts a recognised value back inside another fact's label. |
| 921 | sort.Slice(spans, func(i, j int) bool { return spans[i][1] < spans[j][1] }) |
| 922 | sort.SliceStable(hits, func(i, j int) bool { return hits[i].at[0] < hits[j].at[0] }) |
| 923 | |
| 924 | // **Deduped BEFORE labels are derived, not after.** A label costs a |
| 925 | // backward walk, and a chunk repeating one value ten thousand times |
| 926 | // needs exactly one. The caller dedupes across passages on the same |
| 927 | // key and keeps the first, so doing it here keeps the same |
| 928 | // occurrence and changes no outcome. |
| 929 | seen := map[Kind]map[string]bool{} |
| 930 | var out []Found |
| 931 | for _, k := range order { |
| 932 | if !wanted[k] { |
| 933 | continue |
| 934 | } |
| 935 | for _, h := range hits { |
| 936 | if h.kind != k { |
| 937 | continue |
| 938 | } |
| 939 | if seen[k] == nil { |
| 940 | seen[k] = map[string]bool{} |
| 941 | } |
| 942 | if seen[k][h.canonical] { |
| 943 | continue |
| 944 | } |
| 945 | seen[k][h.canonical] = true |
| 946 | lbl, own, noBox := labelOf(text, h.at[0], spans, heading, title) |
| 947 | out = append(out, Found{ |
| 948 | Kind: h.kind, |
| 949 | Span: h.span, |
| 950 | Canonical: h.canonical, |
| 951 | Label: lbl, |
| 952 | Own: own, |
| 953 | NoBox: noBox, |
| 954 | }) |
| 955 | } |
| 956 | } |
| 957 | return out |
| 958 | } |
| 959 | |
| 960 | // Kinds returns the render order, so a caller applying §8.8's per-kind |
| 961 | // rules produces the same boxes in the same places on every request. |
| 962 | func Kinds() []Kind { return append([]Kind(nil), order...) } |
| 963 | |
| 964 | // --------------------------------------------------------------------- |
| 965 | // What §8.9's model half borrows from this layer. |
| 966 | // |
| 967 | // Exported so the model half uses THIS code rather than a copy of it: a |
| 968 | // second implementation of the label rules is how a model row would come |
| 969 | // to be labelled differently from a validated row sitting beside it, |
| 970 | // with nothing to say which rule was which. |
| 971 | // --------------------------------------------------------------------- |
| 972 | |
| 973 | // RecognisedSpans returns the byte range of every validated fact of every |
| 974 | // kind in text, sorted by end. |
| 975 | // |
| 976 | // §8.9 refuses a model's span that lies inside one of these — a street |
| 977 | // name inside an email address is not an address — and needs the ranges |
| 978 | // of ALL kinds for the same reason Scan does: the answer must not depend |
| 979 | // on which kinds the query happened to name. |
| 980 | func RecognisedSpans(text string) [][2]int { |
| 981 | var spans [][2]int |
| 982 | for _, r := range recognisers { |
| 983 | for _, m := range r.re.FindAllStringIndex(text, -1) { |
| 984 | span := text[m[0]:m[1]] |
| 985 | if r.valid(span, r.canon(span)) { |
| 986 | spans = append(spans, [2]int{m[0], m[1]}) |
| 987 | } |
| 988 | } |
| 989 | } |
| 990 | sort.Slice(spans, func(i, j int) bool { return spans[i][1] < spans[j][1] }) |
| 991 | return spans |
| 992 | } |
| 993 | |
| 994 | // LabelAtSpans applies §8.8's five label rules to a value at byte offset |
| 995 | // start. It returns the label, and whether the kind must render no box |
| 996 | // because the document labelled the value too long to show — in which |
| 997 | // case the label is the over-long one, exactly as Scan reports it, and |
| 998 | // must not be rendered. |
| 999 | // |
| 1000 | // spans are the passage's RecognisedSpans, computed once by the caller. |
| 1001 | // **There is deliberately no variant that computes them itself.** One |
| 1002 | // existed — LabelAt — and a review showed that swapping it in for this |
| 1003 | // function passed the whole suite while recomputing every recogniser per |
| 1004 | // label: 6.8 seconds for a legal-sized model response. Removing it makes |
| 1005 | // that mistake a compile error instead of a timing regression nothing |
| 1006 | // could see. |
| 1007 | func LabelAtSpans(text string, start int, spans [][2]int, heading, title string) (string, bool) { |
| 1008 | l, _, noBox := labelOf(text, start, spans, heading, title) |
| 1009 | return l, noBox |
| 1010 | } |
| 1011 | |
| 1012 | // WordBounded reports whether text[start:end] begins and ends on a word |
| 1013 | // boundary — neither neighbour is a letter or a digit. |
| 1014 | // |
| 1015 | // §8.9 requires it of a model's span, because an exact byte match inside |
| 1016 | // a longer word ("Street" found in "Streetside") reproduces source bytes |
| 1017 | // and names nothing. |
| 1018 | func WordBounded(text string, start, end int) bool { |
| 1019 | if start < 0 || end > len(text) || start >= end { |
| 1020 | return false |
| 1021 | } |
| 1022 | if start > 0 { |
| 1023 | if r, _ := utf8.DecodeLastRuneInString(text[:start]); isWordRune(r) { |
| 1024 | return false |
| 1025 | } |
| 1026 | } |
| 1027 | if end < len(text) { |
| 1028 | if r, _ := utf8.DecodeRuneInString(text[end:]); isWordRune(r) { |
| 1029 | return false |
| 1030 | } |
| 1031 | } |
| 1032 | return true |
| 1033 | } |
| 1034 | |
| 1035 | func isWordRune(r rune) bool { return unicode.IsLetter(r) || unicode.IsDigit(r) } |
| 1036 | |
amadan