amadan

oficina / memoria Public

fact: the Address model kind, and what §8.9 borrows from this layer

Address is §8.9's first model kind: a query can ask for it, and no
recogniser produces one. Eligibility for a model is an explicit
allowlist, never "whatever has no recogniser", and a test holds the two
sets apart — adding a recogniser for Address later goes red and forces
the decision rather than quietly sending a validated kind to a model.

Bare "address" now primes the postal kind; "email address" still means
email, because phrases are matched before single words.

RecognisedSpans, LabelAt and WordBounded are exported so the model half
uses this layer's code rather than a copy: a second implementation of the
label rules is how a model row would come to be labelled differently from
the validated row beside it. LabelAt is tested to agree with Scan.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Paul Campbell pushed by paul@keymail.dev e4e6f00ea97e19dd3de83bf83db2d8c752b95e36 parent c088f9f
2 files changed, +184 −6
  • internal/memoria/fact/fact.go +93 −5
  • internal/memoria/fact/fact_test.go +91 −1
diff --git a/internal/memoria/fact/fact.go b/internal/memoria/fact/fact.go
index 904a104..4b2ae80 100644
--- a/internal/memoria/fact/fact.go
+++ b/internal/memoria/fact/fact.go
@@ -40,12 +40,32 @@ const (
IBAN Kind = "iban"
VAT Kind = "vat"
Email Kind = "email"
+ // Address is a postal address, and it is the first MODEL kind: no
+ // recogniser produces one, because an address has no checksum, no
+ // grammar and no length to validate against. §8.9 sends it to a
+ // model; this package only lets a query ask for it.
+ Address Kind = "address"
)
+// modelKinds is §8.9's allowlist: the kinds that may be sent to a model.
+//
+// **An allowlist, and never "whatever has no recogniser".** Draft 1 of
+// §8.9 phrased eligibility as "where the deterministic layer produced
+// nothing", which invites asking a model for a phone number whenever the
+// box happens to be empty — and §8.8's box is empty for five different
+// reasons, from zero candidates to a collision. Eligibility is a
+// property of the kind. TestNoModelKindHasARecogniser holds the two sets
+// apart, so adding a recogniser for Address later forces a decision
+// rather than quietly sending a validated kind to a model.
+var modelKinds = map[Kind]bool{Address: true}
+
+// ModelKind reports whether k may be sent to a model under §8.9.
+func ModelKind(k Kind) bool { return modelKinds[k] }
+
// order is the order kinds are returned and rendered in, so that a
// two-kind query produces the same two boxes in the same places every
// time. A map's range order would not, and §8.8's gate asserts order.
-var order = []Kind{Eircode, Phone, IBAN, VAT, Email}
+var order = []Kind{Eircode, Phone, IBAN, VAT, Email, Address}
// Found is one validated fact, located in a passage.
type Found struct {
@@ -98,6 +118,7 @@ type phrase struct {
// generous lexicon costs a box that does not appear.
var lexicon = []phrase{
{[]string{"email", "address"}, Email},
+ {[]string{"postal", "address"}, Address},
{[]string{"phone", "number"}, Phone},
{[]string{"bank", "details"}, IBAN},
{[]string{"post", "code"}, Eircode},
@@ -112,14 +133,19 @@ var lexicon = []phrase{
{[]string{"iban"}, IBAN},
{[]string{"vat"}, VAT},
{[]string{"email"}, Email},
+ // Bare "address" is POSTAL. It used to prime nothing, because it is
+ // ambiguous with email and only email had a validator; decision 15's
+ // amendment makes postal a model kind, and "email address" is matched
+ // as a phrase before this single word is ever tried.
+ {[]string{"address"}, Address},
}
// Wanted returns the kinds a query names, in `order`. Empty means no box.
//
-// Deliberately absent: "address", which is ambiguous between postal and
-// email and whose postal reading has no validator (§8.8, decision 15);
-// and "number", which is generic and counts only in combination. Both
-// would prime on ordinary queries that are not asking for a fact.
+// Deliberately absent: "number", which is generic and counts only in
+// combination, and would prime on ordinary queries that are not asking
+// for a fact. ("address" used to be absent too; it is now the postal
+// model kind — see the lexicon.)
func Wanted(q string) []Kind {
toks := tokens(q)
want := map[Kind]bool{}
@@ -833,3 +859,65 @@ func Scan(text, heading, title string, want []Kind) []Found {
// Kinds returns the render order, so a caller applying §8.8's per-kind
// rules produces the same boxes in the same places on every request.
func Kinds() []Kind { return append([]Kind(nil), order...) }
+
+// ---------------------------------------------------------------------
+// What §8.9's model half borrows from this layer.
+//
+// Exported so the model half uses THIS code rather than a copy of it: a
+// second implementation of the label rules is how a model row would come
+// to be labelled differently from a validated row sitting beside it,
+// with nothing to say which rule was which.
+// ---------------------------------------------------------------------
+
+// RecognisedSpans returns the byte range of every validated fact of every
+// kind in text, sorted by end.
+//
+// §8.9 refuses a model's span that lies inside one of these — a street
+// name inside an email address is not an address — and needs the ranges
+// of ALL kinds for the same reason Scan does: the answer must not depend
+// on which kinds the query happened to name.
+func RecognisedSpans(text string) [][2]int {
+ var spans [][2]int
+ for _, r := range recognisers {
+ for _, m := range r.re.FindAllStringIndex(text, -1) {
+ span := text[m[0]:m[1]]
+ if r.valid(span, r.canon(span)) {
+ spans = append(spans, [2]int{m[0], m[1]})
+ }
+ }
+ }
+ sort.Slice(spans, func(i, j int) bool { return spans[i][1] < spans[j][1] })
+ return spans
+}
+
+// LabelAt applies §8.8's five label rules to a value at byte offset start.
+// It returns the label, and whether the kind must render no box because
+// the document labelled the value too long to show.
+func LabelAt(text string, start int, heading, title string) (string, bool) {
+ return labelFor(text, start, RecognisedSpans(text), heading, title)
+}
+
+// WordBounded reports whether text[start:end] begins and ends on a word
+// boundary — neither neighbour is a letter or a digit.
+//
+// §8.9 requires it of a model's span, because an exact byte match inside
+// a longer word ("Street" found in "Streetside") reproduces source bytes
+// and names nothing.
+func WordBounded(text string, start, end int) bool {
+ if start < 0 || end > len(text) || start >= end {
+ return false
+ }
+ if start > 0 {
+ if r, _ := utf8.DecodeLastRuneInString(text[:start]); isWordRune(r) {
+ return false
+ }
+ }
+ if end < len(text) {
+ if r, _ := utf8.DecodeRuneInString(text[end:]); isWordRune(r) {
+ return false
+ }
+ }
+ return true
+}
+
+func isWordRune(r rune) bool { return unicode.IsLetter(r) || unicode.IsDigit(r) }
diff --git a/internal/memoria/fact/fact_test.go b/internal/memoria/fact/fact_test.go
index ef8be3b..c36351e 100644
--- a/internal/memoria/fact/fact_test.go
+++ b/internal/memoria/fact/fact_test.go
@@ -36,6 +36,12 @@ func TestWantedNamesTheKind(t *testing.T) {
{"email address", []Kind{Email}},
{"e-mail", []Kind{Email}},
{"phone and email", []Kind{Phone, Email}},
+ {"address", []Kind{Address}},
+ {"office address", []Kind{Address}},
+ {"postal address", []Kind{Address}},
+ // The collision the phrase-first lexicon exists to prevent.
+ {"email address", []Kind{Email}},
+ {"phone number and postal address", []Kind{Phone, Address}},
} {
if got := Wanted(c.q); !reflect.DeepEqual(got, c.want) {
t.Errorf("Wanted(%q) = %v, want %v", c.q, got, c.want)
@@ -50,7 +56,7 @@ func TestWantedNamesTheKind(t *testing.T) {
// address and an email address, and only one of those has anything
// resembling a validator (§8.8, decision 15).
func TestWantedRefusesGenericWords(t *testing.T) {
- for _, q := range []string{"address", "number", "office address", "account number", "", "revenue"} {
+ for _, q := range []string{"number", "account number", "", "revenue"} {
if got := Wanted(q); len(got) != 0 {
t.Errorf("Wanted(%q) = %v, want nothing", q, got)
}
@@ -538,3 +544,87 @@ func TestScanSortsSpansSoNestingCannotLeakIntoALabel(t *testing.T) {
}
}
}
+
+// §8.9: eligibility for a model is a property of the kind, decided by an
+// allowlist — and the allowlist must never contain a kind the
+// deterministic layer can validate. If somebody adds a recogniser for
+// Address, this goes red and forces the decision the spec says has to be
+// made, instead of a validated kind being sent to a model.
+func TestNoModelKindHasARecogniser(t *testing.T) {
+ recognised := map[Kind]bool{}
+ for _, r := range recognisers {
+ recognised[r.kind] = true
+ }
+ for k := range modelKinds {
+ if recognised[k] {
+ t.Errorf("%s is both a model kind and has a recogniser; §8.9 forbids sending a validated kind to a model", k)
+ }
+ }
+ // And the presence half: the allowlist is not empty, and every
+ // validator-backed kind answers false.
+ if !ModelKind(Address) {
+ t.Error("Address is not a model kind")
+ }
+ for _, k := range []Kind{Eircode, Phone, IBAN, VAT, Email} {
+ if ModelKind(k) {
+ t.Errorf("%s is validator-backed and must never be a model kind", k)
+ }
+ }
+}
+
+// The deterministic layer never produces an Address, whatever the text.
+// If it did, §8.8's box would show a postal address no checksum vouched
+// for, without the 🤖 marking §8.9 requires of anything a model chose.
+func TestScanNeverProducesAnAddress(t *testing.T) {
+ text := "Office address: 12 Main Street, Dublin 2, D02 X285"
+ if got := Scan(text, "Contact", "Tito", []Kind{Address}); len(got) != 0 {
+ t.Errorf("Scan produced %d address facts; only a model may", len(got))
+ }
+}
+
+func TestRecognisedSpansCoverEveryKind(t *testing.T) {
+ text := "Mail info@tito.io or ring +353 1 547 7854, at D02 X285."
+ got := RecognisedSpans(text)
+ var spans []string
+ for _, sp := range got {
+ spans = append(spans, text[sp[0]:sp[1]])
+ }
+ want := []string{"info@tito.io", "+353 1 547 7854", "D02 X285"}
+ if !reflect.DeepEqual(spans, want) {
+ t.Errorf("spans = %q, want %q (sorted by end, every kind)", spans, want)
+ }
+}
+
+// LabelAt must agree with Scan for a value Scan found — that is the
+// whole reason it is exported rather than reimplemented.
+func TestLabelAtAgreesWithScan(t *testing.T) {
+ text := officeChunk
+ for _, f := range Scan(text, "Office Address", "Office", []Kind{Phone}) {
+ start := strings.Index(text, f.Span)
+ got, noBox := LabelAt(text, start, "Office Address", "Office")
+ if got != f.Label || noBox != f.NoBox {
+ t.Errorf("LabelAt(%q) = %q/%v, Scan said %q/%v", f.Span, got, noBox, f.Label, f.NoBox)
+ }
+ }
+}
+
+func TestWordBounded(t *testing.T) {
+ text := "12 Main Street, Streetside"
+ for _, c := range []struct {
+ sub string
+ at int
+ want bool
+ }{
+ {"12 Main Street", 0, true},
+ {"Street", strings.Index(text, "Streetside"), false}, // inside a word
+ {"ain Street", 4, false}, // starts mid-word
+ {"Main", 3, true},
+ } {
+ if got := WordBounded(text, c.at, c.at+len(c.sub)); got != c.want {
+ t.Errorf("WordBounded(%q at %d) = %v, want %v", c.sub, c.at, got, c.want)
+ }
+ }
+ if WordBounded(text, 3, 3) {
+ t.Error("an empty range is not a bounded word")
+ }
+}