amadan

oficina / sheets Public

Formula lexer, AST and precedence-climbing parser

Paul Campbell 575eddcacae2a444ae2482a1c46e28de0a1bcc1f parent b39f7a9
5 files changed, +596 −0
  • internal/sheets/calc/ast.go +43 −0
  • internal/sheets/calc/parse.go +194 −0
  • internal/sheets/calc/parse_call.go +34 −0
  • internal/sheets/calc/parse_test.go +89 −0
  • internal/sheets/calc/token.go +236 −0

Intent

full ledger: core

diff --git a/internal/sheets/calc/ast.go b/internal/sheets/calc/ast.go
new file mode 100644
index 0000000..256cdf9
--- /dev/null
+++ b/internal/sheets/calc/ast.go
@@ -0,0 +1,43 @@
+package calc
+
+// Node is any expression node.
+type Node interface{ node() }
+
+type Num struct{ V float64 }
+type Str struct{ V string }
+type Bool struct{ V bool }
+
+// Ref is a single-cell reference as written. Positions are what the source
+// says; binding to stable ids happens in the reference pass, because
+// relative references are positional offsets and must stay that way until
+// a paste anchor is known.
+type Ref struct {
+ ColPos, RowPos int
+ AbsCol, AbsRow bool
+}
+
+type Range struct{ From, To *Ref }
+
+type Binary struct {
+ Op string
+ Left, Right Node
+}
+
+type Unary struct {
+ Op string
+ X Node
+}
+
+type Call struct {
+ Name string
+ Args []Node
+}
+
+func (*Num) node() {}
+func (*Str) node() {}
+func (*Bool) node() {}
+func (*Ref) node() {}
+func (*Range) node() {}
+func (*Binary) node() {}
+func (*Unary) node() {}
+func (*Call) node() {}
diff --git a/internal/sheets/calc/parse.go b/internal/sheets/calc/parse.go
new file mode 100644
index 0000000..9018d08
--- /dev/null
+++ b/internal/sheets/calc/parse.go
@@ -0,0 +1,194 @@
+package calc
+
+import (
+ "fmt"
+ "strings"
+)
+
+// Parse compiles a formula's source text (without a leading "=") into an
+// AST. Errors report only a byte position and a reason; the formula's own
+// content never appears in an error string.
+//
+// Precedence, loosest to tightest: comparison (= <> < <= > >=), then
+// concatenation (&), then + -, then * /, then ^ (right-associative), then
+// unary minus, then primary. Unary minus binding tighter than ^ matches
+// spreadsheet convention (Excel: -2^2 = 4, not -4), which is the reverse
+// of most programming languages and is easy to get wrong later, so it is
+// pinned here with a test.
+func Parse(src string) (Node, error) {
+ toks, err := lex(src)
+ if err != nil {
+ return nil, err
+ }
+ p := &parser{toks: toks}
+ p.cur = toks[0]
+
+ n, err := p.parseComparison()
+ if err != nil {
+ return nil, err
+ }
+ if p.cur.kind != tEOF {
+ return nil, p.errorf("unexpected trailing input")
+ }
+ return n, nil
+}
+
+type parser struct {
+ toks []token
+ pos int
+ cur token
+}
+
+func (p *parser) advance() {
+ p.pos++
+ if p.pos < len(p.toks) {
+ p.cur = p.toks[p.pos]
+ } else {
+ p.cur = token{kind: tEOF, pos: p.cur.pos}
+ }
+}
+
+func (p *parser) errorf(reason string) error {
+ return fmt.Errorf("calc: parse error at position %d: %s", p.cur.pos, reason)
+}
+
+var comparisonOps = map[string]bool{"=": true, "<>": true, "<": true, "<=": true, ">": true, ">=": true}
+var concatOps = map[string]bool{"&": true}
+var additiveOps = map[string]bool{"+": true, "-": true}
+var termOps = map[string]bool{"*": true, "/": true}
+
+// parseBinaryLevel implements one left-associative precedence level shared
+// by comparison, concatenation, addition and multiplication: parse one
+// operand via next, then fold in any run of same-level operators.
+func (p *parser) parseBinaryLevel(ops map[string]bool, next func() (Node, error)) (Node, error) {
+ left, err := next()
+ if err != nil {
+ return nil, err
+ }
+ for p.cur.kind == tOp && ops[p.cur.op] {
+ op := p.cur.op
+ p.advance()
+ right, err := next()
+ if err != nil {
+ return nil, err
+ }
+ left = &Binary{Op: op, Left: left, Right: right}
+ }
+ return left, nil
+}
+
+func (p *parser) parseComparison() (Node, error) {
+ return p.parseBinaryLevel(comparisonOps, p.parseConcat)
+}
+
+func (p *parser) parseConcat() (Node, error) {
+ return p.parseBinaryLevel(concatOps, p.parseAdditive)
+}
+
+func (p *parser) parseAdditive() (Node, error) {
+ return p.parseBinaryLevel(additiveOps, p.parseTerm)
+}
+
+func (p *parser) parseTerm() (Node, error) {
+ return p.parseBinaryLevel(termOps, p.parsePower)
+}
+
+// parsePower is right-associative: 2^3^2 = 2^(3^2).
+func (p *parser) parsePower() (Node, error) {
+ left, err := p.parseUnary()
+ if err != nil {
+ return nil, err
+ }
+ if p.cur.kind == tOp && p.cur.op == "^" {
+ p.advance()
+ right, err := p.parsePower()
+ if err != nil {
+ return nil, err
+ }
+ return &Binary{Op: "^", Left: left, Right: right}, nil
+ }
+ return left, nil
+}
+
+// parseUnary binds tighter than parsePower: see the precedence note on Parse.
+func (p *parser) parseUnary() (Node, error) {
+ if p.cur.kind == tOp && p.cur.op == "-" {
+ p.advance()
+ x, err := p.parseUnary()
+ if err != nil {
+ return nil, err
+ }
+ return &Unary{Op: "-", X: x}, nil
+ }
+ return p.parsePrimary()
+}
+
+func (p *parser) parsePrimary() (Node, error) {
+ switch p.cur.kind {
+ case tNum:
+ v := p.cur.num
+ p.advance()
+ return &Num{V: v}, nil
+ case tStr:
+ v := p.cur.str
+ p.advance()
+ return &Str{V: v}, nil
+ case tRef:
+ return p.parseRefOrRange()
+ case tLParen:
+ p.advance()
+ inner, err := p.parseComparison()
+ if err != nil {
+ return nil, err
+ }
+ if p.cur.kind != tRParen {
+ return nil, p.errorf("expected closing parenthesis")
+ }
+ p.advance()
+ return inner, nil
+ case tIdent:
+ return p.parseIdentOrCall()
+ default:
+ return nil, p.errorf("unexpected token, want expression")
+ }
+}
+
+func (p *parser) parseRefOrRange() (Node, error) {
+ from := refNode(p.cur.ref)
+ p.advance()
+ if p.cur.kind != tColon {
+ return from, nil
+ }
+ p.advance()
+ if p.cur.kind != tRef {
+ return nil, p.errorf("expected reference after ':'")
+ }
+ to := refNode(p.cur.ref)
+ p.advance()
+ return &Range{From: from, To: to}, nil
+}
+
+func refNode(r refToken) *Ref {
+ return &Ref{ColPos: r.colPos, RowPos: r.rowPos, AbsCol: r.absCol, AbsRow: r.absRow}
+}
+
+func (p *parser) parseIdentOrCall() (Node, error) {
+ name := p.cur.ident
+ p.advance()
+
+ switch strings.ToUpper(name) {
+ case "TRUE":
+ return &Bool{V: true}, nil
+ case "FALSE":
+ return &Bool{V: false}, nil
+ }
+
+ if p.cur.kind != tLParen {
+ return nil, p.errorf("unexpected identifier, want function call")
+ }
+ args, err := p.parseCallArgs()
+ if err != nil {
+ return nil, err
+ }
+ return &Call{Name: strings.ToUpper(name), Args: args}, nil
+}
diff --git a/internal/sheets/calc/parse_call.go b/internal/sheets/calc/parse_call.go
new file mode 100644
index 0000000..d723ca8
--- /dev/null
+++ b/internal/sheets/calc/parse_call.go
@@ -0,0 +1,34 @@
+package calc
+
+// parseCallArgs parses a parenthesized, comma-separated argument list. The
+// current token must be the opening '(' on entry; it consumes through the
+// matching ')'.
+func (p *parser) parseCallArgs() ([]Node, error) {
+ p.advance() // consume '('
+
+ if p.cur.kind == tRParen {
+ p.advance()
+ return nil, nil
+ }
+
+ var args []Node
+ for {
+ arg, err := p.parseComparison()
+ if err != nil {
+ return nil, err
+ }
+ args = append(args, arg)
+
+ if p.cur.kind == tComma {
+ p.advance()
+ continue
+ }
+ break
+ }
+
+ if p.cur.kind != tRParen {
+ return nil, p.errorf("expected closing parenthesis")
+ }
+ p.advance()
+ return args, nil
+}
diff --git a/internal/sheets/calc/parse_test.go b/internal/sheets/calc/parse_test.go
new file mode 100644
index 0000000..524be85
--- /dev/null
+++ b/internal/sheets/calc/parse_test.go
@@ -0,0 +1,89 @@
+package calc
+
+import "testing"
+
+func TestParseArithmeticPrecedence(t *testing.T) {
+ n, err := Parse("1+2*3")
+ if err != nil {
+ t.Fatal(err)
+ }
+ b, ok := n.(*Binary)
+ if !ok || b.Op != "+" {
+ t.Fatalf("root = %T %v, want Binary +", n, n)
+ }
+ r, ok := b.Right.(*Binary)
+ if !ok || r.Op != "*" {
+ t.Fatal("multiplication did not bind tighter than addition")
+ }
+}
+
+func TestParseAbsoluteAndRelativeRefs(t *testing.T) {
+ n, err := Parse("$A$1")
+ if err != nil {
+ t.Fatal(err)
+ }
+ r := n.(*Ref)
+ if !r.AbsCol || !r.AbsRow {
+ t.Fatal("$A$1 did not parse as fully absolute")
+ }
+ n2, _ := Parse("A1")
+ r2 := n2.(*Ref)
+ if r2.AbsCol || r2.AbsRow {
+ t.Fatal("A1 parsed as absolute")
+ }
+ if r2.ColPos != 0 || r2.RowPos != 0 {
+ t.Fatalf("A1 = col %d row %d, want 0,0", r2.ColPos, r2.RowPos)
+ }
+}
+
+func TestParseRangeAndCall(t *testing.T) {
+ n, err := Parse("SUM(A1:B10)")
+ if err != nil {
+ t.Fatal(err)
+ }
+ c, ok := n.(*Call)
+ if !ok || c.Name != "SUM" {
+ t.Fatalf("root = %T, want Call SUM", n)
+ }
+ if _, ok := c.Args[0].(*Range); !ok {
+ t.Fatalf("arg 0 = %T, want Range", c.Args[0])
+ }
+}
+
+func TestParseRejectsGarbage(t *testing.T) {
+ for _, src := range []string{"1+", "(1", "SUM(", "A1:", "*"} {
+ if _, err := Parse(src); err == nil {
+ t.Errorf("Parse(%q) succeeded, want error", src)
+ }
+ }
+}
+
+func TestParsePowerIsRightAssociative(t *testing.T) {
+ // 2^3^2 must parse as 2^(3^2) = 2^9 = 512, not (2^3)^2 = 64.
+ n, err := Parse("2^3^2")
+ if err != nil {
+ t.Fatal(err)
+ }
+ b, ok := n.(*Binary)
+ if !ok || b.Op != "^" {
+ t.Fatalf("root = %T %v, want Binary ^", n, n)
+ }
+ if _, ok := b.Left.(*Num); !ok {
+ t.Fatalf("left = %T, want Num (right-associative grouping)", b.Left)
+ }
+ r, ok := b.Right.(*Binary)
+ if !ok || r.Op != "^" {
+ t.Fatal("right operand should itself be a ^ binary (right-associative)")
+ }
+}
+
+func TestParseFunctionNameCaseInsensitive(t *testing.T) {
+ n, err := Parse("sum(A1)")
+ if err != nil {
+ t.Fatal(err)
+ }
+ c, ok := n.(*Call)
+ if !ok || c.Name != "SUM" {
+ t.Fatalf("root = %T %v, want Call SUM (normalised to upper case)", n, n)
+ }
+}
diff --git a/internal/sheets/calc/token.go b/internal/sheets/calc/token.go
new file mode 100644
index 0000000..6eefb2c
--- /dev/null
+++ b/internal/sheets/calc/token.go
@@ -0,0 +1,236 @@
+package calc
+
+import (
+ "fmt"
+ "regexp"
+ "strconv"
+ "strings"
+)
+
+// tokKind classifies a lexed token. Formula text itself never appears in
+// error messages built around these tokens — only positions and reasons —
+// because a formula can carry sensitive cell content.
+type tokKind int
+
+const (
+ tEOF tokKind = iota
+ tNum
+ tStr
+ tIdent
+ tRef
+ tOp
+ tLParen
+ tRParen
+ tComma
+ tColon
+)
+
+type token struct {
+ kind tokKind
+ pos int // byte offset in the source, for error reporting only
+
+ num float64
+ str string
+ ident string
+ op string
+ ref refToken
+}
+
+// refToken is a lexed single-cell reference, decoded but not yet an AST
+// node (the parser wraps it in *Ref or joins two into *Range).
+type refToken struct {
+ colPos, rowPos int
+ absCol, absRow bool
+}
+
+var refPattern = regexp.MustCompile(`^\$?[A-Za-z]{1,3}\$?[0-9]+$`)
+
+// lex scans src into a token stream terminated by a single tEOF token.
+func lex(src string) ([]token, error) {
+ var toks []token
+ i := 0
+ n := len(src)
+ for i < n {
+ c := src[i]
+ switch {
+ case c == ' ' || c == '\t' || c == '\n' || c == '\r':
+ i++
+ case isDigit(c) || (c == '.' && i+1 < n && isDigit(src[i+1])):
+ tok, next, err := lexNumber(src, i)
+ if err != nil {
+ return nil, err
+ }
+ toks = append(toks, tok)
+ i = next
+ case c == '"':
+ tok, next, err := lexString(src, i)
+ if err != nil {
+ return nil, err
+ }
+ toks = append(toks, tok)
+ i = next
+ case isLetter(c) || c == '$':
+ tok, next, err := lexWord(src, i)
+ if err != nil {
+ return nil, err
+ }
+ toks = append(toks, tok)
+ i = next
+ default:
+ tok, next, err := lexOperator(src, i)
+ if err != nil {
+ return nil, err
+ }
+ toks = append(toks, tok)
+ i = next
+ }
+ }
+ toks = append(toks, token{kind: tEOF, pos: n})
+ return toks, nil
+}
+
+func isDigit(c byte) bool { return c >= '0' && c <= '9' }
+
+func isLetter(c byte) bool {
+ return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')
+}
+
+func isWordChar(c byte) bool {
+ return isLetter(c) || isDigit(c) || c == '$'
+}
+
+func lexNumber(src string, start int) (token, int, error) {
+ i := start
+ n := len(src)
+ for i < n && isDigit(src[i]) {
+ i++
+ }
+ if i < n && src[i] == '.' {
+ i++
+ for i < n && isDigit(src[i]) {
+ i++
+ }
+ }
+ if i < n && (src[i] == 'e' || src[i] == 'E') {
+ j := i + 1
+ if j < n && (src[j] == '+' || src[j] == '-') {
+ j++
+ }
+ if j < n && isDigit(src[j]) {
+ i = j
+ for i < n && isDigit(src[i]) {
+ i++
+ }
+ }
+ }
+ v, err := strconv.ParseFloat(src[start:i], 64)
+ if err != nil {
+ return token{}, 0, fmt.Errorf("calc: malformed number literal at position %d", start)
+ }
+ return token{kind: tNum, pos: start, num: v}, i, nil
+}
+
+func lexString(src string, start int) (token, int, error) {
+ i := start + 1
+ n := len(src)
+ var sb strings.Builder
+ for {
+ if i >= n {
+ return token{}, 0, fmt.Errorf("calc: unterminated string literal starting at position %d", start)
+ }
+ if src[i] == '"' {
+ if i+1 < n && src[i+1] == '"' {
+ sb.WriteByte('"')
+ i += 2
+ continue
+ }
+ i++
+ break
+ }
+ sb.WriteByte(src[i])
+ i++
+ }
+ return token{kind: tStr, pos: start, str: sb.String()}, i, nil
+}
+
+func lexWord(src string, start int) (token, int, error) {
+ i := start
+ n := len(src)
+ for i < n && isWordChar(src[i]) {
+ i++
+ }
+ w := src[start:i]
+
+ if refPattern.MatchString(w) {
+ ref, err := decodeRef(w)
+ if err != nil {
+ return token{}, 0, fmt.Errorf("calc: malformed reference at position %d", start)
+ }
+ return token{kind: tRef, pos: start, ref: ref}, i, nil
+ }
+ if strings.Contains(w, "$") {
+ return token{}, 0, fmt.Errorf("calc: malformed reference at position %d", start)
+ }
+ return token{kind: tIdent, pos: start, ident: w}, i, nil
+}
+
+// decodeRef splits a word already known to match refPattern into a
+// zero-based, absolute-flagged reference.
+func decodeRef(w string) (refToken, error) {
+ idx := 0
+ var r refToken
+ if w[idx] == '$' {
+ r.absCol = true
+ idx++
+ }
+ letterStart := idx
+ for idx < len(w) && isLetter(w[idx]) {
+ idx++
+ }
+ col, err := ParseColLabel(w[letterStart:idx])
+ if err != nil {
+ return refToken{}, err
+ }
+ r.colPos = col
+ if idx < len(w) && w[idx] == '$' {
+ r.absRow = true
+ idx++
+ }
+ row, err := strconv.Atoi(w[idx:])
+ if err != nil {
+ return refToken{}, err
+ }
+ r.rowPos = row - 1
+ return r, nil
+}
+
+func lexOperator(src string, i int) (token, int, error) {
+ c := src[i]
+ switch c {
+ case '+', '-', '*', '/', '^', '&', '=':
+ return token{kind: tOp, pos: i, op: string(c)}, i + 1, nil
+ case '<':
+ if i+1 < len(src) && src[i+1] == '=' {
+ return token{kind: tOp, pos: i, op: "<="}, i + 2, nil
+ }
+ if i+1 < len(src) && src[i+1] == '>' {
+ return token{kind: tOp, pos: i, op: "<>"}, i + 2, nil
+ }
+ return token{kind: tOp, pos: i, op: "<"}, i + 1, nil
+ case '>':
+ if i+1 < len(src) && src[i+1] == '=' {
+ return token{kind: tOp, pos: i, op: ">="}, i + 2, nil
+ }
+ return token{kind: tOp, pos: i, op: ">"}, i + 1, nil
+ case '(':
+ return token{kind: tLParen, pos: i}, i + 1, nil
+ case ')':
+ return token{kind: tRParen, pos: i}, i + 1, nil
+ case ',':
+ return token{kind: tComma, pos: i}, i + 1, nil
+ case ':':
+ return token{kind: tColon, pos: i}, i + 1, nil
+ default:
+ return token{}, 0, fmt.Errorf("calc: unexpected character at position %d", i)
+ }
+}