| 1 | // Command gencorpus regenerates internal/sheets/calc/testdata/corpus/formulas.json |
| 2 | // from a real spreadsheet. Run it when adding functions; commit the |
| 3 | // result. |
| 4 | // |
| 5 | // go run ./tools/gencorpus -out internal/sheets/calc/testdata/corpus/formulas.json |
| 6 | // |
| 7 | // Requires LibreOffice. `soffice` is looked up on PATH by default; pass |
| 8 | // -soffice /path/to/soffice when it is installed somewhere PATH does not |
| 9 | // reach (a user-space install, for instance). CI does not run this — CI |
| 10 | // reads the committed file, which is why the file is hash-pinned by |
| 11 | // calc.TestCorpusIsPinned (internal/sheets/calc/corpus_test.go). After a |
| 12 | // successful run, review the diff, then run |
| 13 | // |
| 14 | // go test ./internal/sheets/calc/ -run TestCorpusIsPinned -v |
| 15 | // |
| 16 | // and paste the printed sha256 into corpusSHA256. |
| 17 | // |
| 18 | // # Mechanism |
| 19 | // |
| 20 | // gencorpus writes a real .ods (OpenDocument spreadsheet) with one |
| 21 | // formula per row in column D. Each cell carries an ODF-syntax |
| 22 | // table:formula attribute (an "of:=" prefix, semicolon argument |
| 23 | // separators — ODF's own formula language, not Excel's comma-separated |
| 24 | // one), which is what makes LibreOffice treat the cell as a formula to |
| 25 | // (re)compute, not literal text. It then shells out to |
| 26 | // `soffice --headless --convert-to csv`, which opens the document, |
| 27 | // recalculates, and exports the computed values as CSV — not the |
| 28 | // formula source. Each output row is paired back up with the case that |
| 29 | // produced it, by row order, into the corpus JSON. |
| 30 | // |
| 31 | // # First real run, and what it found |
| 32 | // |
| 33 | // This tool was written without a LibreOffice to run it against, and its |
| 34 | // first real run (LibreOffice 26.8.0.3, 2026-08-31) was treated as its |
| 35 | // own first test. It found three real defects, all now fixed and all |
| 36 | // documented at the code that fixes them: |
| 37 | // |
| 38 | // - the mimetype zip entry was written with zip.CreateHeader, whose |
| 39 | // data-descriptor flag makes LibreOffice refuse the whole document |
| 40 | // ("source file could not be loaded") — see writeODS; |
| 41 | // - content.xml did not declare xmlns:of, without which every formula |
| 42 | // evaluates to Err:510 — see contentXML; |
| 43 | // - soffice was invoked with no -env:UserInstallation, which can drop |
| 44 | // it into an interactive first-run flow — see runSoffice. |
| 45 | // |
| 46 | // Recalculation on export is not assumed: every formula cell is written |
| 47 | // with a poisoned office:value cache (poisonedCacheValue) that no case |
| 48 | // can legitimately produce, and checkResults fails the run if that value |
| 49 | // comes back. It does not — LibreOffice recalculates on convert-to. |
| 50 | package main |
| 51 | |
| 52 | import ( |
| 53 | "flag" |
| 54 | "fmt" |
| 55 | "os" |
| 56 | "path/filepath" |
| 57 | ) |
| 58 | |
| 59 | // corpusCase is one row to feed LibreOffice: Function names the |
| 60 | // spreadsheet function under test (matched against |
| 61 | // testdata/corpus/volatile-excluded.txt and used for |
| 62 | // TestAgainstReferenceCorpus's per-row skip), Expr is the formula text as a |
| 63 | // person would type it after "=" — Excel/English syntax, comma argument |
| 64 | // separators; toODFFormula (ods.go) translates to ODF's own syntax for |
| 65 | // the file this tool actually writes. |
| 66 | type corpusCase struct { |
| 67 | Function string |
| 68 | Expr string |
| 69 | } |
| 70 | |
| 71 | func main() { |
| 72 | out := flag.String("out", "", "path to write the corpus JSON to") |
| 73 | soffice := flag.String("soffice", "soffice", "path to the LibreOffice `soffice` binary") |
| 74 | flag.Parse() |
| 75 | if *out == "" { |
| 76 | fmt.Fprintln(os.Stderr, "gencorpus: -out is required") |
| 77 | os.Exit(2) |
| 78 | } |
| 79 | |
| 80 | tmpDir, err := os.MkdirTemp("", "gencorpus-*") |
| 81 | if err != nil { |
| 82 | fatal(err) |
| 83 | } |
| 84 | defer os.RemoveAll(tmpDir) |
| 85 | |
| 86 | version, err := sofficeVersion(*soffice, tmpDir) |
| 87 | if err != nil { |
| 88 | fatal(fmt.Errorf("cannot run %q: %w (install LibreOffice, or pass -soffice /path/to/soffice)", *soffice, err)) |
| 89 | } |
| 90 | fmt.Printf("gencorpus: using %s (%s)\n", version, *soffice) |
| 91 | |
| 92 | // Refuse to silently narrow an existing corpus: if *out already has |
| 93 | // more entries than this run is about to produce, something regressed |
| 94 | // (a case dropped from the list above without a function actually |
| 95 | // going away) and overwriting it would replace an honest "hand- |
| 96 | // authored, unverified" corpus — or a real one — with a smaller file |
| 97 | // that still claims full coverage. See cases' own doc comment for the |
| 98 | // concrete incident this guards against. |
| 99 | if existing, err := existingEntryCount(*out); err == nil && existing > len(cases) { |
| 100 | fatal(fmt.Errorf("%s already has %d entries; this run would only produce %d — refusing to narrow the corpus (add the missing cases to gencorpus's own list, don't just let it overwrite)", *out, existing, len(cases))) |
| 101 | } |
| 102 | |
| 103 | odsPath := filepath.Join(tmpDir, "corpus.ods") |
| 104 | if err := writeODS(odsPath, cases); err != nil { |
| 105 | fatal(err) |
| 106 | } |
| 107 | |
| 108 | csvPath, err := runSoffice(*soffice, tmpDir, odsPath) |
| 109 | if err != nil { |
| 110 | fatal(err) |
| 111 | } |
| 112 | results, err := readResults(csvPath) |
| 113 | if err != nil { |
| 114 | fatal(err) |
| 115 | } |
| 116 | if err := checkResults(cases, results); err != nil { |
| 117 | fatal(err) |
| 118 | } |
| 119 | |
| 120 | if err := writeCorpusJSON(*out, cases, results, version); err != nil { |
| 121 | fatal(err) |
| 122 | } |
| 123 | fmt.Printf("gencorpus: wrote %d entries to %s\n", len(cases), *out) |
| 124 | fmt.Println("gencorpus: next: review the diff, then run `go test ./internal/sheets/calc/ -run TestCorpusIsPinned -v` and update corpusSHA256") |
| 125 | } |
| 126 | |
| 127 | func fatal(err error) { |
| 128 | fmt.Fprintln(os.Stderr, "gencorpus:", err) |
| 129 | os.Exit(1) |
| 130 | } |
| 131 | |
amadan