| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "encoding/csv" |
| 5 | "encoding/json" |
| 6 | "fmt" |
| 7 | "os" |
| 8 | "strconv" |
| 9 | "strings" |
| 10 | ) |
| 11 | |
| 12 | // readResults parses the CSV soffice wrote into plain strings, in row |
| 13 | // order — pairing them back up with cases is main's job, by index. The |
| 14 | // value read is field formulaColumnIndex (column D), NOT field 0: |
| 15 | // columns A:C are the fixed support table the lookup cases reference |
| 16 | // (see ods.go's lookupTableRows), so field 0 is "10", "20", "30", then |
| 17 | // nothing. A short row means the formula cell exported nothing at all, |
| 18 | // which checkResults treats as a failed run rather than an empty value. |
| 19 | func readResults(path string) ([]string, error) { |
| 20 | f, err := os.Open(path) |
| 21 | if err != nil { |
| 22 | return nil, err |
| 23 | } |
| 24 | defer f.Close() |
| 25 | |
| 26 | r := csv.NewReader(f) |
| 27 | r.FieldsPerRecord = -1 |
| 28 | rows, err := r.ReadAll() |
| 29 | if err != nil { |
| 30 | return nil, err |
| 31 | } |
| 32 | out := make([]string, len(rows)) |
| 33 | for i, row := range rows { |
| 34 | if len(row) > formulaColumnIndex { |
| 35 | out[i] = row[formulaColumnIndex] |
| 36 | } |
| 37 | } |
| 38 | return out, nil |
| 39 | } |
| 40 | |
| 41 | // corpusValueJSON mirrors calc's own corpusValue wire format (see |
| 42 | // internal/sheets/calc/corpus_test.go) — duplicated here rather than |
| 43 | // imported, since gencorpus is a standalone tool and calc's copy is |
| 44 | // unexported test-only plumbing that this package has no business |
| 45 | // depending on. |
| 46 | type corpusValueJSON struct { |
| 47 | Kind string `json:"kind"` |
| 48 | Num float64 `json:"num,omitempty"` |
| 49 | Str string `json:"str,omitempty"` |
| 50 | Bool bool `json:"bool,omitempty"` |
| 51 | Err string `json:"err,omitempty"` |
| 52 | } |
| 53 | |
| 54 | type corpusEntryJSON struct { |
| 55 | Function string `json:"function"` |
| 56 | Expr string `json:"expr"` |
| 57 | Want corpusValueJSON `json:"want"` |
| 58 | } |
| 59 | |
| 60 | type corpusFileJSON struct { |
| 61 | Note string `json:"note"` |
| 62 | Entries []corpusEntryJSON `json:"entries"` |
| 63 | } |
| 64 | |
| 65 | // classify turns one CSV cell's text into corpusValueJSON. LibreOffice's |
| 66 | // CSV export renders TRUE/FALSE for booleans, the six Excel error |
| 67 | // strings verbatim for errors, a plain decimal for numbers, and anything |
| 68 | // else as text. |
| 69 | func classify(s string) corpusValueJSON { |
| 70 | switch s { |
| 71 | case "TRUE": |
| 72 | return corpusValueJSON{Kind: "bool", Bool: true} |
| 73 | case "FALSE": |
| 74 | return corpusValueJSON{Kind: "bool", Bool: false} |
| 75 | case "#DIV/0!", "#VALUE!", "#REF!", "#NAME?", "#NUM!", "#N/A": |
| 76 | return corpusValueJSON{Kind: "err", Err: s} |
| 77 | } |
| 78 | if n, err := strconv.ParseFloat(s, 64); err == nil { |
| 79 | return corpusValueJSON{Kind: "num", Num: n} |
| 80 | } |
| 81 | return corpusValueJSON{Kind: "str", Str: s} |
| 82 | } |
| 83 | |
| 84 | // writeCorpusJSON writes the corpus file. It does NOT hand the whole |
| 85 | // document to json.MarshalIndent: that puts every field of every entry |
| 86 | // on its own line, so a regeneration that changed one value still |
| 87 | // produces a several-hundred-line diff and the one row a reviewer needs |
| 88 | // to see is buried in it. Each entry is written on a single line |
| 89 | // instead, in the exact spacing the file has always had, so `git diff` |
| 90 | // after a regeneration shows precisely the rows LibreOffice answered |
| 91 | // differently and nothing else. Reviewing that diff is the whole point |
| 92 | // of this tool's workflow (see main's closing message); a format that |
| 93 | // defeats it is not a neutral choice. |
| 94 | func writeCorpusJSON(path string, cases []corpusCase, results []string, version string) error { |
| 95 | note, err := json.Marshal(corpusNote(path, version)) |
| 96 | if err != nil { |
| 97 | return err |
| 98 | } |
| 99 | var b strings.Builder |
| 100 | b.WriteString("{\n \"note\": ") |
| 101 | b.Write(note) |
| 102 | b.WriteString(",\n \"entries\": [\n") |
| 103 | for i, c := range cases { |
| 104 | line, err := entryLine(corpusEntryJSON{Function: c.Function, Expr: c.Expr, Want: classify(results[i])}) |
| 105 | if err != nil { |
| 106 | return err |
| 107 | } |
| 108 | b.WriteString(" " + line) |
| 109 | if i < len(cases)-1 { |
| 110 | b.WriteString(",") |
| 111 | } |
| 112 | b.WriteString("\n") |
| 113 | } |
| 114 | b.WriteString(" ]\n}\n") |
| 115 | return os.WriteFile(path, []byte(b.String()), 0o644) |
| 116 | } |
| 117 | |
| 118 | // entryLine renders one entry as a single JSON object. Every scalar goes |
| 119 | // through json.Marshal rather than fmt, so quoting and float formatting |
| 120 | // are the encoding/json package's problem, not this function's; only the |
| 121 | // whitespace between fields is chosen here. |
| 122 | func entryLine(e corpusEntryJSON) (string, error) { |
| 123 | fn, err := json.Marshal(e.Function) |
| 124 | if err != nil { |
| 125 | return "", err |
| 126 | } |
| 127 | expr, err := json.Marshal(e.Expr) |
| 128 | if err != nil { |
| 129 | return "", err |
| 130 | } |
| 131 | want, err := wantLine(e.Want) |
| 132 | if err != nil { |
| 133 | return "", err |
| 134 | } |
| 135 | return fmt.Sprintf(`{"function": %s, "expr": %s, "want": %s}`, fn, expr, want), nil |
| 136 | } |
| 137 | |
| 138 | // wantLine renders one "want" value. The kind's own field is always |
| 139 | // written, zero value included: corpusValueJSON's omitempty tags would |
| 140 | // drop "num": 0 and "bool": false entirely, leaving a bare {"kind": |
| 141 | // "num"} that reads like a value the tool failed to record rather than |
| 142 | // like the number zero. A corpus is read by people deciding whether it |
| 143 | // is trustworthy; it should not make its own honest answers look like |
| 144 | // gaps. |
| 145 | func wantLine(v corpusValueJSON) (string, error) { |
| 146 | var payload any |
| 147 | switch v.Kind { |
| 148 | case "num": |
| 149 | payload = v.Num |
| 150 | case "str": |
| 151 | payload = v.Str |
| 152 | case "bool": |
| 153 | payload = v.Bool |
| 154 | case "err": |
| 155 | payload = v.Err |
| 156 | default: |
| 157 | return `{"kind": "` + v.Kind + `"}`, nil |
| 158 | } |
| 159 | enc, err := json.Marshal(payload) |
| 160 | if err != nil { |
| 161 | return "", err |
| 162 | } |
| 163 | return fmt.Sprintf(`{"kind": %q, "%s": %s}`, v.Kind, v.Kind, enc), nil |
| 164 | } |
| 165 | |
| 166 | // existingEntryCount reads path's current entry count, if the file |
| 167 | // exists and parses — used by main to refuse a run that would silently |
| 168 | // narrow an already-committed corpus. A missing or unparsable file |
| 169 | // returns a non-nil error, which main treats as "nothing to compare |
| 170 | // against" rather than a reason to fail. |
| 171 | func existingEntryCount(path string) (int, error) { |
| 172 | data, err := os.ReadFile(path) |
| 173 | if err != nil { |
| 174 | return 0, err |
| 175 | } |
| 176 | var cf corpusFileJSON |
| 177 | if err := json.Unmarshal(data, &cf); err != nil { |
| 178 | return 0, err |
| 179 | } |
| 180 | return len(cf.Entries), nil |
| 181 | } |
| 182 | |
| 183 | // corpusNote is the provenance line written into the corpus file |
| 184 | // itself. It names the exact LibreOffice build that computed the values |
| 185 | // (captured from `soffice --version` at run time, not typed in by hand) |
| 186 | // because "generated" without a version is not a reproducible claim, and |
| 187 | // it names the one place where the recorded values are not literally |
| 188 | // what LibreOffice printed. |
| 189 | func corpusNote(path, version string) string { |
| 190 | return "GENERATED by tools/gencorpus from a real headless " + version + |
| 191 | " run: each entry's \"want\" is what that LibreOffice computed for that " + |
| 192 | "exact formula, exported via `--convert-to csv`, not a value anyone " + |
| 193 | "reasoned out. Recalculation on export is proven per run, not assumed: " + |
| 194 | "every formula cell is written with a poisoned office:value cache (999999) " + |
| 195 | "and the run fails if that value comes back. Two honest caveats. (1) The " + |
| 196 | "values come through LibreOffice's CSV export, which renders a number at " + |
| 197 | "roughly 15 significant digits with trailing zeros stripped -- SQRT(2) is " + |
| 198 | "recorded as 1.4142135623731, not the full 1.4142135623730951; " + |
| 199 | "corpus_test.go's valuesMatch compares numbers with a 1e-9 epsilon, so this " + |
| 200 | "is a rendering artifact, not a disagreement. (2) LibreOffice Calc is not Excel. Where the two are known " + |
| 201 | "to differ, the difference is recorded and reasoned about in " + |
| 202 | "internal/sheets/xlsx/oracle_divergences_test.go, cross-checked against " + |
| 203 | "excelize's independent formula engine -- this file records Calc's answer. " + |
| 204 | "Regenerate with `go run ./tools/gencorpus -out " + path + |
| 205 | "` (add -soffice /path/to/soffice if it is not on PATH) whenever a case is " + |
| 206 | "added; commit the result and update corpusSHA256 in " + |
| 207 | "internal/sheets/calc/corpus_test.go." |
| 208 | } |
| 209 | |
amadan