amadan

rastrillo / fiesta Public

Clone 
git clone https://amadan.net/rastrillo/fiesta

Plain git — no account needed to clone.

Download

Download this file

1//go:build fiestaseed
2
3// The Fiesta translation exporter for internal/designsystem.
4// hack/seed/run-translations.sh copies this file into a scratch
5// rastrillo worktree at the given commit and runs TestFiestaTranslations
6// there, with FIESTA_ENGLISH naming the file Fiesta's hack/seed/english
7// wrote: every English string the Fiesta gallery says, by id. Each one
8// is matched byte for byte against the gallery's own tables, prose and
9// sampleProse (merged into prose at init), and then against the
10// framework's base catalogs by their English value, which is where the
11// shell's words and each language's own name live. Under
12// FIESTA_SEED_OUT it writes:
13//
14// - internal/gallery/i18n/<locale>.json for the eleven locales that
15// are not English, every id sorted, the hits carried;
16// - internal/gallery/i18n/samples.<locale>.json for the same eleven,
17// the sample strings (every id samples.<key>), keyed as Rastrillo's
18// sampleProse is, by the key: the English, or the English and its
19// sense ("Archive (verb)"), which is what the lookup matches;
20// - internal/gallery/i18n/money.json, moneySampleMarkup's output for
21// each of the twelve locales, English included: the Money sample is
22// formatted per locale from data and is not translated by key;
23// - internal/gallery/i18n/drafts.json, the entries that are machine
24// drafts nobody has reviewed, each with the English it was drafted
25// from and the locales it was drafted in;
26// - hack/seed/translations-todo.json, the misses with no draft yet,
27// each with its id, its English and the locales it lacks; the file
28// is removed when there are none.
29//
30// A draft already in a catalog is kept on a later run while drafts.json
31// says it was drafted from the English the id has now. A hit replaces
32// it; English that has changed since puts the id back on the todo list.
33package designsystem
34
35import (
36 "bytes"
37 "encoding/json"
38 "errors"
39 "html"
40 "html/template"
41 "io/fs"
42 "os"
43 "path/filepath"
44 "slices"
45 "sort"
46 "strings"
47 "testing"
48
49 "amadan.net/rastrillo/rastrillo"
50)
51
52// translationDraft is one entry of drafts.json.
53type translationDraft struct {
54 English string `json:"english"`
55 Locales []string `json:"locales"`
56}
57
58// translationMiss is one entry of translations-todo.json.
59type translationMiss struct {
60 ID string `json:"id"`
61 English string `json:"english"`
62 Locales []string `json:"locales"`
63}
64
65func TestFiestaTranslations(t *testing.T) {
66 out := os.Getenv("FIESTA_SEED_OUT")
67 englishFile := os.Getenv("FIESTA_ENGLISH")
68 if out == "" || englishFile == "" {
69 t.Skip("FIESTA_SEED_OUT and FIESTA_ENGLISH are set by hack/seed/run-translations.sh")
70 }
71 var english map[string]string
72 readJSON(t, englishFile, &english)
73 if len(english) == 0 {
74 t.Fatalf("%s holds no strings", englishFile)
75 }
76 dir := filepath.Join(out, "internal", "gallery", "i18n")
77 todoFile := filepath.Join(out, "hack", "seed", "translations-todo.json")
78
79 base := rastrillo.BaseCatalogs()
80 // The base catalog's keys by their English value, sorted, so a
81 // string two keys share is looked up in a fixed order.
82 byEnglish := map[string][]string{}
83 for key, en := range base["en"] {
84 byEnglish[en] = append(byEnglish[en], key)
85 }
86 for _, keys := range byEnglish {
87 sort.Strings(keys)
88 }
89 // lookup is Rastrillo's translation of en into locale, and which
90 // table carried it.
91 lookup := func(en, locale string) (string, string) {
92 if row, ok := prose[en]; ok && row[locale] != "" {
93 if _, sample := sampleProse[en]; sample {
94 return row[locale], "sampleProse"
95 }
96 return row[locale], "prose"
97 }
98 for _, key := range byEnglish[en] {
99 if tr := base[locale][key]; tr != "" {
100 return tr, "locales"
101 }
102 }
103 return "", ""
104 }
105
106 var locales []string
107 for _, l := range rastrillo.BaseLocales() {
108 if l != "en" {
109 locales = append(locales, l)
110 }
111 }
112 sort.Strings(locales)
113 old := map[string]map[string]string{}
114 for _, l := range locales {
115 c := map[string]string{}
116 readJSONIfThere(t, filepath.Join(dir, l+".json"), &c)
117 old[l] = c
118 }
119 oldDrafts := map[string]translationDraft{}
120 readJSONIfThere(t, filepath.Join(dir, "drafts.json"), &oldDrafts)
121
122 // A sample id is its key with samplePrefix in front; the key is what
123 // Rastrillo's tables are matched by and what its catalog files the
124 // translation under.
125 const samplePrefix = "samples."
126 keyOf := func(id string) (string, bool) {
127 if k, ok := strings.CutPrefix(id, samplePrefix); ok {
128 return k, true
129 }
130 return english[id], false
131 }
132 oldSamples := map[string]map[string]string{}
133 for _, l := range locales {
134 c := map[string]string{}
135 readJSONIfThere(t, filepath.Join(dir, "samples."+l+".json"), &c)
136 oldSamples[l] = c
137 }
138 oldValue := func(l, id string) string {
139 if k, sample := keyOf(id); sample {
140 return oldSamples[l][k]
141 }
142 return old[l][id]
143 }
144
145 ids := make([]string, 0, len(english))
146 for id := range english {
147 ids = append(ids, id)
148 }
149 sort.Strings(ids)
150 catalogs := map[string]map[string]string{}
151 samples := map[string]map[string]string{}
152 for _, l := range locales {
153 catalogs[l] = map[string]string{}
154 samples[l] = map[string]string{}
155 }
156 put := func(l, id, tr string) {
157 if k, sample := keyOf(id); sample {
158 samples[l][k] = tr
159 return
160 }
161 catalogs[l][id] = tr
162 }
163 drafts := map[string]translationDraft{}
164 var todo []translationMiss
165 carried := map[string]map[string]int{}
166 drafted := map[string]int{}
167 for _, l := range locales {
168 carried[l] = map[string]int{}
169 }
170 sampleCarried := map[string]int{}
171 sampleDrafted := map[string]int{}
172 for _, id := range ids {
173 en := english[id]
174 key, sample := keyOf(id)
175 var missing, kept []string
176 for _, l := range locales {
177 tr, table := lookup(key, l)
178 if table == "" && sample && !strings.Contains(key, "<") && html.UnescapeString(key) != key {
179 // A sample's text as html/template wrote it into the
180 // seed (It&#39;s, &#43;18s) is the same words as
181 // Rastrillo's table writes them plain, and its
182 // translation goes in written the same way.
183 if tr, table = lookup(html.UnescapeString(key), l); table != "" {
184 tr = template.HTMLEscapeString(tr)
185 }
186 }
187 if table != "" {
188 put(l, id, tr)
189 carried[l][table]++
190 if sample {
191 sampleCarried[l]++
192 }
193 continue
194 }
195 d, ok := oldDrafts[id]
196 if ok && d.English == en && slices.Contains(d.Locales, l) && oldValue(l, id) != "" {
197 put(l, id, oldValue(l, id))
198 kept = append(kept, l)
199 drafted[l]++
200 if sample {
201 sampleDrafted[l]++
202 }
203 continue
204 }
205 missing = append(missing, l)
206 }
207 if len(kept) > 0 {
208 drafts[id] = translationDraft{English: en, Locales: kept}
209 }
210 if len(missing) > 0 {
211 todo = append(todo, translationMiss{ID: id, English: en, Locales: missing})
212 }
213 }
214
215 if err := os.MkdirAll(dir, 0o755); err != nil {
216 t.Fatal(err)
217 }
218 for _, l := range locales {
219 writeJSON(t, filepath.Join(dir, l+".json"), catalogs[l])
220 writeJSON(t, filepath.Join(dir, "samples."+l+".json"), samples[l])
221 }
222 money := map[string]string{}
223 for _, l := range rastrillo.BaseLocales() {
224 money[l] = moneySampleMarkup(l)
225 }
226 writeJSON(t, filepath.Join(dir, "money.json"), money)
227 writeJSON(t, filepath.Join(dir, "drafts.json"), drafts)
228 if len(todo) == 0 {
229 if err := os.Remove(todoFile); err != nil && !errors.Is(err, fs.ErrNotExist) {
230 t.Fatal(err)
231 }
232 } else {
233 writeJSON(t, todoFile, todo)
234 }
235
236 t.Logf("%d English strings", len(ids))
237 for _, l := range locales {
238 t.Logf("%-7s carried %3d (prose %d, sampleProse %d, locales %d), drafted %3d, missing %3d",
239 l, carried[l]["prose"]+carried[l]["sampleProse"]+carried[l]["locales"],
240 carried[l]["prose"], carried[l]["sampleProse"], carried[l]["locales"],
241 drafted[l], len(ids)-carried[l]["prose"]-carried[l]["sampleProse"]-carried[l]["locales"]-drafted[l])
242 }
243 for _, l := range locales {
244 t.Logf("%-7s of them samples: carried %3d, drafted %3d", l, sampleCarried[l], sampleDrafted[l])
245 }
246 t.Logf("%d ids on the todo list", len(todo))
247}
248
249// readJSON decodes the file at path into v.
250func readJSON(t *testing.T, path string, v any) {
251 t.Helper()
252 b, err := os.ReadFile(path)
253 if err != nil {
254 t.Fatal(err)
255 }
256 if err := json.Unmarshal(b, v); err != nil {
257 t.Fatalf("%s: %v", path, err)
258 }
259}
260
261// readJSONIfThere is readJSON for a file an earlier run may not have
262// written: a missing file leaves v as it is.
263func readJSONIfThere(t *testing.T, path string, v any) {
264 t.Helper()
265 if _, err := os.Stat(path); errors.Is(err, fs.ErrNotExist) {
266 return
267 }
268 readJSON(t, path, v)
269}
270
271// writeJSON writes v to path indented, with no HTML escaping, so a
272// translation reads in the file as it reads on the page; maps come out
273// sorted by key.
274func writeJSON(t *testing.T, path string, v any) {
275 t.Helper()
276 var b bytes.Buffer
277 enc := json.NewEncoder(&b)
278 enc.SetEscapeHTML(false)
279 enc.SetIndent("", " ")
280 if err := enc.Encode(v); err != nil {
281 t.Fatal(err)
282 }
283 if err := os.WriteFile(path, b.Bytes(), 0o644); err != nil {
284 t.Fatal(err)
285 }
286}
287