carlos / platform Public

Clone
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package provision
2
3// census-propose (#369): census rulings as approvals. The root-read census
4// (TestPlan14RootReadCensus) fails on any change to a unit it tracks or to
5// a package envelope. Most of those changes are approvals, not decisions:
6// the same reads in an edited body, or an envelope fingerprint that moved.
7// This file classifies each discrepancy, writes the overlay that closes the
8// approvable ones (rows, @legacy fingerprints, envelope re-pins, the seal,
9// the chain manifest line), and the approval card a person reads to
10// approve it. Changes that need a disposition (a new unit, a removed one,
11// the scanner's field list) are decisions: it names them and writes
12// nothing.
13//
14// make census-card print the card; writes nothing
15// make census-propose NAME=<branch> CLAIMS=<file> write the overlay and the card
16//
17// CLAIMS has one line per changed unit, "<file>:<unit>\t<what changed>".
18// Rows carry rev3=census-ruling-<NAME> from the start: approval happens
19// before the merge, so there is no PROPOSED row to flip afterwards.
20
21import (
22 "fmt"
23 "os"
24 "path/filepath"
25 "regexp"
26 "sort"
27 "strings"
28 "testing"
29)
30
31// Change classes. The first three are approvals; the rest are decisions.
32const (
33 censusBody = "body" // digest moved, the same reads (offsets aside)
34 censusSites = "sites" // more or fewer reads of kinds the unit already makes
35 censusEnvelope = "envelope" // a package's source fingerprint moved
36 censusReads = "reads" // a kind of read the unit did not make before
37 censusNew = "new" // an unclassified unit: needs a disposition
38 censusRemoved = "removed" // a unit or envelope gone: needs a removal row
39 censusOther = "other" // a census discrepancy no change above explains
40)
41
42// censusDecision: a new kind of read can make a unit's disposition wrong
43// (a registry reader that starts opening files is not the unit that was
44// classified), so it is the owner's call, as a new or removed unit is
45// (Astra review, census-approvals P1).
46func censusDecision(class string) bool {
47 return class == censusReads || class == censusNew || class == censusRemoved || class == censusOther
48}
49
50type censusChange struct {
51 Key, Class string
52 Before classifiedRootRead // the inventory's row; zero for a new unit
53 After rootReadEntry // the scan's; zero for a removed unit
54 // An envelope's fingerprints.
55 OldHash, NewHash string
56}
57
58// kindCounts is a unit's read sites without their offsets.
59func kindCounts(kinds string) map[string]int {
60 m := map[string]int{}
61 if kinds == "" {
62 return m
63 }
64 for _, k := range strings.Split(kinds, ",") {
65 name, _, _ := strings.Cut(k, "@")
66 m[name]++
67 }
68 return m
69}
70
71// kindDiff says how a unit's read sites changed: "+registry:Socket",
72// "-file:Open ×2", or "unchanged (N)".
73func kindDiff(before, after string) string {
74 b, a := kindCounts(before), kindCounts(after)
75 names := map[string]bool{}
76 for k := range b {
77 names[k] = true
78 }
79 for k := range a {
80 names[k] = true
81 }
82 var sorted []string
83 for k := range names {
84 sorted = append(sorted, k)
85 }
86 sort.Strings(sorted)
87 var parts []string
88 for _, k := range sorted {
89 d := a[k] - b[k]
90 sign := "+"
91 if d < 0 {
92 sign, d = "-", -d
93 }
94 switch {
95 case d == 1:
96 parts = append(parts, sign+k)
97 case d > 1:
98 parts = append(parts, fmt.Sprintf("%s%s ×%d", sign, k, d))
99 }
100 }
101 if len(parts) == 0 {
102 n := 0
103 for _, c := range a {
104 n += c
105 }
106 return fmt.Sprintf("unchanged (%d)", n)
107 }
108 return strings.Join(parts, ", ")
109}
110
111func sameKinds(before, after string) bool {
112 return kindDiff(before, after) == kindDiff(after, after)
113}
114
115// newKinds reports whether after reads a kind before did not.
116func newKinds(before, after string) bool {
117 b := kindCounts(before)
118 for k := range kindCounts(after) {
119 if b[k] == 0 {
120 return true
121 }
122 }
123 return false
124}
125
126// classifyRootReads names every discrepancy the census would report under
127// seal, one change per unit or envelope. A discrepancy no unit or envelope
128// change accounts for (a removed declaration back in the tree, the seal
129// itself, the scanner's field list) is an "other" decision carrying the
130// census's own message, so the card never shows less than the census does
131// (Astra review, census-approvals P2).
132func classifyRootReads(got rootReadScan, inv rootReadInventory, seal string) []censusChange {
133 var cs []censusChange
134 seen := map[string]bool{}
135 for _, g := range got.Entries {
136 key := g.File + ":" + g.Unit
137 seen[key] = true
138 w, ok := inv.Entries[key]
139 switch {
140 case !ok:
141 cs = append(cs, censusChange{Key: key, Class: censusNew, After: g})
142 case w.Kinds == g.Kinds && w.Digest == g.Digest && declarationHolds(got, inv, key):
143 case sameKinds(w.Kinds, g.Kinds):
144 cs = append(cs, censusChange{Key: key, Class: censusBody, Before: w, After: g})
145 case newKinds(w.Kinds, g.Kinds):
146 cs = append(cs, censusChange{Key: key, Class: censusReads, Before: w, After: g})
147 default:
148 cs = append(cs, censusChange{Key: key, Class: censusSites, Before: w, After: g})
149 }
150 }
151 for key, w := range inv.Entries {
152 if seen[key] {
153 continue
154 }
155 decl, declared := got.Declarations[key]
156 switch {
157 case w.Class != "legacy" && w.Kinds == "" && declared && decl == w.Digest:
158 // Declared, reads nothing, unchanged: not a discrepancy.
159 case w.Class != "legacy" && w.Kinds == "" && declared:
160 cs = append(cs, censusChange{Key: key, Class: censusBody, Before: w, After: rootReadEntry{File: w.File, Unit: w.Unit, Digest: decl}})
161 default:
162 cs = append(cs, censusChange{Key: key, Class: censusRemoved, Before: w})
163 }
164 }
165 for dir, hash := range got.Coverage {
166 if inv.Coverage[dir] != hash {
167 cs = append(cs, censusChange{Key: dir, Class: censusEnvelope, OldHash: inv.Coverage[dir], NewHash: hash})
168 }
169 }
170 for dir, hash := range inv.Coverage {
171 if _, ok := got.Coverage[dir]; !ok {
172 cs = append(cs, censusChange{Key: dir, Class: censusRemoved, OldHash: hash})
173 }
174 }
175 keys := map[string]bool{}
176 for _, c := range cs {
177 keys[c.Key] = true
178 }
179 for _, msg := range censusDiscrepancies(got, inv, seal) {
180 explained := false
181 for key := range keys {
182 if censusNames(msg, key) {
183 explained = true
184 break
185 }
186 }
187 if !explained {
188 cs = append(cs, censusChange{Key: msg, Class: censusOther})
189 }
190 }
191 sort.Slice(cs, func(i, j int) bool {
192 if cs[i].Class != cs[j].Class {
193 return cs[i].Class < cs[j].Class
194 }
195 return cs[i].Key < cs[j].Key
196 })
197 return cs
198}
199
200// censusNames reports whether a census message names key whole: keys may
201// contain spaces ("file.go:type T"), and an envelope's directory is a
202// prefix of its units' keys, so the key must be bounded by a space before
203// it and a space, a semicolon or the end after it (Astra round 2).
204func censusNames(msg, key string) bool {
205 for i := 0; ; {
206 j := strings.Index(msg[i:], " "+key)
207 if j < 0 {
208 return false
209 }
210 end := i + j + 1 + len(key)
211 if end == len(msg) || msg[end] == ' ' || msg[end] == ';' {
212 return true
213 }
214 i += j + 1
215 }
216}
217
218// declarationHolds is checkRootReadDeclarations's test for one unit.
219func declarationHolds(got rootReadScan, inv rootReadInventory, key string) bool {
220 frozen, ok := inv.Frozen[key]
221 if !ok {
222 return true
223 }
224 cur := inv.Entries[key]
225 if cur.Class == "legacy" {
226 return got.Declarations[key] == frozen.Digest && rootReadLegacyFingerprint(cur) == rootReadLegacyFingerprint(frozen)
227 }
228 return got.Declarations[key] == cur.Digest
229}
230
231type rootReadProposal struct {
232 Overlay, Seal, Card string
233}
234
235var censusName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
236
237// proposeRootReads writes the overlay that closes every change in the scan,
238// re-freezing each changed unit at the disposition it already has with its
239// claim appended, under census-ruling-<name>. It refuses decisions, a
240// change with no claim and a claim for no change, and checks its own work:
241// loaded onto a copy of inv, the overlay must leave the census clean.
242func proposeRootReads(got rootReadScan, inv rootReadInventory, seal, name string, claims map[string]string) (rootReadProposal, error) {
243 if !censusName.MatchString(name) {
244 return rootReadProposal{}, fmt.Errorf("census name %q: want lower-case letters, digits and dashes", name)
245 }
246 cs := classifyRootReads(got, inv, seal)
247 if len(cs) == 0 {
248 return rootReadProposal{}, fmt.Errorf("nothing to propose: the census is clean")
249 }
250 var decisions []string
251 units := map[string]bool{}
252 for _, c := range cs {
253 if censusDecision(c.Class) {
254 decisions = append(decisions, c.Class+" "+c.Key)
255 }
256 if c.Class == censusBody || c.Class == censusSites {
257 units[c.Key] = true
258 }
259 }
260 if len(decisions) > 0 {
261 return rootReadProposal{}, fmt.Errorf("a decision is needed, not an approval: %s; write those rows by hand and ask on #280", strings.Join(decisions, "; "))
262 }
263 for key := range units {
264 if strings.TrimSpace(claims[key]) == "" {
265 return rootReadProposal{}, fmt.Errorf("no claim for %s: say what changed in the CLAIMS file", key)
266 }
267 }
268 for key, claim := range claims {
269 if !units[key] {
270 return rootReadProposal{}, fmt.Errorf("claim for %s: not a change the census sees", key)
271 }
272 if strings.ContainsAny(claim, "\t\n\r") {
273 return rootReadProposal{}, fmt.Errorf("claim for %s: a tab or newline would break the row", key)
274 }
275 }
276
277 ruling := "census-ruling-" + name
278 var frozen, plain, cover []string
279 counts := map[string]int{}
280 for _, c := range cs {
281 counts[c.Class]++
282 if c.Class == censusEnvelope {
283 cover = append(cover, "@coverage\t"+c.Key+"\t"+c.NewHash)
284 continue
285 }
286 e := classifiedRootRead{rootReadEntry: c.After, Class: c.Before.Class, Task: c.Before.Task, Basis: c.Before.Basis + ";rev3=" + ruling + ";basis=" + strings.TrimSpace(claims[c.Key])}
287 row := strings.Join([]string{e.File, e.Unit, e.Class, e.Task, e.Basis, e.Kinds, e.Digest}, "\t")
288 if e.Class == "legacy" {
289 frozen = append(frozen, "@legacy\t"+rootReadLegacyFingerprint(e)+"\t"+row)
290 }
291 plain = append(plain, row)
292 }
293 header := []string{
294 fmt.Sprintf("# Census overlay %s, "+censusProposeMark+". %s, approved by the census owner before the merge; the approval card is docs/superpowers/notes/census-proposal-%s.md.", name, ruling, name),
295 fmt.Sprintf("# %d body, %d sites, %d envelope: each changed unit re-frozen at its existing disposition with its claim appended.", counts[censusBody], counts[censusSites], counts[censusEnvelope]),
296 }
297 overlay := strings.Join(append(append(append(header, frozen...), plain...), cover...), "\n") + "\n"
298
299 after := cloneRootReadInventory(inv)
300 f, err := os.CreateTemp("", "census-overlay-*.tsv")
301 if err != nil {
302 return rootReadProposal{}, err
303 }
304 defer os.Remove(f.Name())
305 if _, err := f.WriteString(overlay); err != nil {
306 f.Close()
307 return rootReadProposal{}, err
308 }
309 if err := f.Close(); err != nil {
310 return rootReadProposal{}, err
311 }
312 if err := loadRootReadInventory(f.Name(), &after, true); err != nil {
313 return rootReadProposal{}, fmt.Errorf("the overlay does not load: %w", err)
314 }
315 newSeal := rootReadLegacySeal(after)
316 if errs := censusDiscrepancies(got, after, newSeal); len(errs) != 0 {
317 return rootReadProposal{}, fmt.Errorf("the overlay leaves discrepancies: %s", strings.Join(errs, "; "))
318 }
319 return rootReadProposal{Overlay: overlay, Seal: newSeal, Card: rootReadCard(name, cs, claims, seal, newSeal)}, nil
320}
321
322func censusDiscrepancies(got rootReadScan, inv rootReadInventory, seal string) []string {
323 errs := append(compareRootReads(got, inv), checkRootReadLegacy(inv, seal)...)
324 return append(errs, checkRootReadDeclarations(got, inv)...)
325}
326
327// rootReadCard is what a person reads to approve the change. newSeal is
328// empty for a dry run (make census-card).
329func rootReadCard(name string, cs []censusChange, claims map[string]string, seal, newSeal string) string {
330 var b strings.Builder
331 fmt.Fprintf(&b, "# Census approval: %s\n\n", name)
332 var decisions []censusChange
333 var envelopes []string
334 var units []censusChange
335 for _, c := range cs {
336 switch {
337 case censusDecision(c.Class):
338 decisions = append(decisions, c)
339 case c.Class == censusEnvelope:
340 envelopes = append(envelopes, "`"+c.Key+"`")
341 default:
342 units = append(units, c)
343 }
344 }
345 switch {
346 case len(cs) == 0:
347 fmt.Fprintf(&b, "Nothing to approve: the census is clean (seal `%s`).\n", seal)
348 return b.String()
349 case len(decisions) == 0:
350 fmt.Fprintf(&b, "Ruling `census-ruling-%s`. Every change below keeps the disposition it already has, so it needs an approval, not a decision.\n\n", name)
351 default:
352 fmt.Fprintf(&b, "Ruling `census-ruling-%s`. %d change(s) need a decision (below), which census-propose cannot write.\n\n", name, len(decisions))
353 }
354 if len(units) > 0 {
355 b.WriteString("| unit | change | read sites | what changed |\n|---|---|---|---|\n")
356 for _, c := range units {
357 claim := strings.TrimSpace(claims[c.Key])
358 if claim == "" {
359 claim = "(no claim yet)"
360 }
361 fmt.Fprintf(&b, "| `%s` `%s` | %s | %s | %s |\n", c.After.File, c.After.Unit, c.Class, kindDiff(c.Before.Kinds, c.After.Kinds), claim)
362 }
363 b.WriteString("\n")
364 }
365 if len(envelopes) > 0 {
366 fmt.Fprintf(&b, "Envelopes re-pinned: %s.\n\n", strings.Join(envelopes, ", "))
367 }
368 if len(decisions) > 0 {
369 b.WriteString("Decisions (write their rows by hand and ask on #280):\n\n")
370 for _, c := range decisions {
371 detail := ""
372 switch c.Class {
373 case censusNew:
374 detail = ": reads " + kindDiff("", c.After.Kinds)
375 case censusReads:
376 detail = ": " + kindDiff(c.Before.Kinds, c.After.Kinds)
377 }
378 fmt.Fprintf(&b, "- %s `%s`%s\n", c.Class, c.Key, detail)
379 }
380 b.WriteString("\n")
381 }
382 if newSeal != "" {
383 fmt.Fprintf(&b, "Seal: `%s` → `%s`.\n\nOverlay: `internal/provision/testdata/%s`, appended to `%s`.\n", seal, newSeal, rootReadOverlayFile(name), rootReadChainManifest)
384 } else {
385 fmt.Fprintf(&b, "Seal now: `%s`. Dry run: nothing written.\n", seal)
386 }
387 return b.String()
388}
389
390// censusProposeMark is in the first line of every overlay census-propose
391// writes: a re-run replaces only a file that carries it.
392const censusProposeMark = "written by census-propose (#369)"
393
394func rootReadOverlayFile(name string) string { return "plan14-root-reads-" + name + ".tsv" }
395
396func cloneRootReadInventory(inv rootReadInventory) rootReadInventory {
397 out := rootReadInventory{Fields: inv.Fields}
398 out.Entries = cloneMap(inv.Entries)
399 out.Frozen = cloneMap(inv.Frozen)
400 out.Removed = cloneMap(inv.Removed)
401 out.Coverage = cloneMap(inv.Coverage)
402 out.Legacy = cloneMap(inv.Legacy)
403 return out
404}
405
406func cloneMap[V any](m map[string]V) map[string]V {
407 if m == nil {
408 return nil
409 }
410 out := make(map[string]V, len(m))
411 for k, v := range m {
412 out[k] = v
413 }
414 return out
415}
416
417// readCensusClaims reads a CLAIMS file: "<file>:<unit>\t<what changed>"
418// per line, blank lines and # comments ignored.
419func readCensusClaims(path string) (map[string]string, error) {
420 raw, err := os.ReadFile(path)
421 if err != nil {
422 return nil, err
423 }
424 claims := map[string]string{}
425 for i, line := range strings.Split(string(raw), "\n") {
426 line = strings.TrimRight(line, "\r")
427 if strings.TrimSpace(line) == "" || strings.HasPrefix(line, "#") {
428 continue
429 }
430 key, claim, ok := strings.Cut(line, "\t")
431 if !ok || !strings.Contains(key, ":") || strings.TrimSpace(claim) == "" {
432 return nil, fmt.Errorf("%s:%d: want <file>:<unit><TAB><what changed>", path, i+1)
433 }
434 if _, dup := claims[key]; dup {
435 return nil, fmt.Errorf("%s:%d: a second claim for %s", path, i+1, key)
436 }
437 claims[key] = claim
438 }
439 return claims, nil
440}
441
442const (
443 censusCardStart = "--- census card ---"
444 censusCardEnd = "--- end census card ---"
445)
446
447// TestPlan14RootReadCensusPropose is census-propose and census-card (see
448// the Makefile). It does nothing unless one of them asked.
449func TestPlan14RootReadCensusPropose(t *testing.T) {
450 name, dry := os.Getenv("CARLOS_CENSUS_PROPOSE"), os.Getenv("CARLOS_CENSUS_CARD") != ""
451 if name == "" && !dry {
452 t.Skip("run by make census-card and make census-propose")
453 }
454 root := censusRoot(t)
455 // A re-run (the code changed again) proposes against the chain before
456 // its own overlay, which must still be the manifest's last; nothing is
457 // changed on disk until the new proposal is built and checked.
458 before := ""
459 if !dry {
460 if !censusName.MatchString(name) {
461 t.Fatalf("census name %q: want lower-case letters, digits and dashes", name)
462 }
463 before = rootReadOverlayFile(name)
464 }
465 got, err := rootReadProduction(root)
466 if err != nil {
467 t.Fatal(err)
468 }
469 inv, seal := rootReadChainBefore(t, root, before)
470 claims := map[string]string{}
471 if path := os.Getenv("CARLOS_CENSUS_CLAIMS"); path != "" {
472 if claims, err = readCensusClaims(path); err != nil {
473 t.Fatal(err)
474 }
475 }
476 if dry {
477 if name == "" {
478 name = "<name>"
479 }
480 fmt.Printf("%s\n%s%s\n", censusCardStart, rootReadCard(name, classifyRootReads(got, inv, seal), claims, seal, ""), censusCardEnd)
481 return
482 }
483 p, err := proposeRootReads(got, inv, seal, name, claims)
484 if err != nil {
485 t.Fatal(err)
486 }
487 if err := writeRootReadProposal(root, name, p); err != nil {
488 t.Fatal(err)
489 }
490 fmt.Printf("%s\n%s%s\n", censusCardStart, p.Card, censusCardEnd)
491}
492
493// writeRootReadProposal writes a checked proposal: the overlay, its line
494// at the end of the chain manifest (replacing its own line on a re-run)
495// and the approval card. It refuses, writing nothing, to replace an
496// overlay census-propose did not write, or one that is not the manifest's
497// last.
498func writeRootReadProposal(root, name string, p rootReadProposal) error {
499 file := rootReadOverlayFile(name)
500 overlay := filepath.Join(root, "internal/provision/testdata", file)
501 if raw, err := os.ReadFile(overlay); err == nil {
502 first, _, _ := strings.Cut(string(raw), "\n")
503 if !strings.Contains(first, censusProposeMark) {
504 return fmt.Errorf("%s exists and census-propose did not write it: propose under a new name", file)
505 }
506 } else if !os.IsNotExist(err) {
507 return err
508 }
509 manifest := filepath.Join(root, rootReadChainManifest)
510 raw, err := os.ReadFile(manifest)
511 if err != nil {
512 return err
513 }
514 lines := strings.Split(strings.TrimRight(string(raw), "\n"), "\n")
515 links, err := readRootReadChainManifest(manifest)
516 if err != nil {
517 return err
518 }
519 for i, l := range links {
520 if l.File != file {
521 continue
522 }
523 if i != len(links)-1 || lines[len(lines)-1] != l.File+"\t"+l.Seal {
524 return fmt.Errorf("%s is not the last overlay in %s: propose under a new name", file, rootReadChainManifest)
525 }
526 lines = lines[:len(lines)-1]
527 }
528 lines = append(lines, file+"\t"+p.Seal)
529 note := filepath.Join(root, "docs/superpowers/notes", "census-proposal-"+name+".md")
530 // Every file is written beside its target first and renamed only once
531 // all three are on disk, the manifest last, so a failed write leaves
532 // the previous proposal whole (Astra round 2). A rename failing between
533 // the others is not rolled back (Astra round 3, declined): the files
534 // are a working tree's, the census fails on any mismatch, and a re-run
535 // recovers, since the manifest still ends in this name's line.
536 writes := []struct{ path, text string }{
537 {overlay, p.Overlay},
538 {note, p.Card},
539 {manifest, strings.Join(lines, "\n") + "\n"},
540 }
541 var temps []string
542 defer func() {
543 for _, tmp := range temps {
544 os.Remove(tmp)
545 }
546 }()
547 for _, w := range writes {
548 f, err := os.CreateTemp(filepath.Dir(w.path), "."+filepath.Base(w.path)+".*")
549 if err != nil {
550 return err
551 }
552 temps = append(temps, f.Name())
553 if _, err := f.WriteString(w.text); err != nil {
554 f.Close()
555 return err
556 }
557 if err := f.Chmod(0o644); err != nil {
558 f.Close()
559 return err
560 }
561 if err := f.Close(); err != nil {
562 return err
563 }
564 }
565 for i, w := range writes {
566 if err := os.Rename(temps[i], w.path); err != nil {
567 return err
568 }
569 }
570 return nil
571}
572