oficina / slides Public

Download this file

1	#!/usr/bin/env python3
2	"""Generate or check the notices embedded in the Slides release."""
3	import hashlib
4	import json
5	import os
6	from pathlib import Path
7	import subprocess
8	import sys
9	import tempfile
10	from html.parser import HTMLParser
11
12	ROOT = Path(__file__).resolve().parent.parent
13	OUT = ROOT / 'internal/slides/credits'
14	ENV = os.environ | {'GOPRIVATE': 'amadan.net/oficina/estilo.git'}
15	EXCLUDE = ','.join(['amadan.net/oficina/estilo.git', 'amadan.net/oficina/home/handoff', 'amadan.net/oficina/home/connect'])
16
17
18	def run(args, cwd=ROOT, env=ENV):
19	return subprocess.check_output(args, cwd=cwd, env=env)
20
21
22	def digest(path):
23	return hashlib.sha256(path.read_bytes()).hexdigest()
24
25
26	def inputs():
27	files = [ROOT / p for p in ['core/Cargo.toml', 'core/Cargo.lock', 'core/pdf/Cargo.toml', 'core/pdf/Cargo.lock', 'rust-toolchain.toml', 'fixtures/package.json', 'fixtures/package-lock.json', 'internal/slides/core/testdata/slides_core.wasm', 'internal/slides/pdfcore/testdata/slides_pdf.wasm']]
28	for pattern in ['internal/slides/static/*.js', 'internal/slides/fonts/files/*', 'internal/slides/static/fonts/*', 'internal/slides/credits/upstream/*']:
29	files.extend(ROOT.glob(pattern))
30	return {str(p.relative_to(ROOT)): digest(p) for p in sorted(files) if p.is_file()}
31
32
33	def notices(directory):
34	result = []
35	for path in sorted(directory.rglob("*")):
36	lower = path.name.lower()
37	if path.suffix.lower() in [".go", ".rs", ".js", ".ts", ".c", ".h", ".py"]:
38	continue
39	if path.is_file() and any(lower == word or lower.startswith(word + '.') or lower.startswith(word + '-') or lower.startswith(word + '_') for word in ['license', 'licence', 'copying', 'notice', 'copyright', 'patents']):
40	if path.is_symlink() or path.stat().st_size > 2 * 1024 * 1024:
41	raise ValueError('notice must be a bounded regular file')
42	text = path.read_bytes().decode('utf-8')
43	if not text.strip():
44	raise ValueError('empty notice')
45	result.append({'Name': str(path.relative_to(directory)), 'Text': text})
46	return result
47
48
49	def go_inventory():
50	with tempfile.TemporaryDirectory() as temp:
51	tool = str(Path(temp) / 'notices')
52	host = json.loads(run(['go', 'env', '-json', 'GOHOSTOS', 'GOHOSTARCH']))
53	run(['go', 'build', '-o', tool, 'amadan.net/oficina/estilo.git/cmd/oficina-notices'], env=ENV | {'GOOS': host['GOHOSTOS'], 'GOARCH': host['GOHOSTARCH'], 'CGO_ENABLED': '0'})
54	return json.loads(run([tool, '-package', './cmd/slides', '-lucide', '-onest', '-exclude', EXCLUDE], env=ENV | {'GOOS': 'linux', 'GOARCH': 'arm64', 'CGO_ENABLED': '0', 'GOWORK': 'off'}))
55
56
57	class NoticeText(HTMLParser):
58	def __init__(self):
59	super().__init__(convert_charrefs=True)
60	self.parts = []
61
62	def handle_data(self, data):
63	self.parts.append(data)
64
65	def handle_starttag(self, tag, attrs):
66	if tag in ['p', 'h1', 'h2', 'h3', 'li', 'pre', 'br']:
67	self.parts.append('\n')
68
69
70	def asset_inventory():
71	fixtures = ROOT / 'fixtures'
72	run(['npm', 'ci', '--ignore-scripts', '--no-audit', '--no-fund'], cwd=fixtures)
73	js = r'''
74	import { build } from 'esbuild';
75	import { readFileSync, readdirSync } from 'node:fs';
76	const inputs = new Set();
77	for (const [name, entry] of [['editor', 'editor-entry.js']]) {
78	const result = await build({entryPoints:[entry],bundle:true,format:'esm',minify:true,target:'es2020',outfile:`../internal/slides/static/${name}.js`,write:false,metafile:true});
79	const original = readdirSync('../internal/slides/static').filter(p => p.startsWith(name+'.') && p.endsWith('.js'));
80	if (original.length !== 1 || !Buffer.from(result.outputFiles.find(p => p.path.endsWith('.js')).contents).equals(readFileSync('../internal/slides/static/'+original[0]))) throw new Error('committed bundle differs from locked source');
81	for (const path of Object.keys(result.metafile.inputs)) if (path.includes('node_modules/')) inputs.add(path);
82	}
83	console.log(JSON.stringify([...inputs]));
84	'''
85	bundled = json.loads(run(['node', '--input-type=module', '-e', js], cwd=fixtures))
86	directories = set()
87	for name in bundled:
88	directory = (fixtures / name).resolve().parent
89	node_root = (fixtures / 'node_modules').resolve()
90	while not (directory / 'package.json').is_file():
91	if directory == node_root or not directory.is_relative_to(node_root):
92	raise ValueError('bundled package provenance is missing')
93	directory = directory.parent
94	if not directory.is_relative_to(node_root):
95	raise ValueError('bundled package escapes node_modules')
96	directories.add(directory)
97	components = []
98	for directory in sorted(directories):
99	package = json.loads((directory / 'package.json').read_text())
100	components.append({'Name': package['name'], 'Version': package['version'], 'SourceURL': 'https://www.npmjs.com/package/' + package['name'] + '/v/' + package['version'], 'Notices': notices(directory)})
101	selected = {}
102	for crate, script, built, embedded in [
103	('core', 'scripts/build-wasm.sh', 'core/target/wasm32-unknown-unknown/release/slides_core.wasm', 'internal/slides/core/testdata/slides_core.wasm'),
104	('core/pdf', 'scripts/build-pdf-wasm.sh', 'core/pdf/target/wasm32-unknown-unknown/release/slides_pdf.wasm', 'internal/slides/pdfcore/testdata/slides_pdf.wasm')]:
105	run(['bash', script])
106	if (ROOT / built).read_bytes() != (ROOT / embedded).read_bytes():
107	raise ValueError('committed Wasm differs from locked source')
108	metadata = json.loads(run(['cargo', 'metadata', '--locked', '--filter-platform', 'wasm32-unknown-unknown', '--format-version', '1'], cwd=ROOT / crate))
109	packages = {p['id']: p for p in metadata['packages']}
110	nodes = {p['id']: p for p in metadata['resolve']['nodes']}
111	pending, seen = [metadata['resolve']['root']], set()
112	while pending:
113	current = pending.pop()
114	if current in seen:
115	continue
116	seen.add(current)
117	pending.extend(d['pkg'] for d in nodes[current]['deps'] if any(k['kind'] != 'dev' for k in d['dep_kinds']))
118	for identity in seen:
119	package = packages[identity]
120	if identity != metadata['resolve']['root']:
121	if package['source'] != 'registry+https://github.com/rust-lang/crates.io-index':
122	raise ValueError('unverified Cargo source replacement')
123	selected[(package['name'], package['version'])] = package
124	for key in sorted(selected):
125	package = selected[key]
126	directory = Path(package['manifest_path']).parent
127	texts = notices(directory)
128	if package['name'] in ['yrs', 'krilla', 'krilla-svg']:
129	expected = '3996e91df00acdb67334695944bc43f60627103f' if package['name'] == 'yrs' else '3ffdf0588cf98050aad6edba51ca70162e1fb5b5'
130	vcs = json.loads((directory / '.cargo_vcs_info.json').read_text())
131	if vcs['git']['sha1'] != expected:
132	raise ValueError('upstream source revision changed')
133	for record in json.loads((OUT / 'upstream/provenance.json').read_text()):
134	prefix = 'yrs-' if package['name'] == 'yrs' else 'krilla-'
135	if not record['file'].startswith(prefix):
136	continue
137	path = OUT / 'upstream' / record['file']
138	if digest(path) != record['sha256']:
139	raise ValueError('original notice checksum changed')
140	texts.append({'Name': record['file'], 'Text': path.read_bytes().decode('utf-8')})
141	components.append({'Name': package['name'], 'Version': package['version'], 'SourceURL': 'https://crates.io/crates/' + package['name'] + '/' + package['version'], 'Notices': texts})
142	sysroot = Path(run(['rustc', '--print', 'sysroot']).decode().strip())
143	rust_version = run(['rustc', '--version']).decode().split()[1]
144	text = NoticeText()
145	text.feed((sysroot / 'share/doc/rust/COPYRIGHT-library.html').read_text())
146	components.append({'Name': 'Rust standard library', 'Version': rust_version, 'SourceURL': 'https://doc.rust-lang.org/' + rust_version + '/COPYRIGHT-library.html', 'Notices': [{'Name': 'Copyright notices', 'Text': ''.join(text.parts)}]})
147	components.append({'Name': 'Liberation Fonts', 'SourceURL': 'https://github.com/liberationfonts/liberation-fonts', 'Notices': notices(ROOT / 'internal/slides/fonts/files')})
148	if any(not item['Notices'] for item in components):
149	raise ValueError('an asset dependency lacks its original notice')
150	return {'inputs': inputs(), 'components': components}
151
152
153	def main():
154	if sys.argv[1:] not in [[], ['--check']]:
155	raise SystemExit('usage: scripts/credits.py [--check]')
156	checking = sys.argv[1:] == ['--check']
157	assets_file = OUT / 'assets.json'
158	assets = json.loads(assets_file.read_text()) if checking else asset_inventory()
159	if assets['inputs'] != inputs():
160	raise SystemExit('credit asset inventory is stale; run scripts/credits.py')
161	inventory = go_inventory()
162	inventory['components'].extend(assets['components'])
163	inventory['components'].sort(key=lambda item: item['Name'])
164	target = OUT / 'notices.json'
165	if checking:
166	if json.loads(target.read_text()) != inventory:
167	raise SystemExit('embedded credits are stale; run scripts/credits.py')
168	else:
169	assets_file.write_text(json.dumps(assets, indent=2) + '\n')
170	target.write_text(json.dumps(inventory, indent=2) + '\n')
171	print('credits: ' + ('checked' if checking else 'generated'))
172
173
174	if __name__ == '__main__':
175	main()
176