| 1 | #!/usr/bin/env node |
| 2 | // 🤖 Build the public prompt registry (status.elevenmessenger.com/prompts.html) |
| 3 | // from local Claude Code transcripts — every AI session behind this codebase, |
| 4 | // published whole, with credentials REDACTED and each redaction annotated. |
| 5 | // |
| 6 | // Why publish transcripts at all: the same transparency leg as the carbon |
| 7 | // tally and the contributors page — this codebase is largely AI-built, and |
| 8 | // the registry is the provenance record. The redaction bar (Paul's call, |
| 9 | // 2026-07-12) is CREDENTIALS-ONLY: secrets, tokens, keys, and the gitignored |
| 10 | // CLAUDE.local.md ops runbook are removed and annotated in place; hostnames |
| 11 | // and ops narrative stay (instance hostnames are already public record via |
| 12 | // Certificate Transparency logs). |
| 13 | // |
| 14 | // node tools/prompt-registry.mjs --out <dir> # build the corpus |
| 15 | // node tools/prompt-registry.mjs --out <dir> --check-only # scan, no write |
| 16 | // --also <path> extra checkout path whose sessions to include |
| 17 | // (repeatable) — e.g. the repo's pre-rename path: |
| 18 | // dormant project dirs escape Claude Code's pruning, |
| 19 | // so the oldest surviving sessions live there |
| 20 | // --contributor <name> the human name sessions publish under (defaults |
| 21 | // to the anonymous machine label) — publishing under |
| 22 | // your name is the same choice as publishing at all |
| 23 | // --personal <file> contributor's personal redaction list: one literal |
| 24 | // string (or /regex/) per line, removed as |
| 25 | // ⟦redacted: personal⟧. A /regex/ line also matches |
| 26 | // itself literally, so a path-shaped literal is |
| 27 | // never silently reinterpreted as a pattern. The |
| 28 | // policy allows a SMALL budget of these for personal |
| 29 | // reasons; the count publishes per session like |
| 30 | // every other category, so the budget is kept |
| 31 | // honest in the open. |
| 32 | // --sessions <file> session-level controls. Sessions are in the |
| 33 | // ledger by default (opting in is what contributing |
| 34 | // means), but per line a contributor may: |
| 35 | // withhold <session-id> <reason> — publish a |
| 36 | // tombstone (date, prompt count, reason) with |
| 37 | // the content withheld |
| 38 | // omit <session-id> — no per-session trace; only |
| 39 | // the aggregate omitted count publishes |
| 40 | // Reason vocabulary (docs/status.md): personal-safety |
| 41 | // (sharing would cause or attract harm to a person), |
| 42 | // security (would reveal secrets whose exposure |
| 43 | // could cause harm, incl. national security), |
| 44 | // third-party (the session centres on an |
| 45 | // identifiable person who didn't opt into the |
| 46 | // ledger), beyond-intent (consequences beyond |
| 47 | // revealing the human's intent — the thing |
| 48 | // transparency is for). |
| 49 | // |
| 50 | // With ANTHROPIC_API_KEY set, each session gets a one-line Haiku summary for |
| 51 | // the index, and each substantial assistant reply gets a one-line gist |
| 52 | // (<session>.sum.json) so the viewer's collapsed "Claude replied" rows tell |
| 53 | // you what's inside before you open them. Both are generated from REDACTED |
| 54 | // text only (nothing unredacted leaves this machine), cached across runs |
| 55 | // (transcripts are append-only, so cached keys never go stale), and passed |
| 56 | // through the same fail-closed verifier as everything else. Without a key the |
| 57 | // index falls back to each session's first prompt and the viewer to each |
| 58 | // reply's first line: the tool never *requires* AI (CLAUDE.md: built to |
| 59 | // outlive it). |
| 60 | // |
| 61 | // Output layout (uploaded to the status bucket under prompts/): |
| 62 | // <dir>/prompts/index.json — session index + redaction tallies |
| 63 | // <dir>/prompts/prompts.json — every prompt (capped text), for the page's |
| 64 | // search / pagination / by-contributor views |
| 65 | // <dir>/prompts/<session>.jsonl — the redacted transcript, original shape |
| 66 | // |
| 67 | // FAIL-CLOSED CONTRACT: after building, every output file is re-scanned with |
| 68 | // the same detector set (plus sentinel phrases from CLAUDE.local.md); any hit |
| 69 | // aborts the build with a non-zero exit. Nothing partial is ever left behind |
| 70 | // for an upload script to grab: outputs are written to a temp dir and moved |
| 71 | // into place only after the whole corpus passes. |
| 72 | // |
| 73 | // Known limits (documented, not hidden): base64 blobs are removed wholesale |
| 74 | // because images can't be text-audited; the high-entropy backstop over-redacts |
| 75 | // (ciphertext samples, wrapped keys, message ids look exactly like tokens — |
| 76 | // losing those from a public copy costs nothing); a std-base64 secret whose |
| 77 | // '/' lands unluckily can split below the backstop's length floor — the |
| 78 | // contextual rules (env assignments, auth headers, url params, known |
| 79 | // prefixes) are the real net for labelled secrets. |
| 80 | import { readFileSync, writeFileSync, readdirSync, existsSync, mkdirSync, rmSync, renameSync, statSync } from "node:fs"; |
| 81 | import { execSync } from "node:child_process"; |
| 82 | import { createHash } from "node:crypto"; |
| 83 | import { join, basename } from "node:path"; |
| 84 | import os from "node:os"; |
| 85 | |
| 86 | const args = process.argv.slice(2); |
| 87 | const argAll = (flag) => args.flatMap((a, i) => (a === flag && args[i + 1] ? [args[i + 1]] : [])); |
| 88 | const checkMode = args.includes("check"); |
| 89 | const outRoot = argAll("--out")[0] || (checkMode ? join(os.homedir(), ".cache", "eleven-registry") : null); |
| 90 | const alsoRoots = argAll("--also"); |
| 91 | const checkOnly = args.includes("--check-only"); |
| 92 | if (!outRoot) { |
| 93 | console.error("usage: node tools/prompt-registry.mjs --out <dir> [check] [--check-only] [--also <path>]… [--contributor <name>]"); |
| 94 | process.exit(1); |
| 95 | } |
| 96 | |
| 97 | const git = (a) => execSync("git " + a, { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }); |
| 98 | |
| 99 | // ---- `check` — the SessionStart staleness nudge ------------------------------ |
| 100 | // Mirrors carbon-tally.mjs `check`: no transcript sweep, no redaction — read |
| 101 | // the last build's generated_at and print only when this machine is sitting on |
| 102 | // unpublished sessions going stale (stdout lands in the agent's context; the |
| 103 | // raw transcripts prune after ~a week). A machine with no corpus at --out |
| 104 | // (default ~/.cache/eleven-registry) stays silent: publishing is opt-in |
| 105 | // (docs/status.md § The prompt registry), so the nudge follows that choice, |
| 106 | // never precedes it. Never fails the session. |
| 107 | const STALE_DAYS = Number(process.env.LCHAT_REGISTRY_STALE_DAYS || 3); |
| 108 | if (checkMode) { |
| 109 | try { |
| 110 | const promptsDir = join(outRoot, "prompts"); |
| 111 | // Newest generated_at across slug corpora (plus the pre-multi flat layout). |
| 112 | const built = Math.max(...readdirSync(promptsDir, { withFileTypes: true }) |
| 113 | .map((d) => (d.isDirectory() ? join(promptsDir, d.name, "index.json") : d.name === "index.json" ? join(promptsDir, d.name) : null)) |
| 114 | .filter((p) => p && existsSync(p)) |
| 115 | .map((p) => Date.parse(JSON.parse(readFileSync(p, "utf8")).generated_at || ""))); |
| 116 | const days = (Date.now() - built) / 86400e3; |
| 117 | if (Number.isFinite(days) && days > STALE_DAYS) { |
| 118 | // Only nag over real material: any of this repo's transcripts touched |
| 119 | // since the last build (mtime scan — same dir resolution as the build). |
| 120 | const root = join(git("rev-parse --path-format=absolute --git-common-dir").trim(), ".."); |
| 121 | const prefix = root.replace(/[^a-zA-Z0-9]/g, "-"); |
| 122 | const projects = join(os.homedir(), ".claude", "projects"); |
| 123 | const grown = readdirSync(projects).filter((d) => d.startsWith(prefix)) |
| 124 | .flatMap((d) => readdirSync(join(projects, d)).filter((f) => f.endsWith(".jsonl")).map((f) => join(projects, d, f))) |
| 125 | .filter((f) => statSync(f).mtimeMs > built).length; |
| 126 | if (grown) console.log("Prompt registry is stale: this machine's last build is " + Math.floor(days) + |
| 127 | " days old and " + grown + " session transcript(s) have been touched since. Local transcripts prune " + |
| 128 | "after ~a week, so, when convenient this session, accrue it: security sweep, build, haiku-subagent " + |
| 129 | "summaries for new sessions, rebuild, publish — docs/status.md § The prompt registry (this machine's " + |
| 130 | "exact flags are in the operator's notes)."); |
| 131 | } |
| 132 | } catch {} // no corpus yet, or unreadable — not a contributor machine; stay quiet |
| 133 | process.exit(0); |
| 134 | } |
| 135 | |
| 136 | // Same public-safe machine label as the carbon tally (never a hostname). |
| 137 | const NAMES = ["ada", "grace", "alan", "edsger", "radia", "hedy", "katherine", "margaret", |
| 138 | "linus", "barbara", "donald", "hypatia", "annie", "vint", "niklaus", "frances"]; |
| 139 | const DOINGS = ["stargazing", "whistling", "humming", "sketching", "wandering", "gardening", |
| 140 | "juggling", "sailing", "painting", "daydreaming", "tinkering", "strolling", "fiddling", |
| 141 | "doodling", "pottering", "moseying"]; |
| 142 | function sourceLabel() { |
| 143 | if (process.env.LCHAT_CARBON_SOURCE) return process.env.LCHAT_CARBON_SOURCE; |
| 144 | const seed = os.userInfo().username + "@" + os.hostname().split(".")[0]; |
| 145 | let h = 2166136261; // FNV-1a |
| 146 | for (const c of seed) { h ^= c.codePointAt(0); h = Math.imul(h, 16777619) >>> 0; } |
| 147 | return NAMES[h % NAMES.length] + "-" + DOINGS[(h >>> 4) % DOINGS.length]; |
| 148 | } |
| 149 | const source = sourceLabel(); |
| 150 | const contributor = argAll("--contributor")[0] || source; |
| 151 | // Multi-contributor layout: each machine publishes under its own slug prefix |
| 152 | // (prompts/<slug>/…) so corpora never clobber each other; the page merges |
| 153 | // every contributor listed in prompts/contributors.json. |
| 154 | const slug = contributor.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "") || source; |
| 155 | |
| 156 | // Session-level controls: withhold (tombstone) / omit (aggregate count only). |
| 157 | const sessionCtl = new Map(); |
| 158 | for (const f of argAll("--sessions")) { |
| 159 | for (const line of readFileSync(f, "utf8").split("\n")) { |
| 160 | const t = line.trim(); |
| 161 | if (!t || t.startsWith("#")) continue; |
| 162 | const [mode, sid, ...rest] = t.split(/\s+/); |
| 163 | if ((mode === "withhold" || mode === "omit") && sid) { |
| 164 | sessionCtl.set(sid, { mode, reason: rest.join(" ") || "unspecified" }); |
| 165 | } else { |
| 166 | console.error("bad --sessions line (want 'withhold <id> <reason>' or 'omit <id>'): " + t); |
| 167 | process.exit(1); |
| 168 | } |
| 169 | } |
| 170 | } |
| 171 | let omittedCount = 0; |
| 172 | |
| 173 | // ---- redaction rules --------------------------------------------------------- |
| 174 | // Each category carries the "why" that the annotation and the registry page |
| 175 | // show. Order matters: structural blocks first (so one marker replaces a whole |
| 176 | // runbook or image, not a thousand token-shaped shreds), the entropy backstop |
| 177 | // last. |
| 178 | const WHY = { |
| 179 | "claude-local-md": "the gitignored CLAUDE.local.md ops runbook (server addresses, token-recovery paths, secret locations) — excluded wholesale", |
| 180 | "private-key": "cryptographic private key material", |
| 181 | "binary-payload": "base64 binary (image/file) removed — binary can't be text-audited for embedded secrets", |
| 182 | "api-key": "provider API key or token (recognised key format)", |
| 183 | "jwt": "signed bearer token (JWT)", |
| 184 | "url-token": "credential-bearing URL parameter or fragment (e.g. an admin link)", |
| 185 | "auth-header": "HTTP authorization header value", |
| 186 | "env-secret": "secret-valued environment assignment (…TOKEN / KEY / SECRET / PASSWORD = …)", |
| 187 | "kv-secret": "password/token field in structured output", |
| 188 | "high-entropy": "long high-entropy string — could be a credential, redacted fail-closed", |
| 189 | "email": "an email address — people's addresses stay out of the public ledger", |
| 190 | "personal": "removed at the contributor's request — the policy allows each contributor a small personal redaction budget, and the count is published", |
| 191 | "session-withheld": "a whole session withheld by its contributor, on the record as a tombstone — allowed where sharing would cause or attract harm to a person (personal-safety), reveal secrets whose exposure could cause harm (security), or carry consequences beyond revealing the human's intent (beyond-intent)", |
| 192 | }; |
| 193 | const mark = (cat) => "⟦redacted: " + cat + "⟧"; |
| 194 | |
| 195 | // The CLAUDE.local.md block as embedded in every session's context reminder: |
| 196 | // from its "Contents of …CLAUDE.local.md…" header to the next top-level |
| 197 | // context section (or end of string). |
| 198 | const LOCAL_MD_BLOCK = /Contents of [^\n]*CLAUDE\.local\.md[^\n]*\n[\s\S]*?(?=\nContents of |\n# userEmail|\n# currentDate|$)/g; |
| 199 | // Sentinels: phrases that live in CLAUDE.local.md and nowhere legitimate. Any |
| 200 | // survivor means the block regex missed a variant — nuke the whole containing |
| 201 | // string, annotated, rather than ship it. |
| 202 | const LOCAL_MD_SENTINELS = ["sync-claude-local.sh", "CLAUDE.local.md (deploy runbook)"]; |
| 203 | |
| 204 | // Personal redactions run FIRST so a personal phrase wins over (and isn't |
| 205 | // split by) the mechanical rules. |
| 206 | const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); |
| 207 | // A slash-wrapped line is a /regex/ — but it ALSO gets a literal rule (first, |
| 208 | // so a whole-literal occurrence is redacted in one piece before the pattern |
| 209 | // nibbles at substrings). A literal that merely LOOKS slash-wrapped (a path, |
| 210 | // say) would otherwise be silently reinterpreted as a pattern that can miss |
| 211 | // its own text — and the fail-closed verifier shares these rules, so it can't |
| 212 | // catch what a rule can't match (this leaked a personal string three times |
| 213 | // before the 2026-07-27 accrual caught it by literal grep). |
| 214 | const personalRules = argAll("--personal").flatMap((f) => |
| 215 | readFileSync(f, "utf8").split("\n").map((l) => l.trim()).filter((l) => l && !l.startsWith("#")) |
| 216 | .flatMap((l) => l.length > 2 && l.startsWith("/") && l.endsWith("/") |
| 217 | ? [["personal", new RegExp(escapeRe(l), "gi")], ["personal", new RegExp(l.slice(1, -1), "gi")]] |
| 218 | : [["personal", new RegExp(escapeRe(l), "gi")]])); |
| 219 | |
| 220 | const RULES = [ |
| 221 | ...personalRules, |
| 222 | ["claude-local-md", LOCAL_MD_BLOCK], |
| 223 | ["private-key", /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)/g], |
| 224 | ["private-key", /\bAGE-SECRET-KEY-1[A-Z0-9]{8,}/g], |
| 225 | ["binary-payload", /(?:data:[a-z0-9.+/-]+;base64,)?[A-Za-z0-9+/=\n]{1024,}/g], |
| 226 | ["api-key", /\bsk-ant-[A-Za-z0-9_-]{8,}/g], |
| 227 | ["api-key", /\b(?:sk|rk|pk)_(?:live|test)_[A-Za-z0-9]{8,}/g], |
| 228 | ["api-key", /\bwhsec_[A-Za-z0-9]{8,}/g], |
| 229 | ["api-key", /\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{20,}/g], |
| 230 | ["api-key", /\bgithub_pat_[A-Za-z0-9_]{20,}/g], |
| 231 | ["api-key", /\bxox[a-z]-[A-Za-z0-9-]{10,}/g], |
| 232 | ["api-key", /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g], |
| 233 | ["jwt", /\beyJ[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_-]{6,}\b/g], |
| 234 | ["url-token", /([#?&](?:token|secret|key|auth|access_token|invite)=)[A-Za-z0-9._~%-]{6,}/gi], |
| 235 | ["email", /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g], |
| 236 | ["auth-header", /((?:Authorization|X-Api-Key|X-Auth-Token)["']?\s*[:=]\s*["']?(?:Bearer\s+|Basic\s+|token\s+)?)(?!(?:Bearer|Basic|token)\b)[A-Za-z0-9._~+/=-]{6,}/gi], |
| 237 | ["env-secret", /\b([A-Z][A-Z0-9_]*(?:TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIALS?|APIKEY|API_KEY|PRIVATE_KEY|KEY))=(["']?)[^\s"']{6,}\2/g], |
| 238 | ["kv-secret", /("(?:password|token|secret|credential|api_key|admin_token)"\s*:\s*")([^"⟦]{6,})(")/gi], |
| 239 | // Backstop: base64url-ish runs ≥36 chars mixing upper+lower+digit (an lchat |
| 240 | // token is exactly 43 such chars). Pure hex (git SHAs) and UUIDs never mix |
| 241 | // case, so they survive. |
| 242 | ["high-entropy", /(?=[A-Za-z0-9_+-]*[A-Z])(?=[A-Za-z0-9_+-]*[a-z])(?=[A-Za-z0-9_+-]*[0-9])\b[A-Za-z0-9_+-]{36,}={0,2}\b/g], |
| 243 | ["high-entropy", /(?=[A-Za-z0-9+/]*[A-Z])(?=[A-Za-z0-9+/]*[a-z])(?=[A-Za-z0-9+/]*[0-9])\b[A-Za-z0-9+/]{44,}={0,2}\b/g], |
| 244 | ]; |
| 245 | |
| 246 | function redactString(s, tally) { |
| 247 | for (const [cat, re] of RULES) { |
| 248 | re.lastIndex = 0; |
| 249 | if (!re.test(s)) continue; |
| 250 | re.lastIndex = 0; |
| 251 | s = s.replace(re, (...m) => { |
| 252 | tally[cat] = (tally[cat] || 0) + 1; |
| 253 | // Rules with capture groups keep the non-secret label (param name, env |
| 254 | // var name, header prefix) and redact only the value. |
| 255 | if (cat === "url-token" || cat === "auth-header") return m[1] + mark(cat); |
| 256 | if (cat === "env-secret") return m[1] + "=" + m[2] + mark(cat) + m[2]; |
| 257 | if (cat === "kv-secret") return m[1] + mark(cat) + m[3]; |
| 258 | return mark(cat); |
| 259 | }); |
| 260 | } |
| 261 | for (const sn of LOCAL_MD_SENTINELS) { |
| 262 | if (s.includes(sn)) { // fail closed: the block regex missed a variant |
| 263 | tally["claude-local-md"] = (tally["claude-local-md"] || 0) + 1; |
| 264 | return mark("claude-local-md"); |
| 265 | } |
| 266 | } |
| 267 | return s; |
| 268 | } |
| 269 | |
| 270 | function walk(v, tally) { |
| 271 | if (typeof v === "string") return redactString(v, tally); |
| 272 | if (Array.isArray(v)) return v.map((x) => walk(x, tally)); |
| 273 | if (v && typeof v === "object") { |
| 274 | const o = {}; |
| 275 | // Keys are content too (tool results serialize hostname→stats maps and |
| 276 | // the like) — redact them the same as values. |
| 277 | for (const [k, val] of Object.entries(v)) o[redactString(k, tally)] = walk(val, tally); |
| 278 | return o; |
| 279 | } |
| 280 | return v; |
| 281 | } |
| 282 | |
| 283 | // Post-build detector: the same rules re-run over every PARSED string of the |
| 284 | // finished files — symmetric with how redaction walked them (running over the |
| 285 | // serialized JSONL instead would manufacture matches out of `\t`/`\\` escape |
| 286 | // sequences the parsed-string redactor never saw). Markers become a single |
| 287 | // space first, so a redacted value must leave nothing behind that still |
| 288 | // detects, and a marker can never join two innocent runs into a match. |
| 289 | function detectStrings(v, hits, path) { |
| 290 | if (typeof v === "string") { |
| 291 | const s = v.replace(/⟦redacted: [a-z-]+⟧/g, " "); |
| 292 | for (const [cat, re] of RULES) { |
| 293 | re.lastIndex = 0; |
| 294 | if (re.test(s)) hits.push(cat + " @" + path); |
| 295 | } |
| 296 | for (const sn of LOCAL_MD_SENTINELS) if (s.includes(sn)) hits.push("sentinel:" + sn + " @" + path); |
| 297 | } else if (Array.isArray(v)) v.forEach((x, i) => detectStrings(x, hits, path + "." + i)); |
| 298 | else if (v && typeof v === "object") { |
| 299 | for (const [k, val] of Object.entries(v)) { |
| 300 | detectStrings(k, hits, path + ".(key)"); // keys are content too |
| 301 | detectStrings(val, hits, path + "." + k); |
| 302 | } |
| 303 | } |
| 304 | } |
| 305 | function detect(text, whole) { |
| 306 | const hits = []; |
| 307 | if (whole) { // index.json is one pretty-printed document, not JSONL |
| 308 | try { detectStrings(JSON.parse(text), hits, "doc"); } catch { hits.push("unparseable doc"); } |
| 309 | return hits; |
| 310 | } |
| 311 | let n = 0; |
| 312 | for (const line of text.split("\n")) { |
| 313 | n++; |
| 314 | if (!line.trim()) continue; |
| 315 | try { detectStrings(JSON.parse(line), hits, "line" + n); } catch { hits.push("unparseable @line" + n); } |
| 316 | if (hits.length) break; // one finding fails the file; no need to keep scanning |
| 317 | } |
| 318 | return hits; |
| 319 | } |
| 320 | |
| 321 | // ---- gather sessions ----------------------------------------------------------- |
| 322 | // Same dir resolution as the carbon tally: the main checkout's sanitized path |
| 323 | // plus prefix-matched worktree dirs — plus any --also paths (a renamed/moved |
| 324 | // checkout leaves its sessions under the old sanitized path). |
| 325 | const repoRoot = join(git("rev-parse --path-format=absolute --git-common-dir").trim(), ".."); |
| 326 | const prefixes = [repoRoot, ...alsoRoots].map((p) => p.replace(/[^a-zA-Z0-9]/g, "-")); |
| 327 | const projectsDir = join(os.homedir(), ".claude", "projects"); |
| 328 | const dirs = readdirSync(projectsDir).filter((d) => prefixes.some((p) => d.startsWith(p))).map((d) => join(projectsDir, d)); |
| 329 | |
| 330 | const files = dirs.flatMap((d) => |
| 331 | readdirSync(d).filter((f) => f.endsWith(".jsonl")).map((f) => join(d, f))); |
| 332 | console.log("transcript dirs: " + dirs.length + ", sessions: " + files.length + " (machine label: " + source + ")"); |
| 333 | |
| 334 | const tmp = outRoot + ".building"; |
| 335 | rmSync(tmp, { recursive: true, force: true }); |
| 336 | mkdirSync(join(tmp, "prompts", slug), { recursive: true }); |
| 337 | // Cache reads prefer the slug prefix, falling back to the pre-multi flat |
| 338 | // layout so an existing corpus's summaries/gists survive the migration. |
| 339 | const priorPath = (f) => { |
| 340 | const p = join(outRoot, "prompts", slug, f); |
| 341 | return existsSync(p) ? p : join(outRoot, "prompts", f); |
| 342 | }; |
| 343 | |
| 344 | // Real user prompts in a user turn: plain-string content, or its text blocks — |
| 345 | // skipping tool results, harness wrappers (<system-reminder>, <command-…>), |
| 346 | // and interrupted-request stubs. The viewer applies the SAME predicate so a |
| 347 | // prompt's ordinal here anchors to the right card there. |
| 348 | const promptTexts = (msg) => { |
| 349 | const c = msg?.message?.content; |
| 350 | const texts = typeof c === "string" ? [c] |
| 351 | : Array.isArray(c) ? c.filter((b) => b.type === "text").map((b) => b.text) : []; |
| 352 | return texts.filter((t) => t && !t.startsWith("<") && !t.startsWith("Caveat:") && !t.startsWith("[Request interrupted")); |
| 353 | }; |
| 354 | |
| 355 | const sessions = []; |
| 356 | const allPrompts = []; |
| 357 | const sessionReplies = new Map(); // id → [{k: "line:block", t}] assistant texts worth a gist |
| 358 | let grandTally = {}; |
| 359 | for (const file of files) { |
| 360 | const id = basename(file, ".jsonl"); |
| 361 | const ctl = sessionCtl.get(id); |
| 362 | if (ctl?.mode === "omit") { omittedCount++; continue; } |
| 363 | const withheld = ctl?.mode === "withhold" ? ctl.reason : null; |
| 364 | const tally = {}; |
| 365 | const outLines = []; |
| 366 | const models = new Set(); // every model that spoke in the session (incl. subagents) |
| 367 | let first = null, last = null, prompts = 0, snippet = null, lastSnippet = null; |
| 368 | for (const line of readFileSync(file, "utf8").split("\n")) { |
| 369 | if (!line.trim()) continue; |
| 370 | let obj; |
| 371 | try { obj = JSON.parse(line); } catch { continue; } // torn tail line of a live session |
| 372 | const red = walk(obj, tally); |
| 373 | if (red.timestamp) { last = red.timestamp; first = first || red.timestamp; } |
| 374 | // "<synthetic>" marks harness-synthesized stubs (errors etc.), not a model |
| 375 | if (red.type === "assistant" && red.message && red.message.model && red.message.model !== "<synthetic>") models.add(red.message.model); |
| 376 | if (red.type === "user" && !red.isMeta) { |
| 377 | for (const t of promptTexts(red)) { |
| 378 | prompts++; |
| 379 | snippet = snippet || t.slice(0, 240); |
| 380 | lastSnippet = t.slice(0, 240); |
| 381 | if (!withheld) allPrompts.push({ c: contributor, s: id, ts: red.timestamp || null, n: prompts, t: t.slice(0, 1200) }); |
| 382 | } |
| 383 | } |
| 384 | // Collect substantial replies for gisting, keyed "parsedLine:block" — the |
| 385 | // viewer counts parsed lines the same way, so keys anchor across both. |
| 386 | if (!withheld && red.type === "assistant" && Array.isArray(red.message?.content)) { |
| 387 | red.message.content.forEach((b, bi) => { |
| 388 | if (b.type === "text" && b.text && b.text.length > 200) { |
| 389 | if (!sessionReplies.has(id)) sessionReplies.set(id, []); |
| 390 | sessionReplies.get(id).push({ k: outLines.length + ":" + bi, t: b.text.slice(0, 1500) }); |
| 391 | } |
| 392 | }); |
| 393 | } |
| 394 | outLines.push(JSON.stringify(red)); |
| 395 | } |
| 396 | if (outLines.length === 0) continue; |
| 397 | if (withheld) { |
| 398 | // Tombstone: the session is on the record — when, how much, and why it's |
| 399 | // withheld — but no content, no snippets, no transcript file. |
| 400 | sessions.push({ id, source, contributor, models: [...models].sort(), |
| 401 | started_at: first, ended_at: last, prompts, withheld }); |
| 402 | continue; |
| 403 | } |
| 404 | const body = outLines.join("\n") + "\n"; |
| 405 | if (!checkOnly) writeFileSync(join(tmp, "prompts", slug, id + ".jsonl"), body); |
| 406 | const redactions = Object.values(tally).reduce((a, b) => a + b, 0); |
| 407 | for (const [k, v] of Object.entries(tally)) grandTally[k] = (grandTally[k] || 0) + v; |
| 408 | sessions.push({ id, source, contributor, models: [...models].sort(), |
| 409 | started_at: first, ended_at: last, prompts, |
| 410 | lines: outLines.length, bytes: Buffer.byteLength(body), redactions, |
| 411 | redacted: tally, first_prompt: snippet, last_prompt: lastSnippet }); |
| 412 | } |
| 413 | sessions.sort((a, b) => (b.started_at || "").localeCompare(a.started_at || "")); |
| 414 | allPrompts.sort((a, b) => (b.ts || "").localeCompare(a.ts || "")); |
| 415 | |
| 416 | // ---- one-line session summaries (Haiku, optional) -------------------------- |
| 417 | // Input is the session's redacted prompts only. Cached by id+line-count so a |
| 418 | // re-run only summarizes new or still-growing sessions. |
| 419 | const HAIKU = "claude-haiku-4-5-20251001"; |
| 420 | const apiKey = process.env.ANTHROPIC_API_KEY; |
| 421 | const prevSummaries = new Map(); |
| 422 | try { |
| 423 | for (const s of JSON.parse(readFileSync(priorPath("index.json"), "utf8")).sessions) { |
| 424 | if (s.summary) prevSummaries.set(s.id + ":" + s.lines, s.summary); |
| 425 | } |
| 426 | } catch {} // first run, or no prior corpus — nothing cached |
| 427 | |
| 428 | async function haiku(content, maxTokens) { |
| 429 | const res = await fetch("https://api.anthropic.com/v1/messages", { |
| 430 | method: "POST", |
| 431 | headers: { "x-api-key": apiKey, "anthropic-version": "2023-06-01", "content-type": "application/json" }, |
| 432 | body: JSON.stringify({ model: HAIKU, max_tokens: maxTokens, messages: [{ role: "user", content }] }), |
| 433 | }); |
| 434 | if (!res.ok) throw new Error("api " + res.status); |
| 435 | return (await res.json()).content?.find((b) => b.type === "text")?.text?.trim() || null; |
| 436 | } |
| 437 | |
| 438 | // THE SUMMARY RULE (applies to any summary generator, API or subagent): if |
| 439 | // the session's prompts reference a GitHub issue/PR URL, the summary ends |
| 440 | // with the single most relevant one verbatim — the page renders it as a |
| 441 | // titled link, so the summary carries its provenance. The URL doesn't count |
| 442 | // toward the length limit (hence the wider slice below). |
| 443 | async function summarize(sess) { |
| 444 | const texts = allPrompts.filter((p) => p.s === sess.id).map((p) => p.t.slice(0, 400)).reverse(); |
| 445 | if (!texts.length) return null; |
| 446 | const out = await haiku( |
| 447 | "One line, at most 90 characters, plain text, no quotes or trailing period: " + |
| 448 | "what was this coding session about? Base it only on the user's prompts below. " + |
| 449 | "If the prompts reference a GitHub issue or pull-request URL, append the single " + |
| 450 | "most relevant one verbatim (full https://github.com/… URL) to the end of the " + |
| 451 | "line — it does not count toward the length limit.\n\n" + |
| 452 | texts.join("\n---\n").slice(0, 8000), 120); |
| 453 | return out ? out.split("\n")[0].slice(0, 220) : null; |
| 454 | } |
| 455 | |
| 456 | // One-line gists of substantial assistant replies, batched 12-per-call. |
| 457 | // Returns k→gist for the batch, or null when the reply didn't parse. |
| 458 | async function gistBatch(batch) { |
| 459 | const out = await haiku( |
| 460 | "Each numbered block below is one assistant reply from a coding session. " + |
| 461 | "For each, write one line (at most 80 characters, plain text) saying what the reply does or concludes. " + |
| 462 | "Return ONLY a JSON array of strings, one per block, same order.\n\n" + |
| 463 | batch.map((r, i) => (i + 1) + ".\n" + r.t).join("\n\n"), 800); |
| 464 | const m = out && out.match(/\[[\s\S]*\]/); |
| 465 | const arr = m ? JSON.parse(m[0]) : null; |
| 466 | if (!Array.isArray(arr) || arr.length !== batch.length) throw new Error("bad batch shape"); |
| 467 | const map = {}; |
| 468 | batch.forEach((r, i) => { if (typeof arr[i] === "string" && arr[i]) map[r.k] = redactString(arr[i].slice(0, 120), {}); }); |
| 469 | return map; |
| 470 | } |
| 471 | |
| 472 | if (apiKey && !checkOnly) { |
| 473 | let done = 0, fresh = 0, failed = 0; |
| 474 | const queue = sessions.filter((s) => !s.withheld); // tombstones get no summary |
| 475 | await Promise.all(Array.from({ length: 4 }, async () => { |
| 476 | for (let s; (s = queue.shift()); ) { |
| 477 | // Carried cache values re-pass through TODAY'S rules — a redaction |
| 478 | // category added later must reach summaries cached before it existed. |
| 479 | const cached = prevSummaries.get(s.id + ":" + s.lines); |
| 480 | if (cached) s.summary = redactString(cached, {}) || undefined; |
| 481 | else { |
| 482 | try { s.summary = redactString((await summarize(s)) || "", {}) || undefined; fresh++; } |
| 483 | catch { failed++; } // no summary — the page falls back to first_prompt |
| 484 | } |
| 485 | if (++done % 50 === 0) console.log("summaries: " + done + "/" + sessions.length); |
| 486 | } |
| 487 | })); |
| 488 | console.log("summaries: " + sessions.length + " sessions (" + fresh + " new, " + failed + " failed)"); |
| 489 | } else if (!checkOnly) { |
| 490 | console.log("summaries: skipped (no ANTHROPIC_API_KEY) — index falls back to first prompts"); |
| 491 | for (const s of sessions) { |
| 492 | if (s.withheld) continue; |
| 493 | const c = prevSummaries.get(s.id + ":" + s.lines); |
| 494 | if (c) s.summary = redactString(c, {}) || undefined; // re-pass today's rules |
| 495 | } |
| 496 | } |
| 497 | |
| 498 | // ---- per-reply gists (<session>.sum.json) ----------------------------------- |
| 499 | // Cached per reply key; a keyless run still carries prior gists forward so a |
| 500 | // rebuild never loses them. Transcripts are append-only, so keys never drift. |
| 501 | if (!checkOnly) { |
| 502 | const loadPrior = (id) => { |
| 503 | try { return JSON.parse(readFileSync(priorPath(id + ".sum.json"), "utf8")).gists || {}; } |
| 504 | catch { return {}; } |
| 505 | }; |
| 506 | let calls = 0, failed = 0, total = 0, cachedN = 0; |
| 507 | const queue = [...sessionReplies.entries()]; |
| 508 | await Promise.all(Array.from({ length: 4 }, async () => { |
| 509 | for (let e; (e = queue.shift()); ) { |
| 510 | const [id, list] = e; |
| 511 | const gists = loadPrior(id); |
| 512 | total += list.length; |
| 513 | cachedN += list.filter((r) => gists[r.k]).length; |
| 514 | if (apiKey) { |
| 515 | const missing = list.filter((r) => !gists[r.k]); |
| 516 | for (let i = 0; i < missing.length; i += 12) { |
| 517 | try { Object.assign(gists, await gistBatch(missing.slice(i, i + 12))); calls++; } |
| 518 | catch { failed++; } // viewer falls back to the reply's first line |
| 519 | } |
| 520 | } |
| 521 | const keep = {}; |
| 522 | for (const r of list) if (gists[r.k]) { |
| 523 | const g = redactString(gists[r.k], {}); // re-pass today's rules |
| 524 | if (g) keep[r.k] = g; |
| 525 | } |
| 526 | if (Object.keys(keep).length) writeFileSync(join(tmp, "prompts", slug, id + ".sum.json"), JSON.stringify({ gists: keep })); |
| 527 | } |
| 528 | })); |
| 529 | console.log(apiKey |
| 530 | ? "reply gists: " + total + " replies (" + cachedN + " cached), " + calls + " batch calls, " + failed + " failed" |
| 531 | : "reply gists: skipped (no ANTHROPIC_API_KEY) — " + cachedN + "/" + total + " carried from cache"); |
| 532 | } |
| 533 | |
| 534 | // ---- GitHub issue titles (optional — needs an authed gh CLI) --------------- |
| 535 | // The repo is private, so viewers can't resolve titles client-side; resolve |
| 536 | // them here, cache across runs, scrub, and let the verifier scan the result. |
| 537 | // URLs come from prompts and summaries only (not whole transcripts). |
| 538 | const ISSUE_RE = /https:\/\/github\.com\/[\w.-]+\/[\w.-]+\/(?:issues|pull)\/\d+/g; |
| 539 | if (!checkOnly) { |
| 540 | const urls = new Set(); |
| 541 | for (const p of allPrompts) for (const m of p.t.matchAll(ISSUE_RE)) urls.add(m[0]); |
| 542 | for (const s of sessions) { |
| 543 | if (s.withheld) continue; |
| 544 | for (const f of [s.summary, s.first_prompt, s.last_prompt]) { |
| 545 | if (f) for (const m of f.matchAll(ISSUE_RE)) urls.add(m[0]); |
| 546 | } |
| 547 | } |
| 548 | let issues = {}; |
| 549 | try { issues = JSON.parse(readFileSync(priorPath("issues.json"), "utf8")); } catch {} |
| 550 | let fetched = 0, failedI = 0; |
| 551 | for (const u of urls) { |
| 552 | if (issues[u]) continue; |
| 553 | const m = /github\.com\/([\w.-]+\/[\w.-]+)\/(?:issues|pull)\/(\d+)/.exec(u); |
| 554 | try { |
| 555 | // /issues/{n} serves PR titles too. |
| 556 | const title = execSync("gh api repos/" + m[1] + "/issues/" + m[2] + " --jq .title", |
| 557 | { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); |
| 558 | if (title) { issues[u] = { repo: m[1], n: Number(m[2]), title: redactString(title, {}).slice(0, 140) }; fetched++; } |
| 559 | } catch { failedI++; } // no gh / no access — the page falls back to a bare link |
| 560 | } |
| 561 | const keepI = {}; |
| 562 | for (const u of urls) if (issues[u]) keepI[u] = issues[u]; |
| 563 | writeFileSync(join(tmp, "prompts", slug, "issues.json"), JSON.stringify(keepI)); |
| 564 | console.log("issue titles: " + Object.keys(keepI).length + "/" + urls.size + " resolved (" + fetched + " fetched, " + failedI + " failed)"); |
| 565 | } |
| 566 | |
| 567 | const index = { |
| 568 | generated_at: new Date().toISOString(), |
| 569 | source, |
| 570 | note: "Every Claude Code session transcript behind this codebase from this machine, credentials redacted and annotated in place. Local transcripts prune after roughly a week, so the registry accrues per run; sessions predating the tool's first run are lost to pruning.", |
| 571 | why: WHY, |
| 572 | totals: { sessions: sessions.length, prompts: sessions.reduce((a, s) => a + s.prompts, 0), |
| 573 | redactions: sessions.reduce((a, s) => a + (s.redactions || 0), 0), |
| 574 | withheld: sessions.filter((s) => s.withheld).length, omitted: omittedCount }, |
| 575 | sessions, |
| 576 | }; |
| 577 | if (!checkOnly) { |
| 578 | writeFileSync(join(tmp, "prompts", slug, "index.json"), JSON.stringify(index, null, 1)); |
| 579 | writeFileSync(join(tmp, "prompts", slug, "prompts.json"), |
| 580 | JSON.stringify({ generated_at: index.generated_at, prompts: allPrompts })); |
| 581 | } |
| 582 | |
| 583 | // ---- contributors manifest ---------------------------------------------------- |
| 584 | // Merge this machine's entry into the LIVE manifest (public read, no creds |
| 585 | // needed) so uploading it never drops other contributors. If the manifest |
| 586 | // can't be reached, only this machine's entry is written and the log says |
| 587 | // not to upload contributors.json from that build. |
| 588 | const MANIFEST_URL = process.env.LCHAT_REGISTRY_MANIFEST_URL || |
| 589 | "https://status.elevenmessenger.com/prompts/contributors.json"; |
| 590 | let manifestLive = false; |
| 591 | if (!checkOnly) { |
| 592 | let manifest = { contributors: [] }; |
| 593 | try { |
| 594 | const r = await fetch(MANIFEST_URL); |
| 595 | if (r.ok) { |
| 596 | const j = await r.json(); |
| 597 | if (Array.isArray(j.contributors)) { manifest = j; manifestLive = true; } |
| 598 | } else if (r.status === 403 || r.status === 404) { |
| 599 | manifestLive = true; // the bucket answered: no manifest yet — first publish |
| 600 | } |
| 601 | } catch {} |
| 602 | manifest.contributors = [...manifest.contributors.filter((c) => c.slug !== slug), { slug, name: contributor }] |
| 603 | .sort((a, b) => a.slug.localeCompare(b.slug)); |
| 604 | writeFileSync(join(tmp, "prompts", "contributors.json"), JSON.stringify(manifest, null, 1)); |
| 605 | } |
| 606 | |
| 607 | // ---- fail-closed verification -------------------------------------------------- |
| 608 | let bad = 0; |
| 609 | if (!checkOnly) { |
| 610 | const files = [ |
| 611 | ...readdirSync(join(tmp, "prompts")).filter((f) => f.endsWith(".json")).map((f) => join(tmp, "prompts", f)), |
| 612 | ...readdirSync(join(tmp, "prompts", slug)).map((f) => join(tmp, "prompts", slug, f)), |
| 613 | ]; |
| 614 | for (const path of files) { |
| 615 | const f = basename(path); |
| 616 | const hits = detect(readFileSync(path, "utf8"), f.endsWith(".json")); |
| 617 | if (hits.length) { bad++; console.error("LEAK " + f + ": " + hits.join(", ")); } |
| 618 | } |
| 619 | if (bad) { |
| 620 | console.error("\n" + bad + " file(s) failed post-redaction scan — corpus NOT published, left in " + tmp); |
| 621 | process.exit(1); |
| 622 | } |
| 623 | // The hash ledger entry: sha256 of every published file under this slug, |
| 624 | // committed to the PUBLIC elevenmessenger/releases repo (its git history is |
| 625 | // the tamper-evidence — see docs/status.md § The prompt registry). Written |
| 626 | // after verification so it hashes exactly what publishes; also uploaded |
| 627 | // with the corpus so the bucket is self-describing. |
| 628 | const hashed = {}; |
| 629 | for (const f of readdirSync(join(tmp, "prompts", slug)).sort()) { |
| 630 | hashed[f] = "sha256:" + createHash("sha256").update(readFileSync(join(tmp, "prompts", slug, f))).digest("hex"); |
| 631 | } |
| 632 | const ledger = { generated_at: index.generated_at, contributor, slug, |
| 633 | totals: index.totals, files: hashed }; |
| 634 | writeFileSync(join(tmp, "prompts", slug, "manifest.sha256.json"), JSON.stringify(ledger, null, 1)); |
| 635 | |
| 636 | rmSync(join(outRoot, "prompts"), { recursive: true, force: true }); |
| 637 | mkdirSync(outRoot, { recursive: true }); |
| 638 | renameSync(join(tmp, "prompts"), join(outRoot, "prompts")); |
| 639 | rmSync(tmp, { recursive: true, force: true }); |
| 640 | } |
| 641 | |
| 642 | const mb = (n) => (n / 1048576).toFixed(1) + " MB"; |
| 643 | console.log("sessions: " + sessions.length + ", prompts: " + index.totals.prompts + |
| 644 | ", redactions: " + index.totals.redactions + " " + JSON.stringify(grandTally)); |
| 645 | console.log("corpus: " + mb(sessions.reduce((a, s) => a + (s.bytes || 0), 0)) + |
| 646 | (checkOnly ? " (check-only, nothing written)" : " → " + join(outRoot, "prompts", slug))); |
| 647 | if (!checkOnly) { |
| 648 | console.log("\nsync (own prefix only, then the manifest):" + |
| 649 | "\n aws s3 sync " + join(outRoot, "prompts", slug) + " s3://<status-bucket>/prompts/" + slug + " --delete --cache-control max-age=300" + |
| 650 | (manifestLive |
| 651 | ? "\n aws s3 cp " + join(outRoot, "prompts", "contributors.json") + " s3://<status-bucket>/prompts/contributors.json --cache-control max-age=300" |
| 652 | : "\n ⚠ manifest built OFFLINE — do NOT upload contributors.json (it would drop other contributors); re-run online first")); |
| 653 | console.log("ledger (public tamper-evidence — commit the hash manifest to elevenmessenger/releases):" + |
| 654 | "\n gh api -X PUT repos/elevenmessenger/releases/contents/prompt-registry/" + slug + ".json" + |
| 655 | " -f message='prompt-registry: accrue " + slug + "'" + |
| 656 | " -f content=\"$(base64 -i " + join(outRoot, "prompts", slug, "manifest.sha256.json") + ")\"" + |
| 657 | " -f sha=\"$(gh api repos/elevenmessenger/releases/contents/prompt-registry/" + slug + ".json --jq .sha 2>/dev/null || true)\""); |
| 658 | } |
| 659 | |
amadan