| 1 | // Command seed ingests text files straight into a memoria database, |
| 2 | // without going through HTTP. |
| 3 | // |
| 4 | // It exists so a fresh instance can be opened on a corpus that is already |
| 5 | // worth searching, and it is the honest ancestor of slice 3's connectors: |
| 6 | // same Store, same chunking, same sealing, no network. Nothing here |
| 7 | // bypasses the trust model — it needs the same DEK the app runs with, and |
| 8 | // what it writes is sealed exactly as the app would seal it. |
| 9 | package main |
| 10 | |
| 11 | import ( |
| 12 | "context" |
| 13 | "encoding/base64" |
| 14 | "flag" |
| 15 | "fmt" |
| 16 | "io/fs" |
| 17 | "log/slog" |
| 18 | "os" |
| 19 | "path/filepath" |
| 20 | "strings" |
| 21 | |
| 22 | "amadan.net/rastrillo/rastrillo/db" |
| 23 | "amadan.net/rastrillo/rastrillo/migrate" |
| 24 | |
| 25 | memoria "amadan.net/oficina/memoria/internal/memoria" |
| 26 | "amadan.net/oficina/memoria/internal/memoria/index" |
| 27 | "amadan.net/oficina/memoria/internal/memoria/seal" |
| 28 | ) |
| 29 | |
| 30 | func main() { |
| 31 | dbPath := flag.String("db", "", "path to the memoria database") |
| 32 | root := flag.String("dir", "", "directory to walk for .md and .txt files") |
| 33 | source := flag.String("source", "seed", "the Source recorded on each document") |
| 34 | flag.Parse() |
| 35 | if *dbPath == "" || *root == "" { |
| 36 | fmt.Fprintln(os.Stderr, "usage: seed --db <path> --dir <path> [--source name]") |
| 37 | os.Exit(2) |
| 38 | } |
| 39 | |
| 40 | dekRaw := os.Getenv("MEMORIA_DEK") |
| 41 | if dekRaw == "" { |
| 42 | fmt.Fprintln(os.Stderr, "MEMORIA_DEK is not set; refusing to seed with a key the app cannot derive") |
| 43 | os.Exit(1) |
| 44 | } |
| 45 | dek, err := base64.StdEncoding.DecodeString(dekRaw) |
| 46 | if err != nil { |
| 47 | fmt.Fprintf(os.Stderr, "MEMORIA_DEK is not valid base64: %v\n", err) |
| 48 | os.Exit(1) |
| 49 | } |
| 50 | sealer, err := seal.New(dek) |
| 51 | if err != nil { |
| 52 | fmt.Fprintf(os.Stderr, "MEMORIA_DEK rejected: %v\n", err) |
| 53 | os.Exit(1) |
| 54 | } |
| 55 | |
| 56 | logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})) |
| 57 | d, err := db.Open(*dbPath, logger) |
| 58 | if err != nil { |
| 59 | fmt.Fprintf(os.Stderr, "open database: %v\n", err) |
| 60 | os.Exit(1) |
| 61 | } |
| 62 | defer d.Close() |
| 63 | |
| 64 | // Apply migrations. Without this, seeding a database the app has |
| 65 | // never opened fails with "no such table: documents" once per file |
| 66 | // and then reports success with nothing seeded — which is a |
| 67 | // plausible empty result, and exactly the shape this codebase treats |
| 68 | // as the worst available failure. |
| 69 | if _, err := migrate.Apply(context.Background(), d, memoria.BootSchema); err != nil { |
| 70 | fmt.Fprintf(os.Stderr, "migrate: %v\n", err) |
| 71 | os.Exit(1) |
| 72 | } |
| 73 | |
| 74 | ix, err := index.Open() |
| 75 | if err != nil { |
| 76 | fmt.Fprintf(os.Stderr, "open index: %v\n", err) |
| 77 | os.Exit(1) |
| 78 | } |
| 79 | defer ix.Close() |
| 80 | |
| 81 | store := memoria.NewStore(d, sealer, ix, nil) |
| 82 | ctx := context.Background() |
| 83 | |
| 84 | var n, skipped, indexed int |
| 85 | err = filepath.WalkDir(*root, func(p string, e fs.DirEntry, err error) error { |
| 86 | if err != nil { |
| 87 | return err |
| 88 | } |
| 89 | if e.IsDir() { |
| 90 | // Anything under a dot-directory or node_modules is not the |
| 91 | // team's writing. |
| 92 | base := e.Name() |
| 93 | if strings.HasPrefix(base, ".") || base == "node_modules" || base == "releases" { |
| 94 | return fs.SkipDir |
| 95 | } |
| 96 | return nil |
| 97 | } |
| 98 | switch strings.ToLower(filepath.Ext(p)) { |
| 99 | case ".md", ".txt", ".markdown": |
| 100 | default: |
| 101 | return nil |
| 102 | } |
| 103 | raw, err := os.ReadFile(p) |
| 104 | if err != nil { |
| 105 | return err |
| 106 | } |
| 107 | if len(strings.TrimSpace(string(raw))) == 0 { |
| 108 | skipped++ |
| 109 | return nil |
| 110 | } |
| 111 | rel, _ := filepath.Rel(*root, p) |
| 112 | title := rel |
| 113 | // The relative path is the stable key, so re-seeding after an |
| 114 | // edit updates the document rather than adding a second copy. |
| 115 | if _, err := store.Ingest(ctx, memoria.IngestSpec{ |
| 116 | Source: *source, Key: rel, Title: title, Body: string(raw), |
| 117 | }); err != nil { |
| 118 | // Fatal, not a skip. An earlier version counted these and |
| 119 | // carried on, so a missing table produced 59 skips and a |
| 120 | // cheerful "0 documents seeded" summary. |
| 121 | return fmt.Errorf("ingest %s: %w", rel, err) |
| 122 | } |
| 123 | n++ |
| 124 | // Report per file only when something actually changed, so a |
| 125 | // re-seed of an unchanged tree is quiet rather than looking like |
| 126 | // 60 successful writes. |
| 127 | if before, _ := store.Count(); before != indexed { |
| 128 | indexed = before |
| 129 | fmt.Printf("seeded %s\n", rel) |
| 130 | } |
| 131 | return nil |
| 132 | }) |
| 133 | if err != nil { |
| 134 | fmt.Fprintf(os.Stderr, "walk: %v\n", err) |
| 135 | os.Exit(1) |
| 136 | } |
| 137 | count, _ := store.Count() |
| 138 | // count is this process's own index, which holds only what THIS run |
| 139 | // chunked. A re-seed of an unchanged tree therefore reports zero, and |
| 140 | // that is success, not failure — so say which it is. |
| 141 | switch { |
| 142 | case count == 0 && n > 0: |
| 143 | fmt.Printf("\n%d documents checked, all unchanged — nothing to re-index\n", n) |
| 144 | default: |
| 145 | fmt.Printf("\n%d documents seeded, %d skipped, %d passages indexed\n", n, skipped, count) |
| 146 | } |
| 147 | } |
| 148 | |
amadan