| 1 | // A document that committed and then failed to reach the index is |
| 2 | // STORED. This file is the gate on the sentence that says so. |
| 3 | // |
| 4 | // These tests live in package memoria rather than in memoriatest, and |
| 5 | // that is the whole reason the file exists here: reaching this state |
| 6 | // means breaking the in-memory index under a running app, and the only |
| 7 | // way to do that from outside would be to grow a production method whose |
| 8 | // only caller is a test. The index is closed directly instead, one field |
| 9 | // away, and nothing is exported to make it possible. |
| 10 | package memoria |
| 11 | |
| 12 | import ( |
| 13 | "bytes" |
| 14 | "context" |
| 15 | "encoding/json" |
| 16 | "io" |
| 17 | "log/slog" |
| 18 | "mime/multipart" |
| 19 | "net/http" |
| 20 | "net/http/httptest" |
| 21 | "path/filepath" |
| 22 | "strings" |
| 23 | "testing" |
| 24 | |
| 25 | "amadan.net/rastrillo/rastrillo/db" |
| 26 | |
| 27 | "amadan.net/oficina/memoria/internal/memoria/seal" |
| 28 | ) |
| 29 | |
| 30 | const lagOrigin = "http://app.test" |
| 31 | |
| 32 | // appWithABrokenIndex builds the whole app and then closes its index, so |
| 33 | // every ingest from here commits its row and fails its refresh. That is |
| 34 | // the real shape of the failure: the transaction is already done when the |
| 35 | // index is touched, deliberately (see Ingest), so a broken index cannot |
| 36 | // roll a document back. |
| 37 | func appWithABrokenIndex(t *testing.T) (http.Handler, *app) { |
| 38 | t.Helper() |
| 39 | mux, a := appWithAWorkingIndex(t) |
| 40 | if err := a.store.ix.Close(); err != nil { |
| 41 | t.Fatalf("closing the index: %v", err) |
| 42 | } |
| 43 | return mux, a |
| 44 | } |
| 45 | |
| 46 | // appWithAWorkingIndex is the same app with its index intact. Delete's |
| 47 | // tests need one: a document has to be stored and indexed before there is |
| 48 | // anything to break, and closing the index at construction (which is all |
| 49 | // appWithABrokenIndex adds) leaves nothing to delete. |
| 50 | func appWithAWorkingIndex(t *testing.T) (http.Handler, *app) { |
| 51 | t.Helper() |
| 52 | logger := slog.New(slog.NewTextHandler(io.Discard, nil)) |
| 53 | d, err := db.Open(filepath.Join(t.TempDir(), "app.db"), logger) |
| 54 | if err != nil { |
| 55 | t.Fatalf("db.Open: %v", err) |
| 56 | } |
| 57 | t.Cleanup(func() { d.Close() }) |
| 58 | dek := make([]byte, 32) |
| 59 | for i := range dek { |
| 60 | dek[i] = byte(i * 7) |
| 61 | } |
| 62 | sealer, err := seal.New(dek) |
| 63 | if err != nil { |
| 64 | t.Fatalf("seal.New: %v", err) |
| 65 | } |
| 66 | mux, a, err := AppForTest(d, lagOrigin, sealer, logger) |
| 67 | if err != nil { |
| 68 | t.Fatalf("AppForTest: %v", err) |
| 69 | } |
| 70 | return mux, a |
| 71 | } |
| 72 | |
| 73 | // postOneFile submits a single file the way the form does. asJSON asks |
| 74 | // for the tray's judgement instead of the redirect; both go through the |
| 75 | // same handler. |
| 76 | func postOneFile(t *testing.T, h http.Handler, name, body string, asJSON bool) *httptest.ResponseRecorder { |
| 77 | t.Helper() |
| 78 | var buf bytes.Buffer |
| 79 | mw := multipart.NewWriter(&buf) |
| 80 | w, err := mw.CreateFormFile("file", name) |
| 81 | if err != nil { |
| 82 | t.Fatal(err) |
| 83 | } |
| 84 | if _, err := w.Write([]byte(body)); err != nil { |
| 85 | t.Fatal(err) |
| 86 | } |
| 87 | mw.Close() |
| 88 | req := httptest.NewRequest(http.MethodPost, "/documents", &buf) |
| 89 | req.Header.Set("Content-Type", mw.FormDataContentType()) |
| 90 | req.Header.Set("Origin", lagOrigin) |
| 91 | if asJSON { |
| 92 | req.Header.Set("Accept", "application/json") |
| 93 | } |
| 94 | rec := httptest.NewRecorder() |
| 95 | h.ServeHTTP(rec, req) |
| 96 | return rec |
| 97 | } |
| 98 | |
| 99 | // A committed document is never called a refusal. |
| 100 | // |
| 101 | // The failure this replaces: Ingest refreshes the index AFTER the |
| 102 | // transaction commits, and any error from that refresh came back as a |
| 103 | // plain error. createDocument read it as a store failure and said "{name} |
| 104 | // could not be stored" — about a row that was in the database, in the |
| 105 | // document list and in the byte count, and that came back searchable on |
| 106 | // the next boot. A refusal the app had not verified, which is the |
| 107 | // cardinal sin in AGENTS.md, asserted on the path most likely to fail |
| 108 | // halfway. Found by a code review of this branch. |
| 109 | // |
| 110 | // The vacuity guard is the first assertion, not the last: the document |
| 111 | // has to actually be in the database, or every sentence checked below is |
| 112 | // being judged against something that was never stored. |
| 113 | // |
| 114 | // MUTATION-TESTED 2026-09-04, each reverted, each observed red: |
| 115 | // |
| 116 | // (a) Ingest's two post-commit returns put back to `return 0, err` — |
| 117 | // the code as it stood before this review → "the reply says 0 |
| 118 | // stored ... and calls it refused: [quarterly-notes.txt could not |
| 119 | // be stored]", plus the message check |
| 120 | // (b) the `errors.Is(err, ErrIndexLag)` case deleted from |
| 121 | // createDocument's upload loop, leaving the error to fall through |
| 122 | // to the refusal branch → the same failure. This is the half that |
| 123 | // would rot first: ErrIndexLag can exist and be ignored. |
| 124 | // (c) `stored++` removed from that case, so the document is named but |
| 125 | // not counted → "the reply says 0 stored and calls it refused: []". |
| 126 | // Worth knowing what this one found on the way: before the |
| 127 | // "nothing happened at all" guard was widened to look at lagging |
| 128 | // too, the same mutation sent the whole submission down the empty |
| 129 | // branch and the app said "There was nothing to store — the box was |
| 130 | // empty and no file was chosen" about a document in the database. |
| 131 | func TestAStoredDocumentIsNeverReportedAsRefused(t *testing.T) { |
| 132 | h, a := appWithABrokenIndex(t) |
| 133 | c := newDropZone() |
| 134 | const name = "quarterly-notes.txt" |
| 135 | |
| 136 | rec := postOneFile(t, h, name, "Revenue was flat across the first region we serve.", true) |
| 137 | if rec.Code != http.StatusOK { |
| 138 | t.Fatalf("the JSON path answered %d, want 200:\n%s", rec.Code, rec.Body.String()) |
| 139 | } |
| 140 | var reply uploadOutcome |
| 141 | if err := json.Unmarshal(rec.Body.Bytes(), &reply); err != nil { |
| 142 | t.Fatalf("the JSON path did not answer JSON (%v):\n%s", err, rec.Body.String()) |
| 143 | } |
| 144 | |
| 145 | // Vacuity first. If the row is not there, everything below is a test |
| 146 | // of the words for a document that genuinely was not stored. |
| 147 | docs, err := a.store.Summaries(context.Background()) |
| 148 | if err != nil { |
| 149 | t.Fatalf("listing documents: %v", err) |
| 150 | } |
| 151 | if len(docs) != 1 { |
| 152 | t.Fatalf("%d documents are in the database, want 1. The index was broken, not the "+ |
| 153 | "database, so this test is no longer about a committed document at all", len(docs)) |
| 154 | } |
| 155 | |
| 156 | if reply.Stored != 1 || len(reply.Refused) != 0 { |
| 157 | t.Errorf("the reply says %d stored and calls it refused: %v. The document is in the "+ |
| 158 | "database — the list above has it — so a refusal here is a refusal this app "+ |
| 159 | "never verified, on the row it is looking at.", reply.Stored, reply.Refused) |
| 160 | } |
| 161 | if want := fill(c.ReasonStore, "{name}", name); strings.Contains(reply.Message, want) { |
| 162 | t.Errorf("the sentence says %q about a document that is stored:\n %q", want, reply.Message) |
| 163 | } |
| 164 | // And it does say the true thing, by name. An outcome that merely |
| 165 | // stopped lying — "Stored 1 document." and nothing else — would leave |
| 166 | // someone searching for a document this process cannot find. |
| 167 | if want := fill(c.ReasonNotSearchable, "{name}", name); !strings.Contains(reply.Message, want) { |
| 168 | t.Errorf("the sentence does not say the document is unsearchable:\n want %q in\n %q", |
| 169 | want, reply.Message) |
| 170 | } |
| 171 | if len(reply.Lagging) != 1 || !strings.Contains(reply.Lagging[0], name) { |
| 172 | t.Errorf("the tray is handed lagging=%v, so it cannot put that caveat in the file's own "+ |
| 173 | "row and would say the word for a clean success instead", reply.Lagging) |
| 174 | } |
| 175 | } |
| 176 | |
| 177 | // The scriptless half of the same claim. It has no JSON to inspect, so |
| 178 | // what is checked is the redirect: a document that stored must not be |
| 179 | // counted behind &refused=. |
| 180 | // |
| 181 | // MUTATION-TESTED 2026-09-04, reverted: Ingest's post-commit returns put |
| 182 | // back to `return 0, err` → "the scriptless path redirected to |
| 183 | // /?stored=0&refused=1 after storing 1 document". |
| 184 | func TestTheScriptlessRedirectDoesNotCountAStoredDocumentAsRefused(t *testing.T) { |
| 185 | h, a := appWithABrokenIndex(t) |
| 186 | rec := postOneFile(t, h, "hosting-costs.txt", "Hosting cost 40 a month before the migration.", false) |
| 187 | if rec.Code != http.StatusSeeOther { |
| 188 | t.Fatalf("the scriptless path answered %d, want 303:\n%s", rec.Code, rec.Body.String()) |
| 189 | } |
| 190 | docs, err := a.store.Summaries(context.Background()) |
| 191 | if err != nil { |
| 192 | t.Fatalf("listing documents: %v", err) |
| 193 | } |
| 194 | if len(docs) != 1 { |
| 195 | t.Fatalf("%d documents are in the database, want 1; this test is not about a "+ |
| 196 | "committed document any more", len(docs)) |
| 197 | } |
| 198 | loc := rec.Header().Get("Location") |
| 199 | if loc != "/add?stored=1" { |
| 200 | t.Errorf("the scriptless path redirected to %s after storing %d document. The counts in "+ |
| 201 | "the URL are the fallback for a lost flash cookie, and a stored document "+ |
| 202 | "counted as refused there is the same lie in fewer words.", loc, len(docs)) |
| 203 | } |
| 204 | } |
| 205 | |
amadan