| 1 | package memoria |
| 2 | |
| 3 | import ( |
| 4 | nativeauth "amadan.net/rastrillo/native/server" |
| 5 | "context" |
| 6 | "encoding/json" |
| 7 | "errors" |
| 8 | "io" |
| 9 | "log/slog" |
| 10 | "mime/multipart" |
| 11 | "net/http" |
| 12 | "net/url" |
| 13 | "strconv" |
| 14 | "strings" |
| 15 | "time" |
| 16 | |
| 17 | "amadan.net/oficina/estilo.git/chrome" |
| 18 | "amadan.net/oficina/estilo.git/sky" |
| 19 | "amadan.net/rastrillo/rastrillo" |
| 20 | "amadan.net/rastrillo/rastrillo/auth" |
| 21 | |
| 22 | "amadan.net/oficina/suite" |
| 23 | "amadan.net/rastrillo/rastrillo/db" |
| 24 | "amadan.net/rastrillo/rastrillo/sessions" |
| 25 | "github.com/go-chi/chi/v5" |
| 26 | |
| 27 | "amadan.net/oficina/memoria/internal/memoria/query" |
| 28 | ) |
| 29 | |
| 30 | // The three upload limits, and the ONE place any of them is written |
| 31 | // down. Every sentence, every hint and every data attribute on the page |
| 32 | // is derived from here — see uploadLimit, dropLimit and fileLimit just |
| 33 | // below, and dropZone, which is what the page and the browser read. |
| 34 | // |
| 35 | // maxUpload caps ONE document, pasted or uploaded. 2 MiB of text is |
| 36 | // about 350,000 words — longer than any document anyone has pasted into |
| 37 | // a box, and small enough that chunking and sealing it stays inside a |
| 38 | // request. The number carries its reason here because AGENTS.md requires |
| 39 | // that of a cap, and because the honest alternative (a background job) |
| 40 | // is slice 3's problem, not slice 0's. |
| 41 | // |
| 42 | // maxDropBytes and maxDropFiles cap a whole SUBMISSION, because one |
| 43 | // submission is now several documents. The per-file cap alone would let |
| 44 | // a drop of twenty-five 2 MiB files ask this handler to seal and index |
| 45 | // 50 MiB inside a single request. Both submission caps are refused UP |
| 46 | // FRONT, before anything at all is stored, so a drop is never |
| 47 | // half-taken by a limit nobody was told about — the half-taken drop is |
| 48 | // this codebase's worst failure shape, and a cap enforced part-way |
| 49 | // through is how you build one. |
| 50 | const ( |
| 51 | maxUpload = 2 << 20 |
| 52 | maxDropBytes = 20 << 20 |
| 53 | maxDropFiles = 25 |
| 54 | |
| 55 | // dropSlack is multipart framing: boundaries, part headers, and the |
| 56 | // title and body fields travelling alongside the files. It is added |
| 57 | // to the BODY cap so a submission carrying exactly maxDropBytes of |
| 58 | // file content still fits — and it is added to ParseMultipartForm's |
| 59 | // memory threshold for a much sharper reason. See createDocument. |
| 60 | dropSlack = 64 << 10 |
| 61 | |
| 62 | // maxInFlight caps how many of a drop's files the tray uploads at |
| 63 | // once. It is the one number here the server does NOT enforce: it |
| 64 | // is a courtesy, and the handler is as safe against twenty-five |
| 65 | // simultaneous posts as against three. It is written here anyway, |
| 66 | // beside the caps, because the browser reads all four off the page |
| 67 | // and a number kept in the script is a number nobody reviewing this |
| 68 | // file would find. |
| 69 | // |
| 70 | // Three, because these instances hibernate. Each upload wakes the |
| 71 | // process, then seals, chunks and indexes inside its own request; |
| 72 | // twenty-five of those at once on a cold instance is a queue nobody |
| 73 | // can see, and the tray would sit at "Storing…" on every row. |
| 74 | maxInFlight = 3 |
| 75 | ) |
| 76 | |
| 77 | // uploadLimit, dropLimit and fileLimit are the three caps said out loud, |
| 78 | // and each is the ONLY place its number is put into words. Every refusal |
| 79 | // and every hint reads them, because a refusal that names a different |
| 80 | // number from the hint above it is a page arguing with itself and |
| 81 | // neither line is obviously the wrong one. |
| 82 | func uploadLimit() string { return strconv.Itoa(maxUpload>>20) + " MiB" } |
| 83 | func dropLimit() string { return strconv.Itoa(maxDropBytes>>20) + " MiB" } |
| 84 | func fileLimit() string { return strconv.Itoa(maxDropFiles) } |
| 85 | |
| 86 | type app struct { |
| 87 | native *nativeauth.Auth |
| 88 | db *db.DB |
| 89 | logger *slog.Logger |
| 90 | store *Store |
| 91 | auth *auth.Auth |
| 92 | home *homeIntegration |
| 93 | // sess reads the session cookie auth mints (the same table, the same |
| 94 | // cookie name), for the pages that want to know who is signed in |
| 95 | // without requiring anybody: the error pages, the sign-in page's |
| 96 | // shortcut, the Open source page. Built once, in wireNative. |
| 97 | sess *sessions.Sessions |
| 98 | // origin is MEMORIA_ORIGIN: the error page's Back check and the return |
| 99 | // cookie's binding. |
| 100 | origin string |
| 101 | // clock is time.Now unless a test sets it (the return cookie's expiry). |
| 102 | clock func() time.Time |
| 103 | // directory is the hub directory origin (HOME_DIRECTORY_ORIGIN) the |
| 104 | // switcher's "All apps" goes to; set only in Home mode, and empty means |
| 105 | // no switcher. |
| 106 | directory string |
| 107 | |
| 108 | syncer *Syncer |
| 109 | // owners is the admission list, and there is exactly one of it. |
| 110 | // |
| 111 | // Three things key on it: requireOwner (may this person reach an |
| 112 | // owner-only route), beginGuard (may this address make the instance |
| 113 | // dial, §3.2), and rastrillo's Authorize at verification (§4.3). This |
| 114 | // branch briefly carried a second field — an `admits` closure — so the |
| 115 | // guard and requireOwner each had their own copy of "is an owner", |
| 116 | // which is the drift suite ruling 7 exists to prevent, arrived at |
| 117 | // inside one struct. One field, one predicate: isOwner. |
| 118 | owners []string |
| 119 | |
| 120 | // allowUnauthenticated is set only by AppForTest. Production cannot |
| 121 | // set it: cmd/memoria never calls that constructor, and |
| 122 | // TestProductionDoesNotUseTheTestConstructor fails if it starts to. |
| 123 | allowUnauthenticated bool |
| 124 | // testIdentity is the address AppForTest presents. Same seam, same |
| 125 | // guard: production never sets it. |
| 126 | testIdentity string |
| 127 | // verifyToken and syncClient let a test drive the connector against |
| 128 | // a fake Notion. Both nil in production, where the real client is |
| 129 | // built from the stored credential. |
| 130 | verifyToken func(ctx context.Context, token string) (string, error) |
| 131 | syncClient SourceClient |
| 132 | } |
| 133 | |
| 134 | // UseFakeSource points the connector at a stand-in Notion for tests. |
| 135 | // Named for what it is; production never calls it. |
| 136 | func (a *app) UseFakeSource(c SourceClient, verify func(ctx context.Context, token string) (string, error)) { |
| 137 | a.syncClient, a.verifyToken = c, verify |
| 138 | } |
| 139 | |
| 140 | // UseAuth attaches auth after construction. App builds the router, and |
| 141 | // the router needs auth's handlers, but auth needs the admission gate |
| 142 | // that only App can supply — so the knot is untied by attaching late. |
| 143 | // |
| 144 | // It takes only auth. An earlier version on this branch also took the |
| 145 | // admission predicate, to stop main handing UseAuth a different one from |
| 146 | // the one NewAuth got — a mutation a review found and the tests missed. |
| 147 | // That problem is now solved further up: there is one owners list, set |
| 148 | // once by SetOwners, and isOwner is the only predicate over it, so there |
| 149 | // is no second value to disagree with. |
| 150 | func (a *app) UseAuth(x *auth.Auth) { a.auth = x } |
| 151 | |
| 152 | // page is the data every template gets. Frame is the chrome (chrome.go): |
| 153 | // the bar, the <title> and whether anybody is signed in. Chunks is on it |
| 154 | // because the layout states it on every signed-in screen: a corpus count |
| 155 | // is how a person tells "nothing matched" from "nothing indexed". |
| 156 | type page struct { |
| 157 | Frame Frame |
| 158 | // ModelOn switches the footer's egress sentence (layout.html). Set on |
| 159 | // EVERY page, because the footer is on every page — the first build |
| 160 | // updated the README and left every page promising "no document ever |
| 161 | // leaves this instance" while a search had just sent passages. |
| 162 | ModelOn bool |
| 163 | Chunks int |
| 164 | Query string |
| 165 | Searched bool |
| 166 | Flash string |
| 167 | FlashKind string |
| 168 | Results []Result |
| 169 | Documents []Summary |
| 170 | Drop dropZone |
| 171 | |
| 172 | // Notices is §6.5's loud withholding: what this page could not show |
| 173 | // and which app it came from. **Never a count of what the reader may |
| 174 | // not see** — a NOT_VISIBLE is omitted silently, and the only value |
| 175 | // that may be silent (§4.1). |
| 176 | // |
| 177 | // A page with results and no notices is a complete answer. A page |
| 178 | // with notices is an honest partial one, which §6.5 calls the normal |
| 179 | // case rather than an error path. What must never happen is the third |
| 180 | // thing: a short page that reads as complete. |
| 181 | Notices []string |
| 182 | |
| 183 | // Facts is §8.8's box, above the results and below the notices. |
| 184 | // Built in the store from the passages §4.1 kept, so a page can only |
| 185 | // ever state a fact drawn from something this reader may read. |
| 186 | // |
| 187 | // **Empty is the ordinary state**, and it renders nothing at all — |
| 188 | // never "no phone number found". The corpus is not the world, and a |
| 189 | // confident absence is this codebase's worst failure shape. |
| 190 | Facts []Fact |
| 191 | |
| 192 | // SearchFailed is not "no results". A search that errored renders |
| 193 | // its own callout in the results region, because "nothing matched" |
| 194 | // and "the search broke" look identical to a reader and this |
| 195 | // codebase treats that shape as the worst available failure. |
| 196 | SearchFailed bool |
| 197 | |
| 198 | // Primed means the first result is the one Enter goes to. Decided |
| 199 | // here rather than in the browser so that one heuristic has one |
| 200 | // home; search.js only honours the attribute the template writes. |
| 201 | Primed bool |
| 202 | } |
| 203 | |
| 204 | // dropZone is the upload form's limits and ALL of its copy, in one |
| 205 | // place. Every user-facing sentence this form can produce is a field |
| 206 | // here: the hint above the input, what the browser says when it refuses |
| 207 | // a dropped file before the post, and what the handler says afterwards |
| 208 | // about what it stored and what it would not take. |
| 209 | // |
| 210 | // Written down once rather than duplicated in JavaScript because a |
| 211 | // client check that has drifted from the server is worse than no client |
| 212 | // check: it refuses what the server would have taken, or promises to |
| 213 | // take what the server will refuse, and either way the page is lying |
| 214 | // about the rule. The browser's half is rendered onto the zone as data |
| 215 | // attributes and read back from there. |
| 216 | // |
| 217 | // The browser's checks are a courtesy and never the guard. Every one of |
| 218 | // them is made again in createDocument, on bytes rather than on a File |
| 219 | // object, because a drop is not the only way to reach that handler. |
| 220 | // |
| 221 | // Placeholders are filled by fill(). {limit}, {total} and {max} are the |
| 222 | // caps and are filled HERE, server-side, so neither the browser nor a |
| 223 | // later reader can put a different number in. {name}, {names} and {n} |
| 224 | // are filled by whichever side knows them: the browser for a file it is |
| 225 | // refusing before the post, the handler for one it refused after. |
| 226 | type dropZone struct { |
| 227 | // The limits, published so the browser enforces the server's own |
| 228 | // numbers. TestTheAdvertisedCapIsTheHandlersCap and its neighbours |
| 229 | // post submissions either side of each rendered boundary. |
| 230 | MaxBytes int |
| 231 | MaxTotalBytes int |
| 232 | MaxFiles int |
| 233 | SniffBytes int |
| 234 | |
| 235 | // Labels and the hint under the input. |
| 236 | TitleHint string |
| 237 | FileLabel string |
| 238 | Hint string |
| 239 | |
| 240 | // Whole-submission refusals: nothing is stored, and the sentence |
| 241 | // says so. Both are checked in the browser AND again in the handler. |
| 242 | TooManyFiles string |
| 243 | TooMuchAtOnce string |
| 244 | |
| 245 | // Drop-only refusals. Neither can reach the handler — a folder never |
| 246 | // becomes a part, and a drop the browser would not hand over never |
| 247 | // becomes a request — so these have no server-side twin. |
| 248 | Folder string |
| 249 | Blocked string |
| 250 | |
| 251 | // MaxInFlight caps how many uploads the tray runs at once. It is a |
| 252 | // courtesy to the instance, not a rule the server enforces — a |
| 253 | // hibernating instance woken by twenty-five simultaneous seal-and- |
| 254 | // index requests answers none of them quickly — so it is published |
| 255 | // alongside the caps rather than written down in the script, for the |
| 256 | // same reason they are: one place, and the page is where the browser |
| 257 | // reads it. |
| 258 | MaxInFlight int |
| 259 | |
| 260 | // RefreshHeader is the header the tray's re-fetch sends so the page |
| 261 | // leaves its notice unspent (notice.go). Published here so the script |
| 262 | // and the server cannot spell it differently. |
| 263 | RefreshHeader string |
| 264 | |
| 265 | // ---- the tray ----------------------------------------------------- |
| 266 | // |
| 267 | // A drop no longer fills the form's input and wait for Store it: each |
| 268 | // file is posted on its own, in the background, and the tray in the |
| 269 | // corner is where its progress and its outcome are said. Every word |
| 270 | // of that is here, for the reason the rest of this struct is here — |
| 271 | // the tray and the scriptless flash must be incapable of saying |
| 272 | // different things about the same file. |
| 273 | |
| 274 | // The tray's own furniture. |
| 275 | TrayTitle string |
| 276 | TrayDismiss string |
| 277 | |
| 278 | // The three states a row passes through, and only three, because |
| 279 | // there are only three things that can honestly be said. Waiting: the |
| 280 | // file is queued behind MaxInFlight others and nothing has been sent. |
| 281 | // Uploading: {pct} is filled from the browser's own progress event, |
| 282 | // which counts BYTES SENT — and is filled empty when the browser |
| 283 | // declines to say, because a percentage nobody measured is a lie with |
| 284 | // a number in it. Storing: the body is sent and the server is sealing, |
| 285 | // chunking and indexing it, which is work no client can measure at |
| 286 | // all, so the bar goes indeterminate and stops claiming anything. |
| 287 | Waiting string |
| 288 | Uploading string |
| 289 | Storing string |
| 290 | |
| 291 | // A row's own result when its file landed. Short because the row has |
| 292 | // already named the file above it; the refusal clauses below name it |
| 293 | // again because they travel into the summary sentence too. |
| 294 | RowStored string |
| 295 | |
| 296 | // Shown once everything settles, when anything was stored or might |
| 297 | // have been. The document list is not on this page, so this points to |
| 298 | // it. For an unconfirmed file the list is the only place to find out. |
| 299 | // The link goes to "/" and nothing rewrites it; the page never moves |
| 300 | // on its own, because someone mid-drop does not want the ground moving. |
| 301 | Listed string |
| 302 | ViewDocs string |
| 303 | |
| 304 | // Fresh is said in the tray's live region when the passage count in |
| 305 | // the footer has been replaced from a re-fetch of this page, because |
| 306 | // that swap is otherwise silent. It claims nothing about any file. |
| 307 | Fresh string |
| 308 | |
| 309 | // The outcome sentence, both halves. The handler composes it after |
| 310 | // storing; the browser composes the refusal half before posting, out |
| 311 | // of the same strings — so a file refused in the page and the same |
| 312 | // file refused by the server are refused in the same words. |
| 313 | StoredOne string |
| 314 | StoredMany string |
| 315 | NoneStored string |
| 316 | RefusedOne string |
| 317 | // RefusedMany's {n} is the count of refusals, and it is the number |
| 318 | // this codebase most needs to be right: a person told "4 stored" |
| 319 | // when five went in has been silently truncated. |
| 320 | RefusedMany string |
| 321 | // RefusedMore only appears when the list of names is too long to |
| 322 | // carry. The count in front of the list is still the true count, so |
| 323 | // a cut-short list never implies it is complete. |
| 324 | RefusedMore string |
| 325 | |
| 326 | // Unsure is the refusal head's twin, for files whose fate is unknown |
| 327 | // rather than decided. It is a separate count on purpose: folding a |
| 328 | // lost answer into "1 refused" would state, on the sentence written |
| 329 | // to stop this app claiming things it has not checked, that a |
| 330 | // document was rejected when it may be sitting in the database. |
| 331 | UnsureOne string |
| 332 | UnsureMany string |
| 333 | |
| 334 | // One reason clause per way a file can be refused. They compose into |
| 335 | // a list: "scan.png is not text; notes.txt is empty". |
| 336 | ReasonTooBig string |
| 337 | ReasonBinary string |
| 338 | ReasonEmpty string |
| 339 | ReasonUnreadable string |
| 340 | ReasonStore string |
| 341 | // ReasonNoAnswer is the one this codebase would otherwise get wrong. |
| 342 | // An upload that dies mid-flight — the network went, the instance |
| 343 | // hibernated, a proxy answered with something that is not ours — is |
| 344 | // NOT a refusal: the request may have been received, sealed and |
| 345 | // stored in full, with only the answer lost. So the clause says the |
| 346 | // one true thing, which is that nobody here knows. |
| 347 | ReasonNoAnswer string |
| 348 | // ReasonNotSearchable is not a refusal either, and it is the twin of |
| 349 | // the mistake above rather than of the ones beside it. The document |
| 350 | // committed and the index refresh after the commit failed, so the row |
| 351 | // is in the database, in the list and in the byte count, and this |
| 352 | // process cannot find it. Saying "could not be stored" about that — |
| 353 | // which is what this handler used to say — is a refusal the app never |
| 354 | // verified, on a document it has. It says the restart out loud because |
| 355 | // the index is rebuilt from the sealed rows at every boot, so a |
| 356 | // restart is the thing that actually fixes it and "yet" on its own |
| 357 | // would promise a recovery that never comes. |
| 358 | ReasonNotSearchable string |
| 359 | |
| 360 | // PasteName is what the paste box is called when it is the thing |
| 361 | // being refused. The title someone typed is content and would then |
| 362 | // travel in a cookie, so the box is named rather than its contents. |
| 363 | PasteName string |
| 364 | // Unnamed covers a part that arrives with no filename at all. It |
| 365 | // cannot be left blank: " is not text" names nothing. |
| 366 | Unnamed string |
| 367 | |
| 368 | // Form-level refusals, reached through /?err= codes. |
| 369 | NothingSubmitted string |
| 370 | Wrong string |
| 371 | |
| 372 | // ---- the delete button --------------------------------------------- |
| 373 | // |
| 374 | // Not the upload form's copy, but here for the reason the rest is |
| 375 | // here: one place, reviewed once, and incapable of drifting from the |
| 376 | // sentence beside it. |
| 377 | // |
| 378 | // DeleteFailed is the only thing a delete ever says. A delete that |
| 379 | // worked says nothing — the row is gone from the list below and the |
| 380 | // list is the answer — and a delete that did not says this, because |
| 381 | // the alternative is what this branch was opened to fix: the error |
| 382 | // swallowed, the redirect issued, and a person who asked for a |
| 383 | // passage to stop being findable left believing it had. |
| 384 | // |
| 385 | // It says "still stored" and stops there. Delete removes the index |
| 386 | // entries before the transaction, so a failure can leave the document |
| 387 | // stored and momentarily unfindable as well as stored and findable; |
| 388 | // the half that is true either way is the half that is said, and the |
| 389 | // document's presence in the list under this sentence is the rest of |
| 390 | // the answer. |
| 391 | DeleteFailed string |
| 392 | } |
| 393 | |
| 394 | // newDropZone builds that. |
| 395 | func newDropZone() dropZone { |
| 396 | return dropZone{ |
| 397 | MaxBytes: maxUpload, |
| 398 | MaxTotalBytes: maxDropBytes, |
| 399 | MaxFiles: maxDropFiles, |
| 400 | SniffBytes: sniffBytes, |
| 401 | |
| 402 | TitleHint: "(optional)", |
| 403 | FileLabel: "…or upload text files", |
| 404 | // Derived, so the hint cannot come to disagree with the caps it |
| 405 | // describes — nor with the refusals, which say them in the same |
| 406 | // words from the same place. |
| 407 | Hint: fill("…or drop them anywhere on this page. Up to {max} files at once, {limit} each and {total} in all.", |
| 408 | "{max}", fileLimit(), "{limit}", uploadLimit(), "{total}", dropLimit()), |
| 409 | |
| 410 | TooManyFiles: fill("That drop had {n} files. Memoria takes {max} at a time, so none were stored. Drop in smaller batches.", |
| 411 | "{max}", fileLimit()), |
| 412 | TooMuchAtOnce: fill("That drop was more than {total} altogether. Memoria takes {total} at a time, so none was stored. Drop fewer files at once.", |
| 413 | "{total}", dropLimit()), |
| 414 | |
| 415 | Folder: "Memoria upload doesn't support folders. Drop the files inside it instead.", |
| 416 | // The browsers that refuse this are old, but the failure is |
| 417 | // silent — the files simply never arrive in the input — and a |
| 418 | // silent nothing is the one outcome that must not happen. |
| 419 | Blocked: "This browser would not let a dropped file into the form, so nothing was taken. Choose the files above instead.", |
| 420 | |
| 421 | MaxInFlight: maxInFlight, |
| 422 | RefreshHeader: refreshHeader, |
| 423 | |
| 424 | TrayTitle: "Uploads", |
| 425 | TrayDismiss: "Dismiss", |
| 426 | |
| 427 | Waiting: "Waiting…", |
| 428 | Uploading: "Uploading {pct}", |
| 429 | Storing: "Storing…", |
| 430 | RowStored: "Stored", |
| 431 | |
| 432 | Listed: "Stored documents are in the document list.", |
| 433 | ViewDocs: "View documents", |
| 434 | Fresh: "The passage count below is up to date.", |
| 435 | |
| 436 | StoredOne: "Stored 1 document.", |
| 437 | StoredMany: "Stored {n} documents.", |
| 438 | NoneStored: "Nothing was stored.", |
| 439 | RefusedOne: "1 refused", |
| 440 | RefusedMany: "{n} refused", |
| 441 | RefusedMore: "and {n} more", |
| 442 | UnsureOne: "1 unconfirmed", |
| 443 | UnsureMany: "{n} unconfirmed", |
| 444 | |
| 445 | ReasonTooBig: fill("{name} is larger than {limit}", "{limit}", uploadLimit()), |
| 446 | ReasonBinary: "{name} is not text", |
| 447 | ReasonEmpty: "{name} is empty", |
| 448 | ReasonUnreadable: "{name} could not be read", |
| 449 | ReasonStore: "{name} could not be stored", |
| 450 | ReasonNoAnswer: "{name} got no answer, so whether it was stored is unknown", |
| 451 | |
| 452 | ReasonNotSearchable: "{name} is stored but will not be found by a search until this instance restarts", |
| 453 | |
| 454 | PasteName: "the pasted text", |
| 455 | Unnamed: "a file with no name", |
| 456 | |
| 457 | NothingSubmitted: "There was nothing to store — the box was empty and no file was chosen.", |
| 458 | Wrong: "That did not work.", |
| 459 | |
| 460 | DeleteFailed: "Document was not deleted. It is still stored.", |
| 461 | } |
| 462 | } |
| 463 | |
| 464 | // fill substitutes {placeholder} pairs. The joining punctuation lives in |
| 465 | // the code that assembles a sentence, never inside a reviewed string. |
| 466 | func fill(s string, pairs ...string) string { |
| 467 | if len(pairs) == 0 { |
| 468 | return s |
| 469 | } |
| 470 | return strings.NewReplacer(pairs...).Replace(s) |
| 471 | } |
| 472 | |
| 473 | // storedPart is the first half of the outcome sentence. |
| 474 | func (c dropZone) storedPart(n int) string { |
| 475 | switch { |
| 476 | case n == 1: |
| 477 | return c.StoredOne |
| 478 | case n > 1: |
| 479 | return fill(c.StoredMany, "{n}", strconv.Itoa(n)) |
| 480 | default: |
| 481 | return c.NoneStored |
| 482 | } |
| 483 | } |
| 484 | |
| 485 | // refusedHead is the second half's opening, without the names. |
| 486 | func (c dropZone) refusedHead(n int) string { |
| 487 | if n == 1 { |
| 488 | return c.RefusedOne |
| 489 | } |
| 490 | return fill(c.RefusedMany, "{n}", strconv.Itoa(n)) |
| 491 | } |
| 492 | |
| 493 | // flashBudget is how many bytes of refusal clauses will be carried. |
| 494 | // |
| 495 | // The sentence travels in a cookie, and a cookie much over 4 KiB is |
| 496 | // dropped by the browser without a word — which would take the names |
| 497 | // with it. Past this point the list is cut short and says how much it |
| 498 | // left out: a list that quietly stops is the same lie as a count that |
| 499 | // quietly stops. |
| 500 | const flashBudget = 2500 |
| 501 | |
| 502 | // fit cuts a list of clauses down to what is left of the flash budget, |
| 503 | // and says how many it left out rather than stopping quietly. used is |
| 504 | // carried across calls because one sentence can hold two lists and the |
| 505 | // cookie holds them both. |
| 506 | func (c dropZone) fit(clauses []string, used *int) []string { |
| 507 | out := make([]string, 0, len(clauses)) |
| 508 | for i, r := range clauses { |
| 509 | if i > 0 && *used+len(r) > flashBudget { |
| 510 | out = append(out, fill(c.RefusedMore, "{n}", strconv.Itoa(len(clauses)-i))) |
| 511 | break |
| 512 | } |
| 513 | out = append(out, r) |
| 514 | *used += len(r) + 2 |
| 515 | } |
| 516 | return out |
| 517 | } |
| 518 | |
| 519 | // outcome composes what a person is told after a submission: how many |
| 520 | // documents were stored, and every refusal by name with its reason. |
| 521 | // |
| 522 | // It is one sentence and it always states both halves. "Stored 4 |
| 523 | // documents." on its own, after five files went in, is the silent |
| 524 | // truncation this codebase treats as its worst failure. |
| 525 | // |
| 526 | // lagging is the third thing that can be true of a file and it is |
| 527 | // counted in stored, not against it: those documents ARE stored, so the |
| 528 | // count is right, and each one is then named with what is wrong with it. |
| 529 | // It carries no count of its own — every clause is a whole sentence |
| 530 | // naming its own file, so there is no number here that could quietly |
| 531 | // disagree with the list beside it. |
| 532 | func (c dropZone) outcome(stored int, refused, lagging []string) (msg, kind string) { |
| 533 | if len(refused) == 0 && len(lagging) == 0 { |
| 534 | return c.storedPart(stored), "good" |
| 535 | } |
| 536 | used := 0 |
| 537 | msg = c.storedPart(stored) |
| 538 | if len(refused) > 0 { |
| 539 | msg += " " + c.refusedHead(len(refused)) + ": " + strings.Join(c.fit(refused, &used), "; ") + "." |
| 540 | } |
| 541 | if len(lagging) > 0 { |
| 542 | msg += " " + strings.Join(c.fit(lagging, &used), "; ") + "." |
| 543 | } |
| 544 | return msg, "danger" |
| 545 | } |
| 546 | |
| 547 | // outcomeCounts is the same sentence with the names left out. It is what |
| 548 | // the page falls back to when the flash cookie did not survive the |
| 549 | // redirect: fewer facts, but never a friendlier set of them — the |
| 550 | // refusal count is still stated, so a submission that lost some files |
| 551 | // cannot render as a clean success. |
| 552 | func (c dropZone) outcomeCounts(stored, refused int) (msg, kind string) { |
| 553 | if refused == 0 { |
| 554 | return c.storedPart(stored), "good" |
| 555 | } |
| 556 | return c.storedPart(stored) + " " + c.refusedHead(refused) + ".", "danger" |
| 557 | } |
| 558 | |
| 559 | // chunkCount returns the indexed passage count, and -1 when it cannot be |
| 560 | // determined. |
| 561 | // |
| 562 | // Not 0. Returning 0 on error made the footer say "0 passages indexed" |
| 563 | // and the no-match page add "so this is a real absence rather than an |
| 564 | // empty index" — asserting, on the line that exists to prevent exactly |
| 565 | // this, that an absence was real when the app had failed to look. |
| 566 | func (a *app) chunkCount() int { |
| 567 | n, err := a.store.Count() |
| 568 | if err != nil { |
| 569 | a.logger.Error("count passages", "err", err) |
| 570 | return -1 |
| 571 | } |
| 572 | return n |
| 573 | } |
| 574 | |
| 575 | func (a *app) index(w http.ResponseWriter, r *http.Request) { |
| 576 | q := strings.TrimSpace(r.URL.Query().Get("q")) |
| 577 | f := frame{} |
| 578 | if q != "" { |
| 579 | // A search's tab says what was searched for (spec §10). |
| 580 | f = titled("“" + q + "”") |
| 581 | } |
| 582 | p := a.pageFor(r, f) |
| 583 | p.Query, p.Searched = q, q != "" |
| 584 | a.runSearch(r, &p) |
| 585 | docs, err := a.store.Summaries(r.Context()) |
| 586 | if err != nil { |
| 587 | a.logger.Error("list documents", "err", err) |
| 588 | } |
| 589 | p.Documents = docs |
| 590 | // Adding content moved to /add. The one message that still lands here |
| 591 | // is a failed delete, and it has a cookie of its own; see notice.go. |
| 592 | if kind, msg, ok := indexNotice.take(w, r); ok { |
| 593 | p.Flash, p.FlashKind = msg, kind |
| 594 | } |
| 595 | render(w, "index", p) |
| 596 | } |
| 597 | |
| 598 | // addContent is GET /add: the page for adding content — pasting, choosing |
| 599 | // or dropping files — with the outcome of the last submission. It used to |
| 600 | // be a section of the index; Paul asked for it to be its own page |
| 601 | // (2026-09-29). POST /documents redirects back here. |
| 602 | func (a *app) addContent(w http.ResponseWriter, r *http.Request) { |
| 603 | drop := newDropZone() |
| 604 | p := a.pageFor(r, titled("Add content")) |
| 605 | p.Drop = drop |
| 606 | if f := r.URL.Query().Get("stored"); f != "" { |
| 607 | p.Flash, p.FlashKind = drop.outcomeCounts(count(f), count(r.URL.Query().Get("refused"))) |
| 608 | } |
| 609 | if f := r.URL.Query().Get("err"); f != "" { |
| 610 | p.Flash, p.FlashKind = flashFor(f, count(r.URL.Query().Get("n"))), "danger" |
| 611 | } |
| 612 | // The cookie carries the sentence that NAMES things, so it wins over |
| 613 | // the counts in the URL, which are counts and nothing else. It is |
| 614 | // this page's own cookie and a visit to any other page leaves it |
| 615 | // alone; see notice.go. |
| 616 | if kind, msg, ok := addNotice.take(w, r); ok { |
| 617 | p.Flash, p.FlashKind = msg, kind |
| 618 | } |
| 619 | render(w, "add", p) |
| 620 | } |
| 621 | |
| 622 | // searchResults renders the results region alone, for search.js. |
| 623 | // |
| 624 | // It is a real route under the same session gate as everything else, and |
| 625 | // it renders the same template the page does. The script that calls it is |
| 626 | // an enhancement: delete search.js and the form below it still submits to |
| 627 | // "/" and still gets these results, inside a page. |
| 628 | func (a *app) searchResults(w http.ResponseWriter, r *http.Request) { |
| 629 | q := strings.TrimSpace(r.URL.Query().Get("q")) |
| 630 | p := page{Query: q, Searched: q != ""} |
| 631 | a.runSearch(r, &p) |
| 632 | // The passage count is only spoken by the no-match copy, and this |
| 633 | // route runs once per keystroke. Counting only when the count will |
| 634 | // be read keeps a per-keystroke query off the database. |
| 635 | if p.Searched && len(p.Results) == 0 { |
| 636 | p.Chunks = a.chunkCount() |
| 637 | } |
| 638 | renderFragment(w, "results", p) |
| 639 | } |
| 640 | |
| 641 | // runSearch fills in Results, SearchFailed and Primed for a query that |
| 642 | // has already been trimmed onto p. |
| 643 | // |
| 644 | // Shared by the page and the fragment on purpose: the two renderings must |
| 645 | // agree about which result is primed, or where Enter goes would depend on |
| 646 | // whether the script had loaded yet. |
| 647 | func (a *app) runSearch(r *http.Request, p *page) { |
| 648 | if !p.Searched { |
| 649 | return |
| 650 | } |
| 651 | // The asker, typed, in lone shape: a lowercased address (§4.1, suite |
| 652 | // §3). The kind travels on the wire so a receiver keyed on person |
| 653 | // refs answers UNKNOWN for the whole call rather than guessing — a |
| 654 | // loud failure instead of a silent mismatch between an address and a |
| 655 | // person ref that happen to collide. |
| 656 | // |
| 657 | // **No identity, no search.** Not an empty page: a page. A reader who |
| 658 | // is not signed in has nobody for a sibling to rule on, and every |
| 659 | // passage would have to be withheld anyway — so this is the same |
| 660 | // refusal, said before the round trip rather than after it. |
| 661 | addr, ok := a.identity(r) |
| 662 | if !ok || addr == "" { |
| 663 | a.logger.Error("search refused: no identity on the request") |
| 664 | p.Searched, p.SearchFailed = false, true |
| 665 | return |
| 666 | } |
| 667 | found, err := a.store.Search(r.Context(), |
| 668 | suite.Subject{Kind: suite.SubjectAddress, Value: addr}, p.Query, 25) |
| 669 | if err != nil { |
| 670 | // No query text in the log line: AGENTS.md, and `carlos logs` is |
| 671 | // storage we cannot grep. |
| 672 | a.logger.Error("search failed", "err", err) |
| 673 | p.Searched, p.SearchFailed = false, true |
| 674 | return |
| 675 | } |
| 676 | p.Results = found.Results |
| 677 | p.Notices = found.Notices |
| 678 | // Copied here, in the method BOTH routes share, for the same reason |
| 679 | // Primed is decided here: the page and the fragment must not be able |
| 680 | // to disagree about whether there is a box. |
| 681 | p.Facts = found.Facts |
| 682 | p.Primed = query.PrimesFirstResult(p.Query, len(found.Results)) |
| 683 | } |
| 684 | |
| 685 | // count reads a non-negative integer out of a query parameter. Anything |
| 686 | // unparseable is 0, which is safe here: these are display counts, and |
| 687 | // the sentence they build is chosen by the same numbers the handler |
| 688 | // redirected with. |
| 689 | func count(s string) int { |
| 690 | n, err := strconv.Atoi(s) |
| 691 | if err != nil || n < 0 { |
| 692 | return 0 |
| 693 | } |
| 694 | return n |
| 695 | } |
| 696 | |
| 697 | // flashFor maps an error code to copy. Codes and counts travel in the |
| 698 | // URL rather than messages, so nothing a person typed and no filename is |
| 699 | // reflected back into the page from there. |
| 700 | // |
| 701 | // Only whole-submission refusals reach this — the ones where nothing was |
| 702 | // stored and no file needs naming, because either nothing was offered or |
| 703 | // none of it was so much as opened. Everything per-file is composed by |
| 704 | // outcome() and travels in the flash cookie. |
| 705 | func flashFor(code string, n int) string { |
| 706 | c := newDropZone() |
| 707 | switch code { |
| 708 | case "empty": |
| 709 | return c.NothingSubmitted |
| 710 | case "toomany": |
| 711 | return fill(c.TooManyFiles, "{n}", strconv.Itoa(n)) |
| 712 | case "toototal": |
| 713 | return c.TooMuchAtOnce |
| 714 | default: |
| 715 | return c.Wrong |
| 716 | } |
| 717 | } |
| 718 | |
| 719 | // ---- the asynchronous answer ------------------------------------------ |
| 720 | |
| 721 | // uploadOutcome is what POST /documents answers when the caller asked for |
| 722 | // JSON. It is the SAME judgement the HTML path makes — the same handler, |
| 723 | // the same caps, the same NUL sniff, the same words — said in a shape the |
| 724 | // tray can put beside one file. |
| 725 | // |
| 726 | // Message is the whole outcome sentence, composed by outcome(), so the |
| 727 | // tray and the flash a scriptless browser sees cannot come to say |
| 728 | // different things about the same file. Stored and Refused are the parts |
| 729 | // that sentence was built from, because the tray has to place a refusal |
| 730 | // against the row it belongs to, and a sentence cannot be taken apart |
| 731 | // again once composed. |
| 732 | // |
| 733 | // What is NOT here: any identifier for the stored document. A drop of |
| 734 | // twenty-five files would otherwise hand the browser twenty-five row ids |
| 735 | // it has no use for, and the tray offers a reload instead. |
| 736 | // |
| 737 | // Lagging is the one that has to be carried separately rather than left |
| 738 | // to the sentence. A file that committed and then failed to reach the |
| 739 | // index is COUNTED IN Stored, because it is stored; if the tray had only |
| 740 | // the count it would put the word for a clean success in that file's row |
| 741 | // and drop the caveat on the floor — "stored" alone, which is the |
| 742 | // confident half-truth this whole struct is arranged against. |
| 743 | // |
| 744 | // A filename can appear in Refused, in Lagging and in Message. That is |
| 745 | // content, and it is allowed here for the same reason it is allowed in |
| 746 | // the flash cookie: this body goes to the one browser that just sent the |
| 747 | // file, is rendered on that person's own screen, and is never a log line, |
| 748 | // an error string or a URL. |
| 749 | type uploadOutcome struct { |
| 750 | Stored int `json:"stored"` |
| 751 | Refused []string `json:"refused"` |
| 752 | Lagging []string `json:"lagging"` |
| 753 | Message string `json:"message"` |
| 754 | Kind string `json:"kind"` |
| 755 | } |
| 756 | |
| 757 | // wantsJSON reports whether the caller asked for the tray's answer rather |
| 758 | // than the browser's redirect. |
| 759 | // |
| 760 | // Deliberately narrow: it looks for an explicit application/json in |
| 761 | // Accept, which a navigating browser never sends — Chrome's and Firefox's |
| 762 | // navigation Accept headers offer html, xml, images and */*, and no JSON. |
| 763 | // So a plain form submission cannot fall down this path by accident, and |
| 764 | // the scriptless post keeps its 303 and its flash cookie. |
| 765 | func wantsJSON(r *http.Request) bool { |
| 766 | return strings.Contains(r.Header.Get("Accept"), "application/json") |
| 767 | } |
| 768 | |
| 769 | // writeOutcome answers the JSON path. |
| 770 | // |
| 771 | // The status is 200 for every judgement, including one where nothing was |
| 772 | // stored, because a refusal IS the answer to the question that was asked: |
| 773 | // the request was received, read and judged, and the words for what |
| 774 | // happened are in the body. That leaves a non-200 meaning exactly one |
| 775 | // thing to the tray — no judgement arrived — which is the distinction the |
| 776 | // row has to draw, since a lost answer is not a refusal and must never be |
| 777 | // reported as one. |
| 778 | func writeOutcome(w http.ResponseWriter, out uploadOutcome) { |
| 779 | if out.Refused == nil { |
| 780 | // An absent list and an empty list are the same fact, and the |
| 781 | // tray should not have to know that JSON spells one of them |
| 782 | // null. |
| 783 | out.Refused = []string{} |
| 784 | } |
| 785 | if out.Lagging == nil { |
| 786 | out.Lagging = []string{} |
| 787 | } |
| 788 | body, err := json.Marshal(out) |
| 789 | if err != nil { |
| 790 | // Nothing above can fail to marshal. If it somehow did, the |
| 791 | // answer is a status with no words rather than half a body: the |
| 792 | // tray reads a non-200 as "no answer" and says so, which is |
| 793 | // true. The error itself carries no content. |
| 794 | http.Error(w, "could not answer", http.StatusInternalServerError) |
| 795 | return |
| 796 | } |
| 797 | w.Header().Set("Content-Type", "application/json; charset=utf-8") |
| 798 | w.Header().Set("X-Content-Type-Options", "nosniff") |
| 799 | w.WriteHeader(http.StatusOK) |
| 800 | w.Write(body) |
| 801 | } |
| 802 | |
| 803 | // createDocument stores one document per file, plus one more for the |
| 804 | // paste box if anything was typed into it. |
| 805 | // |
| 806 | // Every file is judged on its own merits and the good ones are stored |
| 807 | // even when their neighbours are refused. Each is its own Ingest, its |
| 808 | // own transaction and its own post-commit index refresh, so a failure on |
| 809 | // the third file leaves the first two stored and searchable. |
| 810 | // |
| 811 | // The person is then told both halves: how many documents landed, and |
| 812 | // every refusal by name with the reason for that file. Storing some |
| 813 | // while implying all is this codebase's worst failure shape; so is |
| 814 | // refusing some without saying which. |
| 815 | func (a *app) createDocument(w http.ResponseWriter, r *http.Request) { |
| 816 | c := newDropZone() |
| 817 | |
| 818 | // Cap the BODY before parsing it, and give ParseMultipartForm a |
| 819 | // memory threshold at least as large as the body can now be. |
| 820 | // |
| 821 | // The argument to ParseMultipartForm is a MEMORY threshold, not a |
| 822 | // limit. Without the MaxBytesReader a 3 MiB part parsed with err=nil |
| 823 | // and Go had already written the whole thing to a temp file in |
| 824 | // PLAINTEXT — measured, canary present — and a 10 GB upload filled |
| 825 | // the disk. Found by a code review; the comment that used to be here |
| 826 | // described a cap that did not exist. |
| 827 | // |
| 828 | // Multi-file sharpens the second half of that trap, because the |
| 829 | // threshold is spent CUMULATIVELY across parts: Go holds file parts |
| 830 | // in memory until the running total passes it, then spills the rest |
| 831 | // to disk. A threshold of maxUpload with a body of maxDropBytes |
| 832 | // would write 18 MiB of somebody's documents to a temp file in the |
| 833 | // clear on every large drop, with no error raised anywhere. So the |
| 834 | // threshold is the whole body cap, and nothing is ever spilled. |
| 835 | r.Body = http.MaxBytesReader(w, r.Body, maxDropBytes+dropSlack) |
| 836 | if err := r.ParseMultipartForm(maxDropBytes + dropSlack); err != nil { |
| 837 | var tooBig *http.MaxBytesError |
| 838 | if errors.As(err, &tooBig) { |
| 839 | a.refuse(w, r, "toototal", 0) |
| 840 | return |
| 841 | } |
| 842 | a.refuse(w, r, "malformed", 0) |
| 843 | return |
| 844 | } |
| 845 | |
| 846 | files := r.MultipartForm.File["file"] |
| 847 | |
| 848 | // Both submission caps are checked here, before anything is stored, |
| 849 | // so the answer is "none of them were taken" rather than "the first |
| 850 | // eleven were". |
| 851 | if len(files) > maxDropFiles { |
| 852 | a.refuse(w, r, "toomany", len(files)) |
| 853 | return |
| 854 | } |
| 855 | total := 0 |
| 856 | for _, hdr := range files { |
| 857 | total += int(hdr.Size) |
| 858 | } |
| 859 | // The MaxBytesReader above refuses anything well over, but it caps |
| 860 | // the BODY, framing included, and this caps the content. A drop |
| 861 | // sitting between the two would otherwise be taken. |
| 862 | if total > maxDropBytes { |
| 863 | a.refuse(w, r, "toototal", 0) |
| 864 | return |
| 865 | } |
| 866 | |
| 867 | stored := 0 |
| 868 | var refused []string |
| 869 | // Stored, and not searchable in this process. Kept apart from |
| 870 | // refused because the two need opposite sentences: see ErrIndexLag, |
| 871 | // and see the clause itself in dropZone. |
| 872 | var lagging []string |
| 873 | |
| 874 | for _, hdr := range files { |
| 875 | // A filename is content. It is named to the person who dropped |
| 876 | // it and nowhere else: never a log line, never an error string, |
| 877 | // never the redirect URL. |
| 878 | name := displayName(c, hdr.Filename) |
| 879 | raw, reason := a.readUpload(c, hdr, name) |
| 880 | if reason != "" { |
| 881 | refused = append(refused, reason) |
| 882 | continue |
| 883 | } |
| 884 | // The filename is the stable key: re-uploading an edited file |
| 885 | // updates one document instead of leaving its old self indexed |
| 886 | // under the same title. It is the title too — with several files |
| 887 | // in one submission there is no single title field that could |
| 888 | // serve them all, and a title box that silently titled one of |
| 889 | // five would be a worse answer than the filename. |
| 890 | title := strings.TrimSuffix(hdr.Filename, filepathExt(hdr.Filename)) |
| 891 | // AllowTitleOnly stays OFF for people. An empty upload is a |
| 892 | // mistake and refusing it is the useful answer; connectors turn |
| 893 | // it on, because an empty page is a fact about the source. |
| 894 | _, err := a.store.Ingest(r.Context(), IngestSpec{ |
| 895 | Source: "upload", Key: hdr.Filename, Title: title, Body: string(raw), |
| 896 | }) |
| 897 | switch { |
| 898 | case errors.Is(err, ErrIndexLag): |
| 899 | // The row committed and the index did not take it. Counted as |
| 900 | // stored, because it is, and then named — "could not be |
| 901 | // stored" about a document that is in the list is a refusal |
| 902 | // this app never verified. |
| 903 | a.logger.Error("index refresh failed after commit", "source", "upload", "err", err) |
| 904 | stored++ |
| 905 | lagging = append(lagging, fill(c.ReasonNotSearchable, "{name}", name)) |
| 906 | case err != nil: |
| 907 | // No document content in the log line — AGENTS.md's rule, |
| 908 | // and the reason this carries no title and no filename. |
| 909 | a.logger.Error("ingest failed", "source", "upload", "bytes", len(raw), "err", err) |
| 910 | refused = append(refused, fill(c.ReasonStore, "{name}", name)) |
| 911 | default: |
| 912 | stored++ |
| 913 | } |
| 914 | } |
| 915 | |
| 916 | // The paste box is one more document, not something a file |
| 917 | // overwrites. It used to be replaced by any uploaded file without a |
| 918 | // word: real data loss, and the person was told "Stored, chunked and |
| 919 | // indexed." |
| 920 | title := strings.TrimSpace(r.FormValue("title")) |
| 921 | body := strings.TrimSpace(r.FormValue("body")) |
| 922 | if body != "" { |
| 923 | if len(body) > maxUpload { |
| 924 | refused = append(refused, fill(c.ReasonTooBig, "{name}", c.PasteName)) |
| 925 | } else { |
| 926 | _, err := a.store.Ingest(r.Context(), IngestSpec{ |
| 927 | Source: "paste", Key: title, Title: title, Body: body, |
| 928 | }) |
| 929 | switch { |
| 930 | case errors.Is(err, ErrIndexLag): |
| 931 | a.logger.Error("index refresh failed after commit", "source", "paste", "err", err) |
| 932 | stored++ |
| 933 | lagging = append(lagging, fill(c.ReasonNotSearchable, "{name}", c.PasteName)) |
| 934 | case err != nil: |
| 935 | a.logger.Error("ingest failed", "source", "paste", "bytes", len(body), "err", err) |
| 936 | refused = append(refused, fill(c.ReasonStore, "{name}", c.PasteName)) |
| 937 | default: |
| 938 | stored++ |
| 939 | } |
| 940 | } |
| 941 | } |
| 942 | |
| 943 | // Nothing happened at all — no file, no paste. lagging is in the |
| 944 | // condition even though every clause in it is paired with a stored++ |
| 945 | // above: this branch says "there was nothing to store" out loud, and |
| 946 | // it must be structurally incapable of saying that about a document |
| 947 | // sitting in the database. A test that dropped the pairing landed |
| 948 | // exactly here. |
| 949 | if stored == 0 && len(refused) == 0 && len(lagging) == 0 { |
| 950 | a.refuse(w, r, "empty", 0) |
| 951 | return |
| 952 | } |
| 953 | |
| 954 | msg, kind := c.outcome(stored, refused, lagging) |
| 955 | if wantsJSON(r) { |
| 956 | // No flash cookie on this path. It is one-shot and it outlives |
| 957 | // the request that set it, so a tray that has already said this |
| 958 | // out loud would ALSO ambush the next page load with it — a |
| 959 | // notice about a drop that finished five minutes ago. |
| 960 | writeOutcome(w, uploadOutcome{Stored: stored, Refused: refused, Lagging: lagging, Message: msg, Kind: kind}) |
| 961 | return |
| 962 | } |
| 963 | addNotice.set(w, kind, msg) |
| 964 | target := "/add?stored=" + strconv.Itoa(stored) |
| 965 | if len(refused) > 0 { |
| 966 | target += "&refused=" + strconv.Itoa(len(refused)) |
| 967 | } |
| 968 | // The lagging count is deliberately NOT here, and the reason is |
| 969 | // worth writing down. The URL is the fallback for a lost flash |
| 970 | // cookie, and it carries the refusal count because a refusal means a |
| 971 | // document that does not exist. A lagging document does exist, is in |
| 972 | // the list below and is in the byte count, so the fallback sentence |
| 973 | // stays true when it omits it — incomplete, never wrong. Saying it |
| 974 | // here would need a second reviewed sentence with a count in it and |
| 975 | // no name to attach it to, which is more copy for a strictly worse |
| 976 | // version of what the cookie already carries. |
| 977 | http.Redirect(w, r, target, http.StatusSeeOther) |
| 978 | } |
| 979 | |
| 980 | // readUpload reads one part and judges it, returning either its bytes or |
| 981 | // a refusal clause naming the file and saying why. |
| 982 | // |
| 983 | // Every check the browser makes is made again here, on bytes rather than |
| 984 | // on a File object: a drop is not the only way to reach this handler. |
| 985 | func (a *app) readUpload(c dropZone, hdr *multipart.FileHeader, name string) ([]byte, string) { |
| 986 | f, err := hdr.Open() |
| 987 | if err != nil { |
| 988 | return nil, fill(c.ReasonUnreadable, "{name}", name) |
| 989 | } |
| 990 | defer f.Close() |
| 991 | // LimitReader at cap+1 so a file exactly at the cap passes and one |
| 992 | // byte over is detectable, rather than silently truncated — a |
| 993 | // truncated document indexes as a complete one, which is a citation |
| 994 | // that lies. |
| 995 | raw, err := io.ReadAll(io.LimitReader(f, maxUpload+1)) |
| 996 | switch { |
| 997 | case err != nil: |
| 998 | return nil, fill(c.ReasonUnreadable, "{name}", name) |
| 999 | case len(raw) > maxUpload: |
| 1000 | return nil, fill(c.ReasonTooBig, "{name}", name) |
| 1001 | case !looksLikeText(raw): |
| 1002 | return nil, fill(c.ReasonBinary, "{name}", name) |
| 1003 | case strings.TrimSpace(string(raw)) == "": |
| 1004 | return nil, fill(c.ReasonEmpty, "{name}", name) |
| 1005 | } |
| 1006 | return raw, "" |
| 1007 | } |
| 1008 | |
| 1009 | // refuse answers a whole-submission refusal: nothing was stored, and the |
| 1010 | // same sentence goes into the flash cookie and behind the URL code, so a |
| 1011 | // browser that dropped the cookie still gets the words. |
| 1012 | func (a *app) refuse(w http.ResponseWriter, r *http.Request, code string, n int) { |
| 1013 | msg := flashFor(code, n) |
| 1014 | if wantsJSON(r) { |
| 1015 | // Same words, no cookie, no redirect. See writeOutcome for why a |
| 1016 | // refusal is still a 200: it is an answer, and the tray needs a |
| 1017 | // non-200 to mean the one other thing. |
| 1018 | writeOutcome(w, uploadOutcome{Message: msg, Kind: "danger"}) |
| 1019 | return |
| 1020 | } |
| 1021 | addNotice.set(w, "danger", msg) |
| 1022 | target := "/add?err=" + code |
| 1023 | if n > 0 { |
| 1024 | target += "&n=" + strconv.Itoa(n) |
| 1025 | } |
| 1026 | http.Redirect(w, r, target, http.StatusSeeOther) |
| 1027 | } |
| 1028 | |
| 1029 | // displayName cleans a filename for display. Control bytes are the only |
| 1030 | // thing removed — the name is escaped by the template on its way into |
| 1031 | // the page, and shortening it here would produce a name nobody |
| 1032 | // recognises. |
| 1033 | func displayName(c dropZone, s string) string { |
| 1034 | s = strings.Map(func(r rune) rune { |
| 1035 | if r < 0x20 || r == 0x7f { |
| 1036 | return ' ' |
| 1037 | } |
| 1038 | return r |
| 1039 | }, s) |
| 1040 | if s = strings.TrimSpace(s); s == "" { |
| 1041 | return c.Unnamed |
| 1042 | } |
| 1043 | return s |
| 1044 | } |
| 1045 | |
| 1046 | // passage is one rendered passage of a document. |
| 1047 | type passage struct { |
| 1048 | Ordinal int |
| 1049 | Heading string |
| 1050 | Text string |
| 1051 | } |
| 1052 | |
| 1053 | type documentPage struct { |
| 1054 | page |
| 1055 | ID int64 |
| 1056 | Title string |
| 1057 | Source string |
| 1058 | Bytes int |
| 1059 | Revision int |
| 1060 | Passages []passage |
| 1061 | } |
| 1062 | |
| 1063 | // documentID is the {id} in the URL, or false for anything that is not |
| 1064 | // one — which is the same 404 as a document that is not there. |
| 1065 | func documentID(r *http.Request) (int64, bool) { |
| 1066 | id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) |
| 1067 | return id, err == nil |
| 1068 | } |
| 1069 | |
| 1070 | func (a *app) showDocument(w http.ResponseWriter, r *http.Request) { |
| 1071 | id, ok := documentID(r) |
| 1072 | if !ok { |
| 1073 | a.errorPage(w, r, http.StatusNotFound, kindDocument, "") |
| 1074 | return |
| 1075 | } |
| 1076 | title, body, err := a.store.Body(r.Context(), id) |
| 1077 | if err != nil { |
| 1078 | // A row that is not there 404s. It never 403s, so a document |
| 1079 | // that exists but is not yours is indistinguishable from one |
| 1080 | // that does not exist. |
| 1081 | a.errorPage(w, r, http.StatusNotFound, kindDocument, "") |
| 1082 | return |
| 1083 | } |
| 1084 | // Read the STORED chunks. Re-chunking here would derive ordinals |
| 1085 | // from the current chunker while every citation carries the ordinal |
| 1086 | // from the one that ran at ingest. |
| 1087 | stored, err := a.store.Passages(r.Context(), id) |
| 1088 | if err != nil { |
| 1089 | ref := rastrillo.NewRef() |
| 1090 | a.logger.Error("read passages", "id", id, "ref", ref, "err", err) |
| 1091 | a.errorPage(w, r, http.StatusInternalServerError, kindStatus, ref) |
| 1092 | return |
| 1093 | } |
| 1094 | passages := make([]passage, 0, len(stored)) |
| 1095 | for _, p := range stored { |
| 1096 | passages = append(passages, passage{Ordinal: p.Ordinal, Heading: p.Heading, Text: p.Text}) |
| 1097 | } |
| 1098 | source, revision, nbytes, err := a.store.Meta(r.Context(), id) |
| 1099 | if err != nil { |
| 1100 | a.errorPage(w, r, http.StatusNotFound, kindDocument, "") |
| 1101 | return |
| 1102 | } |
| 1103 | _ = body |
| 1104 | render(w, "document", documentPage{ |
| 1105 | page: a.pageFor(r, frame{parts: []string{title}}), |
| 1106 | ID: id, |
| 1107 | Title: title, |
| 1108 | Source: source, |
| 1109 | Bytes: nbytes, |
| 1110 | Revision: revision, |
| 1111 | Passages: passages, |
| 1112 | }) |
| 1113 | } |
| 1114 | |
| 1115 | // confirmDelete is GET /documents/{id}/delete: the question, asked before |
| 1116 | // anything is deleted. The index's Delete used to be a one-click POST |
| 1117 | // with no undo (polish audit §B3). This is a page rather than a dialog so |
| 1118 | // it works with scripts off and leaves nothing live behind a modal; the |
| 1119 | // POST to the same path is the delete. |
| 1120 | func (a *app) confirmDelete(w http.ResponseWriter, r *http.Request) { |
| 1121 | id, ok := documentID(r) |
| 1122 | if !ok { |
| 1123 | a.errorPage(w, r, http.StatusNotFound, kindDocument, "") |
| 1124 | return |
| 1125 | } |
| 1126 | title, _, err := a.store.Body(r.Context(), id) |
| 1127 | if err != nil { |
| 1128 | a.errorPage(w, r, http.StatusNotFound, kindDocument, "") |
| 1129 | return |
| 1130 | } |
| 1131 | render(w, "delete", documentPage{ |
| 1132 | page: a.pageFor(r, frame{parts: []string{"Delete document", title}, context: title}), |
| 1133 | ID: id, |
| 1134 | Title: title, |
| 1135 | }) |
| 1136 | } |
| 1137 | |
| 1138 | func (a *app) deleteDocument(w http.ResponseWriter, r *http.Request) { |
| 1139 | id, ok := documentID(r) |
| 1140 | if !ok { |
| 1141 | a.errorPage(w, r, http.StatusNotFound, kindDocument, "") |
| 1142 | return |
| 1143 | } |
| 1144 | if err := a.store.Delete(r.Context(), id); err != nil { |
| 1145 | // The delete did not happen, and this used to say nothing about |
| 1146 | // that: the error was logged and the redirect issued exactly as |
| 1147 | // if it had. Someone who asked for a passage to stop being |
| 1148 | // findable was left believing it had stopped. |
| 1149 | // |
| 1150 | // No document id in the sentence and no content anywhere near |
| 1151 | // the log line — the id is a coordinate, not a passage, and it |
| 1152 | // is the only thing logged. |
| 1153 | a.logger.Error("delete document", "id", id, "err", err) |
| 1154 | indexNotice.set(w, "danger", newDropZone().DeleteFailed) |
| 1155 | http.Redirect(w, r, "/", http.StatusSeeOther) |
| 1156 | return |
| 1157 | } |
| 1158 | // A delete that worked says nothing. The row is gone from the list |
| 1159 | // on the page this redirects to, and the list is the answer. |
| 1160 | http.Redirect(w, r, "/", http.StatusSeeOther) |
| 1161 | } |
| 1162 | |
| 1163 | // signinPage is the sign-in page: the layout's page around the suite's |
| 1164 | // door. Exactly one of Door, Sent, SignedOut and Confirm is set — the |
| 1165 | // form, the check-your-email notice that replaces it, the page sign-out |
| 1166 | // lands on, or the confirm step an emailed link opens on. |
| 1167 | type signinPage struct { |
| 1168 | page |
| 1169 | Door *chrome.SignIn |
| 1170 | Sent *chrome.SignInNotice |
| 1171 | SignedOut *chrome.SignInNotice |
| 1172 | Confirm *signinConfirm |
| 1173 | } |
| 1174 | |
| 1175 | // signinConfirm is the confirm step's door: the chrome, and the form that |
| 1176 | // posts the emailed token back. |
| 1177 | type signinConfirm struct { |
| 1178 | Door chrome.SignIn |
| 1179 | Action, Token string |
| 1180 | } |
| 1181 | |
| 1182 | // door is the sign-in form for this instance: Memoria's mark and name on |
| 1183 | // its sky, posting to the guarded Begin. localSignin, refused and |
| 1184 | // renderConfirm all start from it, so the three cannot drift apart. |
| 1185 | func (a *app) door(r *http.Request) chrome.SignIn { |
| 1186 | return chrome.SignIn{ |
| 1187 | Icon: assets.Path("static/brand/icon.svg"), App: appName, Sky: sky.For(r.Host), |
| 1188 | Action: "/auth/begin", |
| 1189 | OpenSource: &chrome.Link{Href: "/help/open-source", Label: "Open source"}, |
| 1190 | } |
| 1191 | } |
| 1192 | |
| 1193 | // refusedCopy is what the door says to an address that has just proved |
| 1194 | // itself and is not admitted here. |
| 1195 | const refusedCopy = "That address can’t sign in to Memoria. Use a different address, or ask whoever runs this team to add it." |
| 1196 | |
| 1197 | // refused is auth's Refused hook: Keymail or an emailed link verified the |
| 1198 | // address and Admit said no. It answers with the sign-in door, carrying |
| 1199 | // the refusal, in a 403 — no session is minted, and auth has already |
| 1200 | // ended the attempt. Before this hook it was rastrillo's bare-text 403, |
| 1201 | // "This address is verified but not admitted here.", which is no page. |
| 1202 | // |
| 1203 | // Rendered, not redirected to ?err=…: a query parameter is something |
| 1204 | // anybody can type, and the sentence means "you proved this address and |
| 1205 | // it is not admitted". Only this hook can say it, and auth calls it only |
| 1206 | // after the address is proved, so saying it is no owners-list oracle — |
| 1207 | // beginGuard's ?sent=1 sameness is about the unproved submit, and is |
| 1208 | // untouched. |
| 1209 | func (a *app) refused(w http.ResponseWriter, r *http.Request, _ auth.Identity) { |
| 1210 | door := a.door(r) |
| 1211 | door.Error = refusedCopy |
| 1212 | w.Header().Set("Cache-Control", "no-store") |
| 1213 | renderStatus(w, "signin", signinPage{ |
| 1214 | page: page{Frame: a.frameAs(r, viewer{}, frame{parts: []string{door.Page()}, signinPage: true}), ModelOn: a.modelOn()}, |
| 1215 | Door: &door, |
| 1216 | }, http.StatusForbidden) |
| 1217 | } |
| 1218 | |
| 1219 | // renderConfirm is auth's RenderConfirm: the page an emailed link opens |
| 1220 | // on. GET /auth/verify spends nothing, so a mail-security scanner that |
| 1221 | // fetches every link in an inbound message cannot spend this one; the |
| 1222 | // door's single Sign in button POSTs the token back to d.Action, and that |
| 1223 | // POST signs in. |
| 1224 | // |
| 1225 | // It is the door with the address field taken away — the same mark, sky, |
| 1226 | // heading and Sign in label — so the person who clicked the link sees |
| 1227 | // Memoria, not rastrillo's plain English fallback. auth has already set |
| 1228 | // no-store, Referrer-Policy: strict-origin (the URL is a live credential) |
| 1229 | // and the framing refusals; nothing here replaces them. d.Token is |
| 1230 | // untrusted query input and reaches the page only through html/template. |
| 1231 | func (a *app) renderConfirm(w http.ResponseWriter, r *http.Request, d auth.ConfirmPageData) { |
| 1232 | c := signinConfirm{Door: a.door(r), Action: d.Action, Token: d.Token} |
| 1233 | c.Door.Lede = "Select Sign in to finish signing in." |
| 1234 | render(w, "signin", signinPage{ |
| 1235 | page: page{Frame: a.frameAs(r, viewer{}, frame{parts: []string{c.Door.Page()}, signinPage: true}), ModelOn: a.modelOn()}, |
| 1236 | Confirm: &c, |
| 1237 | }) |
| 1238 | } |
| 1239 | |
| 1240 | // localSignin serves the sign-in page: GET /signin standalone, and in Home |
| 1241 | // mode the local-address form (/signin/local, and /signin whenever |
| 1242 | // connect hands an outcome back: ?sent, ?err, ?force). |
| 1243 | // |
| 1244 | // Already signed in, standalone, it skips the form (spec §9.2): 303 to a |
| 1245 | // safe return_to or the index. The ADMISSION CHECK RUNS FIRST |
| 1246 | // (admissionOf, the gate's own predicate): an address taken off the owners |
| 1247 | // list is signed out and told so here, rather than sent to a page that |
| 1248 | // sends it straight back — the loop Astra found in Docs (#65, finding 1) |
| 1249 | // and Sheets (#39, finding 2). In Home mode this is the local form, where a |
| 1250 | // Home session goes on purpose to link an account, so it never skips. |
| 1251 | // |
| 1252 | // return_to (spec §9.4): rastrillo's auth lands a magic link on a fixed |
| 1253 | // SignedInPath and never reads return_to, so the value cannot ride the |
| 1254 | // emailed link. It is remembered here in a short-lived sealed cookie |
| 1255 | // (return.go) and read back where the link lands. |
| 1256 | func (a *app) localSignin(w http.ResponseWriter, r *http.Request) { |
| 1257 | q := r.URL.Query() |
| 1258 | ret, hasReturn := safeReturn(q) |
| 1259 | if a.home == nil { |
| 1260 | switch _, st := a.admissionOf(r); st { |
| 1261 | case admitted: |
| 1262 | dest := "/" |
| 1263 | if hasReturn { |
| 1264 | dest = ret |
| 1265 | } |
| 1266 | w.Header().Set("Cache-Control", "no-store") |
| 1267 | http.Redirect(w, r, dest, http.StatusSeeOther) |
| 1268 | return |
| 1269 | case removed: |
| 1270 | a.offboard(w, r) |
| 1271 | return |
| 1272 | } |
| 1273 | } |
| 1274 | // Remember an explicit, validated return whenever this form renders — |
| 1275 | // including Home's forced /signin?force=1 entry, which connect hands |
| 1276 | // here at /signin (Astra on #21, finding 1). The fresh-start reset |
| 1277 | // stays where it was: a bare standalone /signin or /signin/local. |
| 1278 | if a.home == nil || r.URL.Path == "/signin/local" || hasReturn { |
| 1279 | switch { |
| 1280 | case hasReturn: |
| 1281 | a.rememberReturn(w, ret) |
| 1282 | case len(q) == 0: |
| 1283 | // A plain /signin is a fresh start: a destination remembered by |
| 1284 | // an earlier, abandoned sign-in must not hijack this one. |
| 1285 | a.forgetReturn(w) |
| 1286 | } |
| 1287 | } |
| 1288 | w.Header().Set("Cache-Control", "no-store") |
| 1289 | forceEmail := q.Get("force") == "1" |
| 1290 | msg, kind := signinOutcome(q) |
| 1291 | icon, view := assets.Path("static/brand/icon.svg"), sky.For(r.Host) |
| 1292 | |
| 1293 | // A sent link is the door's own check-your-email state, not a flash |
| 1294 | // over the form. The same page for an owner and a stranger: beginGuard |
| 1295 | // answers both with ?sent=1, and nothing here reads anything else. |
| 1296 | if kind == "good" { |
| 1297 | // "Use a different address" goes back to the form KEEPING the |
| 1298 | // return (spec §9.2) — the remembered page, not a bare /signin, |
| 1299 | // which would forget it (Astra on Docs #65, finding 2). With Home |
| 1300 | // configured it goes back to the local form, not /signin, which |
| 1301 | // would leave local recovery for a Home exchange (Astra on Slides |
| 1302 | // #14, finding 2). |
| 1303 | again := signinPath |
| 1304 | if a.home != nil { |
| 1305 | again = "/signin/local" |
| 1306 | } |
| 1307 | if remembered, ok := a.rememberedReturn(r); ok { |
| 1308 | again += "?return_to=" + url.QueryEscape(remembered) |
| 1309 | } |
| 1310 | // The door's words; again is where "Use a different address" goes. |
| 1311 | sent := chrome.SignInNotice{Icon: icon, App: appName, Sky: view}.Sent(again) |
| 1312 | render(w, "signin", signinPage{ |
| 1313 | page: page{Frame: a.frameAs(r, viewer{}, frame{parts: []string{sent.Page()}, signinPage: true}), ModelOn: a.modelOn()}, |
| 1314 | Sent: &sent, |
| 1315 | }) |
| 1316 | return |
| 1317 | } |
| 1318 | |
| 1319 | // The door supplies its own heading, label, help and submit; a plain |
| 1320 | // visit carries no lede. |
| 1321 | d := a.door(r) |
| 1322 | door := &d |
| 1323 | if hasReturn { |
| 1324 | door.Return = ret |
| 1325 | } |
| 1326 | if forceEmail { |
| 1327 | door.Hidden = []chrome.Hidden{{Name: "force", Value: "1"}} |
| 1328 | } |
| 1329 | switch kind { |
| 1330 | case "": |
| 1331 | case "warn": |
| 1332 | // ?reauth=1. Not an error, so not the door's negative callout: |
| 1333 | // it stands in the lede, and says why the form is back. The one |
| 1334 | // lede this door carries: a state, never an introduction. |
| 1335 | door.Lede = msg |
| 1336 | case "field": |
| 1337 | // An address that is not one belongs to the field, not a banner. |
| 1338 | door.AddressError = msg |
| 1339 | default: |
| 1340 | door.Error = msg |
| 1341 | } |
| 1342 | render(w, "signin", signinPage{ |
| 1343 | page: page{Frame: a.frameAs(r, viewer{}, frame{parts: []string{door.Page()}, signinPage: true}), ModelOn: a.modelOn()}, |
| 1344 | Door: door, |
| 1345 | }) |
| 1346 | } |
| 1347 | |
| 1348 | // signinOutcome turns the query a sign-in redirect lands with into the |
| 1349 | // sentence the page says. The vocabulary is not ours: rastrillo's auth |
| 1350 | // handlers and beginGuard write it (?sent=1, ?err=rate|address|expired| |
| 1351 | // keymail|1, ?reauth=1), and offboard writes ?err=removed. This page must |
| 1352 | // read exactly those words. It read ?outcome= for months — a spelling |
| 1353 | // nothing ever wrote — so every sign-in, sent or failed, came back to a |
| 1354 | // blank form. The regression test follows real redirects rather than |
| 1355 | // building the query by hand, because a hand-built query is written from |
| 1356 | // the same model as this switch. |
| 1357 | // |
| 1358 | // There is no "not on this instance" sentence, deliberately: beginGuard |
| 1359 | // answers a stranger with ?sent=1 so the page is not an oracle for who |
| 1360 | // the owners are. (err=removed is said only to the person whose own |
| 1361 | // session was just ended, so it tells nobody else anything.) |
| 1362 | func signinOutcome(q url.Values) (msg, kind string) { |
| 1363 | if q.Get("sent") != "" { |
| 1364 | // No sentence: the door's check-your-email state says it. |
| 1365 | return "", "good" |
| 1366 | } |
| 1367 | switch q.Get("err") { |
| 1368 | case "": |
| 1369 | case "rate": |
| 1370 | return "Too many attempts. Wait a few minutes, then try again.", "danger" |
| 1371 | case "address": |
| 1372 | return "That doesn’t look like an email address.", "field" |
| 1373 | case "expired": |
| 1374 | // rastrillo says one thing for unknown, used and expired alike. |
| 1375 | return "That link has expired or was already used. Enter your email to get a new one.", "danger" |
| 1376 | case "removed": |
| 1377 | return "Your access has been removed, so you’ve been signed out. Ask whoever runs this team if that’s wrong.", "danger" |
| 1378 | case "keymail": |
| 1379 | return "Keymail couldn’t finish signing you in. Enter your email and we’ll send you a link instead.", "danger" |
| 1380 | default: |
| 1381 | return "We couldn’t sign you in. Enter your email to get a new link.", "danger" |
| 1382 | } |
| 1383 | if q.Get("reauth") != "" { |
| 1384 | return "Sign in again to continue.", "warn" |
| 1385 | } |
| 1386 | return "", "" |
| 1387 | } |
| 1388 | |
| 1389 | // sniffBytes is how much of an upload looksLikeText inspects. Named |
| 1390 | // because the drop zone inspects exactly the same prefix in the browser, |
| 1391 | // and it reads the number off the page rather than carrying its own copy |
| 1392 | // — see dropZone. |
| 1393 | const sniffBytes = 8192 |
| 1394 | |
| 1395 | // looksLikeText refuses a binary upload. A NUL byte in the first 8 KiB is |
| 1396 | // the cheap, reliable signal; a PDF or DOCX would otherwise be chunked as |
| 1397 | // mojibake and indexed as though it were prose, which produces citations |
| 1398 | // nobody can read. |
| 1399 | func looksLikeText(b []byte) bool { |
| 1400 | n := len(b) |
| 1401 | if n > sniffBytes { |
| 1402 | n = sniffBytes |
| 1403 | } |
| 1404 | for _, c := range b[:n] { |
| 1405 | if c == 0 { |
| 1406 | return false |
| 1407 | } |
| 1408 | } |
| 1409 | return true |
| 1410 | } |
| 1411 | |
| 1412 | func filepathExt(name string) string { |
| 1413 | if i := strings.LastIndex(name, "."); i >= 0 { |
| 1414 | return name[i:] |
| 1415 | } |
| 1416 | return "" |
| 1417 | } |
| 1418 | |
amadan