| 1 | package memoria |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "net/http" |
| 7 | "strings" |
| 8 | "time" |
| 9 | |
| 10 | "amadan.net/oficina/memoria/internal/memoria/notion" |
| 11 | ) |
| 12 | |
| 13 | // connectorPage is what the connector templates render. |
| 14 | type connectorPage struct { |
| 15 | page |
| 16 | Flash string |
| 17 | FlashKind string |
| 18 | Configured bool |
| 19 | Verified string |
| 20 | // Run is the most recent run, if any. |
| 21 | Run *runView |
| 22 | // Pages is how many pages the ledger holds, and Imported how many |
| 23 | // have actually landed. |
| 24 | Pages int |
| 25 | Imported int |
| 26 | // Siblings is the suite apps memoria holds content from, with §8.1's |
| 27 | // breaker verdict for each. Empty in slice 0 and on any instance with |
| 28 | // no roster, which the template treats as "nothing to say" rather |
| 29 | // than as a problem. |
| 30 | Siblings []sourceRow |
| 31 | // CountsKnown is false when the counts could not be determined. The |
| 32 | // template must then say so rather than render a confident zero. |
| 33 | CountsKnown bool |
| 34 | } |
| 35 | |
| 36 | // runView is one run, rendered. |
| 37 | type runView struct { |
| 38 | State string |
| 39 | Running bool |
| 40 | Roots int |
| 41 | Total int |
| 42 | TotalKnown bool |
| 43 | Seen int |
| 44 | Fetched int |
| 45 | Skipped int |
| 46 | Empty int |
| 47 | Failed int |
| 48 | Unread int |
| 49 | ErrorCode string |
| 50 | Message string |
| 51 | // Tone is the callout tone Message is shown in (runTone). |
| 52 | Tone string |
| 53 | Started string |
| 54 | Finished string |
| 55 | } |
| 56 | |
| 57 | // runMessage maps a run to the sentence the page shows. |
| 58 | // |
| 59 | // Every branch here exists because the alternative reads as something it |
| 60 | // is not: an interrupted run is not a finished one, a failed run is not |
| 61 | // an empty workspace, and a run still going has no total yet. |
| 62 | func runMessage(r *SyncRun) string { |
| 63 | switch r.State { |
| 64 | case RunEnumerating, RunFetching: |
| 65 | return "Importing now. This runs while this page is open, and picks up where it stopped if it is interrupted." |
| 66 | case RunDone: |
| 67 | if r.FailedPages > 0 { |
| 68 | return "Import finished, but some pages could not be read. They are listed as failed below rather than left out of the count." |
| 69 | } |
| 70 | return "Import finished." |
| 71 | case RunInterrupted: |
| 72 | return "The import stopped before it finished — usually because this app went to sleep. Nothing was lost. Start it again to carry on from where it stopped." |
| 73 | case RunFailed: |
| 74 | return failMessage(r.ErrorCode) |
| 75 | } |
| 76 | return "" |
| 77 | } |
| 78 | |
| 79 | // runTone is the tone of runMessage's callout (spec §4): info while it |
| 80 | // runs and when it finished cleanly; a warning when it finished with pages |
| 81 | // it could not read, or stopped (interrupted or failed) — both leave the |
| 82 | // reader with less than they asked for, and neither is their mistake. |
| 83 | // |
| 84 | // Info is the EMPTY tone. rastrillo v0.23's tokens.css draws an untoned |
| 85 | // callout as the neutral one and selects no rst-tone~="info", so writing |
| 86 | // the word would be a value nothing reads |
| 87 | // (TestEveryTemplateAttributeIsInTheDesignSystem). |
| 88 | func runTone(r *SyncRun) string { |
| 89 | switch r.State { |
| 90 | case RunEnumerating, RunFetching: |
| 91 | return "" |
| 92 | case RunDone: |
| 93 | if r.FailedPages > 0 { |
| 94 | return "warning" |
| 95 | } |
| 96 | return "" |
| 97 | } |
| 98 | return "warning" |
| 99 | } |
| 100 | |
| 101 | // failMessage maps an error CODE to copy. Codes travel, never messages: |
| 102 | // Notion's own error bodies quote page ids and connection names. |
| 103 | func failMessage(code string) string { |
| 104 | switch code { |
| 105 | case "unauthorized": |
| 106 | return "Notion refused the token. Check it is still valid and still shared with the pages you want." |
| 107 | case "rate_limited": |
| 108 | return "Notion asked us to slow down for longer than we waited. Nothing was lost — start the import again." |
| 109 | case "traversal_failed": |
| 110 | return "The import could not read all of your Notion, so it stopped rather than finish with pages missing. Nothing was lost." |
| 111 | case "network": |
| 112 | return "We could not reach Notion. Nothing was lost — start the import again." |
| 113 | case "database": |
| 114 | // A failure on OUR side, not Notion's. It used to report as |
| 115 | // "network", so the page said "We could not reach Notion" about a |
| 116 | // database write that never left the building. |
| 117 | return "Something went wrong on our side and the import stopped. Nothing was lost — start it again." |
| 118 | case "interrupted": |
| 119 | return "The import stopped before it finished. Start it again to carry on." |
| 120 | } |
| 121 | return "The import did not finish." |
| 122 | } |
| 123 | |
| 124 | func (a *app) connectorBase(r *http.Request, title string) connectorPage { |
| 125 | return connectorPage{page: a.pageFor(r, titled(title))} |
| 126 | } |
| 127 | |
| 128 | func (a *app) connectors(w http.ResponseWriter, r *http.Request) { |
| 129 | p := a.connectorBase(r, "Sources") |
| 130 | if _, err := a.store.Connector(r.Context(), "notion"); err == nil { |
| 131 | p.Configured = true |
| 132 | } |
| 133 | // The suite apps memoria holds content from, and what §8.1's breaker |
| 134 | // says about each. Owner-only, because this is the page that carries |
| 135 | // numbers about what people were refused. |
| 136 | p.Siblings = a.siblingRows(r) |
| 137 | render(w, "connectors", p) |
| 138 | } |
| 139 | |
| 140 | func (a *app) notionConnector(w http.ResponseWriter, r *http.Request) { |
| 141 | p := a.connectorBase(r, "Notion") |
| 142 | if code := r.URL.Query().Get("err"); code != "" { |
| 143 | p.Flash, p.FlashKind = connectorFlash(code), "danger" |
| 144 | } |
| 145 | if r.URL.Query().Get("saved") != "" { |
| 146 | p.Flash, p.FlashKind = "Connected. Start an import when you are ready.", "good" |
| 147 | } |
| 148 | |
| 149 | c, err := a.store.Connector(r.Context(), "notion") |
| 150 | if errors.Is(err, ErrNoConnector) { |
| 151 | render(w, "notion", p) |
| 152 | return |
| 153 | } |
| 154 | if err != nil { |
| 155 | a.logger.Error("read connector", "err", err) |
| 156 | p.Flash, p.FlashKind = "We could not read the connector settings.", "danger" |
| 157 | render(w, "notion", p) |
| 158 | return |
| 159 | } |
| 160 | p.Configured = true |
| 161 | if c.LastVerified != nil { |
| 162 | p.Verified = stamp(*c.LastVerified) |
| 163 | } |
| 164 | |
| 165 | // Counts. If either fails, CountsKnown stays false and the template |
| 166 | // says the number could not be determined — it never renders 0. The |
| 167 | // precedent is chunkCount, which returned 0 on error and made the |
| 168 | // page assert an absence was real when the app had failed to look. |
| 169 | if total, imported, err := a.store.CountPages(r.Context(), c.ID); err != nil { |
| 170 | a.logger.Error("count source pages", "err", err) |
| 171 | } else { |
| 172 | p.Pages, p.Imported, p.CountsKnown = total, imported, true |
| 173 | } |
| 174 | |
| 175 | run, err := a.store.LatestRun(r.Context(), c.ID) |
| 176 | if err != nil { |
| 177 | a.logger.Error("read latest run", "err", err) |
| 178 | } else if run != nil { |
| 179 | p.Run = viewRun(run) |
| 180 | } |
| 181 | render(w, "notion", p) |
| 182 | } |
| 183 | |
| 184 | func viewRun(r *SyncRun) *runView { |
| 185 | v := &runView{ |
| 186 | State: r.State, Roots: r.Roots, Fetched: r.Fetched, Skipped: r.Skipped, |
| 187 | Empty: r.Empty, Failed: r.FailedPages, Unread: r.UnreadBlocks, |
| 188 | ErrorCode: r.ErrorCode, Message: runMessage(r), Tone: runTone(r), |
| 189 | Running: r.State == RunEnumerating || r.State == RunFetching, |
| 190 | Started: stamp(r.StartedAt), |
| 191 | } |
| 192 | // Total is -1 until a run completes. Rendering that as 0 would tell |
| 193 | // the reader the workspace is empty while it is being read. |
| 194 | if r.Total >= 0 { |
| 195 | v.Total, v.TotalKnown = r.Total, true |
| 196 | } |
| 197 | v.Seen = r.Fetched + r.Skipped + r.Empty + r.FailedPages |
| 198 | if r.FinishedAt != nil { |
| 199 | v.Finished = stamp(*r.FinishedAt) |
| 200 | } |
| 201 | return v |
| 202 | } |
| 203 | |
| 204 | func connectorFlash(code string) string { |
| 205 | switch code { |
| 206 | case "token": |
| 207 | return "That did not look like a Notion integration token." |
| 208 | case "verify": |
| 209 | return "Notion would not accept that token. Check it is an internal integration token and that it has been shared with at least one page." |
| 210 | case "inflight": |
| 211 | return "An import is already running." |
| 212 | case "save": |
| 213 | return "We could not save that." |
| 214 | } |
| 215 | return "That did not work." |
| 216 | } |
| 217 | |
| 218 | // saveNotionConnector stores a token after checking Notion accepts it. |
| 219 | // |
| 220 | // Verified on save so a bad token fails at the form rather than part-way |
| 221 | // through an import. The token is never echoed back to the page. |
| 222 | func (a *app) saveNotionConnector(w http.ResponseWriter, r *http.Request) { |
| 223 | if err := r.ParseForm(); err != nil { |
| 224 | http.Redirect(w, r, "/connectors/notion?err=save", http.StatusSeeOther) |
| 225 | return |
| 226 | } |
| 227 | token := strings.TrimSpace(r.PostFormValue("token")) |
| 228 | if token == "" { |
| 229 | http.Redirect(w, r, "/connectors/notion?err=token", http.StatusSeeOther) |
| 230 | return |
| 231 | } |
| 232 | |
| 233 | workspace, err := a.verifyNotion(r.Context(), token) |
| 234 | if err != nil { |
| 235 | // The code, never the error: a Notion error body can quote a |
| 236 | // page id and the connection's name. |
| 237 | a.logger.Error("verify notion token", "code", notion.CodeOf(err)) |
| 238 | http.Redirect(w, r, "/connectors/notion?err=verify", http.StatusSeeOther) |
| 239 | return |
| 240 | } |
| 241 | if err := a.store.SaveConnector(r.Context(), "notion", token, workspace); err != nil { |
| 242 | a.logger.Error("save connector", "err", err) |
| 243 | http.Redirect(w, r, "/connectors/notion?err=save", http.StatusSeeOther) |
| 244 | return |
| 245 | } |
| 246 | http.Redirect(w, r, "/connectors/notion?saved=1", http.StatusSeeOther) |
| 247 | } |
| 248 | |
| 249 | // verifyNotion is the seam a test replaces; production calls the API. |
| 250 | func (a *app) verifyNotion(ctx context.Context, token string) (string, error) { |
| 251 | if a.verifyToken != nil { |
| 252 | return a.verifyToken(ctx, token) |
| 253 | } |
| 254 | return notion.Verify(ctx, token) |
| 255 | } |
| 256 | |
| 257 | // startNotionSync begins a run and redirects to the status page. |
| 258 | func (a *app) startNotionSync(w http.ResponseWriter, r *http.Request) { |
| 259 | c, err := a.store.Connector(r.Context(), "notion") |
| 260 | if err != nil { |
| 261 | http.Redirect(w, r, "/connectors/notion?err=save", http.StatusSeeOther) |
| 262 | return |
| 263 | } |
| 264 | run, err := a.store.StartRun(r.Context(), c.ID) |
| 265 | if errors.Is(err, ErrSyncInFlight) { |
| 266 | http.Redirect(w, r, "/connectors/notion?err=inflight", http.StatusSeeOther) |
| 267 | return |
| 268 | } |
| 269 | if err != nil { |
| 270 | a.logger.Error("start run", "err", err) |
| 271 | http.Redirect(w, r, "/connectors/notion?err=save", http.StatusSeeOther) |
| 272 | return |
| 273 | } |
| 274 | |
| 275 | // context.WithoutCancel because the run outlives this request — and |
| 276 | // unlike Store.Ingest's post-commit index refresh, which is bounded |
| 277 | // and quick, this is a job of minutes. |
| 278 | // |
| 279 | // Say plainly what this is: a request-spawned goroutine that runs |
| 280 | // until the platform sleeps the process, kept alive by the status |
| 281 | // page being open. It is not a background worker; memoria has none. |
| 282 | // An interrupted run is reclaimed at boot and resumes from the |
| 283 | // ledger. |
| 284 | ctx := context.WithoutCancel(r.Context()) |
| 285 | go a.runSync(ctx, c.ID, run.ID) |
| 286 | |
| 287 | http.Redirect(w, r, "/connectors/notion", http.StatusSeeOther) |
| 288 | } |
| 289 | |
| 290 | func (a *app) runSync(ctx context.Context, connectorID, runID int64) { |
| 291 | if a.syncClient != nil { |
| 292 | a.syncer.RunWith(ctx, a.syncClient, connectorID, runID) |
| 293 | return |
| 294 | } |
| 295 | a.syncer.Run(ctx, connectorID, runID) |
| 296 | } |
| 297 | |
| 298 | // disconnectNotion removes the credential and the ledger, and NO |
| 299 | // DOCUMENTS. |
| 300 | // |
| 301 | // What memoria imported, memoria keeps: the content was published into |
| 302 | // memoria and memoria owns it, so removing it is a separate act |
| 303 | // performed in memoria. The confirmation copy says that before the |
| 304 | // button is pressed rather than after. |
| 305 | func (a *app) disconnectNotion(w http.ResponseWriter, r *http.Request) { |
| 306 | if err := a.store.DeleteConnector(r.Context(), "notion"); err != nil { |
| 307 | a.logger.Error("disconnect connector", "err", err) |
| 308 | http.Redirect(w, r, "/connectors/notion?err=save", http.StatusSeeOther) |
| 309 | return |
| 310 | } |
| 311 | http.Redirect(w, r, "/connectors", http.StatusSeeOther) |
| 312 | } |
| 313 | |
| 314 | // waitForSync blocks until no run is active. Test seam, exported so it |
| 315 | // cannot be reached by accident and obvious when it is. |
| 316 | func (a *app) WaitForSync(connectorID int64, d time.Duration) bool { |
| 317 | deadline := time.Now().Add(d) |
| 318 | for time.Now().Before(deadline) { |
| 319 | run, err := a.store.LatestRun(context.Background(), connectorID) |
| 320 | if err == nil && run != nil && run.State != RunEnumerating && run.State != RunFetching { |
| 321 | return true |
| 322 | } |
| 323 | time.Sleep(5 * time.Millisecond) |
| 324 | } |
| 325 | return false |
| 326 | } |
| 327 | |
amadan