| 1 | package memoria |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "fmt" |
| 7 | "strings" |
| 8 | "time" |
| 9 | |
| 10 | "gorm.io/gorm" |
| 11 | |
| 12 | "amadan.net/oficina/memoria/internal/memoria/chunk" |
| 13 | "amadan.net/oficina/memoria/internal/memoria/seal" |
| 14 | ) |
| 15 | |
| 16 | // Run states. |
| 17 | const ( |
| 18 | RunEnumerating = "enumerating" |
| 19 | RunFetching = "fetching" |
| 20 | RunDone = "done" |
| 21 | RunFailed = "failed" |
| 22 | RunInterrupted = "interrupted" |
| 23 | ) |
| 24 | |
| 25 | // Ledger row states. |
| 26 | const ( |
| 27 | PagePending = "pending" |
| 28 | PageFetched = "fetched" |
| 29 | PageEmpty = "empty" |
| 30 | PageFailed = "failed" |
| 31 | ) |
| 32 | |
| 33 | // heartbeatStale is how long a run may go without a heartbeat before a |
| 34 | // LIVE process may reclaim it. A run killed by the platform sleeping is |
| 35 | // reclaimed at boot instead and does not wait for this. |
| 36 | const heartbeatStale = 10 * time.Minute |
| 37 | |
| 38 | // ErrSyncInFlight is returned when a run is already active. |
| 39 | var ErrSyncInFlight = errors.New("a sync is already running") |
| 40 | |
| 41 | // ErrNoConnector is returned when nothing is configured. |
| 42 | var ErrNoConnector = errors.New("no connector is configured") |
| 43 | |
| 44 | // SaveConnector stores or replaces a connector's credential. |
| 45 | // |
| 46 | // Rotation is replacing the row. There is no key history: a replaced |
| 47 | // token has no remaining use, and keeping one would be a credential |
| 48 | // retained for no reason. |
| 49 | func (s *Store) SaveConnector(ctx context.Context, kind, token, workspace string) error { |
| 50 | sealed, err := s.sealer.Seal(seal.PurposeCredential, []byte(token)) |
| 51 | if err != nil { |
| 52 | return err |
| 53 | } |
| 54 | print, err := s.sealer.Fingerprint(seal.PurposeCredential, []byte("workspace\x00"+workspace)) |
| 55 | if err != nil { |
| 56 | return err |
| 57 | } |
| 58 | now := time.Now().UTC() |
| 59 | return s.db.G.WithContext(ctx).Transaction(func(tx *gorm.DB) error { |
| 60 | var existing Connector |
| 61 | res := tx.Where("kind = ?", kind).First(&existing) |
| 62 | if res.Error == nil { |
| 63 | // An explicit allowlist, never a bound request body. |
| 64 | return tx.Model(&Connector{}).Where("id = ?", existing.ID).Updates(map[string]any{ |
| 65 | "sealed_token": sealed, |
| 66 | "workspace_print": print, |
| 67 | "last_verified": now, |
| 68 | }).Error |
| 69 | } |
| 70 | return tx.Create(&Connector{ |
| 71 | Kind: kind, SealedToken: sealed, WorkspacePrint: print, LastVerified: &now, |
| 72 | }).Error |
| 73 | }) |
| 74 | } |
| 75 | |
| 76 | // Connector returns a configured connector, without its token. |
| 77 | func (s *Store) Connector(ctx context.Context, kind string) (*Connector, error) { |
| 78 | var c Connector |
| 79 | if err := s.db.G.WithContext(ctx).Where("kind = ?", kind).First(&c).Error; err != nil { |
| 80 | if errors.Is(err, gorm.ErrRecordNotFound) { |
| 81 | return nil, ErrNoConnector |
| 82 | } |
| 83 | return nil, err |
| 84 | } |
| 85 | return &c, nil |
| 86 | } |
| 87 | |
| 88 | // ConnectorToken opens a connector's credential. The only place the |
| 89 | // plaintext token exists outside the notion client. |
| 90 | func (s *Store) ConnectorToken(ctx context.Context, kind string) (string, error) { |
| 91 | c, err := s.Connector(ctx, kind) |
| 92 | if err != nil { |
| 93 | return "", err |
| 94 | } |
| 95 | tok, err := s.sealer.Open(seal.PurposeCredential, c.SealedToken) |
| 96 | if err != nil { |
| 97 | return "", err |
| 98 | } |
| 99 | return string(tok), nil |
| 100 | } |
| 101 | |
| 102 | // DeleteConnector removes the credential and the ledger. |
| 103 | // |
| 104 | // It deletes NO DOCUMENTS. What memoria imported, memoria keeps — the |
| 105 | // content was published into memoria and memoria owns it, so removing it |
| 106 | // is a separate act performed in memoria. The confirmation copy says so |
| 107 | // in those words rather than leaving it to be discovered. |
| 108 | func (s *Store) DeleteConnector(ctx context.Context, kind string) error { |
| 109 | return s.db.G.WithContext(ctx).Transaction(func(tx *gorm.DB) error { |
| 110 | var c Connector |
| 111 | if err := tx.Where("kind = ?", kind).First(&c).Error; err != nil { |
| 112 | if errors.Is(err, gorm.ErrRecordNotFound) { |
| 113 | return nil |
| 114 | } |
| 115 | return err |
| 116 | } |
| 117 | if err := tx.Where("connector_id = ?", c.ID).Delete(&SourcePage{}).Error; err != nil { |
| 118 | return err |
| 119 | } |
| 120 | if err := tx.Where("connector_id = ?", c.ID).Delete(&SyncRun{}).Error; err != nil { |
| 121 | return err |
| 122 | } |
| 123 | return tx.Where("id = ?", c.ID).Delete(&Connector{}).Error |
| 124 | }) |
| 125 | } |
| 126 | |
| 127 | // StartRun opens a run, or refuses because one is already active. |
| 128 | // |
| 129 | // Single-flight is a conditional insert rather than a mutex, so it |
| 130 | // survives a restart — but "survives a restart" was the bug in an earlier |
| 131 | // design, not the feature. The run is a goroutine; when the platform |
| 132 | // sleeps the process it dies without writing anything, leaving the row |
| 133 | // "fetching" and every later sync refused as already-running, forever, on |
| 134 | // the NORMAL path. ReclaimRuns fixes that at boot; the heartbeat below |
| 135 | // covers a run wedged inside a live process. |
| 136 | func (s *Store) StartRun(ctx context.Context, connectorID int64) (*SyncRun, error) { |
| 137 | var run SyncRun |
| 138 | err := s.db.G.WithContext(ctx).Transaction(func(tx *gorm.DB) error { |
| 139 | var active []SyncRun |
| 140 | if err := tx.Where("connector_id = ? AND state IN ?", connectorID, |
| 141 | []string{RunEnumerating, RunFetching}).Find(&active).Error; err != nil { |
| 142 | return err |
| 143 | } |
| 144 | for _, a := range active { |
| 145 | if a.Heartbeat != nil && time.Since(*a.Heartbeat) > heartbeatStale { |
| 146 | if err := tx.Model(&SyncRun{}).Where("id = ?", a.ID).Updates(map[string]any{ |
| 147 | "state": RunInterrupted, |
| 148 | "error_code": "interrupted", |
| 149 | }).Error; err != nil { |
| 150 | return err |
| 151 | } |
| 152 | continue |
| 153 | } |
| 154 | return ErrSyncInFlight |
| 155 | } |
| 156 | now := time.Now().UTC() |
| 157 | run = SyncRun{ |
| 158 | ConnectorID: connectorID, State: RunEnumerating, |
| 159 | Total: -1, StartedAt: now, Heartbeat: &now, |
| 160 | } |
| 161 | return tx.Create(&run).Error |
| 162 | }) |
| 163 | if err != nil { |
| 164 | return nil, err |
| 165 | } |
| 166 | return &run, nil |
| 167 | } |
| 168 | |
| 169 | // ReclaimRuns marks every run that was in flight as interrupted. |
| 170 | // |
| 171 | // Called at boot, and sound by construction rather than by timeout: a |
| 172 | // process that has just started cannot be running a sync, so anything |
| 173 | // still marked in flight was killed by the platform. |
| 174 | func (s *Store) ReclaimRuns(ctx context.Context) (int, error) { |
| 175 | res := s.db.G.WithContext(ctx).Model(&SyncRun{}). |
| 176 | Where("state IN ?", []string{RunEnumerating, RunFetching}). |
| 177 | Updates(map[string]any{"state": RunInterrupted, "error_code": "interrupted"}) |
| 178 | return int(res.RowsAffected), res.Error |
| 179 | } |
| 180 | |
| 181 | // LatestRun returns the most recent run for a connector. |
| 182 | func (s *Store) LatestRun(ctx context.Context, connectorID int64) (*SyncRun, error) { |
| 183 | var run SyncRun |
| 184 | err := s.db.G.WithContext(ctx).Where("connector_id = ?", connectorID). |
| 185 | Order("id desc").First(&run).Error |
| 186 | if errors.Is(err, gorm.ErrRecordNotFound) { |
| 187 | return nil, nil |
| 188 | } |
| 189 | if err != nil { |
| 190 | return nil, err |
| 191 | } |
| 192 | return &run, nil |
| 193 | } |
| 194 | |
| 195 | // UpdateRun applies an allowlisted patch to a run. |
| 196 | func (s *Store) UpdateRun(ctx context.Context, runID int64, fields map[string]any) error { |
| 197 | allowed := map[string]bool{ |
| 198 | "state": true, "roots": true, "total": true, "fetched": true, |
| 199 | "skipped": true, "empty": true, "failed_pages": true, |
| 200 | "unread_blocks": true, "error_code": true, "heartbeat": true, |
| 201 | "finished_at": true, |
| 202 | } |
| 203 | patch := make(map[string]any, len(fields)) |
| 204 | for k, v := range fields { |
| 205 | if !allowed[k] { |
| 206 | return fmt.Errorf("update run: field %q is not allowlisted", k) |
| 207 | } |
| 208 | patch[k] = v |
| 209 | } |
| 210 | if len(patch) == 0 { |
| 211 | return nil |
| 212 | } |
| 213 | return s.db.G.WithContext(ctx).Model(&SyncRun{}).Where("id = ?", runID).Updates(patch).Error |
| 214 | } |
| 215 | |
| 216 | // Beat records that a run is still alive. |
| 217 | func (s *Store) Beat(ctx context.Context, runID int64) error { |
| 218 | now := time.Now().UTC() |
| 219 | return s.UpdateRun(ctx, runID, map[string]any{"heartbeat": now}) |
| 220 | } |
| 221 | |
| 222 | // LedgerEntry is one page as the sync loop needs it: the coordinate |
| 223 | // opened, and enough state to decide whether it needs work. |
| 224 | type LedgerEntry struct { |
| 225 | ID int64 |
| 226 | PageID string |
| 227 | Path []string |
| 228 | Title string |
| 229 | SeenEdited string |
| 230 | IngestedEdited string |
| 231 | PathPrint string |
| 232 | State string |
| 233 | DocumentID int64 |
| 234 | } |
| 235 | |
| 236 | // SeeSourcePage records a page the enumeration reached. |
| 237 | // |
| 238 | // Upserts on (connector, page print) and writes what enumeration knows: |
| 239 | // the coordinate, the ancestry, the edit time it SAW. It never touches |
| 240 | // IngestedEdited — that column belongs to the fetch phase, and collapsing |
| 241 | // the two into one makes the comparison compare a value against itself |
| 242 | // and skip every edit. |
| 243 | func (s *Store) SeeSourcePage(ctx context.Context, connectorID int64, pageID, title string, path []string, edited string) error { |
| 244 | print, err := s.sealer.Fingerprint(seal.PurposeCoordinate, []byte("page\x00"+pageID)) |
| 245 | if err != nil { |
| 246 | return err |
| 247 | } |
| 248 | sealedID, err := s.sealer.Seal(seal.PurposeCoordinate, []byte(pageID)) |
| 249 | if err != nil { |
| 250 | return err |
| 251 | } |
| 252 | pathStr := chunk.Chunk{Path: path}.PathString() |
| 253 | // Through the shared helper, never inline. Two places computing this |
| 254 | // fingerprint differently would make every comparison false and |
| 255 | // re-ingest the whole workspace on every sync — or make it always |
| 256 | // true and never re-ingest anything. Both fail quietly. |
| 257 | pathPrint, err := s.pathPrint(path, title) |
| 258 | if err != nil { |
| 259 | return err |
| 260 | } |
| 261 | var sealedPath []byte |
| 262 | if pathStr != "" { |
| 263 | if sealedPath, err = s.sealer.Seal(seal.PurposePath, []byte(pathStr)); err != nil { |
| 264 | return err |
| 265 | } |
| 266 | } |
| 267 | now := time.Now().UTC() |
| 268 | |
| 269 | return s.db.G.WithContext(ctx).Transaction(func(tx *gorm.DB) error { |
| 270 | var row SourcePage |
| 271 | res := tx.Where("connector_id = ? AND page_print = ?", connectorID, print).First(&row) |
| 272 | if res.Error == nil { |
| 273 | return tx.Model(&SourcePage{}).Where("id = ?", row.ID).Updates(map[string]any{ |
| 274 | "sealed_page_id": sealedID, |
| 275 | "sealed_path": sealedPath, |
| 276 | "path_print": pathPrint, |
| 277 | "seen_edited": edited, |
| 278 | "last_seen": now, |
| 279 | }).Error |
| 280 | } |
| 281 | return tx.Create(&SourcePage{ |
| 282 | ConnectorID: connectorID, PagePrint: print, SealedPageID: sealedID, |
| 283 | SealedPath: sealedPath, PathPrint: pathPrint, SeenEdited: edited, |
| 284 | State: PagePending, LastSeen: now, |
| 285 | }).Error |
| 286 | }) |
| 287 | } |
| 288 | |
| 289 | // PagesNeedingWork returns the ledger rows the fetch phase must visit. |
| 290 | // |
| 291 | // Three conditions, and the second is the one an earlier design missed |
| 292 | // entirely: an ANCESTOR rename does not change a descendant's own edit |
| 293 | // time, so without comparing the path print a moved or re-parented page |
| 294 | // keeps stale ancestry forever and nothing upstream ever signals it. |
| 295 | func (s *Store) PagesNeedingWork(ctx context.Context, connectorID int64) ([]LedgerEntry, error) { |
| 296 | var rows []SourcePage |
| 297 | if err := s.db.G.WithContext(ctx). |
| 298 | Where("connector_id = ?", connectorID). |
| 299 | Order("id").Find(&rows).Error; err != nil { |
| 300 | return nil, err |
| 301 | } |
| 302 | out := make([]LedgerEntry, 0, len(rows)) |
| 303 | for _, r := range rows { |
| 304 | if r.State == PageFetched && r.SeenEdited == r.IngestedEdited && r.PathPrint == r.ingestedPathPrint() { |
| 305 | continue |
| 306 | } |
| 307 | id, err := s.sealer.Open(seal.PurposeCoordinate, r.SealedPageID) |
| 308 | if err != nil { |
| 309 | return nil, fmt.Errorf("open coordinate %d: %w", r.ID, err) |
| 310 | } |
| 311 | var path []string |
| 312 | if len(r.SealedPath) > 0 { |
| 313 | p, err := s.sealer.Open(seal.PurposePath, r.SealedPath) |
| 314 | if err != nil { |
| 315 | return nil, fmt.Errorf("open path %d: %w", r.ID, err) |
| 316 | } |
| 317 | if s := string(p); s != "" { |
| 318 | path = strings.Split(s, " › ") |
| 319 | } |
| 320 | } |
| 321 | out = append(out, LedgerEntry{ |
| 322 | ID: r.ID, PageID: string(id), Path: path, |
| 323 | SeenEdited: r.SeenEdited, IngestedEdited: r.IngestedEdited, |
| 324 | PathPrint: r.PathPrint, State: r.State, DocumentID: r.DocumentID, |
| 325 | }) |
| 326 | } |
| 327 | return out, nil |
| 328 | } |
| 329 | |
| 330 | // ingestedPathPrint is the path print as of the last successful ingest. |
| 331 | func (r SourcePage) ingestedPathPrint() string { return r.IngestedPath } |
| 332 | |
| 333 | // MarkPage records the outcome of fetching one page. |
| 334 | func (s *Store) MarkPage(ctx context.Context, id int64, state, errorCode string, documentID int64, edited, pathPrint string) error { |
| 335 | return s.db.G.WithContext(ctx).Model(&SourcePage{}).Where("id = ?", id).Updates(map[string]any{ |
| 336 | "state": state, |
| 337 | "error_code": errorCode, |
| 338 | "document_id": documentID, |
| 339 | "ingested_edited": edited, |
| 340 | "ingested_path": pathPrint, |
| 341 | }).Error |
| 342 | } |
| 343 | |
| 344 | // CountPages returns how many ledger rows a connector holds, and how many |
| 345 | // are in each terminal state. |
| 346 | func (s *Store) CountPages(ctx context.Context, connectorID int64) (total, fetched int, err error) { |
| 347 | var t, f int64 |
| 348 | q := s.db.G.WithContext(ctx).Model(&SourcePage{}).Where("connector_id = ?", connectorID) |
| 349 | if err := q.Count(&t).Error; err != nil { |
| 350 | return 0, 0, err |
| 351 | } |
| 352 | // "Imported" means memoria HOLDS a document for this row, which is |
| 353 | // what document_id records — not what state the last fetch ended in. |
| 354 | // |
| 355 | // Counting states instead made a page that imported fine and then |
| 356 | // 404'd on a later run drop out of the total, though the document was |
| 357 | // still there and still searchable. memoria does not delete on |
| 358 | // absence; the count must not imply that it did. |
| 359 | if err := s.db.G.WithContext(ctx).Model(&SourcePage{}). |
| 360 | Where("connector_id = ? AND document_id > 0", connectorID).Count(&f).Error; err != nil { |
| 361 | return 0, 0, err |
| 362 | } |
| 363 | return int(t), int(f), nil |
| 364 | } |
| 365 | |
amadan