amadan
ios+mac: full-history backfill from search — resumable, cancellable (#584) (#792)
The native twin of the web's deep search (#457/#586), sharing one LChatCore
runner across both platforms:
- MessageStore grows a backfill table (additive): per-thread oldest-fetched
watermark + complete marker, plus oldestStoredId — an interrupted walk
RESUMES, a completed thread is never re-walked (forward sync keeps
coverage total from then on)
- HistoryBackfill pages each chat backwards with the before cursor,
decrypting once and filing into the store via BackgroundSync's recipe —
tombstones evict their targets (#582) as pages land; the pure cursor rule
mirrors the web's backfillCursor and is tested against the same cases
(test/store.test.mjs ↔ HistoryBackfillTests)
- user-triggered only, from search: 'Search earlier messages…' appears under
the results on the iOS thread list and the Mac header search; while the
walk runs the #228 time machine narrates ('Searching back to March 2025…')
with Stop; results re-query per filed page, so hits stream in live
swift test green (full suite, 4 new tests); iOS + Mac targets build. No
changelog — native entries wait for the release mint; device verify then too.
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
5 files changed,
+329
−2
ios/LChat/Views/ThreadListView.swift+31 −1ios/LChatCore/Sources/LChatCore/HistoryBackfill.swift+146 −0ios/LChatCore/Sources/LChatCore/MessageStore.swift+54 −0ios/LChatCore/Tests/LChatCoreTests/HistoryBackfillTests.swift+61 −0ios/LChatMac/NativeChatView.swift+37 −1
diff --git a/ios/LChat/Views/ThreadListView.swift b/ios/LChat/Views/ThreadListView.swift| index cb6bd51..a92ce08 100644 |
| --- a/ios/LChat/Views/ThreadListView.swift |
| +++ b/ios/LChat/Views/ThreadListView.swift |
| @@ -239,6 +239,9 @@ struct ThreadListView: View { |
| // ---- search results (conversations + message full-text) ----------------- |
| @State private var searchHits: [(thread: String, message: MessageStore.StoredMessage)] = [] |
| + // The #584 deep walk: fetch-and-file full history so search reaches the |
| + // beginning. One runner per list; its pageTick streams hits in below. |
| + @StateObject private var backfill = HistoryBackfill() |
| private var messageMatches: [(thread: ChatThread, message: MessageStore.StoredMessage)] { |
| searchHits.compactMap { hit in |
| @@ -293,11 +296,38 @@ struct ThreadListView: View { |
| ContentUnavailableView.search(text: search) |
| .listRowBackground(Color.clear) |
| } |
| + // "Search earlier messages" (#584): offered until every chat's |
| + // history has fully backfilled; while running, the #228 time |
| + // machine narrates and Stop cancels (the watermark persists, so |
| + // stopping costs nothing — a later run resumes). |
| + if !search.trimmingCharacters(in: .whitespaces).isEmpty, |
| + !backfill.completed.contains(store.instance.id) { |
| + Section { |
| + if backfill.running { |
| + HStack { |
| + Text(backfill.progress ?? "Searching…") |
| + .font(.footnote) |
| + .foregroundStyle(.secondary) |
| + Spacer() |
| + Button("Stop") { backfill.cancel() } |
| + .font(.footnote.weight(.semibold)) |
| + } |
| + .accessibilityElement(children: .combine) |
| + } else { |
| + Button("Search earlier messages…") { |
| + backfill.start(instance: store.instance) |
| + } |
| + .font(.subheadline) |
| + } |
| + } |
| + } |
| } |
| .listStyle(.insetGrouped) |
| // Debounced FTS against the local store — decrypted once at ingest, |
| // exactly what the SQLite proposal bought us. |
| - .task(id: search) { |
| + // Keyed on the query AND the walk's page counter, so a running deep |
| + // search streams fresh hits into the open results. |
| + .task(id: "\(search)|\(backfill.pageTick)") { |
| let q = search.trimmingCharacters(in: .whitespaces) |
| guard !q.isEmpty else { searchHits = []; return } |
| try? await Task.sleep(nanoseconds: 120_000_000) |
diff --git a/ios/LChatCore/Sources/LChatCore/HistoryBackfill.swift b/ios/LChatCore/Sources/LChatCore/HistoryBackfill.swift| new file mode 100644 |
| index 0000000..42533dd |
| --- /dev/null |
| +++ b/ios/LChatCore/Sources/LChatCore/HistoryBackfill.swift |
| @@ -0,0 +1,146 @@ |
| +import Foundation |
| + |
| +// 🤖 Full-history backfill into the local store (#584) — the native twin of the |
| +// web's runBackfill (#457): page each chat's history backwards with the |
| +// server's `before` cursor, decrypting rows once and filing them into |
| +// MessageStore, persisting an oldest-fetched watermark per thread so an |
| +// interrupted walk RESUMES rather than restarts. A thread that reaches its |
| +// beginning (an empty page) is marked complete and never re-walked — the |
| +// ordinary forward sync keeps coverage total from then on. |
| +// |
| +// User-triggered from search ("Search earlier messages…"), never automatic: |
| +// full history is a deliberate download, not a background surprise. One |
| +// runner instance drives both the iOS thread-list search and the Mac header |
| +// search; views observe `progress` for the #228 time machine ("Searching back |
| +// to March 2025…") and call cancel() for Stop — the watermark survives, so |
| +// stopping costs nothing. |
| +@MainActor |
| +public final class HistoryBackfill: ObservableObject { |
| + @Published public private(set) var running = false |
| + @Published public private(set) var progress: String? |
| + /// Bumps once per filed page — search UIs re-run their query on it, so |
| + /// hits stream in while the walk digs. |
| + @Published public private(set) var pageTick = 0 |
| + /// Instances whose every chat completed — the trigger hides for them. |
| + @Published public private(set) var completed: Set<String> = [] |
| + |
| + private var task: Task<Void, Never>? |
| + |
| + public init() {} |
| + |
| + /// Pure cursor rule (mirrors the web's backfillCursor; unit-tested): |
| + /// nil = the thread is complete, nothing to do; 0 = nothing stored yet, |
| + /// seed the recent page first; otherwise the `before` cursor to resume at. |
| + nonisolated public static func cursor(markOldest: Int64, markComplete: Bool, |
| + oldestStored: Int64) -> Int64? { |
| + if markComplete { return nil } |
| + if markOldest > 0 { return markOldest } |
| + return oldestStored |
| + } |
| + |
| + public func start(instance: SavedInstance) { |
| + guard !running, task == nil else { return } // one walk at a time |
| + running = true |
| + progress = "Searching…" |
| + task = Task { [weak self] in |
| + await self?.walkInstance(instance) |
| + await MainActor.run { |
| + self?.running = false |
| + self?.progress = nil |
| + self?.task = nil |
| + } |
| + } |
| + } |
| + |
| + public func cancel() { |
| + task?.cancel() |
| + } |
| + |
| + private func walkInstance(_ inst: SavedInstance) async { |
| + guard let base = inst.baseURL else { return } |
| + let api = LChatAPI(baseURL: base, token: inst.deviceToken) |
| + var sawIncomplete = false |
| + for t in await api.threads() { |
| + if Task.isCancelled { break } |
| + // Chats only — app-threads' logs are bounded and load whole already. |
| + guard t.kind == "group" || t.kind == "dm", |
| + let wrapped = t.wrapped, !wrapped.isEmpty, |
| + let raw = try? ECIES.unwrap(wrappedB64: wrapped, privPkcs8B64: inst.userPriv), |
| + let key = try? GroupKey(rawKey: raw) else { continue } |
| + let done = await walkThread(inst: inst, api: api, thread: t, key: key) |
| + if !done { sawIncomplete = true } |
| + } |
| + if !Task.isCancelled, !sawIncomplete { |
| + _ = await MainActor.run { completed.insert(inst.id) } |
| + } |
| + } |
| + |
| + /// Walk one thread to its beginning (or cancellation). Returns whether the |
| + /// thread is complete. |
| + private func walkThread(inst: SavedInstance, api: LChatAPI, |
| + thread t: ServerThread, key: GroupKey) async -> Bool { |
| + let mark = await MessageStore.shared.backfillMark(instance: inst.id, thread: t.id) |
| + let oldestStored = await MessageStore.shared.oldestStoredId(instance: inst.id, thread: t.id) |
| + guard var cursor = Self.cursor(markOldest: mark.oldest, markComplete: mark.complete, |
| + oldestStored: oldestStored) else { return true } |
| + if cursor == 0 { |
| + // Empty store for this thread: seed the recent window for a cursor. |
| + let rows = await api.messages(thread: t.id, after: 0) |
| + guard let first = rows.first else { |
| + // No history at all — trivially complete. |
| + await MessageStore.shared.setBackfillMark(instance: inst.id, thread: t.id, |
| + oldest: 0, complete: true) |
| + return true |
| + } |
| + await file(rows, inst: inst, thread: t, key: key) |
| + cursor = first.id |
| + await MessageStore.shared.setBackfillMark(instance: inst.id, thread: t.id, |
| + oldest: cursor, complete: false) |
| + } |
| + while !Task.isCancelled { |
| + let rows = await api.messagesBefore(thread: t.id, before: cursor) |
| + guard let first = rows.first else { |
| + await MessageStore.shared.setBackfillMark(instance: inst.id, thread: t.id, |
| + oldest: cursor, complete: true) |
| + return true |
| + } |
| + await file(rows, inst: inst, thread: t, key: key) |
| + cursor = first.id |
| + await MessageStore.shared.setBackfillMark(instance: inst.id, thread: t.id, |
| + oldest: cursor, complete: false) |
| + await MainActor.run { |
| + self.progress = Self.progressLabel(first.created_at) |
| + self.pageTick += 1 |
| + } |
| + } |
| + return false |
| + } |
| + |
| + /// Decrypt + file one page — BackgroundSync's recipe: tombstones evict |
| + /// their targets (#582), renderable rows upsert (decrypted once, here). |
| + private func file(_ rows: [ServerMessage], inst: SavedInstance, |
| + thread t: ServerThread, key: GroupKey) async { |
| + for id in ChatFeedModel.tombstoneEvictions(rows) { |
| + await MessageStore.shared.delete(instance: inst.id, thread: t.id, id: id) |
| + } |
| + let batch: [MessageStore.StoredMessage] = rows.compactMap { row -> MessageStore.StoredMessage? in |
| + guard (row.tombstone_of ?? 0) <= 0 else { return nil } |
| + guard let p = ChatFeedModel.renderable(row.blob, key: key, |
| + group: t.kind == "group") else { return nil } |
| + return .init(id: row.id, at: row.created_at, |
| + edited: (row.edited_at ?? 0) > 0, |
| + reactions: [], payload: p) |
| + } |
| + guard !batch.isEmpty else { return } |
| + await MessageStore.shared.upsert(instance: inst.id, thread: t.id, batch) |
| + } |
| + |
| + /// "Searching back to March 2025…" — how deep the time machine has dug. |
| + nonisolated static func progressLabel(_ unixSecs: Int64) -> String { |
| + guard unixSecs > 0 else { return "Searching…" } |
| + let f = DateFormatter() |
| + f.dateFormat = "MMMM yyyy" |
| + let month = f.string(from: Date(timeIntervalSince1970: TimeInterval(unixSecs))) |
| + return "Searching back to \(month)…" |
| + } |
| +} |
diff --git a/ios/LChatCore/Sources/LChatCore/MessageStore.swift b/ios/LChatCore/Sources/LChatCore/MessageStore.swift| index c6c15e5..4266692 100644 |
| --- a/ios/LChatCore/Sources/LChatCore/MessageStore.swift |
| +++ b/ios/LChatCore/Sources/LChatCore/MessageStore.swift |
| @@ -97,6 +97,18 @@ public actor MessageStore { |
| CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts |
| USING fts5(text, instance UNINDEXED, thread UNINDEXED, msgid UNINDEXED) |
| """) |
| + // Where each thread's full-history backfill has reached (#584): the |
| + // oldest fetched id, and whether the walk hit the beginning. Additive, |
| + // like every migration here. |
| + exec(handle, """ |
| + CREATE TABLE IF NOT EXISTS backfill( |
| + instance TEXT NOT NULL, |
| + thread TEXT NOT NULL, |
| + oldest INTEGER NOT NULL DEFAULT 0, |
| + complete INTEGER NOT NULL DEFAULT 0, |
| + PRIMARY KEY(instance, thread) |
| + ) |
| + """) |
| // Never in a backup; protected at rest like the keys. |
| try? FileManager.default.setAttributes( |
| [.protectionKey: FileProtectionType.completeUntilFirstUserAuthentication], |
| @@ -158,6 +170,48 @@ public actor MessageStore { |
| } |
| } |
| + // ---- full-history backfill (#584) --------------------------------------- |
| + |
| + /// Where a thread's backwards walk has reached. (0, false) = never started. |
| + public func backfillMark(instance: String, thread: String) -> (oldest: Int64, complete: Bool) { |
| + guard let handle = open() else { return (0, false) } |
| + var stmt: OpaquePointer? |
| + guard sqlite3_prepare_v2(handle, |
| + "SELECT oldest, complete FROM backfill WHERE instance=? AND thread=?", |
| + -1, &stmt, nil) == SQLITE_OK else { return (0, false) } |
| + defer { sqlite3_finalize(stmt) } |
| + bind(stmt, 1, instance) |
| + bind(stmt, 2, thread) |
| + guard sqlite3_step(stmt) == SQLITE_ROW else { return (0, false) } |
| + return (sqlite3_column_int64(stmt, 0), sqlite3_column_int64(stmt, 1) == 1) |
| + } |
| + |
| + public func setBackfillMark(instance: String, thread: String, oldest: Int64, complete: Bool) { |
| + guard let handle = open() else { return } |
| + run(handle, """ |
| + INSERT INTO backfill(instance, thread, oldest, complete) VALUES(?,?,?,?) |
| + ON CONFLICT(instance, thread) DO UPDATE SET oldest=excluded.oldest, complete=excluded.complete |
| + """) { |
| + bind($0, 1, instance); bind($0, 2, thread) |
| + sqlite3_bind_int64($0, 3, oldest); sqlite3_bind_int64($0, 4, complete ? 1 : 0) |
| + } |
| + } |
| + |
| + /// The lowest stored id for a thread — where a first walk starts from. |
| + /// 0 when nothing is stored yet. |
| + public func oldestStoredId(instance: String, thread: String) -> Int64 { |
| + guard let handle = open() else { return 0 } |
| + var stmt: OpaquePointer? |
| + guard sqlite3_prepare_v2(handle, |
| + "SELECT id FROM messages WHERE instance=? AND thread=? ORDER BY id ASC LIMIT 1", |
| + -1, &stmt, nil) == SQLITE_OK else { return 0 } |
| + defer { sqlite3_finalize(stmt) } |
| + bind(stmt, 1, instance) |
| + bind(stmt, 2, thread) |
| + guard sqlite3_step(stmt) == SQLITE_ROW else { return 0 } |
| + return sqlite3_column_int64(stmt, 0) |
| + } |
| + |
| public func delete(instance: String, thread: String, id: Int64) { |
| guard let handle = open() else { return } |
| run(handle, "DELETE FROM messages WHERE instance=? AND thread=? AND id=?") { |
diff --git a/ios/LChatCore/Tests/LChatCoreTests/HistoryBackfillTests.swift b/ios/LChatCore/Tests/LChatCoreTests/HistoryBackfillTests.swift| new file mode 100644 |
| index 0000000..cff0db8 |
| --- /dev/null |
| +++ b/ios/LChatCore/Tests/LChatCoreTests/HistoryBackfillTests.swift |
| @@ -0,0 +1,61 @@ |
| +import XCTest |
| +@testable import LChatCore |
| + |
| +// 🤖 The backfill walk's decision logic (#584) — mirrors the web's |
| +// backfillCursor tests (test/store.test.mjs), so the two walkers can't drift: |
| +// resume from the persisted watermark, seed when nothing is stored, and never |
| +// re-walk a completed thread. Plus the store's mark round-trip. |
| +final class HistoryBackfillTests: XCTestCase { |
| + func testCursorResumesFromWatermarkNotFromScratch() { |
| + XCTAssertEqual(HistoryBackfill.cursor(markOldest: 42, markComplete: false, oldestStored: 900), 42, |
| + "interrupted walk continues at its mark") |
| + XCTAssertEqual(HistoryBackfill.cursor(markOldest: 0, markComplete: false, oldestStored: 900), 900, |
| + "first walk starts at the oldest stored row") |
| + XCTAssertEqual(HistoryBackfill.cursor(markOldest: 0, markComplete: false, oldestStored: 0), 0, |
| + "empty store → seed a recent page first") |
| + } |
| + |
| + func testCursorNeverRewalksACompletedThread() { |
| + XCTAssertNil(HistoryBackfill.cursor(markOldest: 1, markComplete: true, oldestStored: 900)) |
| + XCTAssertNil(HistoryBackfill.cursor(markOldest: 0, markComplete: true, oldestStored: 0), |
| + "empty-thread completion holds too") |
| + } |
| + |
| + func testMarkRoundTripAndOldestStored() async { |
| + let store = MessageStore.shared |
| + let instance = "test-\(UUID().uuidString)" |
| + |
| + var mark = await store.backfillMark(instance: instance, thread: "t1") |
| + XCTAssertEqual(mark.oldest, 0) |
| + XCTAssertFalse(mark.complete, "never-started thread reads (0, false)") |
| + |
| + await store.setBackfillMark(instance: instance, thread: "t1", oldest: 156, complete: false) |
| + mark = await store.backfillMark(instance: instance, thread: "t1") |
| + XCTAssertEqual(mark.oldest, 156) |
| + XCTAssertFalse(mark.complete) |
| + |
| + await store.setBackfillMark(instance: instance, thread: "t1", oldest: 6, complete: true) |
| + mark = await store.backfillMark(instance: instance, thread: "t1") |
| + XCTAssertEqual(mark.oldest, 6) |
| + XCTAssertTrue(mark.complete, "upsert replaces, never duplicates") |
| + |
| + var oldest = await store.oldestStoredId(instance: instance, thread: "t1") |
| + XCTAssertEqual(oldest, 0, "no rows stored yet") |
| + var p = MessagePayload() |
| + p.text = "hello" |
| + await store.upsert(instance: instance, thread: "t1", [ |
| + .init(id: 40, at: 100, edited: false, reactions: [], payload: p), |
| + .init(id: 12, at: 90, edited: false, reactions: [], payload: p), |
| + ]) |
| + oldest = await store.oldestStoredId(instance: instance, thread: "t1") |
| + XCTAssertEqual(oldest, 12) |
| + } |
| + |
| + func testProgressLabelNamesTheMonth() { |
| + // 2025-03-15 (UTC) — the label localises but always carries the year. |
| + let label = HistoryBackfill.progressLabel(1742000000) |
| + XCTAssertTrue(label.hasPrefix("Searching back to"), label) |
| + XCTAssertTrue(label.contains("2025"), label) |
| + XCTAssertEqual(HistoryBackfill.progressLabel(0), "Searching…") |
| + } |
| +} |
diff --git a/ios/LChatMac/NativeChatView.swift b/ios/LChatMac/NativeChatView.swift| index 55d58cd..9ae64b5 100644 |
| --- a/ios/LChatMac/NativeChatView.swift |
| +++ b/ios/LChatMac/NativeChatView.swift |
| @@ -913,6 +913,9 @@ struct NativeChatView: View { |
| // ---- search typeahead (Messages.app-style dropdown) ---------------------- |
| @State private var searchHits: [(thread: String, message: MessageStore.StoredMessage)] = [] |
| + // The #584 deep walk (shared LChatCore runner — same one the iOS list |
| + // uses): fetch-and-file full history so search reaches the beginning. |
| + @StateObject private var backfill = HistoryBackfill() |
| @State private var searchSel = 0 |
| private var searching: Bool { !search.trimmingCharacters(in: .whitespaces).isEmpty } |
| @@ -994,6 +997,37 @@ struct NativeChatView: View { |
| .foregroundStyle(.secondary) |
| .padding(12) |
| } |
| + // "Search earlier messages" (#584): offered until every chat's |
| + // history is local; while the walk runs, the #228 time machine |
| + // narrates and Stop cancels (the watermark persists — a later |
| + // run resumes where this one stopped). |
| + if !backfill.completed.contains(instance.id) { |
| + Divider().padding(.vertical, 4) |
| + if backfill.running { |
| + HStack(spacing: 8) { |
| + Text(backfill.progress ?? "Searching…") |
| + .font(.system(size: 12)) |
| + .foregroundStyle(.secondary) |
| + Spacer(minLength: 0) |
| + Button("Stop") { backfill.cancel() } |
| + .buttonStyle(.plain) |
| + .font(.system(size: 12, weight: .semibold)) |
| + .foregroundStyle(theme.accent.resolved(scheme)) |
| + } |
| + .padding(.horizontal, 12) |
| + .padding(.vertical, 4) |
| + .accessibilityElement(children: .combine) |
| + } else { |
| + Button("Search earlier messages…") { |
| + backfill.start(instance: instance) |
| + } |
| + .buttonStyle(.plain) |
| + .font(.system(size: 12, weight: .semibold)) |
| + .foregroundStyle(theme.accent.resolved(scheme)) |
| + .padding(.horizontal, 12) |
| + .padding(.vertical, 4) |
| + } |
| + } |
| } |
| .padding(.bottom, 6) |
| .frame(maxWidth: .infinity, alignment: .leading) |
| @@ -1003,7 +1037,9 @@ struct NativeChatView: View { |
| .padding(.horizontal, 8) |
| // Debounced FTS against the local store (decrypted at ingest — the |
| // search your SQLite proposal was for). |
| - .task(id: search) { |
| + // Keyed on the query AND the walk's page counter, so a running deep |
| + // search streams fresh hits into the open results. |
| + .task(id: "\(search)|\(backfill.pageTick)") { |
| searchSel = 0 |
| let q = search.trimmingCharacters(in: .whitespaces) |
| guard !q.isEmpty else { searchHits = []; return } |