amadan
Serve snapshot-then-tail from GET /d/{sqid}/updates, plus ?raw=1
updatesGet now fetches the newest usable snapshot and, when it exists and covers past `since`, writes the snapshot's State first and then only the doc_updates rows after its seq — the same total content as the raw log, in fewer bytes, because a snapshot is an ordinary Yjs update the browser's applyServerLog already replays idempotently. ?raw=1 skips the snapshot lookup entirely and returns exactly the doc_updates rows, unchanged from before this task — the escape hatch a browser needs when a snapshot will not decode, since the server has no way to detect that corruption itself. Docs-Seq keeps design D2's rule: max(since, the highest seq in the bytes actually written), now folding in the snapshot's own seq column when one is served. That's still derived from data already in hand, never a second query — the snapshot's bytes contain its whole prefix, so counting its seq is exactly as safe as counting a tail row's. A failed snapshot lookup falls back to the plain log (logged, not a 500), matching store.go's replay: a snapshot is an accelerator, never the source of truth.
2 files changed,
+171
−8
internal/docs/cursor_test.go+102 −0internal/docs/handlers.go+69 −8
diff --git a/internal/docs/cursor_test.go b/internal/docs/cursor_test.go| index 42a68de..4d44b1b 100644 |
| --- a/internal/docs/cursor_test.go |
| +++ b/internal/docs/cursor_test.go |
| @@ -207,6 +207,108 @@ func TestEmptySliceEchoesSinceRatherThanTheCurrentMax(t *testing.T) { |
| } |
| } |
| +// pastCadence builds a document with more updates than the cadence, so a |
| +// snapshot exists, and returns it. |
| +func pastCadence(t *testing.T, app *App, title string) *Document { |
| + t.Helper() |
| + ctx := context.Background() |
| + owner := app.TestPerson(t, "owner@example.com") |
| + doc, err := app.Store().Create(ctx, owner, title) |
| + if err != nil { |
| + t.Fatalf("Create: %v", err) |
| + } |
| + for i := 0; i < snapshotCadence+3; i++ { |
| + if _, err := app.Store().ApplyUpdate(ctx, doc, validUpdate(t)); err != nil { |
| + t.Fatalf("ApplyUpdate %d: %v", i, err) |
| + } |
| + } |
| + return doc |
| +} |
| + |
| +// A client far behind the snapshot receives the snapshot plus the tail |
| +// rather than the whole log, and the header still names the highest seq |
| +// the bytes actually reach. |
| +func TestUpdatesGetServesSnapshotThenTail(t *testing.T) { |
| + app, err := NewForTest(t) |
| + if err != nil { |
| + t.Fatalf("NewForTest: %v", err) |
| + } |
| + doc := pastCadence(t, app, "Snapshot then tail") |
| + |
| + snap := serve(app, app.TestRequest(t, "owner@example.com", http.MethodGet, |
| + "/d/"+doc.Sqid+"/updates?since=1", nil)) |
| + if snap.Code != http.StatusOK { |
| + t.Fatalf("GET = %d, want 200", snap.Code) |
| + } |
| + raw := serve(app, app.TestRequest(t, "owner@example.com", http.MethodGet, |
| + "/d/"+doc.Sqid+"/updates?since=1&raw=1", nil)) |
| + |
| + if snap.Body.Len() >= raw.Body.Len() { |
| + t.Errorf("snapshot response %d bytes, raw log %d — the snapshot bought nothing", |
| + snap.Body.Len(), raw.Body.Len()) |
| + } |
| + if got := docsSeq(t, snap.Result()); got != int64(snapshotCadence+3) { |
| + t.Errorf("Docs-Seq = %d, want %d", got, snapshotCadence+3) |
| + } |
| +} |
| + |
| +// raw=1 is the escape hatch a browser needs when a snapshot will not |
| +// decode: doc_updates rows only, which is where the log's authority is |
| +// actually reachable from. |
| +func TestRawParameterReturnsOnlyLogRows(t *testing.T) { |
| + app, err := NewForTest(t) |
| + if err != nil { |
| + t.Fatalf("NewForTest: %v", err) |
| + } |
| + ctx := context.Background() |
| + doc := pastCadence(t, app, "Raw") |
| + |
| + raw := serve(app, app.TestRequest(t, "owner@example.com", http.MethodGet, |
| + "/d/"+doc.Sqid+"/updates?raw=1", nil)) |
| + if raw.Code != http.StatusOK { |
| + t.Fatalf("GET raw = %d, want 200", raw.Code) |
| + } |
| + |
| + var rows []DocUpdate |
| + if err := app.Store().db.WithContext(ctx). |
| + Where("doc_id = ?", doc.ID).Order("seq ASC").Find(&rows).Error; err != nil { |
| + t.Fatalf("load log: %v", err) |
| + } |
| + var want []byte |
| + for _, r := range rows { |
| + want = append(want, r.UpdateBin...) |
| + } |
| + if !bytes.Equal(raw.Body.Bytes(), want) { |
| + t.Error("raw=1 did not return exactly the concatenated log") |
| + } |
| +} |
| + |
| +// The live-sync design's D2 survives snapshots: the header is still |
| +// derived from the bytes in hand. |
| +func TestDocsSeqStillDerivedWithSnapshots(t *testing.T) { |
| + app, err := NewForTest(t) |
| + if err != nil { |
| + t.Fatalf("NewForTest: %v", err) |
| + } |
| + doc := pastCadence(t, app, "Header") |
| + top := int64(snapshotCadence + 3) |
| + |
| + behind := serve(app, app.TestRequest(t, "owner@example.com", http.MethodGet, |
| + "/d/"+doc.Sqid+"/updates?since=2", nil)) |
| + if got := docsSeq(t, behind.Result()); got != top { |
| + t.Errorf("Docs-Seq behind the snapshot = %d, want %d", got, top) |
| + } |
| + |
| + ahead := serve(app, app.TestRequest(t, "owner@example.com", http.MethodGet, |
| + "/d/"+doc.Sqid+"/updates?since="+strconv.FormatInt(top+5, 10), nil)) |
| + if ahead.Body.Len() != 0 { |
| + t.Errorf("a cursor ahead of the log returned %d bytes, want none", ahead.Body.Len()) |
| + } |
| + if got := docsSeq(t, ahead.Result()); got != top+5 { |
| + t.Errorf("Docs-Seq ahead of the log = %d, want %d (an echo of since)", got, top+5) |
| + } |
| +} |
| + |
| // An unparseable or negative since is the client's error, and saying |
| // so beats guessing at an intent: silently treating it as 0 would |
| // resend the whole log, which is the expensive answer to a question |
diff --git a/internal/docs/handlers.go b/internal/docs/handlers.go| index dce9d6d..4a2a995 100644 |
| --- a/internal/docs/handlers.go |
| +++ b/internal/docs/handlers.go |
| @@ -423,10 +423,25 @@ func (h *handlers) getDocumentForUpdates(w http.ResponseWriter, r *http.Request) |
| return doc, true |
| } |
| -// updatesGet serves GET /d/{sqid}/updates: the document's raw update |
| -// log, concatenated in seq order — what the browser's Yjs doc |
| -// replays to build its own state (Task 6). Owner-scoped exactly like |
| -// show (getDocumentForUpdates), and capped at maxLogBytes above. |
| +// updatesGet serves GET /d/{sqid}/updates: by default, the newest |
| +// usable snapshot (if any covers past `since`) followed by the tail |
| +// of doc_updates rows after it — snapshot-then-tail, not the raw log |
| +// (Task 6). `?raw=1` is the escape hatch that returns exactly the |
| +// doc_updates rows and nothing else, for a browser whose snapshot |
| +// will not decode; the server cannot detect that corruption itself |
| +// (it never calls into the core here), so the browser is the only |
| +// reader that can, and raw=1 is what lets it fall back to the one |
| +// source it can always trust. Owner-scoped exactly like show |
| +// (getDocumentForUpdates), and capped at maxLogBytes above. |
| +// |
| +// Why snapshot-then-tail loses nothing: a snapshot IS an ordinary Yjs |
| +// update, covering every row up to and including its own seq. The |
| +// browser's applyServerLog already loops Y.readUpdate over a |
| +// concatenated body, and Yjs integration is idempotent, so a client |
| +// asking `?since=5` that receives a snapshot reaching seq 256 has not |
| +// skipped rows 6-256 — the snapshot contains them. That is why no |
| +// client change is needed for the happy path; only raw=1 (wired up by |
| +// a later task) exists for the unhappy one. |
| // |
| // The size is summed and checked BEFORE anything is written to w: |
| // writing the 200 status and a partial body first, then discovering |
| @@ -445,9 +460,38 @@ func (h *handlers) updatesGet(w http.ResponseWriter, r *http.Request) { |
| return |
| } |
| + // raw=1 skips the snapshot lookup entirely and behaves exactly as |
| + // this handler always has: doc_updates rows after `since`, nothing |
| + // else. That is deliberate, not merely "don't bother fetching a |
| + // snapshot we'd discard" — it is what makes raw=1 trustworthy as an |
| + // escape hatch: its behavior does not depend on snapshot.go at all. |
| + raw := r.URL.Query().Get("raw") == "1" |
| + |
| + // A snapshot is an accelerator, never the source of truth (the |
| + // snapshots design's D2, echoed in store.go's replay): if the |
| + // lookup itself fails, this falls back to serving the plain log |
| + // rather than 500ing a request the log alone can still answer. |
| + var snapshot *DocSnapshot |
| + if !raw { |
| + snap, err := h.store.newestSnapshot(r.Context(), doc.ID) |
| + if err != nil { |
| + h.logger.Error("docs: load newest snapshot, serving the plain log instead", "sqid", doc.Sqid, "err", err) |
| + } else if snap != nil && snap.Seq > since { |
| + // A snapshot at or behind `since` buys nothing — the |
| + // client already holds everything it covers — so it is |
| + // only used when it actually advances the tail's start. |
| + snapshot = snap |
| + } |
| + } |
| + |
| + tailSince := since |
| + if snapshot != nil { |
| + tailSince = snapshot.Seq |
| + } |
| + |
| var updates []DocUpdate |
| if err := h.store.db.WithContext(r.Context()). |
| - Where("doc_id = ? AND seq > ?", doc.ID, since). |
| + Where("doc_id = ? AND seq > ?", doc.ID, tailSince). |
| Order("seq ASC"). |
| Find(&updates).Error; err != nil { |
| h.logger.Error("docs: load update log", "sqid", doc.Sqid, "err", err) |
| @@ -455,7 +499,10 @@ func (h *handlers) updatesGet(w http.ResponseWriter, r *http.Request) { |
| return |
| } |
| - var total int |
| + total := 0 |
| + if snapshot != nil { |
| + total = len(snapshot.State) |
| + } |
| for _, u := range updates { |
| total += len(u.UpdateBin) |
| } |
| @@ -465,11 +512,18 @@ func (h *handlers) updatesGet(w http.ResponseWriter, r *http.Request) { |
| return |
| } |
| - // Docs-Seq is derived from the rows in hand — max(since, the |
| + // Docs-Seq is derived from the bytes in hand — max(since, the |
| // highest seq actually being written) — and never from a second |
| // MAX(seq) query. See parseSince's own doc comment and the design's |
| - // D2 for the update this loses if that is ever "simplified". |
| + // D2 for the update this loses if that is ever "simplified". A |
| + // snapshot's own seq column is fine to fold in here even though it |
| + // is not itself a doc_updates row: it travels with bytes (its |
| + // State) that contain its whole prefix, so counting it is exactly |
| + // as safe as counting the highest tail row's seq. |
| highest := since |
| + if snapshot != nil && snapshot.Seq > highest { |
| + highest = snapshot.Seq |
| + } |
| for _, u := range updates { |
| if u.Seq > highest { |
| highest = u.Seq |
| @@ -478,6 +532,13 @@ func (h *handlers) updatesGet(w http.ResponseWriter, r *http.Request) { |
| w.Header().Set("Content-Type", "application/octet-stream") |
| w.Header().Set("Docs-Seq", strconv.FormatInt(highest, 10)) |
| + if snapshot != nil { |
| + if _, err := w.Write(snapshot.State); err != nil { |
| + // See below: the client went away mid-response. |
| + h.logger.Error("docs: write snapshot", "sqid", doc.Sqid, "err", err) |
| + return |
| + } |
| + } |
| for _, u := range updates { |
| if _, err := w.Write(u.UpdateBin); err != nil { |
| // The client went away mid-response; nothing left to do |