From 6325d463ade017b55d91f36762f214e799a8bb6c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 15 May 2026 06:12:24 +0000 Subject: [PATCH 1/4] Stabilize cloud sync conflict handling without reloads Agent-Logs-Url: https://github.com/openpatch/hyperbook/sessions/45e590fb-fc98-492a-9032-44da86f76608 --- packages/markdown/assets/cloud.js | 32 +++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/packages/markdown/assets/cloud.js b/packages/markdown/assets/cloud.js index 9838c9fb..ccdde7e8 100644 --- a/packages/markdown/assets/cloud.js +++ b/packages/markdown/assets/cloud.js @@ -159,13 +159,22 @@ hyperbook.cloud = (function () { } if (result.conflict) { - // 409 — stale state, re-fetch - console.log("⚠ Stale state detected, re-fetching from cloud..."); - await loadFromCloud(); + // 409 — stale state, force overwrite with current local snapshot. + // This avoids repeated reload loops and makes sync deterministic. + console.log("⚠ Stale state detected, overwriting cloud state with local snapshot..."); + const snapshotResult = await this.sendSnapshot(); // FIX: only discard the events we tried to send, not any that // arrived concurrently during the async round-trip this.pendingEvents = this.pendingEvents.slice(eventsToSend.length); - window.location.reload(); + this.lastEventId = snapshotResult.lastEventId; + localStorage.setItem( + LAST_EVENT_ID_KEY, + String(this.lastEventId), + ); + this.lastSaveTime = Date.now(); + this.retryCount = 0; + this.updateUI("saved"); + console.log("✓ Conflict resolved by snapshot overwrite"); return; } @@ -308,11 +317,18 @@ hyperbook.cloud = (function () { const result = await this.sendEvents(queued.events, { afterEventId: queued.afterEventId }); if (result.conflict) { - // Conflict — discard remaining queue, re-fetch - console.log("⚠ Offline queue conflict, re-fetching..."); + // Conflict while replaying offline changes — resolve by replacing + // cloud state with the current local snapshot. + console.log("⚠ Offline queue conflict, overwriting cloud state..."); this.offlineQueue = []; - await loadFromCloud(); - window.location.reload(); + const snapshotResult = await this.sendSnapshot(); + this.lastEventId = snapshotResult.lastEventId; + localStorage.setItem( + LAST_EVENT_ID_KEY, + String(this.lastEventId), + ); + this.lastSaveTime = Date.now(); + console.log("✓ Offline conflict resolved by snapshot overwrite"); return; } From 9dd14a5f50b70d52746c825b3ee58e8b637cc722 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 15 May 2026 06:17:11 +0000 Subject: [PATCH 2/4] Harden snapshot overwrite conflict path in cloud sync Agent-Logs-Url: https://github.com/openpatch/hyperbook/sessions/45e590fb-fc98-492a-9032-44da86f76608 --- packages/markdown/assets/cloud.js | 68 +++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 22 deletions(-) diff --git a/packages/markdown/assets/cloud.js b/packages/markdown/assets/cloud.js index ccdde7e8..29a696b3 100644 --- a/packages/markdown/assets/cloud.js +++ b/packages/markdown/assets/cloud.js @@ -162,19 +162,32 @@ hyperbook.cloud = (function () { // 409 — stale state, force overwrite with current local snapshot. // This avoids repeated reload loops and makes sync deterministic. console.log("⚠ Stale state detected, overwriting cloud state with local snapshot..."); - const snapshotResult = await this.sendSnapshot(); - // FIX: only discard the events we tried to send, not any that - // arrived concurrently during the async round-trip - this.pendingEvents = this.pendingEvents.slice(eventsToSend.length); - this.lastEventId = snapshotResult.lastEventId; - localStorage.setItem( - LAST_EVENT_ID_KEY, - String(this.lastEventId), - ); - this.lastSaveTime = Date.now(); - this.retryCount = 0; - this.updateUI("saved"); - console.log("✓ Conflict resolved by snapshot overwrite"); + try { + const snapshotResult = await this.sendSnapshot(); + const snapshotLastEventId = + snapshotResult && snapshotResult.lastEventId !== undefined + ? snapshotResult.lastEventId + : 0; + // FIX: only discard the events we tried to send, not any that + // arrived concurrently during the async round-trip + this.pendingEvents = this.pendingEvents.slice(eventsToSend.length); + this.lastEventId = snapshotLastEventId; + localStorage.setItem( + LAST_EVENT_ID_KEY, + String(this.lastEventId), + ); + this.lastSaveTime = Date.now(); + this.retryCount = 0; + this.updateUI("saved"); + console.log("✓ Conflict resolved by snapshot overwrite"); + } catch (snapshotError) { + console.error( + "Conflict resolution via snapshot failed:", + snapshotError, + ); + this.updateUI("error"); + this.scheduleRetry(); + } return; } @@ -320,15 +333,26 @@ hyperbook.cloud = (function () { // Conflict while replaying offline changes — resolve by replacing // cloud state with the current local snapshot. console.log("⚠ Offline queue conflict, overwriting cloud state..."); - this.offlineQueue = []; - const snapshotResult = await this.sendSnapshot(); - this.lastEventId = snapshotResult.lastEventId; - localStorage.setItem( - LAST_EVENT_ID_KEY, - String(this.lastEventId), - ); - this.lastSaveTime = Date.now(); - console.log("✓ Offline conflict resolved by snapshot overwrite"); + try { + const snapshotResult = await this.sendSnapshot(); + const snapshotLastEventId = + snapshotResult && snapshotResult.lastEventId !== undefined + ? snapshotResult.lastEventId + : 0; + this.offlineQueue = []; + this.lastEventId = snapshotLastEventId; + localStorage.setItem( + LAST_EVENT_ID_KEY, + String(this.lastEventId), + ); + this.lastSaveTime = Date.now(); + console.log("✓ Offline conflict resolved by snapshot overwrite"); + } catch (snapshotError) { + console.error( + "Offline conflict resolution via snapshot failed:", + snapshotError, + ); + } return; } From 03b6beb2e331097fae9ccb7af07c69434955d7bb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 15 May 2026 07:07:42 +0000 Subject: [PATCH 3/4] Harden cloud sync with persisted offline queue and optimistic concurrency Agent-Logs-Url: https://github.com/openpatch/hyperbook/sessions/536d640b-a460-4471-a26a-7fd57227b560 --- packages/markdown/assets/cloud.js | 93 ++++++- packages/markdown/tests/cloudSync.test.js | 237 ++++++++++++++++++ platforms/cloud/routes/store.js | 67 +++++ .../tests/storeOptimisticConcurrency.test.js | 221 ++++++++++++++++ 4 files changed, 611 insertions(+), 7 deletions(-) create mode 100644 packages/markdown/tests/cloudSync.test.js create mode 100644 platforms/cloud/tests/storeOptimisticConcurrency.test.js diff --git a/packages/markdown/assets/cloud.js b/packages/markdown/assets/cloud.js index 29a696b3..600da11c 100644 --- a/packages/markdown/assets/cloud.js +++ b/packages/markdown/assets/cloud.js @@ -14,6 +14,8 @@ hyperbook.cloud = (function () { const AUTH_TOKEN_KEY = "hyperbook_auth_token"; const AUTH_USER_KEY = "hyperbook_auth_user"; const LAST_EVENT_ID_KEY = "hyperbook_last_event_id"; + const OFFLINE_QUEUE_KEY = "hyperbook_offline_queue"; + const STATE_CHECKSUM_KEY = "hyperbook_state_checksum"; const EVENT_BATCH_MAX_SIZE = 512 * 1024; // 512KB const OFFLINE_QUEUE_MAX_SIZE = 100; // FIX: cap offline queue to avoid unbounded memory growth let isLoadingFromCloud = false; @@ -67,17 +69,39 @@ hyperbook.cloud = (function () { localStorage.getItem(LAST_EVENT_ID_KEY) || "0", 10, ); - - this.offlineQueue = []; + this.lastKnownChecksum = localStorage.getItem(STATE_CHECKSUM_KEY); + this.offlineQueue = this.loadOfflineQueue(); this.isOnline = navigator.onLine; this.setupEventListeners(); + if (this.isOnline && this.offlineQueue.length > 0) { + this.processOfflineQueue(); + } } get isDirty() { return this.pendingEvents.length > 0; } + loadOfflineQueue() { + try { + const raw = localStorage.getItem(OFFLINE_QUEUE_KEY); + if (!raw) return []; + const parsed = JSON.parse(raw); + return Array.isArray(parsed) ? parsed : []; + } catch (error) { + console.warn("Failed to parse offline queue from localStorage:", error); + return []; + } + } + + persistOfflineQueue() { + localStorage.setItem( + OFFLINE_QUEUE_KEY, + JSON.stringify(this.offlineQueue), + ); + } + addEvent(event) { if (isLoadingFromCloud || isReadOnlyMode()) return; this.pendingEvents.push(event); @@ -142,6 +166,7 @@ hyperbook.cloud = (function () { timestamp: Date.now(), }); } + this.persistOfflineQueue(); // FIX: only clear the events we snapshotted this.pendingEvents = this.pendingEvents.slice(eventsToSend.length); this.updateUI("offline-queued"); @@ -163,7 +188,7 @@ hyperbook.cloud = (function () { // This avoids repeated reload loops and makes sync deterministic. console.log("⚠ Stale state detected, overwriting cloud state with local snapshot..."); try { - const snapshotResult = await this.sendSnapshot(); + const snapshotResult = await this.sendSnapshot({ force: true }); const snapshotLastEventId = snapshotResult && snapshotResult.lastEventId !== undefined ? snapshotResult.lastEventId @@ -172,10 +197,14 @@ hyperbook.cloud = (function () { // arrived concurrently during the async round-trip this.pendingEvents = this.pendingEvents.slice(eventsToSend.length); this.lastEventId = snapshotLastEventId; + this.lastKnownChecksum = snapshotResult?.stateChecksum || null; localStorage.setItem( LAST_EVENT_ID_KEY, String(this.lastEventId), ); + if (this.lastKnownChecksum) { + localStorage.setItem(STATE_CHECKSUM_KEY, this.lastKnownChecksum); + } this.lastSaveTime = Date.now(); this.retryCount = 0; this.updateUI("saved"); @@ -195,10 +224,14 @@ hyperbook.cloud = (function () { // events that were added while the network request was in flight this.pendingEvents = this.pendingEvents.slice(eventsToSend.length); this.lastEventId = result.lastEventId; + this.lastKnownChecksum = result.stateChecksum || this.lastKnownChecksum; localStorage.setItem( LAST_EVENT_ID_KEY, String(this.lastEventId), ); + if (this.lastKnownChecksum) { + localStorage.setItem(STATE_CHECKSUM_KEY, this.lastKnownChecksum); + } this.lastSaveTime = Date.now(); this.retryCount = 0; this.updateUI("saved"); @@ -228,10 +261,15 @@ hyperbook.cloud = (function () { body: JSON.stringify({ events: events, afterEventId: afterEventId, + ifMatchChecksum: this.lastKnownChecksum || undefined, }), }, ); - return { lastEventId: data.lastEventId, conflict: false }; + return { + lastEventId: data.lastEventId, + stateChecksum: data.stateChecksum, + conflict: false, + }; } catch (error) { if (error.status === 409) { return { conflict: true }; @@ -240,7 +278,8 @@ hyperbook.cloud = (function () { } } - async sendSnapshot() { + async sendSnapshot(options = {}) { + const force = options.force === true; const storeExport = await hyperbook.store.db.export({ prettyJson: false }); const exportData = JSON.parse(await storeExport.text()); @@ -254,10 +293,17 @@ hyperbook.cloud = (function () { origin: window.location.origin, data: { hyperbook: exportData }, }, + ifMatchLastEventId: force ? undefined : this.lastEventId, + ifMatchChecksum: force ? undefined : this.lastKnownChecksum || undefined, + forceOverwrite: force, }), }, ); - return { lastEventId: data.lastEventId, conflict: false }; + return { + lastEventId: data.lastEventId, + stateChecksum: data.stateChecksum, + conflict: false, + }; } scheduleRetry() { @@ -310,10 +356,15 @@ hyperbook.cloud = (function () { // snapshot instead of trying to replay individual events if (this.offlineQueue.length === 1 && this.offlineQueue[0].snapshot) { this.offlineQueue = []; + this.persistOfflineQueue(); try { const result = await this.sendSnapshot(); this.lastEventId = result.lastEventId; + this.lastKnownChecksum = result.stateChecksum || this.lastKnownChecksum; localStorage.setItem(LAST_EVENT_ID_KEY, String(this.lastEventId)); + if (this.lastKnownChecksum) { + localStorage.setItem(STATE_CHECKSUM_KEY, this.lastKnownChecksum); + } this.lastSaveTime = Date.now(); console.log("✓ Offline snapshot flushed"); } catch (error) { @@ -334,17 +385,22 @@ hyperbook.cloud = (function () { // cloud state with the current local snapshot. console.log("⚠ Offline queue conflict, overwriting cloud state..."); try { - const snapshotResult = await this.sendSnapshot(); + const snapshotResult = await this.sendSnapshot({ force: true }); const snapshotLastEventId = snapshotResult && snapshotResult.lastEventId !== undefined ? snapshotResult.lastEventId : 0; this.offlineQueue = []; + this.persistOfflineQueue(); this.lastEventId = snapshotLastEventId; + this.lastKnownChecksum = snapshotResult?.stateChecksum || null; localStorage.setItem( LAST_EVENT_ID_KEY, String(this.lastEventId), ); + if (this.lastKnownChecksum) { + localStorage.setItem(STATE_CHECKSUM_KEY, this.lastKnownChecksum); + } this.lastSaveTime = Date.now(); console.log("✓ Offline conflict resolved by snapshot overwrite"); } catch (snapshotError) { @@ -357,19 +413,25 @@ hyperbook.cloud = (function () { } this.lastEventId = result.lastEventId; + this.lastKnownChecksum = result.stateChecksum || this.lastKnownChecksum; localStorage.setItem( LAST_EVENT_ID_KEY, String(this.lastEventId), ); + if (this.lastKnownChecksum) { + localStorage.setItem(STATE_CHECKSUM_KEY, this.lastKnownChecksum); + } } catch (error) { console.error("Failed to process offline queue:", error); // Keep remaining items in queue this.offlineQueue = this.offlineQueue.slice(i); + this.persistOfflineQueue(); return; } } this.offlineQueue = []; + this.persistOfflineQueue(); this.lastSaveTime = Date.now(); console.log("✓ Offline queue processed"); } @@ -405,10 +467,14 @@ hyperbook.cloud = (function () { this.updateUI("saving"); const result = await this.sendSnapshot(); this.lastEventId = result.lastEventId; + this.lastKnownChecksum = result.stateChecksum || this.lastKnownChecksum; localStorage.setItem( LAST_EVENT_ID_KEY, String(this.lastEventId), ); + if (this.lastKnownChecksum) { + localStorage.setItem(STATE_CHECKSUM_KEY, this.lastKnownChecksum); + } this.updateUI("saved"); } catch (error) { console.error("Manual save failed:", error); @@ -424,6 +490,7 @@ hyperbook.cloud = (function () { this.pendingEvents = []; this.clearTimers(); this.offlineQueue = []; + this.persistOfflineQueue(); this.retryCount = 0; } } @@ -487,6 +554,8 @@ hyperbook.cloud = (function () { localStorage.removeItem(AUTH_TOKEN_KEY); localStorage.removeItem(AUTH_USER_KEY); localStorage.removeItem(LAST_EVENT_ID_KEY); + localStorage.removeItem(OFFLINE_QUEUE_KEY); + localStorage.removeItem(STATE_CHECKSUM_KEY); } /** @@ -636,6 +705,12 @@ hyperbook.cloud = (function () { syncManager.lastEventId = data.lastEventId; } } + if (data.stateChecksum) { + localStorage.setItem(STATE_CHECKSUM_KEY, data.stateChecksum); + if (syncManager) { + syncManager.lastKnownChecksum = data.stateChecksum; + } + } console.log("✓ Store loaded from cloud"); } @@ -886,7 +961,11 @@ hyperbook.cloud = (function () { syncManager.clearTimers(); const result = await syncManager.sendSnapshot(); syncManager.lastEventId = result.lastEventId; + syncManager.lastKnownChecksum = result.stateChecksum || syncManager.lastKnownChecksum; localStorage.setItem(LAST_EVENT_ID_KEY, String(result.lastEventId)); + if (syncManager.lastKnownChecksum) { + localStorage.setItem(STATE_CHECKSUM_KEY, syncManager.lastKnownChecksum); + } }, userToggle, login, diff --git a/packages/markdown/tests/cloudSync.test.js b/packages/markdown/tests/cloudSync.test.js new file mode 100644 index 00000000..df984aaf --- /dev/null +++ b/packages/markdown/tests/cloudSync.test.js @@ -0,0 +1,237 @@ +import { describe, it, expect } from "vitest"; +import fs from "node:fs"; +import vm from "node:vm"; + +const CLOUD_JS_PATH = "/home/runner/work/hyperbook/hyperbook/packages/markdown/assets/cloud.js"; +const AUTH_TOKEN_KEY = "hyperbook_auth_token"; +const OFFLINE_QUEUE_KEY = "hyperbook_offline_queue"; + +function createStorage(seed = {}) { + const data = { ...seed }; + return { + getItem(key) { + return Object.prototype.hasOwnProperty.call(data, key) ? data[key] : null; + }, + setItem(key, value) { + data[key] = String(value); + }, + removeItem(key) { + delete data[key]; + }, + _dump() { + return { ...data }; + }, + }; +} + +function createToken(payload) { + const base64Payload = Buffer.from(JSON.stringify(payload), "utf8").toString("base64"); + return `header.${base64Payload}.sig`; +} + +function createSandbox({ online, storageSeed, fetchImpl }) { + const localStorage = createStorage(storageSeed); + const windowListeners = {}; + const documentListeners = {}; + const tableHooks = {}; + const fetchCalls = []; + let reloadCount = 0; + + const table = { + name: "collapsibles", + hook(name, fn) { + tableHooks[name] = fn; + }, + }; + + const context = { + console, + Blob, + setTimeout, + clearTimeout, + requestAnimationFrame: (cb) => cb(), + navigator: { onLine: online }, + localStorage, + history: { replaceState: () => {} }, + location: { + origin: "http://localhost", + pathname: "/book", + search: "", + hash: "", + reload: () => { + reloadCount += 1; + }, + }, + close: () => {}, + addEventListener(name, fn) { + windowListeners[name] = windowListeners[name] || []; + windowListeners[name].push(fn); + }, + dispatchWindowEvent(name, event = {}) { + const fns = windowListeners[name] || []; + for (const fn of fns) { + fn(event); + } + }, + document: { + hidden: false, + addEventListener(name, fn) { + documentListeners[name] = documentListeners[name] || []; + documentListeners[name].push(fn); + }, + querySelector: () => null, + querySelectorAll: () => [], + getElementById: () => null, + createElement: () => ({ + setAttribute: () => {}, + removeAttribute: () => {}, + addEventListener: () => {}, + toggleAttribute: () => {}, + closest: () => null, + classList: { add: () => {}, remove: () => {} }, + style: {}, + }), + body: { prepend: () => {} }, + }, + fetch: async (url, options = {}) => { + fetchCalls.push({ url, options }); + return fetchImpl(url, options); + }, + atob: (value) => Buffer.from(value, "base64").toString("utf8"), + btoa: (value) => Buffer.from(value, "utf8").toString("base64"), + HYPERBOOK_CLOUD: { id: "test", url: "http://cloud.local" }, + hyperbook: { + i18n: { get: (_k, _m, fallback) => fallback || "" }, + store: { + db: { + tables: [table], + export: async () => + new Blob([ + JSON.stringify({ + formatName: "dexie", + formatVersion: 1, + data: { tables: [] }, + }), + ], { type: "application/json" }), + import: async () => {}, + }, + }, + }, + }; + + context.window = context; + + localStorage.setItem( + AUTH_TOKEN_KEY, + createToken({ id: 1, username: "student1", readonly: false }), + ); + + const script = fs.readFileSync(CLOUD_JS_PATH, "utf8"); + vm.runInNewContext(script, context); + + return { + context, + localStorage, + fetchCalls, + getReloadCount: () => reloadCount, + async waitForSyncHooks() { + for (let i = 0; i < 50; i++) { + if (typeof tableHooks.creating === "function") { + await new Promise((resolve) => setTimeout(resolve, 5)); + return; + } + await new Promise((resolve) => setTimeout(resolve, 1)); + } + throw new Error("Sync hooks were not registered"); + }, + triggerCreateEvent(data = { id: "s1", collapsed: false }) { + tableHooks.creating("s1", data); + }, + }; +} + +function jsonResponse(status, payload) { + return { + status, + ok: status >= 200 && status < 300, + async json() { + return payload; + }, + }; +} + +describe("cloud sync conflict/offline flows", () => { + it("resolves event conflict via snapshot without reload", async () => { + const sandbox = createSandbox({ + online: true, + storageSeed: {}, + fetchImpl: async (url) => { + if (url.endsWith("/api/store/test")) { + return jsonResponse(404, { error: "No store data found" }); + } + if (url.endsWith("/api/store/test/events")) { + return jsonResponse(409, { error: "Stale state" }); + } + if (url.endsWith("/api/store/test/snapshot")) { + return jsonResponse(200, { lastEventId: 0, stateChecksum: "checksum-1" }); + } + return jsonResponse(500, { error: "Unexpected request" }); + }, + }); + + await sandbox.waitForSyncHooks(); + sandbox.triggerCreateEvent(); + await sandbox.context.hyperbook.cloud.save(); + + const eventCalls = sandbox.fetchCalls.filter((c) => c.url.endsWith("/events")); + const snapshotCalls = sandbox.fetchCalls.filter((c) => c.url.endsWith("/snapshot")); + + expect(eventCalls.length).toBe(1); + expect(snapshotCalls.length).toBe(1); + expect(sandbox.getReloadCount()).toBe(0); + }); + + it("persists offline queue and flushes on reconnect without reload", async () => { + const firstLoad = createSandbox({ + online: false, + storageSeed: {}, + fetchImpl: async (url) => { + if (url.endsWith("/api/store/test")) { + return jsonResponse(404, { error: "No store data found" }); + } + return jsonResponse(500, { error: "Unexpected request while offline" }); + }, + }); + + await firstLoad.waitForSyncHooks(); + firstLoad.triggerCreateEvent(); + await firstLoad.context.hyperbook.cloud.save(); + + const persisted = firstLoad.localStorage.getItem(OFFLINE_QUEUE_KEY); + expect(persisted).toBeTruthy(); + expect(JSON.parse(persisted).length).toBeGreaterThan(0); + + const secondLoad = createSandbox({ + online: true, + storageSeed: firstLoad.localStorage._dump(), + fetchImpl: async (url) => { + if (url.endsWith("/api/store/test")) { + return jsonResponse(404, { error: "No store data found" }); + } + if (url.endsWith("/api/store/test/events")) { + return jsonResponse(200, { lastEventId: 1, stateChecksum: "checksum-2" }); + } + return jsonResponse(500, { error: "Unexpected request" }); + }, + }); + + await secondLoad.waitForSyncHooks(); + secondLoad.context.dispatchWindowEvent("online"); + await new Promise((resolve) => setTimeout(resolve, 0)); + + const eventCalls = secondLoad.fetchCalls.filter((c) => c.url.endsWith("/events")); + expect(eventCalls.length).toBeGreaterThan(0); + expect(secondLoad.getReloadCount()).toBe(0); + expect(secondLoad.localStorage.getItem(OFFLINE_QUEUE_KEY)).toBe("[]"); + }); +}); diff --git a/platforms/cloud/routes/store.js b/platforms/cloud/routes/store.js index 64c6f294..8a6d21b0 100644 --- a/platforms/cloud/routes/store.js +++ b/platforms/cloud/routes/store.js @@ -1,9 +1,17 @@ var express = require("express"); +var crypto = require("crypto"); var db = require("../lib/db"); var middleware = require("../middleware/auth"); var router = express.Router(); +function computeChecksum(data) { + return crypto + .createHash("sha256") + .update(JSON.stringify(data || null)) + .digest("hex"); +} + // GET /api/store/:hyperbookId — fetch current state (snapshot + event replay) router.get( "/:hyperbookId", @@ -33,6 +41,7 @@ router.get( res.json({ snapshot: state.data, lastEventId: state.lastEventId, + stateChecksum: computeChecksum(state.data), updatedAt: state.updatedAt, }); } catch (error) { @@ -57,6 +66,7 @@ router.post( var userId = req.user.id; var events = req.body.events; var afterEventId = req.body.afterEventId; + var ifMatchChecksum = req.body.ifMatchChecksum; if (!Array.isArray(events) || events.length === 0) { res.status(400).json({ error: "Events array required" }); @@ -89,11 +99,26 @@ router.post( return; } + if (ifMatchChecksum) { + var currentState = await db.reconstructState(userId, hyperbook.id); + var serverChecksum = computeChecksum(currentState ? currentState.data : null); + if (ifMatchChecksum !== serverChecksum) { + res.status(409).json({ + error: "Stale state checksum mismatch", + serverLastEventId: serverLatest, + serverChecksum: serverChecksum, + }); + return; + } + } + var lastEventId = await db.appendEvents(userId, hyperbook.id, events); + var updatedState = await db.reconstructState(userId, hyperbook.id); res.json({ success: true, lastEventId: lastEventId, + stateChecksum: computeChecksum(updatedState ? updatedState.data : null), }); } catch (error) { console.error("Append events error:", error); @@ -116,6 +141,13 @@ router.post( var hyperbookId = req.params.hyperbookId; var userId = req.user.id; var data = req.body.data; + var ifMatchLastEventId = req.body.ifMatchLastEventId; + var ifMatchChecksum = req.body.ifMatchChecksum; + var forceOverwrite = req.body.forceOverwrite === true; + + if (ifMatchLastEventId !== null && ifMatchLastEventId !== undefined) { + ifMatchLastEventId = Number(ifMatchLastEventId); + } if (!data) { res.status(400).json({ error: "Snapshot data required" }); @@ -132,6 +164,40 @@ router.post( return; } + var currentLatest = await db.getLatestEventId(userId, hyperbook.id); + var latestSnapshot = await db.getLatestSnapshot(userId, hyperbook.id); + var serverLatest = Math.max( + currentLatest, + latestSnapshot ? latestSnapshot.last_event_id : 0 + ); + + if (!forceOverwrite) { + if ( + ifMatchLastEventId !== null && + ifMatchLastEventId !== undefined && + ifMatchLastEventId !== serverLatest + ) { + res.status(409).json({ + error: "Stale state — re-fetch required", + serverLastEventId: serverLatest, + }); + return; + } + + if (ifMatchChecksum) { + var currentState = await db.reconstructState(userId, hyperbook.id); + var serverChecksum = computeChecksum(currentState ? currentState.data : null); + if (ifMatchChecksum !== serverChecksum) { + res.status(409).json({ + error: "Stale state checksum mismatch", + serverLastEventId: serverLatest, + serverChecksum: serverChecksum, + }); + return; + } + } + } + var snapshotId = await db.replaceWithSnapshot( userId, hyperbook.id, @@ -142,6 +208,7 @@ router.post( success: true, snapshotId: snapshotId, lastEventId: 0, + stateChecksum: computeChecksum(data), }); } catch (error) { console.error("Save snapshot error:", error); diff --git a/platforms/cloud/tests/storeOptimisticConcurrency.test.js b/platforms/cloud/tests/storeOptimisticConcurrency.test.js new file mode 100644 index 00000000..9d2ecfb2 --- /dev/null +++ b/platforms/cloud/tests/storeOptimisticConcurrency.test.js @@ -0,0 +1,221 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +function resetCloudModules() { + ["../app", "../lib/db", "../lib/auth", "../middleware/auth", "../routes/store"].forEach( + (mod) => { + try { + delete require.cache[require.resolve(mod)]; + } catch (_e) { + // ignore + } + }, + ); +} + +describe("store API optimistic concurrency", () => { + let dbFile; + let db; + let auth; + let app; + let server; + let baseUrl; + let token; + + async function api(pathname, options = {}) { + const response = await fetch(`${baseUrl}${pathname}`, { + ...options, + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${token}`, + ...(options.headers || {}), + }, + }); + + const body = await response.json(); + return { status: response.status, body }; + } + + beforeEach(async () => { + dbFile = path.join( + os.tmpdir(), + `hyperbook-cloud-${Date.now()}-${Math.random().toString(16).slice(2)}.sqlite`, + ); + + process.env.DATABASE_PATH = dbFile; + process.env.JWT_SECRET = "test-secret"; + process.env.BASE_URL = "http://localhost:3000"; + + resetCloudModules(); + + db = require("../lib/db"); + await db.initializeDatabase(); + + await db.runAsync("INSERT INTO hyperbooks (slug, name) VALUES (?, ?)", [ + "test", + "Test Hyperbook", + ]); + await db.runAsync( + "INSERT INTO users (id, username, password, role) VALUES (?, ?, ?, ?)", + [1, "student1", "pass", "student"], + ); + + auth = require("../lib/auth"); + token = auth.generateToken({ id: 1, username: "student1", role: "student" }); + + app = require("../app"); + server = app.listen(0); + await new Promise((resolve) => server.once("listening", resolve)); + + const address = server.address(); + baseUrl = `http://127.0.0.1:${address.port}`; + }); + + afterEach(async () => { + if (server) { + await new Promise((resolve) => server.close(resolve)); + } + resetCloudModules(); + + try { + fs.unlinkSync(dbFile); + } catch (_e) { + // ignore cleanup errors + } + }); + + it("returns checksum on GET and snapshot responses", async () => { + const snapshot = await api("/api/store/test/snapshot", { + method: "POST", + body: JSON.stringify({ + data: { + version: 1, + data: { hyperbook: { formatName: "dexie", formatVersion: 1, data: { tables: [] } } }, + }, + }), + }); + + expect(snapshot.status).toBe(200); + expect(snapshot.body.stateChecksum).toBeTruthy(); + + const loaded = await api("/api/store/test"); + expect(loaded.status).toBe(200); + expect(loaded.body.stateChecksum).toBe(snapshot.body.stateChecksum); + }); + + it("rejects stale snapshot if ifMatchLastEventId mismatches", async () => { + const initialSnapshot = await api("/api/store/test/snapshot", { + method: "POST", + body: JSON.stringify({ + data: { + version: 1, + data: { hyperbook: { formatName: "dexie", formatVersion: 1, data: { tables: [] } } }, + }, + }), + }); + expect(initialSnapshot.status).toBe(200); + + const events = await api("/api/store/test/events", { + method: "POST", + body: JSON.stringify({ + afterEventId: 0, + events: [ + { + table: "collapsibles", + op: "create", + primKey: "s1", + data: { id: "s1", collapsed: false }, + }, + ], + }), + }); + expect(events.status).toBe(200); + + const staleSnapshot = await api("/api/store/test/snapshot", { + method: "POST", + body: JSON.stringify({ + ifMatchLastEventId: 0, + data: { + version: 1, + data: { hyperbook: { formatName: "dexie", formatVersion: 1, data: { tables: [] } } }, + }, + }), + }); + + expect(staleSnapshot.status).toBe(409); + expect(staleSnapshot.body.serverLastEventId).toBe(events.body.lastEventId); + }); + + it("rejects stale snapshot if checksum mismatches unless forceOverwrite=true", async () => { + const initialSnapshot = await api("/api/store/test/snapshot", { + method: "POST", + body: JSON.stringify({ + data: { + version: 1, + data: { + hyperbook: { + formatName: "dexie", + formatVersion: 1, + data: { + tables: [ + { + name: "tabs", + schema: "id", + rowCount: 1, + rows: [{ id: "t1", active: "a" }], + }, + ], + }, + }, + }, + }, + }), + }); + expect(initialSnapshot.status).toBe(200); + + const stale = await api("/api/store/test/snapshot", { + method: "POST", + body: JSON.stringify({ + ifMatchLastEventId: 0, + ifMatchChecksum: "deadbeef", + data: { + version: 1, + data: { + hyperbook: { + formatName: "dexie", + formatVersion: 1, + data: { tables: [] }, + }, + }, + }, + }), + }); + + expect(stale.status).toBe(409); + expect(stale.body.serverChecksum).toBeTruthy(); + + const forced = await api("/api/store/test/snapshot", { + method: "POST", + body: JSON.stringify({ + ifMatchLastEventId: 999, + ifMatchChecksum: "deadbeef", + forceOverwrite: true, + data: { + version: 1, + data: { + hyperbook: { + formatName: "dexie", + formatVersion: 1, + data: { tables: [] }, + }, + }, + }, + }), + }); + + expect(forced.status).toBe(200); + expect(forced.body.stateChecksum).toBeTruthy(); + }); +}); From cf48be2f8d0cc649babb103d72cc228971b784c1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 15 May 2026 07:11:34 +0000 Subject: [PATCH 4/4] Address review feedback on cloud sync tests and route helpers Agent-Logs-Url: https://github.com/openpatch/hyperbook/sessions/536d640b-a460-4471-a26a-7fd57227b560 --- packages/markdown/tests/cloudSync.test.js | 3 +- platforms/cloud/routes/store.js | 37 +++++++++++------------ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/packages/markdown/tests/cloudSync.test.js b/packages/markdown/tests/cloudSync.test.js index df984aaf..32dce8bc 100644 --- a/packages/markdown/tests/cloudSync.test.js +++ b/packages/markdown/tests/cloudSync.test.js @@ -1,8 +1,9 @@ import { describe, it, expect } from "vitest"; import fs from "node:fs"; import vm from "node:vm"; +import { fileURLToPath } from "node:url"; -const CLOUD_JS_PATH = "/home/runner/work/hyperbook/hyperbook/packages/markdown/assets/cloud.js"; +const CLOUD_JS_PATH = fileURLToPath(new URL("../assets/cloud.js", import.meta.url)); const AUTH_TOKEN_KEY = "hyperbook_auth_token"; const OFFLINE_QUEUE_KEY = "hyperbook_offline_queue"; diff --git a/platforms/cloud/routes/store.js b/platforms/cloud/routes/store.js index 8a6d21b0..240020c0 100644 --- a/platforms/cloud/routes/store.js +++ b/platforms/cloud/routes/store.js @@ -12,6 +12,19 @@ function computeChecksum(data) { .digest("hex"); } +function hasProvided(value) { + return value !== null && value !== undefined; +} + +async function getServerSyncHead(userId, hyperbookId) { + var currentLatest = await db.getLatestEventId(userId, hyperbookId); + var latestSnapshot = await db.getLatestSnapshot(userId, hyperbookId); + return Math.max( + currentLatest, + latestSnapshot ? latestSnapshot.last_event_id : 0 + ); +} + // GET /api/store/:hyperbookId — fetch current state (snapshot + event replay) router.get( "/:hyperbookId", @@ -84,14 +97,9 @@ router.post( } // Validate afterEventId matches server's latest - var currentLatest = await db.getLatestEventId(userId, hyperbook.id); - var latestSnapshot = await db.getLatestSnapshot(userId, hyperbook.id); - var serverLatest = Math.max( - currentLatest, - latestSnapshot ? latestSnapshot.last_event_id : 0 - ); + var serverLatest = await getServerSyncHead(userId, hyperbook.id); - if (afterEventId !== null && afterEventId !== undefined && afterEventId !== serverLatest) { + if (hasProvided(afterEventId) && afterEventId !== serverLatest) { res.status(409).json({ error: "Stale state — re-fetch required", serverLastEventId: serverLatest, @@ -145,7 +153,7 @@ router.post( var ifMatchChecksum = req.body.ifMatchChecksum; var forceOverwrite = req.body.forceOverwrite === true; - if (ifMatchLastEventId !== null && ifMatchLastEventId !== undefined) { + if (hasProvided(ifMatchLastEventId)) { ifMatchLastEventId = Number(ifMatchLastEventId); } @@ -164,19 +172,10 @@ router.post( return; } - var currentLatest = await db.getLatestEventId(userId, hyperbook.id); - var latestSnapshot = await db.getLatestSnapshot(userId, hyperbook.id); - var serverLatest = Math.max( - currentLatest, - latestSnapshot ? latestSnapshot.last_event_id : 0 - ); + var serverLatest = await getServerSyncHead(userId, hyperbook.id); if (!forceOverwrite) { - if ( - ifMatchLastEventId !== null && - ifMatchLastEventId !== undefined && - ifMatchLastEventId !== serverLatest - ) { + if (hasProvided(ifMatchLastEventId) && ifMatchLastEventId !== serverLatest) { res.status(409).json({ error: "Stale state — re-fetch required", serverLastEventId: serverLatest,