From f41d952ef9f0b519e2838ad6e4fc6258ea315e00 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Tue, 16 Jun 2026 17:26:42 +0200 Subject: [PATCH 01/17] fix: initial attempt --- client/components/Editor/Editor.stories.tsx | 64 +- .../Editor/plugins/collaborative/cursors.ts | 463 ++++++----- .../Editor/plugins/collaborative/document.ts | 274 ++----- .../Editor/plugins/collaborative/index.ts | 6 +- .../Editor/plugins/discussions/fastForward.ts | 3 +- .../Editor/plugins/discussions/firebase.ts | 118 ++- .../Editor/plugins/discussions/plugin.ts | 11 +- client/components/Editor/types.ts | 6 +- .../Editor/utils/__test__/changes.test.ts | 8 +- client/components/Editor/utils/changes.ts | 4 +- client/components/Editor/utils/firebase.ts | 4 +- client/components/Editor/utils/firebaseDoc.ts | 7 +- client/components/Editor/utils/index.ts | 6 +- client/components/Editor/utils/view.ts | 24 +- .../MinimalEditor/MinimalEditor.tsx | 1 + .../DashboardCustomScripts.tsx | 2 +- client/containers/Pub/PubContextProvider.tsx | 1 - client/containers/Pub/PubDocument/PubBody.tsx | 20 +- .../Discussion/DiscussionReanchor.tsx | 4 +- client/containers/Pub/usePubBodyState.ts | 8 +- client/containers/Pub/usePubCollabState.ts | 35 +- client/utils/firebaseClient.ts | 39 +- client/webpack/webpackConfig.dev.js | 22 +- client/webpack/webpackConfig.prod.js | 8 +- infra/docker-compose.dev.yml | 12 +- infra/stack.yml | 14 +- package.json | 10 +- patches/@pubpub__deposit-utils.patch | 17 + pnpm-lock.yaml | 719 +++++++----------- server/apiRoutes.ts | 4 + server/collab/api.ts | 95 +++ server/collab/authority.ts | 130 ++++ server/collab/discussionPositions.ts | 63 ++ server/collab/featureFlag.ts | 43 ++ server/collab/presence.ts | 22 + server/collabCommit/model.ts | 53 ++ server/communityTemplate/applyTemplate.ts | 4 +- server/discussion/utils.ts | 43 +- server/draft/model.ts | 10 +- server/envSchema.ts | 12 +- server/hub/api.ts | 8 +- server/models.ts | 3 + server/pub/__tests__/api.test.ts | 2 +- server/pubHistory/queries.ts | 18 +- server/release/__tests__/api.test.ts | 1 - server/release/queries.ts | 84 +- server/sequelize.ts | 4 +- server/server.ts | 13 +- server/spamTag/commentSpam.ts | 2 +- server/submission/abstract.ts | 18 +- server/submission/queries.ts | 2 +- server/user/__tests__/api.test.ts | 10 +- server/utils/__tests__/ssr.test.tsx | 4 +- server/utils/citations/structuredCitations.ts | 2 +- server/utils/firebaseAdmin.ts | 401 ++++------ server/utils/firebaseTools.ts | 86 +-- server/utils/queryHelpers/pubEnrich.ts | 17 +- stubstub/firebase.ts | 11 +- stubstub/global/setup.ts | 6 +- stubstub/stub.ts | 6 +- tools/bootstrapCheckpoints.ts | 11 +- tools/cleanupFirebase.ts | 34 +- tools/coldStorage.ts | 10 +- tools/index.js | 1 + tools/localdb.ts | 2 +- tools/migrateFirebasePaths.ts | 7 +- tools/migrateFirebaseToPostgres.ts | 306 ++++++++ tools/migrateRedshift.ts | 2 +- ..._16_addCollabCommitTableAndDraftVersion.js | 92 +++ .../2026_06_16_removeFirebasePath.js | 26 + tsconfig.json | 4 +- utils/api/schemas/draft.ts | 2 +- utils/caching/__tests__/purge.test.ts | 4 +- workers/tasks/export/html.tsx | 2 +- workers/tasks/export/notes.ts | 2 +- workers/tasks/export/pandoc.ts | 2 +- workers/tasks/export/styles/buildCss.mts | 2 - workers/tasks/export/types.ts | 2 +- workers/tasks/import/bulk/directives/pub.ts | 13 +- workers/tasks/import/rules.ts | 2 +- 80 files changed, 1991 insertions(+), 1622 deletions(-) create mode 100644 patches/@pubpub__deposit-utils.patch create mode 100644 server/collab/api.ts create mode 100644 server/collab/authority.ts create mode 100644 server/collab/discussionPositions.ts create mode 100644 server/collab/featureFlag.ts create mode 100644 server/collab/presence.ts create mode 100644 server/collabCommit/model.ts create mode 100644 tools/migrateFirebaseToPostgres.ts create mode 100644 tools/migrations/2026_06_16_addCollabCommitTableAndDraftVersion.js create mode 100644 tools/migrations/2026_06_16_removeFirebasePath.js diff --git a/client/components/Editor/Editor.stories.tsx b/client/components/Editor/Editor.stories.tsx index 6cca88ce7d..51295c331e 100644 --- a/client/components/Editor/Editor.stories.tsx +++ b/client/components/Editor/Editor.stories.tsx @@ -3,9 +3,6 @@ import React, { useState } from 'react'; import { storiesOf } from '@storybook/react'; -import firebase from 'firebase/app'; -import 'firebase/auth'; -import 'firebase/database'; import Editor from 'components/Editor'; import { @@ -17,7 +14,6 @@ import { removeLocalHighlight, setLocalHighlight, } from 'components/Editor/utils'; -import { getFirebaseConfig } from 'utils/editor/firebaseConfig'; import initialContent from 'utils/storybook/initialDocs/plainDoc'; const editorWrapperStyle = { @@ -39,19 +35,8 @@ const clientData = { canEdit: true, }; -const initFirebase = (rootKey) => { - const firebaseAppName = `App-${rootKey}`; - /* Check if we've already initialized an Firebase App with the */ - /* same name in this local environment */ - // @ts-expect-error ts-migrate(2769) FIXME: No overload matches this call. - const existingApp = firebase.apps.reduce((prev, curr) => { - return curr.name === firebaseAppName ? curr : prev; - }, undefined); - - /* Use the existing Firebase App or initialize a new one */ - const firebaseApp = existingApp || firebase.initializeApp(getFirebaseConfig(), firebaseAppName); - const database = firebase.database(firebaseApp); - return database.ref(`${rootKey}`); +const initFirebase = (_rootKey) => { + return null; }; const cursorCommands = { @@ -62,8 +47,7 @@ const cursorCommands = { }; const rootKey = 'firebase-testing'; -// @ts-expect-error ts-migrate(2554) FIXME: Expected 1 arguments, but got 2. -const draftRef = initFirebase(rootKey, ''); +const draftRef = initFirebase(rootKey); const newDiscussionId = String(Math.floor(Math.random() * 999999)); const CursorOptionsDemoPub = () => { @@ -161,21 +145,19 @@ storiesOf('Editor', module) onChange={(evt) => { updatechangeObject(evt); }} - collaborativeOptions={{ - pubId: 'storybook-pub-id', - firebaseRef: draftRef as any, - clientData, - initialDocKey: -1, - // onClientChange: () => {}, - onStatusChange: (status) => console.info('collab status is', status), - }} - /> - - ); - }; - return ; - }) - .add('collaborative2', () => { + collaborativeOptions={{ + pubId: 'storybook-pub-id', + clientData, + initialDocKey: -1, + onStatusChange: (status) => console.info('collab status is', status), + }} + /> + + ); + }; + return ; +}) +.add('collaborative2', () => { const Thing = () => { const [changeObject, _updatechangeObject] = useState({}); return ( @@ -237,14 +219,12 @@ storiesOf('Editor', module) }, 1000); } }} - collaborativeOptions={{ - pubId: 'storybook-pub-id', - firebaseRef: draftRef as any, - clientData, - initialDocKey: -1, - // onClientChange: () => {}, - onStatusChange: (status) => console.info('collab status is', status), - }} + collaborativeOptions={{ + pubId: 'storybook-pub-id', + clientData, + initialDocKey: -1, + onStatusChange: (status) => console.info('collab status is', status), + }} /> ); diff --git a/client/components/Editor/plugins/collaborative/cursors.ts b/client/components/Editor/plugins/collaborative/cursors.ts index 8bcc6ab96f..96c1fe2950 100644 --- a/client/components/Editor/plugins/collaborative/cursors.ts +++ b/client/components/Editor/plugins/collaborative/cursors.ts @@ -1,291 +1,262 @@ -import type firebase from 'firebase'; - -import { compressSelectionJSON, uncompressSelectionJSON } from 'prosemirror-compress-pubpub'; +import { uncompressSelectionJSON } from 'prosemirror-compress-pubpub'; import { AllSelection, Plugin, PluginKey, Selection } from 'prosemirror-state'; import { Decoration, DecorationSet, type EditorView } from 'prosemirror-view'; export const cursorsPluginKey = new PluginKey('cursors'); -const createTransactionManager = (clientId: string, view: EditorView) => { - return { - transactionCallback: (metaType: string) => { - return (snapshot: firebase.database.DataSnapshot) => { - const snapshotKey = snapshot.key; - const snapshotVal = snapshot.val(); - if (snapshotVal && snapshotKey !== clientId) { - const trans = view.state.tr; - trans.setMeta(metaType, { - ...snapshotVal, - id: snapshot.key, - }); - view.dispatch(trans); - } - }; - }, - }; -}; +const generateCursorDecorations = (cursorData: any, editorState: any, localClientId: string) => { + if (cursorData.id === localClientId) { + return []; + } -export default (schema, props, collabDocPluginKey) => { - const generateCursorDecorations = (cursorData, editorState) => { - const { localClientId } = collabDocPluginKey.getState(editorState); - if (cursorData.id === localClientId) { - return []; - } + let selection: Selection; + try { + // handle both compressed (legacy) and uncompressed selection formats + const selJSON = cursorData.selection?.a !== undefined + ? uncompressSelectionJSON(cursorData.selection) + : cursorData.selection; - /* Invalid selections can happen if an item is synced before the corresponding changes from that */ - /* remote editor. This try-catch is a safegaurd against that scenario. We simply ignore the */ - /* decoration, and wait for the proper position to sync. */ - let selection; - try { - selection = Selection.fromJSON( - editorState.doc, - uncompressSelectionJSON(cursorData.selection), - ); - } catch (_err) { - return []; - } + selection = Selection.fromJSON(editorState.doc, selJSON); + } catch (_err) { + return []; + } - /* Classnames must begin with letter, so append one single uuid's may not. */ - const formattedDataId = `c-${cursorData.id}`; - const elem = document.createElement('span'); - elem.className = `collab-cursor ${formattedDataId}`; - - /* Add Vertical Bar */ - const innerChildBar = document.createElement('span'); - innerChildBar.className = 'inner-bar'; - elem.appendChild(innerChildBar); - - const style = document.createElement('style'); - elem.appendChild(style); - let innerStyle = ''; - - /* Add small circle at top of bar */ - const innerChildCircleSmall = document.createElement('span'); - innerChildCircleSmall.className = `inner-circle-small ${formattedDataId}`; - innerChildBar.appendChild(innerChildCircleSmall); - - /* Add wrapper for hover items at top of bar */ - const hoverItemsWrapper = document.createElement('span'); - hoverItemsWrapper.className = 'hover-wrapper'; - innerChildBar.appendChild(hoverItemsWrapper); - - /* Add Large Circle for hover */ - const innerChildCircleBig = document.createElement('span'); - innerChildCircleBig.className = 'inner-circle-big'; - hoverItemsWrapper.appendChild(innerChildCircleBig); - - /* If Initials exist - add to hover items wrapper */ - if (cursorData.initials) { - const innerCircleInitials = document.createElement('span'); - innerCircleInitials.className = `initials ${formattedDataId}`; - innerStyle += `.initials.${formattedDataId}::after { content: "${cursorData.initials}"; } `; - hoverItemsWrapper.appendChild(innerCircleInitials); - } - /* If Image exists - add to hover items wrapper */ - if (cursorData.image) { - const innerCircleImage = document.createElement('span'); - innerCircleImage.className = `image ${formattedDataId}`; - innerStyle += `.image.${formattedDataId}::after { background-image: url('${cursorData.image}'); } `; - hoverItemsWrapper.appendChild(innerCircleImage); - } + const formattedDataId = `c-${cursorData.id}`; + const elem = document.createElement('span'); + elem.className = `collab-cursor ${formattedDataId}`; - /* If name exists - add to hover items wrapper */ - if (cursorData.name) { - const innerCircleName = document.createElement('span'); - innerCircleName.className = `name ${formattedDataId}`; - innerStyle += `.name.${formattedDataId}::after { content: "${cursorData.name}"; } `; - if (cursorData.cursorColor) { - innerCircleName.style.backgroundColor = cursorData.cursorColor; - } - hoverItemsWrapper.appendChild(innerCircleName); - } + const innerChildBar = document.createElement('span'); + innerChildBar.className = 'inner-bar'; + elem.appendChild(innerChildBar); + + const style = document.createElement('style'); + elem.appendChild(style); + let innerStyle = ''; + + const innerChildCircleSmall = document.createElement('span'); + innerChildCircleSmall.className = `inner-circle-small ${formattedDataId}`; + innerChildBar.appendChild(innerChildCircleSmall); + + const hoverItemsWrapper = document.createElement('span'); + hoverItemsWrapper.className = 'hover-wrapper'; + innerChildBar.appendChild(hoverItemsWrapper); + + const innerChildCircleBig = document.createElement('span'); + innerChildCircleBig.className = 'inner-circle-big'; + hoverItemsWrapper.appendChild(innerChildCircleBig); - /* If cursor color provided - override defaults */ + if (cursorData.initials) { + const innerCircleInitials = document.createElement('span'); + innerCircleInitials.className = `initials ${formattedDataId}`; + innerStyle += `.initials.${formattedDataId}::after { content: "${cursorData.initials}"; } `; + hoverItemsWrapper.appendChild(innerCircleInitials); + } + + if (cursorData.image) { + const innerCircleImage = document.createElement('span'); + innerCircleImage.className = `image ${formattedDataId}`; + innerStyle += `.image.${formattedDataId}::after { background-image: url('${cursorData.image}'); } `; + hoverItemsWrapper.appendChild(innerCircleImage); + } + + if (cursorData.name) { + const innerCircleName = document.createElement('span'); + innerCircleName.className = `name ${formattedDataId}`; + innerStyle += `.name.${formattedDataId}::after { content: "${cursorData.name}"; } `; if (cursorData.cursorColor) { - innerChildBar.style.backgroundColor = cursorData.cursorColor; - innerChildCircleSmall.style.backgroundColor = cursorData.cursorColor; - innerChildCircleBig.style.backgroundColor = cursorData.cursorColor; - innerStyle += `.name.${formattedDataId}::after { background-color: ${cursorData.cursorColor} !important; } `; + innerCircleName.style.backgroundColor = cursorData.cursorColor; } - style.innerHTML = innerStyle; - // console.timeEnd('redner2'); - const selectionFrom = selection.from; - const selectionTo = selection.to; - const selectionHead = selection.head; + hoverItemsWrapper.appendChild(innerCircleName); + } + + if (cursorData.cursorColor) { + innerChildBar.style.backgroundColor = cursorData.cursorColor; + innerChildCircleSmall.style.backgroundColor = cursorData.cursorColor; + innerChildCircleBig.style.backgroundColor = cursorData.cursorColor; + innerStyle += `.name.${formattedDataId}::after { background-color: ${cursorData.cursorColor} !important; } `; + } + + style.innerHTML = innerStyle; + + const selectionFrom = selection.from; + const selectionTo = selection.to; + const selectionHead = selection.head; + + const decorations: Decoration[] = []; + + decorations.push( + Decoration.widget(selectionHead, elem, { + key: `cursor-widget-${cursorData.id}`, + }) as any, + ); - const decorations = []; + if (selectionFrom !== selectionTo) { decorations.push( - // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'Decoration<{ key: string; lastAc... Remove this comment to see the full error message - Decoration.widget(selectionHead, elem, { - key: `cursor-widget-${cursorData.id}`, - lastActive: cursorData.lastActive, - }), + Decoration.inline( + selectionFrom, + selectionTo, + { + class: `cursor-range ${formattedDataId}`, + style: `background-color: ${cursorData.backgroundColor || 'rgba(0, 25, 150, 0.2)'};`, + }, + { key: `cursor-inline-${cursorData.id}` }, + ) as any, ); + } - if (selectionFrom !== selectionTo) { - decorations.push( - // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'Decoration<{ key: string; lastAc... Remove this comment to see the full error message - Decoration.inline( - selectionFrom, - selectionTo, - { - class: `cursor-range ${formattedDataId}`, - style: `background-color: ${ - cursorData.backgroundColor || 'rgba(0, 25, 150, 0.2)' - };`, - }, - { key: `cursor-inline-${cursorData.id}`, lastActive: cursorData.lastActive }, - ), - ); - } - return decorations; - }; + return decorations; +}; + +export default (schema: any, props: any, collabDocPluginKey: PluginKey) => { + let abortController: AbortController | null = null; + let currentIndicators: Map = new Map(); return new Plugin({ key: cursorsPluginKey, state: { - init: (config, editorState) => { + init: (_config: any, editorState: any) => { return { cursorDecorations: DecorationSet.create(editorState.doc, []), }; }, - apply: (transaction, pluginState, prevEditorState, editorState) => { + apply: (transaction: any, pluginState: any, _prevEditorState: any, editorState: any) => { if (props.isReadOnly) { return pluginState; } - const { localClientId, localClientData } = collabDocPluginKey.getState(editorState); - /* Remove Stale Cursors */ - pluginState.cursorDecorations.find().forEach((decoration) => { - const expirationTime = 1000 * 60 * 5; /* 5 minutes */ - const isExpired = - decoration.spec.lastActive + expirationTime < new Date().getTime(); - if (isExpired) { - props.collaborativeOptions.firebaseRef - .child('cursors') - .child( - decoration.spec.key - .replace('cursor-inline-', '') - .replace('cursor-widget-', ''), - ) - .remove(); - } - }); - - /* Cursor Decorations to Remove */ - /* We want to remove any explicitly deleted cursor decorations */ - /* and decorations for cursors which are being newly updated */ - const cursorDecorationsToRemove = pluginState.cursorDecorations - .find() - .filter((decoration) => { - const setData = transaction.getMeta('setCursor') || {}; - const setId = setData.id; - const removeData = transaction.getMeta('removeCursor') || {}; - const removedId = removeData.id; - - const decorationId = decoration.spec.key - .replace('cursor-inline-', '') - .replace('cursor-widget-', ''); - - const isRemoved = removedId === decorationId; - const isSet = setId === decorationId; - return isRemoved || isSet; - }); - - /* Cursor Decorations to Add */ - const setCursorData = transaction.getMeta('setCursor'); - const cursorDecorationsToAdd = setCursorData - ? generateCursorDecorations(setCursorData, editorState) - : []; - - /* Remove, Map, and Add Cursors */ - const nextCursorDecorations = pluginState.cursorDecorations - .remove(cursorDecorationsToRemove) - .map(transaction.mapping, transaction.doc) - .add(editorState.doc, cursorDecorationsToAdd); - - /* Set Cursor data */ - const { selection: prevSelection = {} as Selection } = prevEditorState; - const selection = editorState.selection || {}; - const needsToInit = !((prevSelection as any).a || prevSelection.anchor); - const isPointer = transaction.getMeta('pointer'); - const isNotSelectAll = selection instanceof AllSelection === false; - const isCursorChange = - !transaction.docChanged && - (selection.anchor !== prevSelection.anchor || - selection.head !== prevSelection.head); - if (isNotSelectAll && (needsToInit || isPointer || isCursorChange)) { - const anchorEqual = prevSelection.anchor === selection.anchor; - const headEqual = prevSelection.head === selection.head; - if (!prevSelection.anchor || !anchorEqual || !headEqual) { - const newCursorData = { - ...localClientData, - selection, - }; - - /* lastActive has to be rounded to the nearest minute (or some larger value) - If it is updated every millisecond, firebase will see it as constant changes and you'll get a - loop of updates triggering millisecond updates. The lastActive is updated anytime a client - makes or receives changes. A client will be active even if they have a tab open and are 'watching'. */ - const smoothingTimeFactor = 1000 * 60; - newCursorData.lastActive = - Math.round(new Date().getTime() / smoothingTimeFactor) * - smoothingTimeFactor; - - const firebaseCursorData = { - ...newCursorData, - selection: needsToInit - ? { - a: 1, - h: 1, - t: 'text', - } - : compressSelectionJSON(selection.toJSON()), - }; - - props.collaborativeOptions.firebaseRef - .child('cursors') - .child(localClientId) - .set(firebaseCursorData); + + const indicatorsUpdate = transaction.getMeta('presenceIndicators'); + + if (indicatorsUpdate) { + const { localClientId } = collabDocPluginKey.getState(editorState); + const allDecorations: Decoration[] = []; + + for (const [_id, indicator] of currentIndicators) { + const decos = generateCursorDecorations( + indicator, + editorState, + localClientId, + ); + allDecorations.push(...decos); } + + return { + cursorDecorations: DecorationSet.create(editorState.doc, allDecorations), + }; } return { - cursorDecorations: nextCursorDecorations, + cursorDecorations: pluginState.cursorDecorations.map( + transaction.mapping, + transaction.doc, + ), }; }, }, - view: (view) => { - /* Retrieve and Listen to Cursors */ - if (!props.isReadOnly) { - const { localClientId } = collabDocPluginKey.getState(view.state); - const transactionManager = createTransactionManager(localClientId, view); - const cursorsRef = props.collaborativeOptions.firebaseRef.child('cursors'); - cursorsRef.child(localClientId).onDisconnect().remove(); - cursorsRef.on('child_added', transactionManager.transactionCallback('setCursor')); - cursorsRef.on('child_changed', transactionManager.transactionCallback('setCursor')); - cursorsRef.on( - 'child_removed', - transactionManager.transactionCallback('removeCursor'), - ); + view: (view: EditorView) => { + if (props.isReadOnly || !props.collaborativeOptions) { + return { destroy: () => {} }; } + const { pubId } = props.collaborativeOptions; + const { localClientId, localClientData } = collabDocPluginKey.getState(view.state); + + abortController = new AbortController(); + let polling = true; + + let presenceRef = Math.random().toString(36).slice(2); + + const sendPresenceUpdate = () => { + const { selection } = view.state; + + if (selection instanceof AllSelection) { + return; + } + + presenceRef = Math.random().toString(36).slice(2); + + const indicator = { + clientId: localClientId, + ref: presenceRef, + ...localClientData, + selection: selection.toJSON(), + }; + + fetch(`/api/pubs/${pubId}/presence/${localClientId}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(indicator), + signal: abortController!.signal, + }).catch(() => {}); + }; + + const pollPresence = async () => { + let refs: Record = {}; + + while (polling && !abortController!.signal.aborted) { + try { + const response = await fetch(`/api/pubs/${pubId}/presence`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ clientId: localClientId, refs }), + signal: abortController!.signal, + }); + + if (!response.ok) { + await new Promise((r) => setTimeout(r, 3000)); + continue; + } + + const indicators = await response.json(); + + if (indicators && typeof indicators === 'object') { + for (const [id, indicator] of Object.entries(indicators) as any) { + if (id !== localClientId && indicator) { + currentIndicators.set(id, { id, ...indicator }); + refs[id] = indicator.ref ?? ''; + } + } + + const { tr } = view.state; + tr.setMeta('presenceIndicators', true); + view.dispatch(tr); + } + } catch (e: any) { + if (e.name === 'AbortError') break; + await new Promise((r) => setTimeout(r, 3000)); + } + } + }; + + pollPresence(); + + let lastAnchor = -1; + let lastHead = -1; + + const checkSelectionChange = () => { + const { anchor, head } = view.state.selection; + + if (anchor !== lastAnchor || head !== lastHead) { + lastAnchor = anchor; + lastHead = head; + sendPresenceUpdate(); + } + }; + + const interval = setInterval(checkSelectionChange, 1000); + return { destroy: () => { - const { localClientId } = collabDocPluginKey.getState(view.state); - const cursorsRef = props.collaborativeOptions.firebaseRef.child('cursors'); - cursorsRef.child(localClientId).remove(); - cursorsRef.off('child_added'); - cursorsRef.off('child_changed'); - cursorsRef.off('child_removed'); + polling = false; + clearInterval(interval); + abortController?.abort(); + currentIndicators.clear(); }, }; }, props: { - decorations: (editorState) => { - const cursorDecorations = cursorsPluginKey - .getState(editorState) - .cursorDecorations.find(); - return DecorationSet.create(editorState.doc, cursorDecorations); + decorations: (editorState: any) => { + const { cursorDecorations } = cursorsPluginKey.getState(editorState); + return cursorDecorations; }, }, }); diff --git a/client/components/Editor/plugins/collaborative/document.ts b/client/components/Editor/plugins/collaborative/document.ts index 8590dd18f1..06be74a23a 100644 --- a/client/components/Editor/plugins/collaborative/document.ts +++ b/client/components/Editor/plugins/collaborative/document.ts @@ -1,39 +1,16 @@ -import type firebase from 'firebase'; import type { Schema } from 'prosemirror-model'; import type { DefinitelyHas } from 'types'; import type { PluginsOptions } from '../../types'; -import { receiveTransaction, sendableSteps } from 'prosemirror-collab'; -import { uncompressStepJSON } from 'prosemirror-compress-pubpub'; -import { Plugin, type PluginKey } from 'prosemirror-state'; -import { Step } from 'prosemirror-transform'; - import { - createFirebaseChange, - getFirebaseConnectionMonitorRef, - storeCheckpoint, -} from '../../utils'; - -/* -Rough pipeline: -Client types changes -Client sets ongoingTransaction=true and writes a transation -if that transaction succeeds - set ongoingTransaction=false -if that transaction fails because of error - throw error -if that transaction fails beause there is already a keyable with that id - process pending stored keyables - -When a remote change is made and synced to client, store that keyable -Attempt to process all stored keyables - don't process if there is an ongoing transaction - -If there is an ongoing transaction, it will eventually finish and trigger a new receiveCollabChanges - or, it will fail and that will cause processStoredKeyables to fire. -*/ + CollabClient, + LongPollListener, + collab, + receiveCommitTransaction, +} from '@pitter-patter/collab-client'; +import { Plugin, type PluginKey } from 'prosemirror-state'; const noop = () => {}; @@ -44,188 +21,84 @@ export default ( localClientId: string, ) => { const { collaborativeOptions, isReadOnly, onError = noop } = options; - const { - pubId, - firebaseRef: ref, - onStatusChange = noop, - onUpdateLatestKey = noop, - } = collaborativeOptions; - let view; - let mostRecentRemoteKey = collaborativeOptions.initialDocKey; - let ongoingTransaction = false; - let hasLoadedChangesOnce = false; - let listeningOn: null | firebase.database.Query = null; - let pendingRemoteKeyables = []; - /* sendCollabChanges is called only from the main Editor */ - /* disppatchTransaction view spec paramater. sendCollabChanges */ - /* is called on every transaction, but it quickly exits if the */ - /* transaction is not of the right type (meta), or if a firebase */ - /* transaction is already in progress. */ + const { pubId, onStatusChange = noop, onUpdateLatestKey = noop } = collaborativeOptions; - /* If the firebase transaction commit fails because the keyable key */ - /* already exists, we either 1) have the transaction in pendingRemoteKeyables */ - /* or we are about to receive a new firebase child. Both cases will result in */ - /* collab.receiveTransaction being called, which will dispatch a transaction */ - /* triggering sendCollabChanges to be called again, thus syncing our local */ - /* uncommitted steps. */ - const sendCollabChanges = (newState) => { - const sendable = sendableSteps(newState); + let view: any; + let collabClient: CollabClient | null = null; + let abortController: AbortController | null = null; - if (isReadOnly || ongoingTransaction || !sendable) { - return null; - } + const commitListener = new LongPollListener( + new URL(`/api/pubs/${pubId}/commits`, window.location.origin), + ); - ongoingTransaction = true; - return ref - .child('changes') - .child(String(mostRecentRemoteKey + 1)) - .transaction( - (existingRemoteSteps) => { - onStatusChange('saving'); - if (existingRemoteSteps) { - /* Returning undefined causes firebase transaction to abort. */ - /* https://firebase.google.com/docs/reference/js/firebase.database.Reference#transactionupdate:-function */ - return undefined; - } - return createFirebaseChange(sendable.steps, localClientId); - }, - undefined, - false, - ) - .then((transactionResult) => { - const { committed, snapshot } = transactionResult; - ongoingTransaction = false; - if (committed) { - onStatusChange('saved'); - - /* If multiple of saveEveryNSteps, update checkpoint */ - const saveEveryNSteps = 100; - if (snapshot.key && snapshot.key % saveEveryNSteps === 0) { - storeCheckpoint(pubId, newState.doc, snapshot.key); - } - } - - processStoredKeyables(); - }) - .catch((err) => { - console.error('Error in firebase transaction:', err); - onError(err); - }); - }; + const sendCollabChanges = (newState: any) => { + if (isReadOnly || !collabClient) { + return; + } - const extractSnapshot = (snapshotVal) => { - const compressedStepsJSON = snapshotVal.s; - const newSteps = compressedStepsJSON.map((compressedStepJSON) => { - return Step.fromJSON(schema, uncompressStepJSON(compressedStepJSON)); + onStatusChange('saving'); + collabClient.send(newState).catch((e) => { + console.error('Error sending collab commit:', e); + onError(e); }); - const newClientIds = new Array(newSteps.length).fill(snapshotVal.cId); - return { - steps: newSteps, - clientIds: newClientIds, - }; }; - /* Iterate over pendingRemoteKeyables if there is no ongoing */ - /* firebase transaction. If there is an ongoing firebase transaction */ - /* it will either fail, causing this function to be called again, or it */ - /* will succeed, which will cause a new keyable child to sync, triggering */ - /* receiveCollabChanges, and thus this function. */ - const processStoredKeyables = () => { - if (ongoingTransaction) { - return null; - } - pendingRemoteKeyables.forEach((snapshot) => { - try { - // @ts-expect-error ts-migrate(2339) FIXME: Property 'val' does not exist on type 'never'. - const { steps, clientIds } = extractSnapshot(snapshot.val()); - const trans = receiveTransaction(view.state, steps, clientIds); - // @ts-expect-error ts-migrate(2339) FIXME: Property 'key' does not exist on type 'never'. - mostRecentRemoteKey = Number(snapshot.key); - view.dispatch(trans); - onUpdateLatestKey(mostRecentRemoteKey); - } catch (err) { - console.error('Error in recieveCollabChanges:', err); - onError(err as Error); - } - }); - pendingRemoteKeyables = []; - const sendable = sendableSteps(view.state); - if (sendable) { - sendCollabChanges(view.state); - } - return null; - }; + const startCollab = (initialState: any) => { + const collabConfig = { + sendCommit: async (commit: any) => { + const response = await fetch(`/api/pubs/${pubId}/commits`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(commit.toJSON()), + }); - /* This is called everytime firebase has a new keyable child */ - /* We store the new keyable in pendingRemoteKeyables, and then */ - /* process all existing stored keyables. */ - const receiveCollabChanges = (snapshot) => { - // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'any' is not assignable to parame... Remove this comment to see the full error message - pendingRemoteKeyables.push(snapshot); - processStoredKeyables(); - }; + if (response.status === 409) { + throw new Error('Too much contention'); + } - const loadDocument = () => { - getFirebaseConnectionMonitorRef(ref).on('value', (snapshot) => { - const isConnected = snapshot.val(); - if (isConnected) { - if (hasLoadedChangesOnce) { - onStatusChange('connected'); + if (!response.ok) { + throw new Error(`Commit failed: ${response.status}`); } - } else { - onStatusChange('disconnected'); - } - }); - return ref - .child('changes') - .orderByKey() - .startAt(String(mostRecentRemoteKey + 1)) - .once('value') - .then((changesSnapshot) => { - const snapshotVal = changesSnapshot.val() || {}; - const allSteps: any[] = []; - const allStepClientIds: any[] = []; - const keys = Object.keys(snapshotVal); + onStatusChange('saved'); + onUpdateLatestKey(commit.version); + }, - /* Uncompress steps and add stepClientIds */ - Object.keys(snapshotVal).forEach((key) => { - const { steps, clientIds } = extractSnapshot(snapshotVal[key]); - allSteps.push(...steps); - allStepClientIds.push(...clientIds); - }); + receiveCommits: (commits: any[]) => { + if (!view) { + return; + } - /* We have to use .reduce here rather than simply calling */ - /* Math.max(keys) because sometimes the keys array is larger */ - /* than the allowed input size of Math.max() */ - mostRecentRemoteKey = keys.length - ? keys.map((k) => Number(k)).reduce((a, b) => Math.max(a, b), 0) - : mostRecentRemoteKey; + let currentState = view.state; + + for (const commit of commits) { + const tr = receiveCommitTransaction(currentState, commit); + view.dispatch(tr); + currentState = view.state; + } - const trans = receiveTransaction(view.state, allSteps, allStepClientIds); - view.dispatch(trans); - onUpdateLatestKey(mostRecentRemoteKey); + if (commits.length > 0) { + const lastCommit = commits[commits.length - 1]; + onUpdateLatestKey(lastCommit.version); + } + }, - /* Set finishedLoading flag */ - const finishedLoadingTrans = view.state.tr; - finishedLoadingTrans.setMeta('finishedLoading', true); - view.dispatch(finishedLoadingTrans); - onStatusChange('connected'); - hasLoadedChangesOnce = true; + listener: commitListener, + }; - /* Listen to Changes */ - listeningOn = ref - .child('changes') - .orderByKey() - .startAt(String(mostRecentRemoteKey + 1)); + collabClient = new CollabClient(collabConfig); + abortController = new AbortController(); - return listeningOn!.on('child_added', (snapshot) => { - receiveCollabChanges(snapshot); - }); - }) - .catch((err) => { - console.error('In loadDocument Error with ', err, err.message); + collabClient + .listen(initialState, abortController.signal) + .catch((e) => { + if (e.name !== 'AbortError') { + console.error('Collab listener error:', e); + onError(e); + } }); + + onStatusChange('connected'); }; return new Plugin({ @@ -242,7 +115,6 @@ export default ( apply: (transaction, pluginState) => { return { isLoaded: transaction.getMeta('finishedLoading') || pluginState.isLoaded, - mostRecentRemoteKey, localClientId, localClientData: collaborativeOptions.clientData, sendCollabChanges, @@ -251,12 +123,18 @@ export default ( }, view: (initView) => { view = initView; - loadDocument(); + + // mark as loaded immediately since the doc comes from the server already + const finishedLoadingTrans = view.state.tr; + finishedLoadingTrans.setMeta('finishedLoading', true); + view.dispatch(finishedLoadingTrans); + + startCollab(view.state); + return { destroy: () => { - listeningOn?.off('child_added'); - getFirebaseConnectionMonitorRef(ref).off('value'); - pendingRemoteKeyables = []; + abortController?.abort(); + collabClient = null; }, }; }, diff --git a/client/components/Editor/plugins/collaborative/index.ts b/client/components/Editor/plugins/collaborative/index.ts index af5aa9f81c..330a55dcfc 100644 --- a/client/components/Editor/plugins/collaborative/index.ts +++ b/client/components/Editor/plugins/collaborative/index.ts @@ -1,4 +1,4 @@ -import { collab } from 'prosemirror-collab'; +import { collab } from '@pitter-patter/collab-client'; import { PluginKey } from 'prosemirror-state'; import { generateHash } from 'utils/hashes'; @@ -9,14 +9,14 @@ import buildDocument from './document'; export const collabDocPluginKey = new PluginKey('collaborative'); export default (schema, props) => { - if (!props.collaborativeOptions?.firebaseRef) { + if (!props.collaborativeOptions) { return []; } const localClientId = `${props.collaborativeOptions.clientData.id}-${generateHash(6)}`; return [ - collab({ clientID: localClientId }), + collab({ version: props.collaborativeOptions.initialDocKey }), buildDocument(schema, props, collabDocPluginKey, localClientId), buildCursors(schema, props, collabDocPluginKey), ]; diff --git a/client/components/Editor/plugins/discussions/fastForward.ts b/client/components/Editor/plugins/discussions/fastForward.ts index 3621f6424d..b35dca9364 100644 --- a/client/components/Editor/plugins/discussions/fastForward.ts +++ b/client/components/Editor/plugins/discussions/fastForward.ts @@ -1,4 +1,3 @@ -import type firebase from 'firebase'; import type { Node } from 'prosemirror-model'; import type { Step } from 'prosemirror-transform'; @@ -9,7 +8,7 @@ import { flattenOnce } from 'utils/arrays'; import { getStepsInChangeRange } from '../../utils'; import { mapDiscussionThroughSteps } from './util'; -type Reference = firebase.database.Reference; +type Reference = any; const getFastForwardedDiscussion = ( discussion: DiscussionInfo, diff --git a/client/components/Editor/plugins/discussions/firebase.ts b/client/components/Editor/plugins/discussions/firebase.ts index 4727710eec..8abbb2cc6d 100644 --- a/client/components/Editor/plugins/discussions/firebase.ts +++ b/client/components/Editor/plugins/discussions/firebase.ts @@ -1,89 +1,85 @@ -import type firebase from 'firebase'; - import type { - CompressedDiscussionInfo, - DiscussionInfo, Discussions, DiscussionsHandler, + NullableDiscussions, RemoteDiscussions, } from './types'; -import { compressSelectionJSON, uncompressSelectionJSON } from 'prosemirror-compress-pubpub'; +/** + * Connects to the server-side discussion position sync via polling. + * Replaces the previous Firebase-based implementation. + * + * Discussion positions are stored in Postgres and broadcast via Valkey pub/sub. + * This client polls the server endpoint for updates and posts local changes. + */ +export const connectToRemoteDiscussions = (pubId: string): RemoteDiscussions => { + let onDiscussions: null | DiscussionsHandler = null; + let pollInterval: ReturnType | null = null; + let lastKnownDiscussions: NullableDiscussions = {}; -type Reference = firebase.database.Reference; -type DataSnapshot = firebase.database.DataSnapshot; + const fetchDiscussions = async () => { + try { + const response = await fetch(`/api/pubs/${pubId}/discussions/positions`); -const uncompressDiscussionInfo = (compressed: CompressedDiscussionInfo): DiscussionInfo => { - const { selection: compressedSelection } = compressed; - const selection = compressedSelection && uncompressSelectionJSON(compressedSelection); - return { ...compressed, selection: selection ?? null }; -}; + if (!response.ok) { + return; + } -const compressDiscussionInfo = (uncompressed: DiscussionInfo): CompressedDiscussionInfo => { - const { selection: uncompressedSelection } = uncompressed; - const selection = uncompressedSelection && compressSelectionJSON(uncompressedSelection); - return { ...uncompressed, selection: selection ?? null }; -}; + const discussions = (await response.json()) as NullableDiscussions; -export const connectToFirebaseDiscussions = (discussionsRef: Reference): RemoteDiscussions => { - let onDiscussions: null | DiscussionsHandler = null; - let disconnect: null | (() => void) = null; + const hasChanges = Object.keys(discussions).some((id) => { + const remote = discussions[id]; + const local = lastKnownDiscussions[id]; - const childAddedHandler = (snapshot: DataSnapshot) => { - const discussion = snapshot.val(); - if (discussion) { - onDiscussions?.({ [snapshot.key!]: uncompressDiscussionInfo(discussion) }); - } - }; + if (!remote && !local) return false; + if (!remote || !local) return true; - const childRemovedHandler = (snapshot: DataSnapshot) => { - onDiscussions?.({ [snapshot.key!]: null }); + return ( + remote.currentKey !== local.currentKey || + remote.selection?.anchor !== local.selection?.anchor || + remote.selection?.head !== local.selection?.head + ); + }); + + if (hasChanges) { + lastKnownDiscussions = discussions; + onDiscussions?.(discussions); + } + } catch (_err) { + // non-fatal, will retry on next poll + } }; const sendDiscussions = (discussions: Discussions) => { - Object.entries(discussions).forEach(([id, discussion]) => { - discussionsRef - .child(id) - .transaction((existingDiscussion: null | CompressedDiscussionInfo) => { - if ( - !existingDiscussion || - discussion.currentKey > existingDiscussion.currentKey - ) { - return compressDiscussionInfo(discussion); - } - return undefined; - }); + fetch(`/api/pubs/${pubId}/discussions/positions`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(discussions), + }).catch(() => { + // non-fatal }); }; - const connectHandler = (handler: DiscussionsHandler) => { + const receiveDiscussions = (handler: DiscussionsHandler) => { onDiscussions = handler; }; - const connect = () => { - discussionsRef.on('child_added', childAddedHandler); - discussionsRef.on('child_removed', childRemovedHandler); - return () => { - discussionsRef.off('child_added', childAddedHandler); - discussionsRef.off('child_removed', childRemovedHandler); - }; - }; + // start polling + fetchDiscussions(); + pollInterval = setInterval(fetchDiscussions, 3000); - discussionsRef.once('value').then((snapshot) => { - const discussionsById = snapshot.val(); - if (discussionsById) { - const uncompressedDiscussionsById: Discussions = {}; - Object.entries(discussionsById).forEach(([id, discussion]) => { - uncompressedDiscussionsById[id] = uncompressDiscussionInfo(discussion as any); - }); - onDiscussions?.(uncompressedDiscussionsById); + const disconnect = () => { + if (pollInterval) { + clearInterval(pollInterval); + pollInterval = null; } - disconnect = connect(); - }); + + onDiscussions = null; + }; return { sendDiscussions, - receiveDiscussions: connectHandler, - disconnect: () => disconnect?.(), + receiveDiscussions, + disconnect, }; }; diff --git a/client/components/Editor/plugins/discussions/plugin.ts b/client/components/Editor/plugins/discussions/plugin.ts index 139da07238..1a754abb53 100644 --- a/client/components/Editor/plugins/discussions/plugin.ts +++ b/client/components/Editor/plugins/discussions/plugin.ts @@ -9,8 +9,7 @@ import { DecorationSet, type EditorView } from 'prosemirror-view'; import { getDiscussionsFromAnchors } from './anchors'; import { getDecorationsForDiscussions, getDecorationsForUpdateResult } from './decorations'; import { createDiscussionsState } from './discussionsState'; -import { createFastForwarder } from './fastForward'; -import { connectToFirebaseDiscussions } from './firebase'; +import { connectToRemoteDiscussions } from './firebase'; export const discussionsPluginKey = new PluginKey('discussions'); @@ -22,10 +21,8 @@ type PluginState = { }; const createPlugin = (discussionsOptions: DiscussionsOptions, initialDoc: Node) => { - const { discussionAnchors, draftRef, initialHistoryKey } = discussionsOptions; - const discussionsRef = draftRef?.child('discussions'); - const remote = discussionsRef && connectToFirebaseDiscussions(discussionsRef); - const fastForward = draftRef && createFastForwarder(draftRef); + const { discussionAnchors, pubId, initialHistoryKey } = discussionsOptions; + const remote = pubId ? connectToRemoteDiscussions(pubId) : null; const initialDiscussions = getDiscussionsFromAnchors(discussionAnchors); let editorView: null | EditorView = null; @@ -35,7 +32,7 @@ const createPlugin = (discussionsOptions: DiscussionsOptions, initialDoc: Node) initialHistoryKey, initialDoc, remoteDiscussions: remote || null, - fastForwardDiscussions: fastForward || null, + fastForwardDiscussions: null, onUpdateDiscussions: (updateResult: DiscussionsUpdateResult) => { if (editorView) { const { tr } = editorView.state; diff --git a/client/components/Editor/types.ts b/client/components/Editor/types.ts index 2a0f0bf04e..bb41c3c272 100644 --- a/client/components/Editor/types.ts +++ b/client/components/Editor/types.ts @@ -1,4 +1,3 @@ -import type firebase from 'firebase'; import type { Node, Schema } from 'prosemirror-model'; import type { EditorState, Plugin, Transaction } from 'prosemirror-state'; @@ -43,14 +42,13 @@ export type CollaborativeOptions = { id: null | string; }; pubId: string; - firebaseRef: firebase.database.Reference; initialDocKey: number; onStatusChange?: (status: CollaborativeEditorStatus) => unknown; onUpdateLatestKey?: (key: number) => unknown; }; export type DiscussionsOptions = { - draftRef?: null | firebase.database.Reference; + pubId?: string | null; initialHistoryKey: number; discussionAnchors: DiscussionAnchor[]; }; @@ -88,7 +86,7 @@ export type OnEditFn = ( export type CompressedChange = { s: Record[]; - t: { '.sv': string }; // This is a special Firebase value + t: number | { '.sv': string }; cId: string; id: string; }; diff --git a/client/components/Editor/utils/__test__/changes.test.ts b/client/components/Editor/utils/__test__/changes.test.ts index e4dead3f43..e6396457b4 100644 --- a/client/components/Editor/utils/__test__/changes.test.ts +++ b/client/components/Editor/utils/__test__/changes.test.ts @@ -1,9 +1,13 @@ import { editorSchema } from 'components/Editor'; -import { editFirebaseDraft } from 'stubstub'; import { getStepsInChangeRange } from '../changes'; -describe('getStepsInChangeRange', () => { +const editFirebaseDraft = (): any => { + throw new Error('Firebase tests are disabled post-migration'); +}; + +// these tests require a live firebase connection and are disabled post-migration +describe.skip('getStepsInChangeRange', () => { it('returns expected steps', async () => { const editor = await editFirebaseDraft(); editor.transform((tr, schema) => { diff --git a/client/components/Editor/utils/changes.ts b/client/components/Editor/utils/changes.ts index 3315e43050..94fa485ae6 100644 --- a/client/components/Editor/utils/changes.ts +++ b/client/components/Editor/utils/changes.ts @@ -1,11 +1,11 @@ -import type firebase from 'firebase'; import type { Node, Schema } from 'prosemirror-model'; import { uncompressStepJSON } from 'prosemirror-compress-pubpub'; import { Step } from 'prosemirror-transform'; +/** @deprecated legacy firebase utility, only used by migration tools */ export const getStepsInChangeRange = async ( - draftRef: firebase.database.Reference, + draftRef: any, schema: Schema, startIndex: number, endIndex: number, diff --git a/client/components/Editor/utils/firebase.ts b/client/components/Editor/utils/firebase.ts index f693f25bfa..0995136a77 100644 --- a/client/components/Editor/utils/firebase.ts +++ b/client/components/Editor/utils/firebase.ts @@ -1,4 +1,3 @@ -import type firebase from 'firebase'; import type { Node } from 'prosemirror-model'; import type { Step } from 'prosemirror-transform'; @@ -55,6 +54,7 @@ export const createFirebaseChange = ( }; }; -export const getFirebaseConnectionMonitorRef = (ref: firebase.database.Reference) => { +/** @deprecated legacy firebase utility */ +export const getFirebaseConnectionMonitorRef = (ref: any) => { return ref.root.child('.info/connected'); }; diff --git a/client/components/Editor/utils/firebaseDoc.ts b/client/components/Editor/utils/firebaseDoc.ts index 3d34e75147..96b32ba6b6 100644 --- a/client/components/Editor/utils/firebaseDoc.ts +++ b/client/components/Editor/utils/firebaseDoc.ts @@ -1,5 +1,3 @@ -import type firebase from 'firebase'; - import { uncompressStateJSON, uncompressStepJSON } from 'prosemirror-compress-pubpub'; import { Node, type Schema } from 'prosemirror-model'; import { Step } from 'prosemirror-transform'; @@ -7,8 +5,9 @@ import { Step } from 'prosemirror-transform'; import { getEmptyDoc } from './doc'; import { flattenKeyables } from './firebase'; -type Reference = firebase.database.Reference; -type Query = firebase.database.Query; +// legacy firebase types -- these files are only used by migration tools +type Reference = any; +type Query = any; type CheckpointMap = Record; diff --git a/client/components/Editor/utils/index.ts b/client/components/Editor/utils/index.ts index dbb49ac04d..41b317193e 100644 --- a/client/components/Editor/utils/index.ts +++ b/client/components/Editor/utils/index.ts @@ -1,7 +1,5 @@ export * from './changes'; export * from './doc'; -export * from './firebase'; -export * from './firebaseDoc'; export * from './media'; export * from './misc'; export * from './nodes'; @@ -12,3 +10,7 @@ export * from './renderStatic'; export * from './schema'; export * from './selection'; export * from './view'; + +// legacy firebase exports -- only used by migration tools, not the app bundle +export { storeCheckpoint, flattenKeyables, createFirebaseChange } from './firebase'; +export { getFirebaseDoc, getFirstKeyAndTimestamp, getLatestKeyAndTimestamp } from './firebaseDoc'; diff --git a/client/components/Editor/utils/view.ts b/client/components/Editor/utils/view.ts index c988a56a1c..7c13f78aa2 100644 --- a/client/components/Editor/utils/view.ts +++ b/client/components/Editor/utils/view.ts @@ -151,7 +151,7 @@ export const getLocalHighlightText = (editorView, highlightId) => { }; }; -export const reanchorDiscussion = (editorView, firebaseRef, discussionId) => { +export const reanchorDiscussion = (editorView, pubId: string, discussionId: string) => { const collabPlugin = editorView.state.collaborative$ || {}; const newCurrentKey = collabPlugin.mostRecentRemoteKey; const selection = editorView.state.selection; @@ -161,17 +161,19 @@ export const reanchorDiscussion = (editorView, firebaseRef, discussionId) => { const transaction = editorView.state.tr; transaction.setMeta('removeDiscussion', { id: discussionId }); editorView.dispatch(transaction); - firebaseRef - .child('discussions') - .child(discussionId) - .update({ - currentKey: newCurrentKey, - selection: { - a: newAnchor, - h: newHead, - t: 'text', + + fetch(`/api/pubs/${pubId}/discussions/positions`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + [discussionId]: { + currentKey: newCurrentKey, + selection: { type: 'text', anchor: newAnchor, head: newHead }, }, - }); + }), + }).catch((err) => { + console.error('Failed to reanchor discussion:', err); + }); }; export const focus = (editorView) => { diff --git a/client/components/MinimalEditor/MinimalEditor.tsx b/client/components/MinimalEditor/MinimalEditor.tsx index 6d8e40f9a6..692e584bff 100644 --- a/client/components/MinimalEditor/MinimalEditor.tsx +++ b/client/components/MinimalEditor/MinimalEditor.tsx @@ -62,6 +62,7 @@ const MinimalEditor = (props: Props) => { }, []); useEffect(() => { + // @ts-expect-error shh needs to be like this for webpack import('../FormattingBar').then(({ buttons, FormattingBar: FormattingBarComponent }) => { setFormattingBar(() => (innerProps) => ( diff --git a/client/containers/DashboardCustomScripts/DashboardCustomScripts.tsx b/client/containers/DashboardCustomScripts/DashboardCustomScripts.tsx index 0a0d8c852f..eaaefbba22 100644 --- a/client/containers/DashboardCustomScripts/DashboardCustomScripts.tsx +++ b/client/containers/DashboardCustomScripts/DashboardCustomScripts.tsx @@ -26,7 +26,7 @@ const DashboardCustomScripts = (props: Props) => { import( /* webpackChunkName: "@monaco-editor/react" */ '@monaco-editor/react' - ).then(({ default: EditorComponent }) => setEditor(EditorComponent)); + ).then(({ default: EditorComponent }) => setEditor(EditorComponent as any)); }, []); const renderLoading = () => { diff --git a/client/containers/Pub/PubContextProvider.tsx b/client/containers/Pub/PubContextProvider.tsx index 069a2688b2..bc831213ba 100644 --- a/client/containers/Pub/PubContextProvider.tsx +++ b/client/containers/Pub/PubContextProvider.tsx @@ -49,7 +49,6 @@ const shimPubContextProps = { }, collabData: { editorChangeObject: {} }, historyData: {}, - firebaseDraftRef: null, updateLocalData: null as any, updatePubData: null as any, submissionState: {}, diff --git a/client/containers/Pub/PubDocument/PubBody.tsx b/client/containers/Pub/PubDocument/PubBody.tsx index ae7da374b4..d93ad34830 100644 --- a/client/containers/Pub/PubDocument/PubBody.tsx +++ b/client/containers/Pub/PubDocument/PubBody.tsx @@ -37,7 +37,7 @@ const PubBody = (props: Props) => { noteManager, updateCollabData, historyData: { setLatestHistoryKey }, - collabData: { status, firebaseDraftRef, localCollabUser }, + collabData: { status, localCollabUser }, pubBodyState: { editorKey, initialContent, @@ -83,18 +83,16 @@ const PubBody = (props: Props) => { [updateCollabData], ); - const collaborativeOptions = includeCollabPlugin && - !!firebaseDraftRef && { - pubId: pubData.id, - initialDocKey: initialHistoryKey, - firebaseRef: firebaseDraftRef, - clientData: localCollabUser, - onStatusChange: handleStatusChange, - onUpdateLatestKey: setLatestHistoryKey, - }; + const collaborativeOptions = includeCollabPlugin && { + pubId: pubData.id, + initialDocKey: initialHistoryKey, + clientData: localCollabUser, + onStatusChange: handleStatusChange, + onUpdateLatestKey: setLatestHistoryKey, + }; const discussionOptions = includeDiscussionsPlugin && { - draftRef: firebaseDraftRef, + pubId: pubData.id, initialHistoryKey, discussionAnchors: discussionAnchors || [], }; diff --git a/client/containers/Pub/PubDocument/PubDiscussions/Discussion/DiscussionReanchor.tsx b/client/containers/Pub/PubDocument/PubDiscussions/Discussion/DiscussionReanchor.tsx index 98cf274ec9..f35db32c79 100644 --- a/client/containers/Pub/PubDocument/PubDiscussions/Discussion/DiscussionReanchor.tsx +++ b/client/containers/Pub/PubDocument/PubDiscussions/Discussion/DiscussionReanchor.tsx @@ -14,7 +14,7 @@ type Props = { const DiscussionReanchor = (props: Props) => { const { discussionData } = props; - const { collabData } = usePubContext(); + const { collabData, pubData } = usePubContext(); const [isActive, setIsActive] = useState(false); const { selection } = collabData.editorChangeObject!; @@ -22,7 +22,7 @@ const DiscussionReanchor = (props: Props) => { const handleReanchor = () => { const { view } = collabData.editorChangeObject!; - reanchorDiscussion(view, collabData.firebaseDraftRef, discussionData.id); + reanchorDiscussion(view, pubData.id, discussionData.id); setIsActive(false); }; const [bodyElement, setBodyElement] = useState(null); diff --git a/client/containers/Pub/usePubBodyState.ts b/client/containers/Pub/usePubBodyState.ts index a16e429973..cbdd260ab3 100644 --- a/client/containers/Pub/usePubBodyState.ts +++ b/client/containers/Pub/usePubBodyState.ts @@ -32,8 +32,8 @@ export const usePubBodyState = (options: Options): PubBodyState => { }, submissionState, historyData: { currentKey, isViewingHistory, historyDoc, historyDocEditorKey }, - collabData: { firebaseDraftRef }, } = options; + const { scopeData: { activePermissions: { canEdit, canEditDraft }, @@ -79,6 +79,7 @@ export const usePubBodyState = (options: Options): PubBodyState => { if (submissionState) { const submissionPreviewDoc = submissionState?.submissionPreviewDoc; + if (submissionPreviewDoc) { return { editorKey: `submission-preview-${currentKey}`, @@ -91,6 +92,7 @@ export const usePubBodyState = (options: Options): PubBodyState => { canCreateAnchoredDiscussions: false, }; } + if (submissionState.selectedTab === 'instructions') { return { editorKey: '', @@ -133,11 +135,11 @@ export const usePubBodyState = (options: Options): PubBodyState => { } return { - editorKey: firebaseDraftRef ? 'ready' : 'unready', + editorKey: 'ready', isReadOnly: !(canEdit || canEditDraft), initialContent: initialDoc, initialHistoryKey: initialDocKey, - includeCollabPlugin: !!firebaseDraftRef, + includeCollabPlugin: true, includeDiscussionsPlugin: true, discussionAnchors, canCreateAnchoredDiscussions: true, diff --git a/client/containers/Pub/usePubCollabState.ts b/client/containers/Pub/usePubCollabState.ts index 7291dbaf22..91c19dbcd8 100644 --- a/client/containers/Pub/usePubCollabState.ts +++ b/client/containers/Pub/usePubCollabState.ts @@ -1,11 +1,8 @@ -import type firebase from 'firebase'; - import type { EditorChangeObject } from 'components/Editor'; import type { LoginData, Maybe, PubPageData } from 'types'; -import { useCallback, useEffect } from 'react'; +import { useCallback } from 'react'; -import { initFirebase } from 'client/utils/firebaseClient'; import { useIdlyUpdatedState } from 'client/utils/useIdlyUpdatedState'; import { getRandomColor } from 'utils/colors'; import { usePageContext } from 'utils/hooks'; @@ -27,7 +24,6 @@ export type PubCollabState = { status: PubCollabStatus; localCollabUser: CollabUser; remoteCollabUsers: CollabUser[]; - firebaseDraftRef: null | firebase.database.Reference; }; type Options = { @@ -49,7 +45,6 @@ const getLocalCollabUser = (canEdit: boolean, loginData: LoginData) => { export const usePubCollabState = (options: Options) => { const { pubData } = options; - const { draft, firebaseToken } = pubData; const { loginData, scopeData: { @@ -59,40 +54,12 @@ export const usePubCollabState = (options: Options) => { const [collabState, updateCollabState] = useIdlyUpdatedState(() => { return { - // TODO(ian): Verify that there are no unchecked property accesses on this - // editorChangeObject and then remove this cast. editorChangeObject: {} as unknown as null, status: 'connecting', localCollabUser: getLocalCollabUser(canEdit || canEditDraft, loginData), remoteCollabUsers: [], - firebaseDraftRef: null, }; }); - const syncRemoteCollabUsers = useCallback( - (snapshot: firebase.database.DataSnapshot) => { - const users = snapshot.val() as null | CollabUser[]; - if (users) { - const remoteCollabUsers = Object.values(users).filter( - (user: any) => user.id !== loginData.id, - ); - updateCollabState({ remoteCollabUsers }); - } - }, - [updateCollabState, loginData], - ); - - useEffect(() => { - if (draft && firebaseToken) { - initFirebase(draft.firebasePath, firebaseToken).then((firebaseDraftRef) => { - if (!firebaseDraftRef) { - return; - } - updateCollabState({ firebaseDraftRef }); - firebaseDraftRef?.child('cursors').on('value', syncRemoteCollabUsers); - }); - } - }, [draft, firebaseToken, updateCollabState, syncRemoteCollabUsers]); - return [collabState, updateCollabState] as const; }; diff --git a/client/utils/firebaseClient.ts b/client/utils/firebaseClient.ts index 4955880a72..262438c773 100644 --- a/client/utils/firebaseClient.ts +++ b/client/utils/firebaseClient.ts @@ -1,31 +1,10 @@ -import firebase from 'firebase/app'; -import 'firebase/auth'; -import 'firebase/database'; - -import { getFirebaseConfig } from 'utils/editor/firebaseConfig'; - -export const initFirebase = async (rootKey: string, authToken: string) => { - const firebaseAppName = `App-${rootKey}`; - /* Check if we've already initialized an Firebase App with the */ - /* same name in this local environment */ - // @ts-expect-error ts-migrate(2769) FIXME: No overload matches this call. - const existingApp = firebase.apps.reduce((prev, curr) => { - return curr.name === firebaseAppName ? curr : prev; - }, undefined); - - /* Use the existing Firebase App or initialize a new one */ - const firebaseApp = existingApp || firebase.initializeApp(getFirebaseConfig(), firebaseAppName); - - const database = firebase.database(firebaseApp); - - /* Authenticate with the server-generated token */ - try { - const auth = await firebase.auth(firebaseApp); - await auth.signOut(); - await auth.signInWithCustomToken(authToken); - return database.ref(rootKey); - } catch (err) { - console.error('Error authenticating firebase', err); - return null; - } +/** + * @deprecated Firebase is no longer used for collaborative editing. + * This module is kept as a stub during the migration period. + * Remove after full migration is confirmed. + */ + +export const initFirebase = async (_rootKey: string, _authToken: string) => { + console.warn('initFirebase called but Firebase has been removed. This is a no-op.'); + return null; }; diff --git a/client/webpack/webpackConfig.dev.js b/client/webpack/webpackConfig.dev.js index dce1744790..12d9b16d51 100644 --- a/client/webpack/webpackConfig.dev.js +++ b/client/webpack/webpackConfig.dev.js @@ -32,6 +32,9 @@ module.exports = { types: resolve(__dirname, '../../types'), facets: resolve(__dirname, '../../facets'), 'prosemirror-state': require.resolve('prosemirror-state'), + '@pitter-patter/collab-client': resolve(__dirname, '../../node_modules/@pitter-patter/collab-client/dist/index.js'), + '@stepwisehq/prosemirror-collab-commit/collab-commit': resolve(__dirname, '../../node_modules/@stepwisehq/prosemirror-collab-commit/dist/collab-commit.js'), + '@stepwisehq/prosemirror-collab-commit': resolve(__dirname, '../../node_modules/@stepwisehq/prosemirror-collab-commit/dist/index.js'), }, }, devtool: 'eval', @@ -58,17 +61,16 @@ module.exports = { }, module: { rules: [ - { - test: /\.(m|c)?js$/, - // this module includes nullish coalescing and optional chaining, which are not supported by webpack 4 - include: /node_modules\/@marsidev\/react-turnstile|node_modules\/(.pnpm\/)?altcha.*|node_modules\/(.pnpm\/)?react-kapsule.*|node_modules\/(.pnpm\/)?react-force-graph.*|node_modules\/(.pnpm\/)?force-graph.*|node_modules\/(.pnpm\/)?float-tooltip.*/, - type: 'javascript/auto', - loader: 'esbuild-loader', - /** @type {import('esbuild-loader').LoaderOptions} */ - options: { - target: 'es6' - }, + { + test: /\.(m|c)?js$/, + include: /node_modules\/@marsidev\/react-turnstile|node_modules\/(.pnpm\/)?altcha.*|node_modules\/(.pnpm\/)?react-kapsule.*|node_modules\/(.pnpm\/)?react-force-graph.*|node_modules\/(.pnpm\/)?force-graph.*|node_modules\/(.pnpm\/)?float-tooltip.*|node_modules\/(.pnpm\/)?@pitter-patter.*|node_modules\/(.pnpm\/)?@stepwisehq.*/, + type: 'javascript/auto', + loader: 'esbuild-loader', + /** @type {import('esbuild-loader').LoaderOptions} */ + options: { + target: 'es6' }, + }, { test: /\.((c|m)?js|jsx|ts|tsx)$/, include: [ diff --git a/client/webpack/webpackConfig.prod.js b/client/webpack/webpackConfig.prod.js index ad74846d8e..9d8aa61ee1 100644 --- a/client/webpack/webpackConfig.prod.js +++ b/client/webpack/webpackConfig.prod.js @@ -33,6 +33,9 @@ module.exports = { types: resolve(__dirname, '../../types'), facets: resolve(__dirname, '../../facets'), 'prosemirror-state': require.resolve('prosemirror-state'), + '@pitter-patter/collab-client': resolve(__dirname, '../../node_modules/@pitter-patter/collab-client/dist/index.js'), + '@stepwisehq/prosemirror-collab-commit/collab-commit': resolve(__dirname, '../../node_modules/@stepwisehq/prosemirror-collab-commit/dist/collab-commit.js'), + '@stepwisehq/prosemirror-collab-commit': resolve(__dirname, '../../node_modules/@stepwisehq/prosemirror-collab-commit/dist/index.js'), }, }, devtool: 'source-map', @@ -46,9 +49,8 @@ module.exports = { module: { rules: [ { - test: /\.(m|c)?js$/, - // this module includes nullish coalescing and optional chaining, which are not supported by webpack 4 - include: /node_modules\/@marsidev\/react-turnstile|node_modules\/(.pnpm\/)?altcha.*|node_modules\/(.pnpm\/)?react-kapsule.*|node_modules\/(.pnpm\/)?react-force-graph.*|node_modules\/(.pnpm\/)?force-graph.*|node_modules\/(.pnpm\/)?float-tooltip.*/, + test: /\.(m|c)?js$/, + include: /node_modules\/@marsidev\/react-turnstile|node_modules\/(.pnpm\/)?altcha.*|node_modules\/(.pnpm\/)?react-kapsule.*|node_modules\/(.pnpm\/)?react-force-graph.*|node_modules\/(.pnpm\/)?force-graph.*|node_modules\/(.pnpm\/)?float-tooltip.*|node_modules\/(.pnpm\/)?@pitter-patter.*|node_modules\/(.pnpm\/)?@stepwisehq.*/, type: 'javascript/auto', loader: 'esbuild-loader', /** @type {import('esbuild-loader').LoaderOptions} */ diff --git a/infra/docker-compose.dev.yml b/infra/docker-compose.dev.yml index c3511a261f..0f7e924208 100644 --- a/infra/docker-compose.dev.yml +++ b/infra/docker-compose.dev.yml @@ -23,9 +23,11 @@ services: PORT: 3000 DATABASE_URL: postgres://appuser:apppassword@db:5432/appdb CLOUDAMQP_URL: amqp://appuser:apppassword@rabbitmq:5672/appvhost + VALKEY_URL: redis://valkey:6379 depends_on: - db - rabbitmq + - valkey ports: - "${WEB_PORT:-9876}:3000" networks: [appnet] @@ -86,7 +88,13 @@ services: - "${DB_PORT:-5439}:5432" networks: [appnet] - + valkey: + image: valkey/valkey:9-alpine + volumes: + - valkeydata:/data + ports: + - "${VALKEY_PORT:-6380}:6379" + networks: [appnet] # cron: # build: @@ -113,4 +121,4 @@ networks: volumes: pgdata: rabbitmqdata: - + valkeydata: diff --git a/infra/stack.yml b/infra/stack.yml index ff3bba3a86..50d29bf1f7 100644 --- a/infra/stack.yml +++ b/infra/stack.yml @@ -28,8 +28,8 @@ services: NODE_ENV: production PORT: '3000' APP_COMMIT: '${APP_COMMIT:-}' - # Reuse the existing env var name, but point it at the in-swarm broker: CLOUDAMQP_URL: 'amqp://appuser:apppassword@rabbitmq:5672/appvhost' + VALKEY_URL: 'redis://valkey:6379' networks: [appnet] depends_on: [db] # note: ignored by swarm, fix later? healthcheck: @@ -63,6 +63,7 @@ services: NODE_ENV: production APP_COMMIT: '${APP_COMMIT:-}' CLOUDAMQP_URL: 'amqp://appuser:apppassword@rabbitmq:5672/appvhost' + VALKEY_URL: 'redis://valkey:6379' command: ['pnpm', 'run', 'workers-prod'] networks: [appnet] deploy: @@ -96,6 +97,16 @@ services: restart_policy: condition: any + valkey: + image: valkey/valkey:9-alpine + volumes: + - valkeydata:/data + networks: [appnet] + deploy: + replicas: 1 + restart_policy: + condition: any + db: image: postgres:16 tmpfs: @@ -190,5 +201,6 @@ networks: volumes: pgdata: rabbitmqdata: + valkeydata: caddy_data: caddy_config: diff --git a/package.json b/package.json index 363c55ff07..0889dc5225 100644 --- a/package.json +++ b/package.json @@ -97,12 +97,17 @@ "@lezer/rust": "^1.0.0", "@lezer/xml": "^1.0.0", "@monaco-editor/react": "4.1.1", + "@pitter-patter/collab-client": "^0.1.0", + "@pitter-patter/collab-server": "^0.1.0", + "@pitter-patter/presence-client": "^0.1.0", + "@pitter-patter/presence-server": "^0.1.0", "@popperjs/core": "^2.11.5", "@pubpub/deposit-utils": "^0.1.10", "@pubpub/prosemirror-pandoc": "^1.1.5", "@pubpub/prosemirror-reactive": "^0.2.0", "@sentry/node": "^7.77.0", "@sentry/react": "^7.77.0", + "@stepwisehq/prosemirror-collab-commit": "^1.0.5", "@ts-rest/core": "^3.30.5", "@ts-rest/express": "^3.30.5", "@ts-rest/open-api": "^3.30.5", @@ -153,7 +158,6 @@ "express-sslify": "^1.2.0", "file-saver": "^2.0.2", "filesize": "^4.1.2", - "firebase": "^7.5.2", "firebase-admin": "^9.4.2", "fs-extra": "^8.1.0", "fuzzysearch": "^1.0.3", @@ -200,7 +204,6 @@ "prompt": "^1.0.0", "prop-types": "^15.7.2", "prosemirror-changeset": "^2.2.0", - "prosemirror-collab": "^1.2.2", "prosemirror-commands": "^1.3.1", "prosemirror-compress-pubpub": "0.0.3", "prosemirror-gapcursor": "^1.1.5", @@ -380,7 +383,8 @@ "validate-with-xmllint" ], "patchedDependencies": { - "reakit": "patches/reakit.patch" + "reakit": "patches/reakit.patch", + "@pubpub/deposit-utils": "patches/@pubpub__deposit-utils.patch" } } } diff --git a/patches/@pubpub__deposit-utils.patch b/patches/@pubpub__deposit-utils.patch new file mode 100644 index 0000000000..2740ce7743 --- /dev/null +++ b/patches/@pubpub__deposit-utils.patch @@ -0,0 +1,17 @@ +diff --git a/package.json b/package.json +index ff9f6ae0f025e5528dd1448730e4f25c10284efb..d022c3ed4245e1b153cddab1b0eccef8e84fa86d 100644 +--- a/package.json ++++ b/package.json +@@ -21,10 +21,12 @@ + "exports": { + ".": "./index.js", + "./crossref": { ++ "types": "./dist/dts/crossref/index.d.ts", + "import": "./dist/esm/crossref/index.js", + "require": "./dist/cjs/crossref/index.js" + }, + "./datacite": { ++ "types": "./dist/dts/datacite/index.d.ts", + "import": "./dist/esm/datacite/index.js", + "require": "./dist/cjs/datacite/index.js" + } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 778ab5e698..f03e44e2b1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5,6 +5,9 @@ settings: excludeLinksFromLockfile: false patchedDependencies: + '@pubpub/deposit-utils': + hash: 6fef8933046b7751abda1bd2ed0422094c79d7828b07f7d5afba26a47c696d89 + path: patches/@pubpub__deposit-utils.patch reakit: hash: 31488077229fe3d19a71c66458b888c58eab18010632a29e3b5c485df77242a8 path: patches/reakit.patch @@ -142,12 +145,24 @@ importers: '@monaco-editor/react': specifier: 4.1.1 version: 4.1.1(monaco-editor@0.21.3)(react-dom@16.14.0(react@16.14.0))(react@16.14.0) + '@pitter-patter/collab-client': + specifier: ^0.1.0 + version: 0.1.0(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0) + '@pitter-patter/collab-server': + specifier: ^0.1.0 + version: 0.1.0(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0) + '@pitter-patter/presence-client': + specifier: ^0.1.0 + version: 0.1.0(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0)(prosemirror-view@1.41.6) + '@pitter-patter/presence-server': + specifier: ^0.1.0 + version: 0.1.0 '@popperjs/core': specifier: ^2.11.5 version: 2.11.8 '@pubpub/deposit-utils': specifier: ^0.1.10 - version: 0.1.10 + version: 0.1.10(patch_hash=6fef8933046b7751abda1bd2ed0422094c79d7828b07f7d5afba26a47c696d89) '@pubpub/prosemirror-pandoc': specifier: ^1.1.5 version: 1.1.5 @@ -160,6 +175,9 @@ importers: '@sentry/react': specifier: ^7.77.0 version: 7.120.4(react@16.14.0) + '@stepwisehq/prosemirror-collab-commit': + specifier: ^1.0.5 + version: 1.0.5 '@ts-rest/core': specifier: ^3.30.5 version: 3.52.1(@types/node@24.11.0)(zod@3.22.4) @@ -310,12 +328,9 @@ importers: filesize: specifier: ^4.1.2 version: 4.2.1 - firebase: - specifier: ^7.5.2 - version: 7.24.0 firebase-admin: specifier: ^9.4.2 - version: 9.12.0(@firebase/app-compat@0.5.8)(@firebase/app-types@0.6.1) + version: 9.12.0(@firebase/app-compat@0.5.8)(@firebase/app-types@0.7.0) fs-extra: specifier: ^8.1.0 version: 8.1.0 @@ -451,9 +466,6 @@ importers: prosemirror-changeset: specifier: ^2.2.0 version: 2.4.0 - prosemirror-collab: - specifier: ^1.2.2 - version: 1.3.1 prosemirror-commands: specifier: ^1.3.1 version: 1.7.1 @@ -2279,22 +2291,10 @@ packages: resolution: {integrity: sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} - '@firebase/analytics-types@0.4.0': - resolution: {integrity: sha512-Jj2xW+8+8XPfWGkv9HPv/uR+Qrmq37NPYT352wf7MvE9LrstpLVmFg3LqG6MCRr5miLAom5sen2gZ+iOhVDeRA==} - - '@firebase/analytics@0.6.0': - resolution: {integrity: sha512-6qYEOPUVYrMhqvJ46Z5Uf1S4uULd6d7vGpMP5Qz+u8kIWuOQGcPdJKQap+Hla6Rq164or9gC2HRXuYXKlgWfpw==} - peerDependencies: - '@firebase/app': 0.x - '@firebase/app-types': 0.x - '@firebase/app-compat@0.5.8': resolution: {integrity: sha512-4De6SUZ36zozl9kh5rZSxKWULpgty27rMzZ6x+xkoo7+NWyhWyFdsdvhFsWhTw/9GGj0wXIcbTjwHYCUIUuHyg==} engines: {node: '>=20.0.0'} - '@firebase/app-types@0.6.1': - resolution: {integrity: sha512-L/ZnJRAq7F++utfuoTKX4CLBG5YR7tFO3PLzG1/oXXKEezJ0kRL3CMRoueBEmTCzVb/6SIs2Qlaw++uDgi5Xyg==} - '@firebase/app-types@0.6.3': resolution: {integrity: sha512-/M13DPPati7FQHEQ9Minjk1HGLm/4K4gs9bR4rzLCWJg64yGtVC0zNg9gDpkw9yc2cvol/mNFxqTtd4geGrwdw==} @@ -2305,35 +2305,12 @@ packages: resolution: {integrity: sha512-WiE9uCGRLUnShdjb9iP20sA3ToWrBbNXr14/N5mow7Nls9dmKgfGaGX5cynLvrltxq2OrDLh1VDNaUgsnS/k/g==} engines: {node: '>=20.0.0'} - '@firebase/app@0.6.11': - resolution: {integrity: sha512-FH++PaoyTzfTAVuJ0gITNYEIcjT5G+D0671La27MU8Vvr6MTko+5YUZ4xS9QItyotSeRF4rMJ1KR7G8LSyySiA==} - - '@firebase/auth-interop-types@0.1.5': - resolution: {integrity: sha512-88h74TMQ6wXChPA6h9Q3E1Jg6TkTHep2+k63OWg3s0ozyGVMeY+TTOti7PFPzq5RhszQPQOoCi59es4MaRvgCw==} - peerDependencies: - '@firebase/app-types': 0.x - '@firebase/util': 0.x - '@firebase/auth-interop-types@0.1.6': resolution: {integrity: sha512-etIi92fW3CctsmR9e3sYM3Uqnoq861M0Id9mdOPF6PWIg38BXL5k4upCNBggGUpLIS0H1grMOvy/wn1xymwe2g==} peerDependencies: '@firebase/app-types': 0.x '@firebase/util': 1.x - '@firebase/auth-types@0.10.1': - resolution: {integrity: sha512-/+gBHb1O9x/YlG7inXfxff/6X3BPZt4zgBv4kql6HEmdzNQCodIRlEYnI+/da+lN+dha7PjaFH7C7ewMmfV7rw==} - peerDependencies: - '@firebase/app-types': 0.x - '@firebase/util': 0.x - - '@firebase/auth@0.15.0': - resolution: {integrity: sha512-IFuzhxS+HtOQl7+SZ/Mhaghy/zTU7CENsJFWbC16tv2wfLZbayKF5jYGdAU3VFLehgC8KjlcIWd10akc3XivfQ==} - peerDependencies: - '@firebase/app': 0.x - - '@firebase/component@0.1.19': - resolution: {integrity: sha512-L0S3g8eqaerg8y0zox3oOHSTwn/FE8RbcRHiurnbESvDViZtP5S5WnhuAPd7FnFxa8ElWK0z1Tr3ikzWDv1xdQ==} - '@firebase/component@0.5.13': resolution: {integrity: sha512-hxhJtpD8Ppf/VU2Rlos6KFCEV77TGIGD5bJlkPK1+B/WUe0mC6dTjW7KhZtXTc+qRBp9nFHWcsIORnT8liHP9w==} @@ -2346,9 +2323,6 @@ packages: peerDependencies: '@firebase/app-compat': 0.x - '@firebase/database-types@0.5.2': - resolution: {integrity: sha512-ap2WQOS3LKmGuVFKUghFft7RxXTyZTDr0Xd8y2aqmWsbJVjgozi0huL/EUMgTjGFrATAjcf2A7aNs8AKKZ2a8g==} - '@firebase/database-types@0.7.3': resolution: {integrity: sha512-dSOJmhKQ0nL8O4EQMRNGpSExWCXeHtH57gGg0BfNAdWcKhC8/4Y+qfKLfWXzyHvrSecpLmO0SmAi/iK2D5fp5A==} @@ -2358,44 +2332,6 @@ packages: '@firebase/database@0.12.8': resolution: {integrity: sha512-JBQVfFLzfhxlQbl4OU6ov9fdsddkytBQdtSSR49cz48homj38ccltAhK6seum+BI7f28cV2LFHF9672lcN+qxA==} - '@firebase/database@0.6.13': - resolution: {integrity: sha512-NommVkAPzU7CKd1gyehmi3lz0K78q0KOfiex7Nfy7MBMwknLm7oNqKovXSgQV1PCLvKXvvAplDSFhDhzIf9obA==} - - '@firebase/firestore-types@1.14.0': - resolution: {integrity: sha512-WF8IBwHzZDhwyOgQnmB0pheVrLNP78A8PGxk1nxb/Nrgh1amo4/zYvFMGgSsTeaQK37xMYS/g7eS948te/dJxw==} - peerDependencies: - '@firebase/app-types': 0.x - - '@firebase/firestore@1.18.0': - resolution: {integrity: sha512-maMq4ltkrwjDRusR2nt0qS4wldHQMp+0IDSfXIjC+SNmjnWY/t/+Skn9U3Po+dB38xpz3i7nsKbs+8utpDnPSw==} - engines: {node: ^8.13.0 || >=10.10.0} - peerDependencies: - '@firebase/app': 0.x - '@firebase/app-types': 0.x - - '@firebase/functions-types@0.3.17': - resolution: {integrity: sha512-DGR4i3VI55KnYk4IxrIw7+VG7Q3gA65azHnZxo98Il8IvYLr2UTBlSh72dTLlDf25NW51HqvJgYJDKvSaAeyHQ==} - - '@firebase/functions@0.5.1': - resolution: {integrity: sha512-yyjPZXXvzFPjkGRSqFVS5Hc2Y7Y48GyyMH+M3i7hLGe69r/59w6wzgXKqTiSYmyE1pxfjxU4a1YqBDHNkQkrYQ==} - peerDependencies: - '@firebase/app': 0.x - '@firebase/app-types': 0.x - - '@firebase/installations-types@0.3.4': - resolution: {integrity: sha512-RfePJFovmdIXb6rYwtngyxuEcWnOrzdZd9m7xAW0gRxDIjBT20n3BOhjpmgRWXo/DAxRmS7bRjWAyTHY9cqN7Q==} - peerDependencies: - '@firebase/app-types': 0.x - - '@firebase/installations@0.4.17': - resolution: {integrity: sha512-AE/TyzIpwkC4UayRJD419xTqZkKzxwk0FLht3Dci8WI2OEKHSwoZG9xv4hOBZebe+fDzoV2EzfatQY8c/6Avig==} - peerDependencies: - '@firebase/app': 0.x - '@firebase/app-types': 0.x - - '@firebase/logger@0.2.6': - resolution: {integrity: sha512-KIxcUvW/cRGWlzK9Vd2KB864HlUnCfdTH0taHE0sXW5Xl7+W68suaeau1oKNEqmc3l45azkd4NzXTCWZRZdXrw==} - '@firebase/logger@0.3.2': resolution: {integrity: sha512-lzLrcJp9QBWpo40OcOM9B8QEtBw2Fk1zOZQdvv+rWS6gKmhQBCEMc4SMABQfWdjsylBcDfniD1Q+fUX1dcBTXA==} @@ -2403,53 +2339,6 @@ packages: resolution: {integrity: sha512-cGskaAvkrnh42b3BA3doDWeBmuHFO/Mx5A83rbRDYakPjO9bJtRL3dX7javzc2Rr/JHZf4HlterTW2lUkfeN4g==} engines: {node: '>=20.0.0'} - '@firebase/messaging-types@0.5.0': - resolution: {integrity: sha512-QaaBswrU6umJYb/ZYvjR5JDSslCGOH6D9P136PhabFAHLTR4TWjsaACvbBXuvwrfCXu10DtcjMxqfhdNIB1Xfg==} - peerDependencies: - '@firebase/app-types': 0.x - - '@firebase/messaging@0.7.1': - resolution: {integrity: sha512-iev/ST9v0xd/8YpGYrZtDcqdD9J6ZWzSuceRn8EKy5vIgQvW/rk2eTQc8axzvDpQ36ZfphMYuhW6XuNrR3Pd2Q==} - peerDependencies: - '@firebase/app': 0.x - '@firebase/app-types': 0.x - - '@firebase/performance-types@0.0.13': - resolution: {integrity: sha512-6fZfIGjQpwo9S5OzMpPyqgYAUZcFzZxHFqOyNtorDIgNXq33nlldTL/vtaUZA8iT9TT5cJlCrF/jthKU7X21EA==} - - '@firebase/performance@0.4.2': - resolution: {integrity: sha512-irHTCVWJ/sxJo0QHg+yQifBeVu8ZJPihiTqYzBUz/0AGc51YSt49FZwqSfknvCN2+OfHaazz/ARVBn87g7Ex8g==} - peerDependencies: - '@firebase/app': 0.x - '@firebase/app-types': 0.x - - '@firebase/polyfill@0.3.36': - resolution: {integrity: sha512-zMM9oSJgY6cT2jx3Ce9LYqb0eIpDE52meIzd/oe/y70F+v9u1LDqk5kUF5mf16zovGBWMNFmgzlsh6Wj0OsFtg==} - - '@firebase/remote-config-types@0.1.9': - resolution: {integrity: sha512-G96qnF3RYGbZsTRut7NBX0sxyczxt1uyCgXQuH/eAfUCngxjEGcZQnBdy6mvSdqdJh5mC31rWPO4v9/s7HwtzA==} - - '@firebase/remote-config@0.1.28': - resolution: {integrity: sha512-4zSdyxpt94jAnFhO8toNjG8oMKBD+xTuBIcK+Nw8BdQWeJhEamgXlupdBARUk1uf3AvYICngHH32+Si/dMVTbw==} - peerDependencies: - '@firebase/app': 0.x - '@firebase/app-types': 0.x - - '@firebase/storage-types@0.3.13': - resolution: {integrity: sha512-pL7b8d5kMNCCL0w9hF7pr16POyKkb3imOW7w0qYrhBnbyJTdVxMWZhb0HxCFyQWC0w3EiIFFmxoz8NTFZDEFog==} - peerDependencies: - '@firebase/app-types': 0.x - '@firebase/util': 0.x - - '@firebase/storage@0.3.43': - resolution: {integrity: sha512-Jp54jcuyimLxPhZHFVAhNbQmgTu3Sda7vXjXrNpPEhlvvMSq4yuZBR6RrZxe/OrNVprLHh/6lTCjwjOVSo3bWA==} - peerDependencies: - '@firebase/app': 0.x - '@firebase/app-types': 0.x - - '@firebase/util@0.3.2': - resolution: {integrity: sha512-Dqs00++c8rwKky6KCKLLY2T1qYO4Q+X5t+lF7DInXDNF4ae1Oau35bkD+OpJ9u7l1pEv7KHowP6CUKuySCOc8g==} - '@firebase/util@1.13.0': resolution: {integrity: sha512-0AZUyYUfpMNcztR5l09izHwXkZpghLgCUaAGjtMwXnCg3bj4ml5VgiwqOMOxJ+Nw4qN/zJAaOQBcJ7KGkWStqQ==} engines: {node: '>=20.0.0'} @@ -2457,9 +2346,6 @@ packages: '@firebase/util@1.5.2': resolution: {integrity: sha512-YvBH2UxFcdWG2HdFnhxZptPl2eVFlpOyTH66iDo13JPEYraWzWToZ5AMTtkyRHVmu7sssUpQlU9igy1KET7TOw==} - '@firebase/webchannel-wrapper@0.4.0': - resolution: {integrity: sha512-8cUA/mg0S+BxIZ72TdZRsXKBP5n5uRcE3k29TZhZw6oIiHBt9JA7CTb/4pE1uKtE/q5NeTY2tBDcagoZ+1zjXQ==} - '@gar/promisify@1.1.3': resolution: {integrity: sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==} @@ -2499,18 +2385,10 @@ packages: resolution: {integrity: sha512-n2FjE7NAOYyshogdc7KQOl/VZb4sneqPjWouSyia9CMDdMhRX5+RIbqalNmC7LOLzuLAN89VlF2HvG8na9G+zQ==} engines: {node: '>=14'} - '@grpc/grpc-js@1.14.3': - resolution: {integrity: sha512-Iq8QQQ/7X3Sac15oB6p0FmUg/klxQvXLeileoqrTRGJYLV+/9tubbr9ipz0GKHjmXVsgFPo/+W+2cA8eNcR+XA==} - engines: {node: '>=12.10.0'} - '@grpc/grpc-js@1.6.12': resolution: {integrity: sha512-JmvQ03OTSpVd9JTlj/K3IWHSz4Gk/JMLUTtW7Zb0KvO1LcOYGATh5cNuRYzCAeDR3O8wq+q8FZe97eO9MBrkUw==} engines: {node: ^8.13.0 || >=10.10.0} - '@grpc/proto-loader@0.5.6': - resolution: {integrity: sha512-DT14xgw3PSzPxwS13auTEwxhMMOoz33DPUKNtmYK/QYbBSpLXJy78FGGs5yVoxVobEqPm4iW9MOIoz0A3bLTRQ==} - engines: {node: '>=6'} - '@grpc/proto-loader@0.6.13': resolution: {integrity: sha512-FjxPYDRTn6Ec3V0arm1FtSpmP6V50wuph2yILpyvTKzjc76oDdoihXqM1DzOW5ubvCC8GivfCnNtfaRE8myJ7g==} engines: {node: '>=6'} @@ -2521,10 +2399,26 @@ packages: engines: {node: '>=6'} hasBin: true - '@grpc/proto-loader@0.8.0': - resolution: {integrity: sha512-rc1hOQtjIWGxcxpb9aHAfLpIctjEnsDehj0DAiVfBlmT84uvR0uUtN2hEi/ecvWVjXUGf5qPF4qEgiLOx1YIMQ==} - engines: {node: '>=6'} - hasBin: true + '@handlewithcare/react-prosemirror@3.1.6': + resolution: {integrity: sha512-j8vK5b7e5gphrgz8OGbZu2Q3rpITkbnFsiAqNL0IUXnwxc/VvgvneLsj9EeJGqGNe61v3CxZDP0G/Tj1CCU2Lg==} + engines: {node: '>=16.9'} + peerDependencies: + '@tiptap/core': ^3.15.3 + '@tiptap/pm': ^3.15.3 + '@tiptap/react': ^3.15.3 + prosemirror-model: ^1.0.0 + prosemirror-state: ^1.0.0 + prosemirror-view: 1.41.7 + react: '>=17 <20' + react-dom: '>=17 <20' + react-reconciler: '>=0.26.1 <=0.33.0' + peerDependenciesMeta: + '@tiptap/core': + optional: true + '@tiptap/pm': + optional: true + '@tiptap/react': + optional: true '@humanwhocodes/config-array@0.13.0': resolution: {integrity: sha512-DZLEEqFWQFiyK6h5YIeynKx7JlvCYWL0cImfSRXZ9l4Sg2efkFGTuFf6vzXjK1cq6IYkU+Eg/JizXw+TD2vRNw==} @@ -2580,9 +2474,6 @@ packages: '@jridgewell/trace-mapping@0.3.9': resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} - '@js-sdsl/ordered-map@4.4.2': - resolution: {integrity: sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==} - '@jsep-plugin/assignment@1.3.0': resolution: {integrity: sha512-VVgV+CXrhbMI3aSusQyclHkenWSAm95WaiKrMxRFam3JSUiIaQjoMIw2sEs/OX4XifnqeQUN4DYbJjlA8EfktQ==} engines: {node: '>= 10.16.0'} @@ -2860,6 +2751,34 @@ packages: '@pdf-lib/upng@1.0.1': resolution: {integrity: sha512-dQK2FUMQtowVP00mtIksrlZhdFXQZPC+taih1q4CvPZ5vqdxR/LKBaFg0oAfzd1GlHZXXSPdQfzQnt+ViGvEIQ==} + '@pitter-patter/collab-client@0.1.0': + resolution: {integrity: sha512-OszIplkYRS4ttck7MhPU/oCNrK2YDjnd6nZ6cuYUah7IfR19HRYkfa4d2vv4ys8PGSYsIp8251yscNivK5ORng==} + peerDependencies: + prosemirror-model: ^1.0.0 + prosemirror-state: ^1.0.0 + prosemirror-transform: ^1.0.0 + + '@pitter-patter/collab-server@0.1.0': + resolution: {integrity: sha512-0esnVrkLyXRbByhn43hWR78ZwI1g4xKlL/Y6jx0cS1ACGQG2O+I2eKJqqcPXN3kmx1Jj2eB9RU2T1e2nOodWVA==} + peerDependencies: + prosemirror-model: ^1.0.0 + prosemirror-state: ^1.0.0 + prosemirror-transform: ^1.0.0 + + '@pitter-patter/presence-client@0.1.0': + resolution: {integrity: sha512-B2b7nXf0j8p2kesB4+Z2dCP1+qGsqklBiT3AyKEIw5PhzyIjw16HVsb2P5b3/zfuITUcqsEN5/B080sPkKHzTQ==} + peerDependencies: + prosemirror-model: ^1.0.0 + prosemirror-state: ^1.0.0 + prosemirror-transform: ^1.0.0 + prosemirror-view: ^1.0.0 + + '@pitter-patter/presence-server@0.1.0': + resolution: {integrity: sha512-0zQSikcCXKYFwUXG8sxmZjQ8rUgIIHpOOZFgrcTpCQtbxqaDeWskStxtMKvg8+bf0dLP4lF3ji8p2DcBmaf9fQ==} + + '@pitter-patter/refs@0.1.0': + resolution: {integrity: sha512-/zChXwSRr+IEOyCk+D2LHYXFvBnsAWyNKXR8B+wWPTW40A1sQF9lSoemeSyR4KaRtT56aa0jTw3+Rsj/687Bgg==} + '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -2932,6 +2851,42 @@ packages: '@pubpub/prosemirror-reactive@0.2.0': resolution: {integrity: sha512-xwnlQbwFyvzUhxKvDwmPrR6Y2KH6G5GZTCh8JVypMeLrXzR4seDNjIGqDVa8APt5RCBjOOp1a3YRYcNuWdzbhg==} + '@redis/bloom@5.12.1': + resolution: {integrity: sha512-PUUfv+ms7jgPSBVoo/DN4AkPHj4D5TZSd6SbJX7egzBplkYUcKmHRE8RKia7UtZ8bSQbLguLvxVO+asKtQfZWA==} + engines: {node: '>= 18.19.0'} + peerDependencies: + '@redis/client': ^5.12.1 + + '@redis/client@5.12.1': + resolution: {integrity: sha512-7aPGWeqA3uFm43o19umzdl16CEjK/JQGtSXVPevplTaOU3VJA/rseBC1QvYUz9lLDIMBimc4SW/zrW4S89BaCA==} + engines: {node: '>= 18.19.0'} + peerDependencies: + '@node-rs/xxhash': ^1.1.0 + '@opentelemetry/api': '>=1 <2' + peerDependenciesMeta: + '@node-rs/xxhash': + optional: true + '@opentelemetry/api': + optional: true + + '@redis/json@5.12.1': + resolution: {integrity: sha512-eOze75esLve4vfqDel7aMX08CNaiLLQS2fV8mpRN9NxPe1rVR4vQyYiW/OgtGUysF6QOr9ANhfxABKNOJfXdKg==} + engines: {node: '>= 18.19.0'} + peerDependencies: + '@redis/client': ^5.12.1 + + '@redis/search@5.12.1': + resolution: {integrity: sha512-ItlxbxC9cKI6IU1TLWoczwJCRb6TdmkEpWv05UrPawqaAnWGRu3rcIqsc5vN483T2fSociuyV1UkWIL5I4//2w==} + engines: {node: '>= 18.19.0'} + peerDependencies: + '@redis/client': ^5.12.1 + + '@redis/time-series@5.12.1': + resolution: {integrity: sha512-c6JL6E3EcZJuNqKFz+KM+l9l5mpcQiKvTwgA3blt5glWJ8hjDk0yeHN3beE/MpqYIQ8UEX44ItQzgkE/gCBELQ==} + engines: {node: '>= 18.19.0'} + peerDependencies: + '@redis/client': ^5.12.1 + '@remirror/core-constants@0.7.4': resolution: {integrity: sha512-3fd7uzg3s9CiDmpSP8tZdhBxpIO5DGvxPSDSobZ7iJ6VgsQgw80dTmgK71eFh8l4Dx4QdPYqx9onrtXLjbIrdw==} @@ -3586,6 +3541,9 @@ packages: '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} + '@stepwisehq/prosemirror-collab-commit@1.0.5': + resolution: {integrity: sha512-zuKaN4oSa5d7E7R2XTVK1vDEhq4yIW8k8eU7kv9zMqrUR5WvJsgkw58uAwVhRBDhDqPo+IUwxz4PwMGCelWKHA==} + '@storybook/addon-knobs@6.4.0': resolution: {integrity: sha512-DiH1/5e2AFHoHrncl1qLu18ZHPHzRMMPvOLFz8AWvvmc+VCqTdIaE+tdxKr3e8rYylKllibgvDOzrLjfTNjF+Q==} deprecated: deprecating @storybook/addon-knobs in favor of @storybook/addon-controls @@ -5463,6 +5421,10 @@ packages: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} engines: {node: '>=6'} + cluster-key-slot@1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + code-block-writer@12.0.0: resolution: {integrity: sha512-q4dMFMlXtKR3XNBHyMHt/3pwYNA69EDk00lloMOaaUMKPUXBw6lpXtbu3MMVG6/uOihGnRDOlkyqsONEUj60+w==} @@ -5660,10 +5622,6 @@ packages: core-js@3.48.0: resolution: {integrity: sha512-zpEHTy1fjTMZCKLHUZoVeylt9XrzaIN2rbPXEt0k+q7JE5CkCZdo6bNq55bn24a69CH7ErAVLKijxJja4fw+UQ==} - core-js@3.6.5: - resolution: {integrity: sha512-vZVEEwZoIsI+vPEuoF9Iqf5H7/M3eeQqWlQnYa8FSKKePuYTf5MWnxb5SDAzCa60b3JBRS5g9b+Dq7b1y/RCrA==} - deprecated: core-js@<3.23.3 is no longer maintained and not recommended for usage due to the number of issues. Because of the V8 engine whims, feature detection in old core-js versions could cause a slowdown up to 100x even if nothing is polyfilled. Some versions have web compatibility issues. Please, upgrade your dependencies to the actual version of core-js. - core-util-is@1.0.2: resolution: {integrity: sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==} @@ -6159,9 +6117,6 @@ packages: dom-serializer@2.0.0: resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==} - dom-storage@2.1.0: - resolution: {integrity: sha512-g6RpyWXzl0RR6OTElHKBl7nwnK87GUyZMYC7JWsB/IA73vpqK2K6LT39x4VepLxlSsWBFrPVLnsSR5Jyty0+2Q==} - dom-walk@0.1.2: resolution: {integrity: sha512-6QvTW9mrGeIegrFXdtQi9pk7O/nSK6lSdXW2eqUspN5LWD7UTji2Fqw5V2YLjBpHEoU9Xl/eUWNpDeZvoyOv2w==} @@ -6644,10 +6599,6 @@ packages: fastq@1.20.1: resolution: {integrity: sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==} - faye-websocket@0.11.3: - resolution: {integrity: sha512-D2y4bovYpzziGgbHYtGCMjlJM36vAl/y+xUyn1C+FVx8szd1E+86KwVw6XvYSzOP8iMpm1X0I4xJD+QtUb36OA==} - engines: {node: '>=0.8.0'} - faye-websocket@0.11.4: resolution: {integrity: sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g==} engines: {node: '>=0.8.0'} @@ -6759,10 +6710,6 @@ packages: resolution: {integrity: sha512-AtA7OH5RbIFGoc0gZOQgaYC6cdjdhZv4w3XgWoupkPKO1HY+0GzixOuXDa75kFeoVyhIyo4PkLg/GAC1dC1P6w==} engines: {node: '>=10.13.0'} - firebase@7.24.0: - resolution: {integrity: sha512-j6jIyGFFBlwWAmrlUg9HyQ/x+YpsPkc/TTkbTyeLwwAJrpAmmEHNPT6O9xtAnMV4g7d3RqLL/u9//aZlbY4rQA==} - engines: {node: ^8.13.0 || >=10.10.0} - flat-cache@3.2.0: resolution: {integrity: sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==} engines: {node: ^10.12.0 || >=12.0.0} @@ -7406,9 +7353,6 @@ packages: resolution: {integrity: sha512-4aFq7wvWyMHKgxsH8QQtGpvbASCf+eM3wPRLI6R+MgAnTCZ6STYsRvttLvRWK0Nfif5piF394St3HeJDaljGPA==} engines: {node: '>= 6'} - idb@3.0.2: - resolution: {integrity: sha512-+FLa/0sTXqyux0o6C+i2lOR0VoS60LU/jzUo5xjfY6+7sEEgy4Gz1O7yFBXvjd7N0NyIGWIRg8DcQSLEG+VSPw==} - idb@7.1.1: resolution: {integrity: sha512-gchesWBzyvGHRO9W8tzUWFDycow5gwjvFKfyV9FF32Y7F50yZMp7mP+T2mJIWFx49zicqyC4uefHM17o6xKIVQ==} @@ -8830,10 +8774,6 @@ packages: resolution: {integrity: sha512-tmPX422rYgofd4epzrNoOXiE8XFZYOcCq1vD7MAXCDO+O+zndlA2ztdKKMa+EeuBG5tHETpr4ml4RGgpqDCCAg==} engines: {node: '>= 0.10.5'} - node-fetch@2.6.1: - resolution: {integrity: sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==} - engines: {node: 4.x || >=6.0.0} - node-fetch@2.6.13: resolution: {integrity: sha512-StxNAxh15zr77QvvkmveSQ8uCQ4+v5FkvNTj0OESmiHu+VRi/gXArXtkWMElOsOUNLtUEvI4yS+rdtOHZTwlQA==} engines: {node: 4.x || >=6.0.0} @@ -9561,9 +9501,6 @@ packages: bluebird: optional: true - promise-polyfill@8.1.3: - resolution: {integrity: sha512-MG5r82wBzh7pSKDRa9y+vllNHz3e3d4CNj1PQE4BQYxLme0gKYYBm9YENq+UkEikyZ0XbiGWxYlVw3Rl9O/U8g==} - promise.allsettled@1.0.7: resolution: {integrity: sha512-hezvKvQQmsFkOdrZfYxUxkyxl8mgFQeT259Ajj9PXdbg9VzBCWrItOev72JyWxkCD5VSSqAeHmlN3tWx4DlmsA==} engines: {node: '>= 0.4'} @@ -9593,9 +9530,6 @@ packages: prosemirror-changeset@2.4.0: resolution: {integrity: sha512-LvqH2v7Q2SF6yxatuPP2e8vSUKS/L+xAU7dPDC4RMyHMhZoGDfBC74mYuyYF4gLqOEG758wajtyhNnsTkuhvng==} - prosemirror-collab@1.3.1: - resolution: {integrity: sha512-4SnynYR9TTYaQVXd/ieUvsVV4PDMBzrq2xPUWutHivDuOshZXqQ5rGbZM84HEaXKbLdItse7weMGOUdDVcLKEQ==} - prosemirror-commands@1.7.1: resolution: {integrity: sha512-rT7qZnQtx5c0/y/KlYaGvtG411S97UaL6gdp6RIZ23DLHanMYLyfGBV5DtSnZdthQql7W+lEVbpSfwtO8T+L2w==} @@ -9830,6 +9764,11 @@ packages: peerDependencies: react: ^16.14.0 + react-dom@19.2.7: + resolution: {integrity: sha512-t0BRVXvbiE/o20Hfw669rLbMCDWtYZLvmJigy2f0MxsXF+71pxhR3xOkspmsO8h3ZlNzyibAmtCa3l4lYKk6gQ==} + peerDependencies: + react: ^19.2.7 + react-dropzone@10.2.2: resolution: {integrity: sha512-U5EKckXVt6IrEyhMMsgmHQiWTGLudhajPPG77KFSvgsMqNEHSyGpqWvOMc5+DhEah/vH4E1n+J5weBNLd5VtyA==} engines: {node: '>= 8'} @@ -9888,6 +9827,12 @@ packages: peerDependencies: react: ^16.0.0 + react-reconciler@0.32.0: + resolution: {integrity: sha512-2NPMOzgTlG0ZWdIf3qG+dcbLSoAc/uLfOwckc3ofy5sSK0pLJqnQLpUFxvGcN2rlXSjnVtGeeFLNimCQEj5gOQ==} + engines: {node: '>=0.10.0'} + peerDependencies: + react: ^19.1.0 + react-redux@7.2.9: resolution: {integrity: sha512-Gx4L3uM182jEEayZfRbI/G11ZpYdNAnBs70lFVMNdHJI76XYtR+7m0MN+eAs7UHBPhWXcnFPaS+9owSCJQHNpQ==} peerDependencies: @@ -9968,6 +9913,10 @@ packages: resolution: {integrity: sha512-0X2CImDkJGApiAlcf0ODKIneSwBPhqJawOa5wCtKbu7ZECrmS26NvtSILynQ66cgkT/RJ4LidJOc3bUESwmU8g==} engines: {node: '>=0.10.0'} + react@19.2.7: + resolution: {integrity: sha512-HNe9WslTbXmFK8o8cmwgAeJFSBvt1bPdHCVKtaaV+WlAN36mpT4hcRpwbf3fY56ar2oIXzsBpOAiIRHAdY0OlQ==} + engines: {node: '>=0.10.0'} + reactcss@1.2.3: resolution: {integrity: sha512-KiwVUcFu1RErkI97ywr8nvx8dNOpT03rbnma0SSalTYjkrPYaEajR4a/MRt6DZ46K6arDRbWMNHF+xH7G7n/8A==} peerDependencies: @@ -10068,6 +10017,10 @@ packages: resolution: {integrity: sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg==} engines: {node: '>=8'} + redis@5.12.1: + resolution: {integrity: sha512-LDsoVvb/CpoV9EN3FXvgvSHNJWuCIzl9MiO3ppOevuGLpSGJhwfQjpEwfFJcQvNSddHADDdZaWx0HnmMxRXG7g==} + engines: {node: '>= 18.19.0'} + redux@4.2.1: resolution: {integrity: sha512-LAUYz4lc+Do8/g7aeRa8JkyDErK6ekstQaqWQrNRW//MY1TvCEpMtpTWvlQ+FPbWCx+Xixu/6SHt5N0HR+SB4w==} @@ -10358,6 +10311,12 @@ packages: scheduler@0.19.1: resolution: {integrity: sha512-n/zwRWRYSUj0/3g/otKDRPMh6qv2SYMWNq85IEa8iZyAv8od9zDYpGSnpBEjNgcMNq6Scbu5KfIPxNF72R/2EA==} + scheduler@0.26.0: + resolution: {integrity: sha512-NlHwttCI/l5gCPR3D1nNXtWABUmBwvZpEQiD4IXSbIDq8BzLIK/7Ir5gTFSGZDUu37K5cMNp0hFtzO38sC7gWA==} + + scheduler@0.27.0: + resolution: {integrity: sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==} + schema-utils@1.0.0: resolution: {integrity: sha512-i27Mic4KovM/lnGsy8whRCHhc7VicJajAjTrYg11K9zfZXnYIt4k5F+kZkwjnrhKzLic/HLU4j11mjsz2G/75g==} engines: {node: '>= 4'} @@ -11836,9 +11795,6 @@ packages: resolution: {integrity: sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==} engines: {node: '>=0.8.0'} - whatwg-fetch@2.0.4: - resolution: {integrity: sha512-dcQ1GWpOD/eEQ97k66aiEVpNnapVj90/+R+SXTPYGHpYBBypfKJEQjLrvMZ7YXbKm21gXd4NcuxUTjiv1YtLng==} - whatwg-url@5.0.0: resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} @@ -11993,10 +11949,6 @@ packages: xmlchars@2.2.0: resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==} - xmlhttprequest@1.8.0: - resolution: {integrity: sha512-58Im/U0mlVBLM38NdZjHyhuMtCqa61469k2YP/AaPbvCoV9aQGUpbJBj1QRm2ytRiVQBD/fsw7L2bJGDVQswBA==} - engines: {node: '>=0.4.0'} - xtend@4.0.2: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} @@ -14096,19 +14048,6 @@ snapshots: '@eslint/js@8.57.1': {} - '@firebase/analytics-types@0.4.0': {} - - '@firebase/analytics@0.6.0(@firebase/app-types@0.6.1)(@firebase/app@0.6.11)': - dependencies: - '@firebase/analytics-types': 0.4.0 - '@firebase/app': 0.6.11 - '@firebase/app-types': 0.6.1 - '@firebase/component': 0.1.19 - '@firebase/installations': 0.4.17(@firebase/app-types@0.6.1)(@firebase/app@0.6.11) - '@firebase/logger': 0.2.6 - '@firebase/util': 0.3.2 - tslib: 1.14.1 - '@firebase/app-compat@0.5.8': dependencies: '@firebase/app': 0.14.8 @@ -14117,8 +14056,6 @@ snapshots: '@firebase/util': 1.13.0 tslib: 2.8.1 - '@firebase/app-types@0.6.1': {} - '@firebase/app-types@0.6.3': {} '@firebase/app-types@0.7.0': {} @@ -14131,44 +14068,11 @@ snapshots: idb: 7.1.1 tslib: 2.8.1 - '@firebase/app@0.6.11': + '@firebase/auth-interop-types@0.1.6(@firebase/app-types@0.7.0)(@firebase/util@1.5.2)': dependencies: - '@firebase/app-types': 0.6.1 - '@firebase/component': 0.1.19 - '@firebase/logger': 0.2.6 - '@firebase/util': 0.3.2 - dom-storage: 2.1.0 - tslib: 1.14.1 - xmlhttprequest: 1.8.0 - - '@firebase/auth-interop-types@0.1.5(@firebase/app-types@0.6.1)(@firebase/util@0.3.2)': - dependencies: - '@firebase/app-types': 0.6.1 - '@firebase/util': 0.3.2 - - '@firebase/auth-interop-types@0.1.6(@firebase/app-types@0.6.1)(@firebase/util@1.5.2)': - dependencies: - '@firebase/app-types': 0.6.1 + '@firebase/app-types': 0.7.0 '@firebase/util': 1.5.2 - '@firebase/auth-types@0.10.1(@firebase/app-types@0.6.1)(@firebase/util@0.3.2)': - dependencies: - '@firebase/app-types': 0.6.1 - '@firebase/util': 0.3.2 - - '@firebase/auth@0.15.0(@firebase/app-types@0.6.1)(@firebase/app@0.6.11)(@firebase/util@0.3.2)': - dependencies: - '@firebase/app': 0.6.11 - '@firebase/auth-types': 0.10.1(@firebase/app-types@0.6.1)(@firebase/util@0.3.2) - transitivePeerDependencies: - - '@firebase/app-types' - - '@firebase/util' - - '@firebase/component@0.1.19': - dependencies: - '@firebase/util': 0.3.2 - tslib: 1.14.1 - '@firebase/component@0.5.13': dependencies: '@firebase/util': 1.5.2 @@ -14179,11 +14083,11 @@ snapshots: '@firebase/util': 1.13.0 tslib: 2.8.1 - '@firebase/database-compat@0.1.8(@firebase/app-compat@0.5.8)(@firebase/app-types@0.6.1)': + '@firebase/database-compat@0.1.8(@firebase/app-compat@0.5.8)(@firebase/app-types@0.7.0)': dependencies: '@firebase/app-compat': 0.5.8 '@firebase/component': 0.5.13 - '@firebase/database': 0.12.8(@firebase/app-types@0.6.1) + '@firebase/database': 0.12.8(@firebase/app-types@0.7.0) '@firebase/database-types': 0.9.7 '@firebase/logger': 0.3.2 '@firebase/util': 1.5.2 @@ -14191,10 +14095,6 @@ snapshots: transitivePeerDependencies: - '@firebase/app-types' - '@firebase/database-types@0.5.2': - dependencies: - '@firebase/app-types': 0.6.1 - '@firebase/database-types@0.7.3': dependencies: '@firebase/app-types': 0.6.3 @@ -14204,9 +14104,9 @@ snapshots: '@firebase/app-types': 0.7.0 '@firebase/util': 1.5.2 - '@firebase/database@0.12.8(@firebase/app-types@0.6.1)': + '@firebase/database@0.12.8(@firebase/app-types@0.7.0)': dependencies: - '@firebase/auth-interop-types': 0.1.6(@firebase/app-types@0.6.1)(@firebase/util@1.5.2) + '@firebase/auth-interop-types': 0.1.6(@firebase/app-types@0.7.0)(@firebase/util@1.5.2) '@firebase/component': 0.5.13 '@firebase/logger': 0.3.2 '@firebase/util': 1.5.2 @@ -14215,64 +14115,6 @@ snapshots: transitivePeerDependencies: - '@firebase/app-types' - '@firebase/database@0.6.13(@firebase/app-types@0.6.1)': - dependencies: - '@firebase/auth-interop-types': 0.1.5(@firebase/app-types@0.6.1)(@firebase/util@0.3.2) - '@firebase/component': 0.1.19 - '@firebase/database-types': 0.5.2 - '@firebase/logger': 0.2.6 - '@firebase/util': 0.3.2 - faye-websocket: 0.11.3 - tslib: 1.14.1 - transitivePeerDependencies: - - '@firebase/app-types' - - '@firebase/firestore-types@1.14.0(@firebase/app-types@0.6.1)': - dependencies: - '@firebase/app-types': 0.6.1 - - '@firebase/firestore@1.18.0(@firebase/app-types@0.6.1)(@firebase/app@0.6.11)': - dependencies: - '@firebase/app': 0.6.11 - '@firebase/app-types': 0.6.1 - '@firebase/component': 0.1.19 - '@firebase/firestore-types': 1.14.0(@firebase/app-types@0.6.1) - '@firebase/logger': 0.2.6 - '@firebase/util': 0.3.2 - '@firebase/webchannel-wrapper': 0.4.0 - '@grpc/grpc-js': 1.14.3 - '@grpc/proto-loader': 0.5.6 - node-fetch: 2.6.1 - tslib: 1.14.1 - - '@firebase/functions-types@0.3.17': {} - - '@firebase/functions@0.5.1(@firebase/app-types@0.6.1)(@firebase/app@0.6.11)': - dependencies: - '@firebase/app': 0.6.11 - '@firebase/app-types': 0.6.1 - '@firebase/component': 0.1.19 - '@firebase/functions-types': 0.3.17 - '@firebase/messaging-types': 0.5.0(@firebase/app-types@0.6.1) - node-fetch: 2.6.1 - tslib: 1.14.1 - - '@firebase/installations-types@0.3.4(@firebase/app-types@0.6.1)': - dependencies: - '@firebase/app-types': 0.6.1 - - '@firebase/installations@0.4.17(@firebase/app-types@0.6.1)(@firebase/app@0.6.11)': - dependencies: - '@firebase/app': 0.6.11 - '@firebase/app-types': 0.6.1 - '@firebase/component': 0.1.19 - '@firebase/installations-types': 0.3.4(@firebase/app-types@0.6.1) - '@firebase/util': 0.3.2 - idb: 3.0.2 - tslib: 1.14.1 - - '@firebase/logger@0.2.6': {} - '@firebase/logger@0.3.2': dependencies: tslib: 2.8.1 @@ -14281,71 +14123,6 @@ snapshots: dependencies: tslib: 2.8.1 - '@firebase/messaging-types@0.5.0(@firebase/app-types@0.6.1)': - dependencies: - '@firebase/app-types': 0.6.1 - - '@firebase/messaging@0.7.1(@firebase/app-types@0.6.1)(@firebase/app@0.6.11)': - dependencies: - '@firebase/app': 0.6.11 - '@firebase/app-types': 0.6.1 - '@firebase/component': 0.1.19 - '@firebase/installations': 0.4.17(@firebase/app-types@0.6.1)(@firebase/app@0.6.11) - '@firebase/messaging-types': 0.5.0(@firebase/app-types@0.6.1) - '@firebase/util': 0.3.2 - idb: 3.0.2 - tslib: 1.14.1 - - '@firebase/performance-types@0.0.13': {} - - '@firebase/performance@0.4.2(@firebase/app-types@0.6.1)(@firebase/app@0.6.11)': - dependencies: - '@firebase/app': 0.6.11 - '@firebase/app-types': 0.6.1 - '@firebase/component': 0.1.19 - '@firebase/installations': 0.4.17(@firebase/app-types@0.6.1)(@firebase/app@0.6.11) - '@firebase/logger': 0.2.6 - '@firebase/performance-types': 0.0.13 - '@firebase/util': 0.3.2 - tslib: 1.14.1 - - '@firebase/polyfill@0.3.36': - dependencies: - core-js: 3.6.5 - promise-polyfill: 8.1.3 - whatwg-fetch: 2.0.4 - - '@firebase/remote-config-types@0.1.9': {} - - '@firebase/remote-config@0.1.28(@firebase/app-types@0.6.1)(@firebase/app@0.6.11)': - dependencies: - '@firebase/app': 0.6.11 - '@firebase/app-types': 0.6.1 - '@firebase/component': 0.1.19 - '@firebase/installations': 0.4.17(@firebase/app-types@0.6.1)(@firebase/app@0.6.11) - '@firebase/logger': 0.2.6 - '@firebase/remote-config-types': 0.1.9 - '@firebase/util': 0.3.2 - tslib: 1.14.1 - - '@firebase/storage-types@0.3.13(@firebase/app-types@0.6.1)(@firebase/util@0.3.2)': - dependencies: - '@firebase/app-types': 0.6.1 - '@firebase/util': 0.3.2 - - '@firebase/storage@0.3.43(@firebase/app-types@0.6.1)(@firebase/app@0.6.11)': - dependencies: - '@firebase/app': 0.6.11 - '@firebase/app-types': 0.6.1 - '@firebase/component': 0.1.19 - '@firebase/storage-types': 0.3.13(@firebase/app-types@0.6.1)(@firebase/util@0.3.2) - '@firebase/util': 0.3.2 - tslib: 1.14.1 - - '@firebase/util@0.3.2': - dependencies: - tslib: 1.14.1 - '@firebase/util@1.13.0': dependencies: tslib: 2.8.1 @@ -14354,8 +14131,6 @@ snapshots: dependencies: tslib: 2.8.1 - '@firebase/webchannel-wrapper@0.4.0': {} - '@gar/promisify@1.1.3': {} '@google-cloud/firestore@4.15.1': @@ -14441,22 +14216,12 @@ snapshots: - encoding - supports-color - '@grpc/grpc-js@1.14.3': - dependencies: - '@grpc/proto-loader': 0.8.0 - '@js-sdsl/ordered-map': 4.4.2 - '@grpc/grpc-js@1.6.12': dependencies: '@grpc/proto-loader': 0.7.15 '@types/node': 24.11.0 optional: true - '@grpc/proto-loader@0.5.6': - dependencies: - lodash.camelcase: 4.3.0 - protobufjs: 6.11.4 - '@grpc/proto-loader@0.6.13': dependencies: '@types/long': 4.0.2 @@ -14474,12 +14239,15 @@ snapshots: yargs: 17.7.2 optional: true - '@grpc/proto-loader@0.8.0': + '@handlewithcare/react-prosemirror@3.1.6(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-view@1.41.6)(react-dom@19.2.7(react@19.2.7))(react-reconciler@0.32.0(react@19.2.7))(react@19.2.7)': dependencies: - lodash.camelcase: 4.3.0 - long: 5.3.2 - protobufjs: 7.5.4 - yargs: 17.7.2 + classnames: 2.5.1 + prosemirror-model: 1.25.4 + prosemirror-state: 1.4.4 + prosemirror-view: 1.41.6 + react: 19.2.7 + react-dom: 19.2.7(react@19.2.7) + react-reconciler: 0.32.0(react@19.2.7) '@humanwhocodes/config-array@0.13.0': dependencies: @@ -14544,8 +14312,6 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 - '@js-sdsl/ordered-map@4.4.2': {} - '@jsep-plugin/assignment@1.3.0(jsep@1.4.0)': dependencies: jsep: 1.4.0 @@ -14919,6 +14685,51 @@ snapshots: dependencies: pako: 1.0.11 + '@pitter-patter/collab-client@0.1.0(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0)': + dependencies: + '@stepwisehq/prosemirror-collab-commit': 1.0.5 + prosemirror-model: 1.25.4 + prosemirror-state: 1.4.4 + prosemirror-transform: 1.11.0 + + '@pitter-patter/collab-server@0.1.0(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0)': + dependencies: + '@stepwisehq/prosemirror-collab-commit': 1.0.5 + prosemirror-model: 1.25.4 + prosemirror-state: 1.4.4 + prosemirror-transform: 1.11.0 + redis: 5.12.1 + transitivePeerDependencies: + - '@node-rs/xxhash' + - '@opentelemetry/api' + + '@pitter-patter/presence-client@0.1.0(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0)(prosemirror-view@1.41.6)': + dependencies: + '@handlewithcare/react-prosemirror': 3.1.6(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-view@1.41.6)(react-dom@19.2.7(react@19.2.7))(react-reconciler@0.32.0(react@19.2.7))(react@19.2.7) + '@pitter-patter/collab-client': 0.1.0(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0) + '@pitter-patter/refs': 0.1.0 + classnames: 2.5.1 + prosemirror-model: 1.25.4 + prosemirror-state: 1.4.4 + prosemirror-transform: 1.11.0 + prosemirror-view: 1.41.6 + react: 19.2.7 + react-dom: 19.2.7(react@19.2.7) + react-reconciler: 0.32.0(react@19.2.7) + transitivePeerDependencies: + - '@tiptap/core' + - '@tiptap/pm' + - '@tiptap/react' + + '@pitter-patter/presence-server@0.1.0': + dependencies: + redis: 5.12.1 + transitivePeerDependencies: + - '@node-rs/xxhash' + - '@opentelemetry/api' + + '@pitter-patter/refs@0.1.0': {} + '@pkgjs/parseargs@0.11.0': optional: true @@ -14941,30 +14752,40 @@ snapshots: '@popperjs/core@2.11.8': {} - '@protobufjs/aspromise@1.1.2': {} + '@protobufjs/aspromise@1.1.2': + optional: true - '@protobufjs/base64@1.1.2': {} + '@protobufjs/base64@1.1.2': + optional: true - '@protobufjs/codegen@2.0.4': {} + '@protobufjs/codegen@2.0.4': + optional: true - '@protobufjs/eventemitter@1.1.0': {} + '@protobufjs/eventemitter@1.1.0': + optional: true '@protobufjs/fetch@1.1.0': dependencies: '@protobufjs/aspromise': 1.1.2 '@protobufjs/inquire': 1.1.0 + optional: true - '@protobufjs/float@1.0.2': {} + '@protobufjs/float@1.0.2': + optional: true - '@protobufjs/inquire@1.1.0': {} + '@protobufjs/inquire@1.1.0': + optional: true - '@protobufjs/path@1.1.2': {} + '@protobufjs/path@1.1.2': + optional: true - '@protobufjs/pool@1.1.0': {} + '@protobufjs/pool@1.1.0': + optional: true - '@protobufjs/utf8@1.1.0': {} + '@protobufjs/utf8@1.1.0': + optional: true - '@pubpub/deposit-utils@0.1.10': + '@pubpub/deposit-utils@0.1.10(patch_hash=6fef8933046b7751abda1bd2ed0422094c79d7828b07f7d5afba26a47c696d89)': dependencies: validate-with-xmllint: 1.2.1 @@ -14976,6 +14797,26 @@ snapshots: prosemirror-state: 1.4.4 prosemirror-view: 1.41.6 + '@redis/bloom@5.12.1(@redis/client@5.12.1)': + dependencies: + '@redis/client': 5.12.1 + + '@redis/client@5.12.1': + dependencies: + cluster-key-slot: 1.1.2 + + '@redis/json@5.12.1(@redis/client@5.12.1)': + dependencies: + '@redis/client': 5.12.1 + + '@redis/search@5.12.1(@redis/client@5.12.1)': + dependencies: + '@redis/client': 5.12.1 + + '@redis/time-series@5.12.1(@redis/client@5.12.1)': + dependencies: + '@redis/client': 5.12.1 + '@remirror/core-constants@0.7.4': dependencies: '@babel/runtime': 7.28.6 @@ -15865,6 +15706,10 @@ snapshots: '@standard-schema/spec@1.1.0': {} + '@stepwisehq/prosemirror-collab-commit@1.0.5': + dependencies: + prosemirror-state: 1.4.4 + '@storybook/addon-knobs@6.4.0(@storybook/addons@6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0))(@storybook/api@6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0))(@storybook/components@6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0))(@storybook/core-events@6.5.16)(@storybook/theming@6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0))(react-dom@16.14.0(react@16.14.0))(react@16.14.0)': dependencies: '@storybook/addons': 6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0) @@ -16885,7 +16730,8 @@ snapshots: '@types/lodash@4.17.23': {} - '@types/long@4.0.2': {} + '@types/long@4.0.2': + optional: true '@types/mdast@3.0.15': dependencies: @@ -18567,6 +18413,8 @@ snapshots: clsx@2.1.1: {} + cluster-key-slot@1.1.2: {} + code-block-writer@12.0.0: {} code-point-at@1.1.0: {} @@ -18775,8 +18623,6 @@ snapshots: core-js@3.48.0: {} - core-js@3.6.5: {} - core-util-is@1.0.2: {} core-util-is@1.0.3: {} @@ -19341,8 +19187,6 @@ snapshots: domhandler: 5.0.3 entities: 4.5.0 - dom-storage@2.1.0: {} - dom-walk@0.1.2: {} dom4@2.1.6: {} @@ -20103,10 +19947,6 @@ snapshots: dependencies: reusify: 1.1.0 - faye-websocket@0.11.3: - dependencies: - websocket-driver: 0.7.4 - faye-websocket@0.11.4: dependencies: websocket-driver: 0.7.4 @@ -20227,9 +20067,9 @@ snapshots: transitivePeerDependencies: - supports-color - firebase-admin@9.12.0(@firebase/app-compat@0.5.8)(@firebase/app-types@0.6.1): + firebase-admin@9.12.0(@firebase/app-compat@0.5.8)(@firebase/app-types@0.7.0): dependencies: - '@firebase/database-compat': 0.1.8(@firebase/app-compat@0.5.8)(@firebase/app-types@0.6.1) + '@firebase/database-compat': 0.1.8(@firebase/app-compat@0.5.8)(@firebase/app-types@0.7.0) '@firebase/database-types': 0.7.3 '@types/node': 24.11.0 dicer: 0.3.1 @@ -20245,23 +20085,6 @@ snapshots: - encoding - supports-color - firebase@7.24.0: - dependencies: - '@firebase/analytics': 0.6.0(@firebase/app-types@0.6.1)(@firebase/app@0.6.11) - '@firebase/app': 0.6.11 - '@firebase/app-types': 0.6.1 - '@firebase/auth': 0.15.0(@firebase/app-types@0.6.1)(@firebase/app@0.6.11)(@firebase/util@0.3.2) - '@firebase/database': 0.6.13(@firebase/app-types@0.6.1) - '@firebase/firestore': 1.18.0(@firebase/app-types@0.6.1)(@firebase/app@0.6.11) - '@firebase/functions': 0.5.1(@firebase/app-types@0.6.1)(@firebase/app@0.6.11) - '@firebase/installations': 0.4.17(@firebase/app-types@0.6.1)(@firebase/app@0.6.11) - '@firebase/messaging': 0.7.1(@firebase/app-types@0.6.1)(@firebase/app@0.6.11) - '@firebase/performance': 0.4.2(@firebase/app-types@0.6.1)(@firebase/app@0.6.11) - '@firebase/polyfill': 0.3.36 - '@firebase/remote-config': 0.1.28(@firebase/app-types@0.6.1)(@firebase/app@0.6.11) - '@firebase/storage': 0.3.43(@firebase/app-types@0.6.1)(@firebase/app@0.6.11) - '@firebase/util': 0.3.2 - flat-cache@3.2.0: dependencies: flatted: 3.3.3 @@ -21150,8 +20973,6 @@ snapshots: dependencies: postcss: 7.0.39 - idb@3.0.2: {} - idb@7.1.1: {} ieee754@1.2.1: {} @@ -21976,7 +21797,8 @@ snapshots: lodash-es@4.17.23: {} - lodash.camelcase@4.3.0: {} + lodash.camelcase@4.3.0: + optional: true lodash.clonedeep@4.5.0: {} @@ -22036,9 +21858,11 @@ snapshots: dependencies: '@sinonjs/commons': 1.8.6 - long@4.0.0: {} + long@4.0.0: + optional: true - long@5.3.2: {} + long@5.3.2: + optional: true loose-envify@1.4.0: dependencies: @@ -22597,8 +22421,6 @@ snapshots: dependencies: minimatch: 3.1.2 - node-fetch@2.6.1: {} - node-fetch@2.6.13: dependencies: whatwg-url: 5.0.0 @@ -23359,8 +23181,6 @@ snapshots: optionalDependencies: bluebird: 3.7.2 - promise-polyfill@8.1.3: {} - promise.allsettled@1.0.7: dependencies: array.prototype.map: 1.0.8 @@ -23414,10 +23234,6 @@ snapshots: dependencies: prosemirror-transform: 1.11.0 - prosemirror-collab@1.3.1: - dependencies: - prosemirror-state: 1.4.4 - prosemirror-commands@1.7.1: dependencies: prosemirror-model: 1.25.4 @@ -23572,6 +23388,7 @@ snapshots: '@types/long': 4.0.2 '@types/node': 24.11.0 long: 4.0.0 + optional: true protobufjs@7.5.4: dependencies: @@ -23587,6 +23404,7 @@ snapshots: '@protobufjs/utf8': 1.1.0 '@types/node': 24.11.0 long: 5.3.2 + optional: true proxy-addr@2.0.7: dependencies: @@ -23807,6 +23625,11 @@ snapshots: react: 16.14.0 scheduler: 0.19.1 + react-dom@19.2.7(react@19.2.7): + dependencies: + react: 19.2.7 + scheduler: 0.27.0 + react-dropzone@10.2.2(react@16.14.0): dependencies: attr-accept: 2.2.5 @@ -23876,6 +23699,11 @@ snapshots: react: 16.14.0 scheduler: 0.18.0 + react-reconciler@0.32.0(react@19.2.7): + dependencies: + react: 19.2.7 + scheduler: 0.26.0 + react-redux@7.2.9(react-dom@16.14.0(react@16.14.0))(react@16.14.0): dependencies: '@babel/runtime': 7.28.6 @@ -23991,6 +23819,8 @@ snapshots: object-assign: 4.1.1 prop-types: 15.8.1 + react@19.2.7: {} + reactcss@1.2.3(react@16.14.0): dependencies: lodash: 4.17.23 @@ -24142,6 +23972,17 @@ snapshots: indent-string: 4.0.0 strip-indent: 3.0.0 + redis@5.12.1: + dependencies: + '@redis/bloom': 5.12.1(@redis/client@5.12.1) + '@redis/client': 5.12.1 + '@redis/json': 5.12.1(@redis/client@5.12.1) + '@redis/search': 5.12.1(@redis/client@5.12.1) + '@redis/time-series': 5.12.1(@redis/client@5.12.1) + transitivePeerDependencies: + - '@node-rs/xxhash' + - '@opentelemetry/api' + redux@4.2.1: dependencies: '@babel/runtime': 7.28.6 @@ -24535,6 +24376,10 @@ snapshots: loose-envify: 1.4.0 object-assign: 4.1.1 + scheduler@0.26.0: {} + + scheduler@0.27.0: {} + schema-utils@1.0.0: dependencies: ajv: 6.12.6 @@ -26306,8 +26151,6 @@ snapshots: websocket-extensions@0.1.4: {} - whatwg-fetch@2.0.4: {} - whatwg-url@5.0.0: dependencies: tr46: 0.0.3 @@ -26483,8 +26326,6 @@ snapshots: xmlchars@2.2.0: {} - xmlhttprequest@1.8.0: {} - xtend@4.0.2: {} y18n@4.0.3: {} diff --git a/server/apiRoutes.ts b/server/apiRoutes.ts index 727d9d7aad..e99d7a2cf5 100644 --- a/server/apiRoutes.ts +++ b/server/apiRoutes.ts @@ -7,6 +7,8 @@ import { router as analyticsImpactRouter } from './analytics/impactApi'; import { router as apiDocsRouter } from './apiDocs/api'; import { router as captchaRouter } from './captcha/api'; import { router as citationRouter } from './citation/api'; +import { router as collabRouter } from './collab/api'; +import { router as collabDiscussionPositionsRouter } from './collab/discussionPositions'; import { router as communityBanRouter } from './communityBan/api'; import { router as communityServicesRouter } from './communityServices/api'; import { router as communityTemplateRouter } from './communityTemplate/api'; @@ -50,6 +52,8 @@ const apiRouter = Router() .use(activityItemRouter) .use(captchaRouter) .use(citationRouter) + .use(collabRouter) + .use(collabDiscussionPositionsRouter) .use(communityServicesRouter) .use(customScriptRouter) .use(discussionRouter) diff --git a/server/collab/api.ts b/server/collab/api.ts new file mode 100644 index 0000000000..1a17686cb8 --- /dev/null +++ b/server/collab/api.ts @@ -0,0 +1,95 @@ +import type { PresenceIndicator } from '@pitter-patter/presence-server'; + +import { TooMuchContentionError } from '@pitter-patter/collab-server'; +import { Router } from 'express'; + +import { Draft, Pub } from 'server/models'; +import { wrap } from 'server/wrap'; + +import { collabAuthority } from './authority'; +import { presenceAuthority } from './presence'; + +export const router = Router(); + +const getDraftIdForPub = async (pubId: string): Promise => { + const pub = await Pub.findOne({ + where: { id: pubId }, + include: [{ model: Draft, as: 'draft' }], + attributes: ['id'], + }); + + return pub?.draft?.id ?? null; +}; + +// receive a commit from a client +router.post( + '/api/pubs/:pubId/commits', + wrap(async (req, res) => { + const draftId = await getDraftIdForPub(req.params.pubId); + + if (!draftId) { + return res.status(404).json({ error: 'Pub or draft not found' }); + } + + try { + await collabAuthority.receiveCommit(draftId, req.body); + } catch (e) { + if (e instanceof TooMuchContentionError) { + return res.status(409).json(null); + } + throw e; + } + + return res.status(204).send(null); + }), +); + +// long-poll for new commits +router.get( + '/api/pubs/:pubId/commits', + wrap(async (req, res) => { + const draftId = await getDraftIdForPub(req.params.pubId); + + if (!draftId) { + return res.status(404).json({ error: 'Pub or draft not found' }); + } + + const version = parseInt(req.query.version as string, 10); + + if (Number.isNaN(version)) { + return res.status(400).json({ error: 'Missing or invalid version query parameter' }); + } + + const commits = await collabAuthority.listenForCommit(draftId, version); + return res.status(200).json(commits); + }), +); + +// update presence indicator +router.post( + '/api/pubs/:pubId/presence/:clientId', + wrap(async (req, res) => { + const indicator = req.body as PresenceIndicator; + await presenceAuthority.updatePresence(req.params.pubId, indicator); + return res.status(204).send(null); + }), +); + +// long-poll for presence updates +router.post( + '/api/pubs/:pubId/presence', + wrap(async (req, res) => { + const { refs, clientId } = req.body as { + refs: Record | undefined; + clientId: string; + }; + + const presence = await presenceAuthority.listenForPresence( + req.params.pubId, + clientId, + refs, + ); + + return res.status(200).json(presence); + }), +); diff --git a/server/collab/authority.ts b/server/collab/authority.ts new file mode 100644 index 0000000000..1176517ebd --- /dev/null +++ b/server/collab/authority.ts @@ -0,0 +1,130 @@ +import type { Transaction } from 'sequelize'; + +import { CollabAuthority, RedisBroadcastManager } from '@pitter-patter/collab-server'; +import { Op } from 'sequelize'; + +import { editorSchema } from 'client/components/Editor/utils'; +import { env } from 'server/env'; +import { CollabCommit, Draft, DraftCheckpoint } from 'server/models'; +import { sequelize } from 'server/sequelize'; + +const broadcastManager = new RedisBroadcastManager({ + redisUrl: env.VALKEY_URL ?? 'redis://localhost:6379', +}); + +export const connectCollabRedis = async () => { + await broadcastManager.connect(); + console.log('[collab] collab broadcast redis connected'); +}; + +export const collabAuthority = new CollabAuthority({ + schema: editorSchema, + broadcastManager, + + runWithTransaction: async (callback) => { + return sequelize.transaction((tr) => callback(tr)); + }, + + getDoc: async (tr, docId) => { + const draft = await Draft.findOne({ + where: { id: docId }, + ...(tr && { lock: tr.LOCK.UPDATE }), + transaction: tr ?? undefined, + }); + + if (!draft) { + throw new Error(`Draft not found: ${docId}`); + } + + const checkpoint = await DraftCheckpoint.findOne({ + where: { draftId: docId }, + transaction: tr ?? undefined, + }); + + if (!checkpoint) { + const emptyDoc = editorSchema.topNodeType.createAndFill()!; + + return { + docJSON: emptyDoc.toJSON(), + version: 0, + lastUpdatedTimestamp: Date.now(), + }; + } + + return { + docJSON: checkpoint.doc, + version: draft.version, + lastUpdatedTimestamp: draft.latestKeyAt?.valueOf() ?? Date.now(), + }; + }, + + saveDoc: async (tr, docId, docJSON, version) => { + await Draft.update( + { version, latestKeyAt: new Date() }, + { where: { id: docId }, transaction: tr }, + ); + + const existing = await DraftCheckpoint.findOne({ + where: { draftId: docId }, + transaction: tr, + }); + + if (existing) { + await existing.update( + { doc: docJSON, historyKey: version, timestamp: Date.now() }, + { transaction: tr }, + ); + } else { + await DraftCheckpoint.create( + { draftId: docId, doc: docJSON, historyKey: version, timestamp: Date.now() }, + { transaction: tr }, + ); + } + }, + + saveCommit: async (tr, docId, commitRef, commitVersion, commitSteps) => { + await CollabCommit.create( + { + draftId: docId, + ref: commitRef, + version: commitVersion, + steps: commitSteps, + }, + { transaction: tr }, + ); + }, + + getCommit: async (tr, docId, commitRef) => { + const commit = await CollabCommit.findOne({ + where: { draftId: docId, ref: commitRef }, + transaction: tr ?? undefined, + }); + + if (!commit) { + return null; + } + + return { + ref: commit.ref, + version: commit.version, + steps: commit.steps, + }; + }, + + getCommits: async (tr, docId, version) => { + const commits = await CollabCommit.findAll({ + where: { + draftId: docId, + version: { [Op.gt]: version }, + }, + order: [['version', 'ASC']], + transaction: tr ?? undefined, + }); + + return commits.map((c) => ({ + ref: c.ref, + version: c.version, + steps: c.steps, + })); + }, +}); diff --git a/server/collab/discussionPositions.ts b/server/collab/discussionPositions.ts new file mode 100644 index 0000000000..58065f3dda --- /dev/null +++ b/server/collab/discussionPositions.ts @@ -0,0 +1,63 @@ +import { Router } from 'express'; + +import { Draft, DraftCheckpoint, Pub } from 'server/models'; +import { wrap } from 'server/wrap'; + +export const router = Router(); + +// get current discussion positions for a pub's draft +router.get( + '/api/pubs/:pubId/discussions/positions', + wrap(async (req, res) => { + const pub = await Pub.findOne({ + where: { id: req.params.pubId }, + include: [{ model: Draft, as: 'draft' }], + attributes: ['id'], + }); + + if (!pub?.draft) { + return res.status(404).json({}); + } + + const checkpoint = await DraftCheckpoint.findOne({ + where: { draftId: pub.draft.id }, + }); + + return res.status(200).json(checkpoint?.discussions ?? {}); + }), +); + +// update discussion positions for a pub's draft +router.post( + '/api/pubs/:pubId/discussions/positions', + wrap(async (req, res) => { + const pub = await Pub.findOne({ + where: { id: req.params.pubId }, + include: [{ model: Draft, as: 'draft' }], + attributes: ['id'], + }); + + if (!pub?.draft) { + return res.status(404).json({}); + } + + const discussions = req.body; + + if (!discussions || typeof discussions !== 'object') { + return res.status(400).json({ error: 'Invalid discussions payload' }); + } + + const checkpoint = await DraftCheckpoint.findOne({ + where: { draftId: pub.draft.id }, + }); + + if (checkpoint) { + // merge incoming discussion positions with existing ones + const existing = checkpoint.discussions ?? {}; + const merged = { ...existing, ...discussions }; + await checkpoint.update({ discussions: merged }); + } + + return res.status(204).send(null); + }), +); diff --git a/server/collab/featureFlag.ts b/server/collab/featureFlag.ts new file mode 100644 index 0000000000..227511ca99 --- /dev/null +++ b/server/collab/featureFlag.ts @@ -0,0 +1,43 @@ +import { FeatureFlag, FeatureFlagCommunity } from 'server/models'; + +const PITTER_PATTER_FLAG_NAME = 'pitterPatterCollab'; + +/** + * Check if Pitter Patter collab is enabled for a given community. + * Uses the existing FeatureFlag system for gradual rollout. + * + * During migration: + * - Create a FeatureFlag named "pitterPatterCollab" + * - Add specific communities via FeatureFlagCommunity to opt them in + * - Or set enabledCommunitiesFraction to 1.0 to enable globally + * + * When the flag does not exist, Pitter Patter is assumed to be enabled + * (post-migration default). + */ +export const isPitterPatterEnabled = async (communityId: string): Promise => { + const flag = await FeatureFlag.findOne({ + where: { name: PITTER_PATTER_FLAG_NAME }, + include: [{ model: FeatureFlagCommunity, as: 'communities' }], + }); + + // if no flag exists, the migration is complete and everyone uses Pitter Patter + if (!flag) { + return true; + } + + // check if the community is explicitly opted in + const communityOptedIn = flag.communities?.some( + (fc) => fc.communityId === communityId, + ); + + if (communityOptedIn) { + return true; + } + + // check fraction-based rollout + if (flag.enabledCommunitiesFraction && flag.enabledCommunitiesFraction >= 1.0) { + return true; + } + + return false; +}; diff --git a/server/collab/presence.ts b/server/collab/presence.ts new file mode 100644 index 0000000000..2fae57e4d2 --- /dev/null +++ b/server/collab/presence.ts @@ -0,0 +1,22 @@ +import { + PresenceAuthority, + RedisPresenceBroadcastManager, + RedisPresencePersistenceManager, +} from '@pitter-patter/presence-server'; + +import { env } from 'server/env'; + +const redisUrl = env.VALKEY_URL ?? 'redis://localhost:6379'; + +const presenceBroadcaster = new RedisPresenceBroadcastManager({ redisUrl }); +const presencePersister = new RedisPresencePersistenceManager({ redisUrl }); + +export const presenceAuthority = new PresenceAuthority({ + persistenceManager: presencePersister, + broadcastManager: presenceBroadcaster, +}); + +export const connectPresenceRedis = async () => { + await Promise.all([presenceBroadcaster.connect(), presencePersister.connect()]); + console.log('[collab] presence redis connected'); +}; diff --git a/server/collabCommit/model.ts b/server/collabCommit/model.ts new file mode 100644 index 0000000000..52c0f4f57e --- /dev/null +++ b/server/collabCommit/model.ts @@ -0,0 +1,53 @@ +import type { CreationOptional, InferAttributes, InferCreationAttributes } from 'sequelize'; + +import type { SerializedModel } from 'types'; + +import { + AllowNull, + BelongsTo, + Column, + DataType, + Default, + Index, + Model, + PrimaryKey, + Table, +} from 'sequelize-typescript'; + +import { Draft } from '../models'; + +@Table +export class CollabCommit extends Model< + InferAttributes, + InferCreationAttributes +> { + public declare toJSON: (this: M) => SerializedModel; + + @Default(DataType.UUIDV4) + @PrimaryKey + @Column(DataType.UUID) + declare id: CreationOptional; + + @AllowNull(false) + @Index({ unique: true, name: 'collab_commits_draft_version_unique' }) + @Index({ name: 'collab_commits_draft_ref_idx' }) + @Column(DataType.UUID) + declare draftId: string; + + @AllowNull(false) + @Index({ unique: true, name: 'collab_commits_draft_version_unique' }) + @Column(DataType.INTEGER) + declare version: number; + + @AllowNull(false) + @Index({ name: 'collab_commits_draft_ref_idx' }) + @Column(DataType.TEXT) + declare ref: string; + + @AllowNull(false) + @Column(DataType.JSONB) + declare steps: Record[]; + + @BelongsTo(() => Draft, { as: 'draft', foreignKey: 'draftId', onDelete: 'CASCADE' }) + declare draft?: Draft; +} diff --git a/server/communityTemplate/applyTemplate.ts b/server/communityTemplate/applyTemplate.ts index deb8a2351a..6333bcf865 100644 --- a/server/communityTemplate/applyTemplate.ts +++ b/server/communityTemplate/applyTemplate.ts @@ -317,8 +317,8 @@ async function applyStarterPubs( actorId: string, ) { // Lazy import to avoid circular dependency (resolved once by Node.js module cache) - const { createPub } = await import('server/pub/queries'); - const { upsertDraftCheckpoint } = await import('server/draftCheckpoint/queries'); + const { createPub } = await import('server/pub/queries.js'); + const { upsertDraftCheckpoint } = await import('server/draftCheckpoint/queries.js'); for (const pubDef of pubDefs) { const collectionIds: string[] = []; diff --git a/server/discussion/utils.ts b/server/discussion/utils.ts index 9f08646c3b..2a6e19e7c1 100644 --- a/server/discussion/utils.ts +++ b/server/discussion/utils.ts @@ -1,11 +1,13 @@ import type { DiscussionInfo } from 'components/Editor/plugins/discussions/types'; import type * as types from 'types'; -import { jsonToNode } from 'components/Editor'; -import { createFastForwarder } from 'components/Editor/plugins/discussions/fastForward'; +import { Step } from 'prosemirror-transform'; + +import { editorSchema, jsonToNode } from 'client/components/Editor/utils'; +import { mapDiscussionThroughSteps } from 'client/components/Editor/plugins/discussions/util'; import { createDiscussionAnchor } from 'server/discussionAnchor/queries'; import { Discussion, DiscussionAnchor, Doc, Release } from 'server/models'; -import { getPubDraftRef } from 'server/utils/firebaseAdmin'; +import { getPubDraft, getStepsBetweenVersions } from 'server/utils/firebaseAdmin'; import { indexByProperty } from 'utils/arrays'; type ExtendedDiscussionInfo = DiscussionInfo & { @@ -17,7 +19,9 @@ const getDiscussions = async (discussionIds: string[], pubId: string) => { where: { id: discussionIds, pubId }, include: [{ model: DiscussionAnchor, as: 'anchors' }], })) as types.DefinitelyHas[]; + const discussionInfoValues: ExtendedDiscussionInfo[] = []; + discussions.forEach(({ anchors, id: discussionId }) => { const firstAnchor = anchors.reduce( (curr, next) => (curr && curr.historyKey < next.historyKey ? curr : next), @@ -27,6 +31,7 @@ const getDiscussions = async (discussionIds: string[], pubId: string) => { (curr, next) => (curr && curr.historyKey > next.historyKey ? curr : next), null as null | types.DiscussionAnchor, ); + if (firstAnchor?.selection && latestAnchor?.selection) { const { historyKey: initKey, @@ -49,6 +54,7 @@ const getDiscussions = async (discussionIds: string[], pubId: string) => { }); } }); + return indexByProperty(discussionInfoValues, 'discussionId'); }; @@ -58,9 +64,11 @@ const getLatestReleaseInfo = async (pubId: string) => { include: [{ model: Doc, as: 'doc' }], order: [['historyKey', 'DESC']], })) as types.DefinitelyHas; + if (!release) { throw new Error('Pub does not have a Release'); } + return { doc: jsonToNode(release.doc.content), historyKey: release.historyKey }; }; @@ -69,15 +77,38 @@ export const createDiscussionAnchorsForLatestRelease = async ( discussionIds: string[], ) => { const { doc, historyKey } = await getLatestReleaseInfo(pubId); - const draftRef = await getPubDraftRef(pubId); - const fastForward = createFastForwarder(draftRef); + const { draft } = await getPubDraft(pubId); const discussions = await getDiscussions(discussionIds, pubId); - const fastForwardedDiscussions = await fastForward(discussions, doc, historyKey); + + // get all steps from the release historyKey to the current draft version + const stepGroups = await getStepsBetweenVersions( + draft.id, + historyKey, + draft.version, + editorSchema, + ); + + const allSteps = stepGroups.reduce((acc, group) => [...acc, ...group], [] as Step[]); + + // fast-forward each discussion through the steps + const fastForwardedDiscussions: Record = {}; + + for (const [id, info] of Object.entries(discussions)) { + if (!info.selection) { + fastForwardedDiscussions[id] = null; + continue; + } + + const mapped = mapDiscussionThroughSteps(info, allSteps); + fastForwardedDiscussions[id] = mapped; + } + return Promise.all( Object.values(discussions).map(async (extendedDiscussionInfo) => { const { originalText, originalTextPrefix, originalTextSuffix, discussionId } = extendedDiscussionInfo; const fastForwardedDiscussionInfo = fastForwardedDiscussions[discussionId]; + if (fastForwardedDiscussionInfo?.selection) { const { selection } = fastForwardedDiscussionInfo; await createDiscussionAnchor({ diff --git a/server/draft/model.ts b/server/draft/model.ts index 270280fec4..d389b6d5f9 100644 --- a/server/draft/model.ts +++ b/server/draft/model.ts @@ -3,7 +3,6 @@ import type { CreationOptional, InferAttributes, InferCreationAttributes } from import type { SerializedModel } from 'types'; import { - AllowNull, Column, DataType, Default, @@ -11,7 +10,6 @@ import { PrimaryKey, Table, } from 'sequelize-typescript'; -// import { Pub } from '../models'; @Table export class Draft extends Model, InferCreationAttributes> { @@ -25,7 +23,11 @@ export class Draft extends Model, InferCreationAttributes @Column(DataType.DATE) declare latestKeyAt: Date | null; - @AllowNull(false) + // kept nullable during Firebase→PitterPatter migration; will be removed after full cutover @Column(DataType.STRING) - declare firebasePath: string; + declare firebasePath: string | null; + + @Default(0) + @Column(DataType.INTEGER) + declare version: CreationOptional; } diff --git a/server/envSchema.ts b/server/envSchema.ts index b65adcd987..1223050485 100644 --- a/server/envSchema.ts +++ b/server/envSchema.ts @@ -71,11 +71,11 @@ export const envSchema = z.object({ // ── Auth / Signing ────────────────────────────────────────────────── JWT_SIGNING_SECRET: z.string().min(1).describe('Secret used to sign JWT tokens'), - // ── Firebase ───────────────────────────────────────────────────────── + // ── Firebase (legacy, only needed for migration tooling) ───────────── FIREBASE_SERVICE_ACCOUNT_BASE64: z .string() - .min(1) - .describe('Base64-encoded Firebase service-account JSON'), + .optional() + .describe('Base64-encoded Firebase service-account JSON (only needed for migration)'), FIREBASE_TEST_DB_URL: z .string() .url() @@ -134,6 +134,12 @@ export const envSchema = z.object({ // ── Message Queues ────────────────────────────────────────────────── CLOUDAMQP_URL: z.string().describe('CloudAMQP (RabbitMQ) connection URL'), + // ── Valkey / Redis ────────────────────────────────────────────────── + VALKEY_URL: z + .string() + .url() + .describe('Valkey (Redis-compatible) URL for collab broadcast and presence'), + // ── Zotero ────────────────────────────────────────────────────────── ZOTERO_CLIENT_KEY: z.string().describe('Zotero OAuth1 consumer key'), ZOTERO_CLIENT_SECRET: z.string().describe('Zotero OAuth1 consumer secret'), diff --git a/server/hub/api.ts b/server/hub/api.ts index afc99c8b6a..ca07d2f8a2 100644 --- a/server/hub/api.ts +++ b/server/hub/api.ts @@ -120,7 +120,7 @@ router.get('/api/hubs/brand-helper', async (req, res, next) => { // Check if user is a manager of any hub const userId = initialData.loginData.id; if (!userId) throw new ForbiddenError(); - const { HubManager } = await import('server/hubManager/model'); + const { HubManager } = await import('server/hubManager/model.js'); const mgr = await HubManager.findOne({ where: { userId } }); if (!mgr) throw new ForbiddenError(); } @@ -142,7 +142,7 @@ router.get('/api/hubs/brand-helper/proxy-image', async (req, res, next) => { if (!initialData.loginData.isSuperAdmin) { const userId = initialData.loginData.id; if (!userId) throw new ForbiddenError(); - const { HubManager } = await import('server/hubManager/model'); + const { HubManager } = await import('server/hubManager/model.js'); const mgr = await HubManager.findOne({ where: { userId } }); if (!mgr) throw new ForbiddenError(); } @@ -768,7 +768,7 @@ const requirePubManager = async (req, pubId: string) => { router.get('/api/pubs/:pubId/curating-hubs', async (req, res, next) => { try { await requirePubManager(req, req.params.pubId); - const { getHubsForPub } = await import('server/hubPub/queries'); + const { getHubsForPub } = await import('server/hubPub/queries.js'); const orgs = await getHubsForPub(req.params.pubId); return res.status(200).json(orgs); } catch (err) { @@ -780,7 +780,7 @@ router.get('/api/pubs/:pubId/curating-hubs', async (req, res, next) => { router.delete('/api/pubs/:pubId/curating-hubs/:hubId', async (req, res, next) => { try { await requirePubManager(req, req.params.pubId); - const { removePubFromHub } = await import('server/hubPub/queries'); + const { removePubFromHub } = await import('server/hubPub/queries.js'); await removePubFromHub(req.params.hubId, req.params.pubId); return res.status(200).json({ success: true }); } catch (err) { diff --git a/server/models.ts b/server/models.ts index 19b4e2c735..8b7eb3d9f8 100644 --- a/server/models.ts +++ b/server/models.ts @@ -19,6 +19,7 @@ import { DepositTarget } from './depositTarget/model'; import { Discussion } from './discussion/model'; import { DiscussionAnchor } from './discussionAnchor/model'; import { Doc } from './doc/model'; +import { CollabCommit } from './collabCommit/model'; import { Draft } from './draft/model'; import { DraftCheckpoint } from './draftCheckpoint/model'; import { EmailChangeToken } from './emailChangeToken/model'; @@ -74,6 +75,7 @@ sequelize.addModels([ Collection, CollectionAttribution, CollectionPub, + CollabCommit, Commenter, Community, CrossrefDepositRecord, @@ -173,6 +175,7 @@ export { Collection, CollectionAttribution, CollectionPub, + CollabCommit, Commenter, Community, CrossrefDepositRecord, diff --git a/server/pub/__tests__/api.test.ts b/server/pub/__tests__/api.test.ts index 21aec92588..4bbfe0b953 100644 --- a/server/pub/__tests__/api.test.ts +++ b/server/pub/__tests__/api.test.ts @@ -588,7 +588,7 @@ describe('GET /api/pubs', () => { vi.mock('utils/import/uploadAndConvertImages', async () => { if (process.env.INTEGRATION) { - return import('utils/import/uploadAndConvertImages'); + return import('utils/import/uploadAndConvertImages.js'); } return { uploadAndConvertImages: (files) => files, diff --git a/server/pubHistory/queries.ts b/server/pubHistory/queries.ts index 3a92c4d588..f10685673c 100644 --- a/server/pubHistory/queries.ts +++ b/server/pubHistory/queries.ts @@ -1,7 +1,7 @@ import { Node, Slice } from 'prosemirror-model'; -import { editorSchema, jsonToNode } from 'client/components/Editor'; -import { editFirebaseDraftByRef, getPubDraftDoc, getPubDraftRef } from 'server/utils/firebaseAdmin'; +import { editorSchema, jsonToNode } from 'client/components/Editor/utils'; +import { editDraft, getPubDraftDoc } from 'server/utils/firebaseAdmin'; import { assert } from 'utils/assert'; type RestorePubOptions = { @@ -13,20 +13,10 @@ type RestorePubOptions = { export const restorePubDraftToHistoryKey = async (options: RestorePubOptions) => { const { pubId, userId, historyKey } = options; assert(typeof historyKey === 'number' && historyKey >= 0); - const pubDraftRef = await getPubDraftRef(pubId); + const { doc } = await getPubDraftDoc(pubId, historyKey); - // Get the actual current state via the PG-checkpoint-aware path so we know - // the real document and key. Without this, cold-stored pubs (where Firebase - // was wiped) would see key=-1 and the restore change would be written at - // key 0 — far below the checkpoint key — leaving the pub permanently stuck - // in historical mode. - const currentState = await getPubDraftDoc(pubId, null); - const currentDoc = Node.fromJSON(editorSchema, currentState.doc); - const editor = await editFirebaseDraftByRef(pubDraftRef, userId, editorSchema, { - doc: currentDoc, - key: currentState.mostRecentRemoteKey, - }); + const editor = await editDraft(pubId, userId, editorSchema); editor.transform((tr, schema) => { const currentDoc = editor.getDoc(); diff --git a/server/release/__tests__/api.test.ts b/server/release/__tests__/api.test.ts index 058abb6d39..1ea7d71e45 100644 --- a/server/release/__tests__/api.test.ts +++ b/server/release/__tests__/api.test.ts @@ -114,7 +114,6 @@ describe('/api/releases', () => { const { community, pub, pubAdmin } = models; const agent = await login(pubAdmin); const pubEditor = await editPub(pub.id); - await pubEditor.clearChanges(); const { body: release } = await agent .post('/api/releases') .send( diff --git a/server/release/queries.ts b/server/release/queries.ts index a99b4a3982..f76e6ef707 100644 --- a/server/release/queries.ts +++ b/server/release/queries.ts @@ -1,11 +1,9 @@ -import type firebase from 'firebase'; - import type { DefinitelyHas, DocJson, Maybe, Release as ReleaseType } from 'types'; import { StepMap } from 'prosemirror-transform'; import { Op } from 'sequelize'; -import { editorSchema, getStepsInChangeRange } from 'components/Editor'; +import { editorSchema } from 'components/Editor/utils'; import { createPubReleasedActivityItem } from 'server/activityItem/queries'; import { createUpdatedDiscussionAnchorForNewSteps } from 'server/discussionAnchor/queries'; import { createDoc } from 'server/doc/queries'; @@ -14,42 +12,38 @@ import { createLatestPubExports } from 'server/export/queries'; import { Discussion, DiscussionAnchor, Doc, Draft, Pub, Release } from 'server/models'; import { sequelize } from 'server/sequelize'; import { defer } from 'server/utils/deferred'; -import { getPubDraftDoc, getPubDraftRef } from 'server/utils/firebaseAdmin'; +import { getPubDraft, getPubDraftDoc, getStepsBetweenVersions } from 'server/utils/firebaseAdmin'; type ReleaseErrorReason = 'merge-failed' | 'duplicate-release'; export class ReleaseQueryError extends Error { - // biome-ignore lint/complexity/noUselessConstructor: shhhhhh constructor(reason: ReleaseErrorReason) { super(reason); } } const getStepsSinceLastRelease = async ( - draftRef: firebase.database.Reference, + draftId: string, previousRelease: Maybe, currentHistoryKey: number, ) => { - if (previousRelease) { - const { historyKey: previousHistoryKey } = previousRelease; - return getStepsInChangeRange( - draftRef, - editorSchema, - previousHistoryKey + 1, - currentHistoryKey, - ); + if (!previousRelease) { + return []; } - return []; + + const { historyKey: previousHistoryKey } = previousRelease; + + return getStepsBetweenVersions(draftId, previousHistoryKey, currentHistoryKey, editorSchema); }; /** * Map a discussion selection through an array of StepMap ranges (from DraftCheckpoint.stepMaps). - * Returns the new selection, or null if it was deleted. */ const mapSelectionThroughStoredStepMaps = ( selection: { anchor: number; head: number } | null, stepMapRanges: number[][], ) => { if (!selection || selection.anchor === selection.head) return null; + let from = Math.min(selection.anchor, selection.head); let to = Math.max(selection.anchor, selection.head); @@ -57,6 +51,7 @@ const mapSelectionThroughStoredStepMaps = ( const map = new StepMap(ranges); from = map.map(from, 1); to = map.map(to, -1); + if (from >= to || from === 0) return null; } @@ -71,8 +66,8 @@ const createDiscussionAnchorsForRelease = async ( ) => { if (!previousRelease) return; - const draftRef = await getPubDraftRef(pubId, sequelizeTransaction); - const steps = await getStepsSinceLastRelease(draftRef, previousRelease, currentHistoryKey); + const { draft } = await getPubDraft(pubId); + const steps = await getStepsSinceLastRelease(draft.id, previousRelease, currentHistoryKey); const flatSteps = steps.reduce((a, b) => [...a, ...b], []); const discussions = await Discussion.findAll({ @@ -80,6 +75,7 @@ const createDiscussionAnchorsForRelease = async ( attributes: ['id'], transaction: sequelizeTransaction, }); + const existingAnchors = await DiscussionAnchor.findAll({ where: { discussionId: { [Op.in]: discussions.map((d) => d.id) }, @@ -90,7 +86,6 @@ const createDiscussionAnchorsForRelease = async ( if (existingAnchors.length === 0) return; - // If we got steps from Firebase, use them directly if (flatSteps.length > 0) { await Promise.all( existingAnchors.map((anchor) => @@ -105,17 +100,9 @@ const createDiscussionAnchorsForRelease = async ( return; } - // No steps from Firebase covering the full range — try stored stepMaps from - // the DraftCheckpoint (cold-stored draft) and compose with any new Firebase - // changes that happened after the stepMaps were captured. - const pub = await Pub.findOne({ - where: { id: pubId }, - include: [{ model: Draft, as: 'draft' }], - transaction: sequelizeTransaction, - }); - if (!pub?.draft) return; + // no commits in the range -- try stored stepMaps from checkpoint + const pgCheckpoint = await getDraftCheckpoint(draft.id, sequelizeTransaction); - const pgCheckpoint = await getDraftCheckpoint(pub.draft.id, sequelizeTransaction); if (!pgCheckpoint?.stepMaps?.length || pgCheckpoint.stepMapToKey == null) { console.warn( `[release] No steps or stepMaps available for pub ${pubId}, skipping anchor mapping`, @@ -123,31 +110,23 @@ const createDiscussionAnchorsForRelease = async ( return; } - // Compose stored stepMaps (release→stepMapToKey) with any new Firebase - // steps (stepMapToKey+1→currentHistoryKey) that happened after thaw. let allStepMapRanges = pgCheckpoint.stepMaps!; + if (pgCheckpoint.stepMapToKey < currentHistoryKey) { - try { - const newStepsByChange = await getStepsInChangeRange( - draftRef, - editorSchema, - pgCheckpoint.stepMapToKey + 1, - currentHistoryKey, - ); - const newFlatSteps = newStepsByChange.reduce((a, b) => [...a, ...b], []); - const newRanges = newFlatSteps.map((s) => - Array.from((s.getMap() as any).ranges as number[]), - ); - allStepMapRanges = [...allStepMapRanges, ...newRanges]; - } catch (err) { - console.warn( - `[release] Could not get Firebase steps ${pgCheckpoint.stepMapToKey + 1}→${currentHistoryKey}, using stored stepMaps only`, - err, - ); - } + const additionalSteps = await getStepsBetweenVersions( + draft.id, + pgCheckpoint.stepMapToKey, + currentHistoryKey, + editorSchema, + ); + + const additionalFlatSteps = additionalSteps.reduce((a, b) => [...a, ...b], []); + const newRanges = additionalFlatSteps.map((s) => + Array.from((s.getMap() as any).ranges as number[]), + ); + allStepMapRanges = [...allStepMapRanges, ...newRanges]; } - // Use composed stepMaps to map anchors await Promise.all( existingAnchors.map(async (anchor) => { try { @@ -155,6 +134,7 @@ const createDiscussionAnchorsForRelease = async ( anchor.selection, allStepMapRanges, ); + await DiscussionAnchor.create( { historyKey: currentHistoryKey, @@ -199,6 +179,7 @@ export const createRelease = async ({ doc: nextDoc, historyData: { currentKey }, } = await getPubDraftDoc(pubId, providedHistoryKey ?? null); + const historyKey = providedHistoryKey ?? currentKey; if (mostRecentRelease && mostRecentRelease.historyKey === historyKey) { @@ -207,6 +188,7 @@ export const createRelease = async ({ const release = await sequelize.transaction(async (txn) => { const docModel = await createDoc(nextDoc, txn); + const [nextRelease] = await Promise.all([ Release.create( { @@ -221,12 +203,14 @@ export const createRelease = async ({ ), createDiscussionAnchorsForRelease(pubId, mostRecentRelease, historyKey, txn), ]); + return nextRelease; }); if (createExports) { await createLatestPubExports(pubId); } + defer(async () => { await createPubReleasedActivityItem(userId, release.id); }); diff --git a/server/sequelize.ts b/server/sequelize.ts index c4c2455276..3a27307da0 100644 --- a/server/sequelize.ts +++ b/server/sequelize.ts @@ -128,7 +128,7 @@ export const sequelizeSyncPromise: Promise = installSearchTriggers, backfillPubSearchVectors, backfillCommunitySearchVectors, - } = await import('server/search2/searchTriggers'); + } = await import('./search2/searchTriggers.js'); await installSearchTriggers(); // Run backfill in the background so it doesn't block app startup. @@ -144,7 +144,7 @@ export const sequelizeSyncPromise: Promise = // Create analytics materialized views (idempotent — no-ops if they exist). // Refresh is handled by the nightly cron, not at startup, because it can // take several minutes and would delay deploys. - const { createSummaryViews } = await import('server/analytics/summaryViews'); + const { createSummaryViews } = await import('./analytics/summaryViews.js'); await createSummaryViews(); })() : Promise.resolve(); diff --git a/server/server.ts b/server/server.ts index abbdb30eac..ad672c7e79 100755 --- a/server/server.ts +++ b/server/server.ts @@ -219,13 +219,15 @@ process.on('uncaughtException', (err) => { /** Same as Heroku's default timeout */ const TIMEOUT_MS = env.REQUEST_TIMEOUT_MS; +const ignoredPaths = /\/api\/analytics|\/api\/ev|\/api\/pubs\/.*?\/presence/; appRouter.use((req, res, next) => { // don't abort requests in test environment if (env.NODE_ENV === 'test') { return next(); } - if (req.path.includes('/api/analytics')) { + if (ignoredPaths.test(req.path)) { + console.log('ignoring path', req.path); return next(); } @@ -378,6 +380,15 @@ export const startServer = async () => { console.warn('[OIDC] Discovery failed at startup (will retry on demand):', err.message); }); + // connect collab redis clients + const { connectCollabRedis } = await import('./collab/authority.js'); + const { connectPresenceRedis } = await import('./collab/presence.js'); + + await Promise.all([connectCollabRedis(), connectPresenceRedis()]).catch((err) => { + console.error('[collab] Failed to connect to Redis/Valkey:', err.message); + console.error('[collab] Collaborative editing will not work until Redis is available.'); + }); + await sequelizeSyncPromise; return app.listen( port, diff --git a/server/spamTag/commentSpam.ts b/server/spamTag/commentSpam.ts index 6c7437a0f3..0c29392f57 100644 --- a/server/spamTag/commentSpam.ts +++ b/server/spamTag/commentSpam.ts @@ -2,7 +2,7 @@ import type { DocJson, NewAccountLinkCommentTriggerSource, UserSpamTagFields } f import { type Mark, Node } from 'prosemirror-model'; -import { editorSchema } from 'client/components/Editor'; +import { editorSchema } from 'client/components/Editor/utils'; import { env } from 'server/env'; import { SpamTag, User } from 'server/models'; import { contextFromUser, notify } from 'server/spamTag/notifications'; diff --git a/server/submission/abstract.ts b/server/submission/abstract.ts index d3dce1d090..becbb32eb4 100644 --- a/server/submission/abstract.ts +++ b/server/submission/abstract.ts @@ -1,28 +1,24 @@ import type { DocJson } from 'types'; -import { Fragment, Node } from 'prosemirror-model'; +import { Fragment } from 'prosemirror-model'; -import { editorSchema, isEmptyDoc, jsonToNode } from 'client/components/Editor'; -import { editFirebaseDraftByRef, getPubDraftDoc, getPubDraftRef } from 'server/utils/firebaseAdmin'; +import { editorSchema, isEmptyDoc, jsonToNode } from 'client/components/Editor/utils'; +import { editDraft } from 'server/utils/firebaseAdmin'; export const appendAbstractToPubDraft = async (pubId: string, abstract: null | DocJson) => { if (abstract && !isEmptyDoc(abstract)) { - const pubDraftRef = await getPubDraftRef(pubId); - const currentState = await getPubDraftDoc(pubId, null); - const currentDoc = Node.fromJSON(editorSchema, currentState.doc); - const editor = await editFirebaseDraftByRef(pubDraftRef, 'submissions', editorSchema, { - doc: currentDoc, - key: currentState.mostRecentRemoteKey, - }); + const editor = await editDraft(pubId, 'submissions', editorSchema); + editor.transform((tr, schema) => { const abstractNode = jsonToNode(abstract, schema); const h1Node = schema.node('heading', { level: 1 }, schema.text('Abstract')); const frag = Fragment.from(h1Node).append(abstractNode.content); tr.insert(0, frag); }); + const committed = await editor.writeChange(); + if (!committed) { - // Someone may be editing the document... throw new Error('Failed to append abstract!'); } } diff --git a/server/submission/queries.ts b/server/submission/queries.ts index 222cde6c56..7590d5761c 100644 --- a/server/submission/queries.ts +++ b/server/submission/queries.ts @@ -1,4 +1,4 @@ -import { getEmptyDoc } from 'client/components/Editor'; +import { getEmptyDoc } from 'client/components/Editor/utils'; import { Collection, Submission, SubmissionWorkflow } from 'server/models'; import { createPub } from 'server/pub/queries'; import { defer } from 'server/utils/deferred'; diff --git a/server/user/__tests__/api.test.ts b/server/user/__tests__/api.test.ts index be521af579..14e4c8f4ed 100644 --- a/server/user/__tests__/api.test.ts +++ b/server/user/__tests__/api.test.ts @@ -50,14 +50,14 @@ const { deleteSessionsForUser } = vitest.hoisted(() => { }; }); -vitest.mock(import('server/utils/session'), async (importOriginal) => { +vitest.mock(import('server/utils/session.js'), async (importOriginal) => { const og = await importOriginal(); return { ...og, deleteSessionsForUser: deleteSessionsForUser, }; }); -vitest.mock(import('server/spamTag/notifications'), async (importOriginal) => { +vitest.mock(import('server/spamTag/notifications/index.js'), async (importOriginal) => { const og = await importOriginal(); return { ...og, @@ -115,7 +115,7 @@ describe('/api/users', () => { .expect(403); const createdUser = await User.findOne({ where: { email: spamSignup.email } }); expect(createdUser).toBeDefined(); - const { getSpamTagForUser } = await import('server/spamTag/userQueries'); + const { getSpamTagForUser } = await import('server/spamTag/userQueries.js'); const spamTag = await getSpamTagForUser(createdUser!.id); expect(spamTag?.status).toBe('confirmed-spam'); await agent @@ -143,7 +143,7 @@ describe('/api/users', () => { if (!createdUser) { throw new Error('Expected user to be created'); } - const { getSpamTagForUser } = await import('server/spamTag/userQueries'); + const { getSpamTagForUser } = await import('server/spamTag/userQueries.js'); const spamTag = await getSpamTagForUser(createdUser.id); expect(spamTag).toBeNull(); await agent @@ -169,7 +169,7 @@ describe('/api/users', () => { .expect(403); const createdUser = await User.findOne({ where: { email: restrictedSignup.email } }); expect(createdUser).toBeDefined(); - const { getSpamTagForUser } = await import('server/spamTag/userQueries'); + const { getSpamTagForUser } = await import('server/spamTag/userQueries.js'); const spamTag = await getSpamTagForUser(createdUser!.id); expect(spamTag?.status).toEqual('confirmed-spam'); diff --git a/server/utils/__tests__/ssr.test.tsx b/server/utils/__tests__/ssr.test.tsx index ca9b4ec86c..465e5adbcd 100644 --- a/server/utils/__tests__/ssr.test.tsx +++ b/server/utils/__tests__/ssr.test.tsx @@ -72,7 +72,7 @@ describe('generateMetaComponents', () => { }; }); - const ssrModule = await import('../ssr'); + const ssrModule = await import('../ssr.js'); expect(ssrModule.generateMetaComponents(props as any)).toMatchInlineSnapshot(` [ @@ -226,7 +226,7 @@ describe('generateMetaComponents', () => { }; }); - const ssrModule = await import('../ssr'); + const ssrModule = await import('../ssr.js'); expect(ssrModule.generateMetaComponents(props as any)).toContainEqual( , ); diff --git a/server/utils/citations/structuredCitations.ts b/server/utils/citations/structuredCitations.ts index 26f01c3803..73a610dd2b 100644 --- a/server/utils/citations/structuredCitations.ts +++ b/server/utils/citations/structuredCitations.ts @@ -7,7 +7,7 @@ import crypto from 'crypto'; import fs from 'fs'; import path from 'path'; -import { getNotesByKindFromDoc, jsonToNode } from 'components/Editor'; +import { getNotesByKindFromDoc, jsonToNode } from 'components/Editor/utils'; import { type CitationInlineStyleKind, type CitationStyleKind, diff --git a/server/utils/firebaseAdmin.ts b/server/utils/firebaseAdmin.ts index a669f2ee14..6c8f1d72df 100644 --- a/server/utils/firebaseAdmin.ts +++ b/server/utils/firebaseAdmin.ts @@ -1,69 +1,15 @@ -import type firebase from 'firebase'; import type { Schema } from 'prosemirror-model'; import type { DocJson, PubDraftInfo } from 'types'; -import firebaseAdmin from 'firebase-admin'; -import { uncompressStepJSON } from 'prosemirror-compress-pubpub'; import { Node } from 'prosemirror-model'; import { Step, Transform } from 'prosemirror-transform'; +import { Op } from 'sequelize'; -import { - editorSchema, - getFirebaseDoc, - getFirstKeyAndTimestamp, - getLatestKeyAndTimestamp, -} from 'components/Editor'; -import { createFirebaseChange, flattenKeyables } from 'components/Editor/utils'; +import { editorSchema } from 'components/Editor/utils'; import { getDraftCheckpoint } from 'server/draftCheckpoint/queries'; -import { env } from 'server/env'; -import { Draft, Pub } from 'server/models'; +import { CollabCommit, Draft, Pub } from 'server/models'; import { expect } from 'utils/assert'; -import { getFirebaseConfig } from 'utils/editor/firebaseConfig'; - -const getFirebaseApp = () => { - if (firebaseAdmin.apps.length > 0) { - return firebaseAdmin.apps[0]; - } - if (env.NODE_ENV === 'test') { - if (env.FIREBASE_TEST_DB_URL) { - return firebaseAdmin.initializeApp({ - databaseURL: env.FIREBASE_TEST_DB_URL, - }); - } - return null; - } - const serviceAccount = JSON.parse( - Buffer.from(env.FIREBASE_SERVICE_ACCOUNT_BASE64, 'base64').toString(), - ); - // biome-ignore lint/suspicious/noConsole: shhhhhh - console.log(`Firebase App will use: ${getFirebaseConfig().databaseURL}`); - return firebaseAdmin.initializeApp( - { - credential: firebaseAdmin.credential.cert(serviceAccount), - databaseURL: getFirebaseConfig().databaseURL, - }, - 'firebase-pub-new', - ); -}; - -const firebaseApp = getFirebaseApp(); -const database = firebaseApp && firebaseApp.database(); - -export const getDatabaseRef = (key: string): firebase.database.Reference => { - return database?.ref(key) as unknown as firebase.database.Reference; -}; - -export const getPubDraftRef = async (pubId: string, sequelizeTransaction: any = null) => { - const pub = expect( - await Pub.findOne({ - where: { id: pubId }, - include: [{ model: Draft, as: 'draft' }], - transaction: sequelizeTransaction, - }), - ); - return getDatabaseRef(expect(pub.draft).firebasePath); -}; export const getPubDraft = async (pubId: string, sequelizeTransaction: any = null) => { const pub = expect( @@ -73,23 +19,22 @@ export const getPubDraft = async (pubId: string, sequelizeTransaction: any = nul transaction: sequelizeTransaction, }), ); - const draft = expect(pub.draft); - return { draft, draftRef: getDatabaseRef(draft.firebasePath) }; + + return { draft: expect(pub.draft) }; }; -const maybeAddKeyTimestampPair = (key, timestamp) => { - if (typeof key === 'number' && key >= 0) { +const maybeAddKeyTimestampPair = (key: number, timestamp: number | null) => { + if (typeof key === 'number' && key >= 0 && timestamp) { return { [key]: timestamp }; } return null; }; /** - * Apply Firebase changes on top of a checkpoint doc to produce the current document. - * Used when loading from a Postgres checkpoint with Firebase changes layered on top. + * Apply commits from Postgres on top of a checkpoint doc to produce the current document. */ -const applyFirebaseChangesOnDoc = async ( - draftRef: firebase.database.Reference, +const applyCommitsOnDoc = async ( + draftId: string, checkpointDoc: DocJson, checkpointKey: number, checkpointTimestamp: number | null, @@ -97,44 +42,35 @@ const applyFirebaseChangesOnDoc = async ( ) => { const versionBound = historyKey ?? Infinity; - const getChanges = draftRef - .child('changes') - .orderByKey() - .startAt(String(checkpointKey + 1)) - .endAt(String(versionBound)) - .once('value'); - - const getMerges = draftRef - .child('merges') - .orderByKey() - .startAt(String(checkpointKey + 1)) - .endAt(String(versionBound)) - .once('value'); - - const [changesSnapshot, mergesSnapshot] = await Promise.all([getChanges, getMerges]); - - const allKeyables = { - ...changesSnapshot.val(), - ...mergesSnapshot.val(), + const whereClause: any = { + draftId, + version: { [Op.gt]: checkpointKey }, }; - const flattenedChanges = flattenKeyables(allKeyables); - const stepsJson = flattenedChanges.flatMap((change) => change.s.map(uncompressStepJSON)); + if (versionBound !== Infinity) { + whereClause.version = { [Op.gt]: checkpointKey, [Op.lte]: versionBound }; + } + + const commits = await CollabCommit.findAll({ + where: whereClause, + order: [['version', 'ASC']], + }); - const keys = Object.keys(allKeyables); - const currentKey = keys.length - ? keys.map((k) => parseInt(k, 10)).reduce((a, b) => Math.max(a, b)) - : checkpointKey; + const allStepsJson = commits.flatMap((commit) => commit.steps); + + const currentKey = commits.length > 0 ? commits[commits.length - 1].version : checkpointKey; const currentTimestamp = - flattenedChanges.length > 0 - ? flattenedChanges[flattenedChanges.length - 1].t + commits.length > 0 + ? (commits[commits.length - 1].createdAt?.valueOf() ?? checkpointTimestamp) : checkpointTimestamp; let doc = Node.fromJSON(editorSchema, checkpointDoc); - for (const stepJson of stepsJson) { + + for (const stepJson of allStepsJson) { const step = Step.fromJSON(editorSchema, stepJson); const { failed, doc: nextDoc } = step.apply(doc); + if (failed) { console.error(`Failed with: ${failed}`); } else if (nextDoc) { @@ -146,123 +82,78 @@ const applyFirebaseChangesOnDoc = async ( doc, key: currentKey, timestamp: currentTimestamp as number, - hasFirebaseChanges: stepsJson.length > 0, }; }; export const getPubDraftDoc = async ( - pubIdOrRef: string | firebase.database.Reference, + pubId: string, historyKey: null | number = null, ): Promise => { - // If called with a raw ref (no pub context), fall back to Firebase-only path - if (typeof pubIdOrRef !== 'string') { - return getPubDraftDocFromFirebase(pubIdOrRef, historyKey); - } - - const pubId = pubIdOrRef; - const { draft, draftRef } = await getPubDraft(pubId); - - // Always try Postgres checkpoint first — but only if the requested historyKey - // is at or after the checkpoint. If the user is browsing history before the - // checkpoint, we need the Firebase path which has older changes/checkpoints. + const { draft } = await getPubDraft(pubId); const pgCheckpoint = await getDraftCheckpoint(draft.id); - if (pgCheckpoint && (historyKey === null || historyKey >= pgCheckpoint.historyKey)) { - // Sequelize returns BIGINT as string — coerce to number for Date use - const pgTimestamp = pgCheckpoint.timestamp ? Number(pgCheckpoint.timestamp) : null; - const { - doc, - key: currentKey, - timestamp: currentTimestamp, - } = await applyFirebaseChangesOnDoc( - draftRef, - pgCheckpoint.doc as DocJson, - pgCheckpoint.historyKey, - pgTimestamp, - historyKey, - ); - - // If the checkpoint has frozen discussions (from cold storage), thaw them - // back into Firebase so the collaborative discussions plugin works. - if (pgCheckpoint.discussions) { - const existingDiscussions = await draftRef.child('discussions').once('value'); - if (!existingDiscussions.val()) { - await draftRef.child('discussions').set(pgCheckpoint.discussions); - } - } - // Gather timestamps for history UI - const [ - { timestamp: firstTimestamp, key: firstKey }, - { timestamp: latestTimestamp, key: latestKey }, - ] = await Promise.all([ - getFirstKeyAndTimestamp(draftRef).catch(() => ({ - timestamp: currentTimestamp, - key: currentKey, - })), - getLatestKeyAndTimestamp(draftRef).catch(() => ({ - timestamp: currentTimestamp, - key: currentKey, - })), - ]); - - // Use the Postgres checkpoint key as the "first" if Firebase has nothing earlier - const effectiveFirstKey = firstKey >= 0 ? firstKey : pgCheckpoint.historyKey; - const effectiveFirstTimestamp = firstKey >= 0 ? firstTimestamp : pgCheckpoint.timestamp; - const effectiveLatestKey = latestKey >= 0 ? Math.max(latestKey, currentKey) : currentKey; - const effectiveLatestTimestamp = latestKey >= 0 ? latestTimestamp : currentTimestamp; + if (!pgCheckpoint) { + // no checkpoint exists, return empty doc at version 0 + const emptyDoc = editorSchema.topNodeType.createAndFill()!; return { - doc: doc.toJSON() as DocJson, - size: doc.content.size, - mostRecentRemoteKey: currentKey, - firstTimestamp: effectiveFirstTimestamp as number, - latestTimestamp: effectiveLatestTimestamp as number, + doc: emptyDoc.toJSON() as DocJson, + size: emptyDoc.content.size, + mostRecentRemoteKey: 0, + firstTimestamp: Date.now(), + latestTimestamp: Date.now(), historyData: { - timestamps: { - ...maybeAddKeyTimestampPair(effectiveFirstKey, effectiveFirstTimestamp), - ...maybeAddKeyTimestampPair(currentKey, currentTimestamp), - ...maybeAddKeyTimestampPair(effectiveLatestKey, effectiveLatestTimestamp), - }, - currentKey, - latestKey: effectiveLatestKey, + timestamps: {}, + currentKey: 0, + latestKey: 0, }, }; } - // No PG checkpoint — fall back to Firebase-only path (legacy drafts) - return getPubDraftDocFromFirebase(draftRef, historyKey); -}; + const pgTimestamp = pgCheckpoint.timestamp ? Number(pgCheckpoint.timestamp) : null; -/** - * Original Firebase-only path for loading a draft doc. - */ -const getPubDraftDocFromFirebase = async ( - pubIdOrRef: string | firebase.database.Reference, - historyKey: null | number = null, -): Promise => { - const draftRef = typeof pubIdOrRef === 'string' ? await getPubDraftRef(pubIdOrRef) : pubIdOrRef; - const [ - { doc, key: currentKey, timestamp: currentTimestamp, checkpointMap }, - { timestamp: firstTimestamp, key: firstKey }, - { timestamp: latestTimestamp, key: latestKey }, - ] = await Promise.all([ - getFirebaseDoc(draftRef, editorSchema, historyKey), - getFirstKeyAndTimestamp(draftRef), - getLatestKeyAndTimestamp(draftRef), + const { + doc, + key: currentKey, + timestamp: currentTimestamp, + } = await applyCommitsOnDoc( + draft.id, + pgCheckpoint.doc as DocJson, + pgCheckpoint.historyKey, + pgTimestamp, + historyKey, + ); + + // get the first and latest commit timestamps for history UI + const [firstCommit, latestCommit] = await Promise.all([ + CollabCommit.findOne({ + where: { draftId: draft.id }, + order: [['version', 'ASC']], + attributes: ['version', 'createdAt'], + }), + CollabCommit.findOne({ + where: { draftId: draft.id }, + order: [['version', 'DESC']], + attributes: ['version', 'createdAt'], + }), ]); + const firstKey = firstCommit?.version ?? pgCheckpoint.historyKey; + const firstTimestamp = firstCommit?.createdAt?.valueOf() ?? pgTimestamp ?? Date.now(); + const latestKey = latestCommit?.version ?? currentKey; + const latestTimestamp = latestCommit?.createdAt?.valueOf() ?? currentTimestamp; + return { doc: doc.toJSON() as DocJson, size: doc.content.size, mostRecentRemoteKey: currentKey, - firstTimestamp, - latestTimestamp, + firstTimestamp: firstTimestamp as number, + latestTimestamp: latestTimestamp as number, historyData: { timestamps: { - ...checkpointMap, - ...maybeAddKeyTimestampPair(firstKey, firstTimestamp), + ...maybeAddKeyTimestampPair(firstKey, firstTimestamp as number), ...maybeAddKeyTimestampPair(currentKey, currentTimestamp), - ...maybeAddKeyTimestampPair(latestKey, latestTimestamp), + ...maybeAddKeyTimestampPair(latestKey, latestTimestamp as number), }, currentKey, latestKey, @@ -275,45 +166,19 @@ export const getLatestKeyInPubDraft = async (pubId: string) => { return Math.max(mostRecentRemoteKey, historyData.latestKey); }; -const getFirebaseDraftPathParts = (draftPath: string) => { - const draftPathMatch = draftPath.match(/drafts\/draft-(.*)/); - if (draftPathMatch) { - const draftId = draftPathMatch[1]; - return { draftId: `draft-${draftId}` }; - } - if (draftPath.includes('/')) { - const [pubIdPart, branchIdPart] = draftPath.split('/'); - if (pubIdPart.startsWith('pub-') && branchIdPart.startsWith('branch-')) { - return { pubId: pubIdPart, branchId: branchIdPart }; - } - } - return null; -}; - -export const getFirebaseToken = ( - clientId: string, - clientData: { canEdit: boolean; canView: boolean; draftPath: string }, -) => { - const { draftPath } = clientData; - const hasValidPrefix = ['pub-', 'drafts/'].some((prefix) => draftPath.startsWith(prefix)); - if (!hasValidPrefix) { - throw new Error( - `Will not create Firebase token for potentially dangerous draft path ${draftPath}`, - ); - } - const tokenData = { ...clientData, ...getFirebaseDraftPathParts(draftPath) }; - return firebaseAdmin.auth(firebaseApp!).createCustomToken(clientId, tokenData); -}; +/** + * Programmatically apply edits to a draft's document. Used by server-side operations + * like imports and migrations that need to modify the doc without a client. + */ +export const editDraft = async (pubId: string, clientId: string, schema: Schema = editorSchema) => { + const { draft } = await getPubDraft(pubId); + const checkpoint = await getDraftCheckpoint(draft.id); -export const editFirebaseDraftByRef = async ( - ref: firebase.database.Reference, - clientId: string, - schema: Schema = editorSchema, - initialState?: { doc: Node; key: number }, -) => { - const fetchDoc = async () => getFirebaseDoc(ref, schema); + let doc = checkpoint + ? Node.fromJSON(schema, checkpoint.doc) + : schema.topNodeType.createAndFill()!; - let { doc, key: currentKey } = initialState ?? (await fetchDoc()); + let currentVersion = draft.version; let pendingSteps: Step[] = []; const api = { @@ -324,42 +189,72 @@ export const editFirebaseDraftByRef = async ( pendingSteps.push(...tr.steps); return api; }, + writeChange: async (): Promise => { - const change = createFirebaseChange(pendingSteps, clientId); - const { committed } = await ref.child(`changes/${currentKey + 1}`).transaction( - (existingContent) => { - if (existingContent) { - // Don't overwrite -- bail instead - return undefined; - } - return change; - }, - undefined, - false, - ); - if (committed) { - ++currentKey; + if (pendingSteps.length === 0) { + return true; + } + + const { collabAuthority } = await import('server/collab/authority.js'); + + try { + const commitData = { + steps: pendingSteps.map((s) => s.toJSON()), + version: currentVersion, + clientId, + ref: `server-${Date.now()}-${Math.random().toString(36).slice(2)}`, + }; + + await collabAuthority.receiveCommit(draft.id, commitData); + currentVersion++; pendingSteps = []; + return true; + } catch (_err) { + return false; } - return committed; - }, - clearChanges: async () => { - await ref.child(`changes`).remove(); - const refetch = await fetchDoc(); - doc = refetch.doc; - currentKey = refetch.key; - pendingSteps = []; - }, - getDoc: () => { - return doc; - }, - getKey: () => { - return currentKey; - }, - getRef: () => { - return ref; }, + + getDoc: () => doc, + getKey: () => currentVersion, }; return api; }; + +/** + * Get steps from the commit table between two versions (exclusive start, inclusive end). + */ +export const getStepsBetweenVersions = async ( + draftId: string, + fromVersion: number, + toVersion: number, + schema: Schema = editorSchema, +): Promise => { + const commits = await CollabCommit.findAll({ + where: { + draftId, + version: { [Op.gt]: fromVersion, [Op.lte]: toVersion }, + }, + order: [['version', 'ASC']], + }); + + return commits.map((commit) => + commit.steps.map((stepJson: any) => Step.fromJSON(schema, stepJson)), + ); +}; + +// legacy utility for migration tools that still interact with firebase-admin directly +export const getDatabaseRef = (path: string) => { + try { + const firebaseAdmin = require('firebase-admin'); + const app = firebaseAdmin.apps[0]; + + if (!app) { + return null; + } + + return app.database().ref(path); + } catch { + return null; + } +}; diff --git a/server/utils/firebaseTools.ts b/server/utils/firebaseTools.ts index bccf91cdb7..5794b883e3 100644 --- a/server/utils/firebaseTools.ts +++ b/server/utils/firebaseTools.ts @@ -1,101 +1,55 @@ import type { DocJson } from 'types'; import { Fragment, Node, Slice } from 'prosemirror-model'; -import { ReplaceStep } from 'prosemirror-transform'; -import { buildSchema, createFirebaseChange } from 'client/components/Editor'; +import { buildSchema } from 'client/components/Editor/utils'; -import { getPubDraftDoc, getPubDraftRef } from './firebaseAdmin'; - -type Reference = Awaited>; +import { editDraft, getPubDraftDoc } from './firebaseAdmin'; const documentSchema = buildSchema(); -const makeReplaceStepFromTo = ({ - from, - to, - slice, - client = 'api', -}: { - from: number; - to: number; - slice: Slice; - client?: string; -}) => { - const replaceStep = new ReplaceStep(from, to, slice); - const change = createFirebaseChange([replaceStep], client); - return change; -}; - -const appendNewChange = async ({ - draftRef, - from, - to, - slice, - client = 'api', - baseKey, -}: { - draftRef: Reference; - from: number; - to: number; - slice: Slice; - client?: string; - baseKey?: number; -}) => { - const latestChange = (await draftRef.child('changes').limitToLast(1).once('value')).val(); - const latestFirebaseKey = latestChange ? Number(Object.keys(latestChange)[0]) : null; - - // Use the highest of the Firebase key and the provided base key (from PG checkpoint) - const latestKey = Math.max(latestFirebaseKey ?? -1, baseKey ?? -1); - const key = latestKey >= 0 ? latestKey + 1 : 0; - - const change = makeReplaceStepFromTo({ from, to, slice, client }); - await draftRef.child('changes').child(key.toString()).set(change); -}; - export const writeDocumentToPubDraft = async ( pubId: string, document: DocJson, options?: { method?: 'replace' | 'overwrite' | 'append' | 'prepend' }, ) => { const { method = 'replace' } = options || {}; - const draftRef = await getPubDraftRef(pubId); const hydratedDocument = Node.fromJSON(documentSchema, document); - const documentFragment = Fragment.from(hydratedDocument.content); const slice = new Slice(documentFragment, 0, 0); - const doc = hydratedDocument.toJSON() as DocJson; - const { size, doc: originalDoc, mostRecentRemoteKey } = await getPubDraftDoc(pubId, null); + const { size, doc: originalDoc } = await getPubDraftDoc(pubId, null); + + const editor = await editDraft(pubId, 'api', documentSchema); + switch (method) { - case 'overwrite': { - const change = makeReplaceStepFromTo({ from: 0, to: 0, slice }); - // this removes the old data - await draftRef.child('changes').set({ [mostRecentRemoteKey + 1]: change }); + case 'overwrite': + case 'replace': { + editor.transform((tr) => { + tr.replace(0, size, slice); + }); + await editor.writeChange(); return doc; } + case 'prepend': { - appendNewChange({ from: 0, to: 0, slice, draftRef, baseKey: mostRecentRemoteKey }); + editor.transform((tr) => { + tr.replace(0, 0, slice); + }); + await editor.writeChange(); return { ...originalDoc, content: [...doc.content, ...originalDoc.content], } as DocJson; } - case 'replace': { - appendNewChange({ from: 0, to: size, slice, draftRef, baseKey: mostRecentRemoteKey }); - return doc; - } default: { - appendNewChange({ - from: size, - to: size, - slice, - draftRef, - baseKey: mostRecentRemoteKey, + editor.transform((tr) => { + tr.replace(size, size, slice); }); + await editor.writeChange(); return { ...originalDoc, content: [...originalDoc.content, ...doc.content], diff --git a/server/utils/queryHelpers/pubEnrich.ts b/server/utils/queryHelpers/pubEnrich.ts index b4ed64039f..f0d64c7a82 100644 --- a/server/utils/queryHelpers/pubEnrich.ts +++ b/server/utils/queryHelpers/pubEnrich.ts @@ -12,7 +12,7 @@ import { Op } from 'sequelize'; import { Doc, Draft, PubEdge } from 'server/models'; import { generateCitationHtml, getStructuredCitationsForPub } from 'server/utils/citations'; -import { getFirebaseToken, getPubDraftDoc } from 'server/utils/firebaseAdmin'; +import { getPubDraftDoc } from 'server/utils/firebaseAdmin'; import { expect } from 'utils/assert'; import { getPubEdgeIncludes } from './pubEdgeOptions'; @@ -63,16 +63,13 @@ export const getPubRelease = async ( }; }; -export const getPubFirebaseToken = async (pubData: SanitizedPubData, initialData: InitialData) => { - const { canView, canViewDraft, canEdit, canEditDraft } = - initialData.scopeData.activePermissions; - const firebaseToken = await getFirebaseToken(initialData.loginData.id || 'anon', { - canEdit: canEdit || canEditDraft, - canView: canView || canViewDraft, - draftPath: pubData.draft?.firebasePath!, - }); +/** + * @deprecated Firebase token is no longer needed. Collab uses Pitter Patter with Postgres. + * Kept as a stub for any callers that haven't been updated yet. + */ +export const getPubFirebaseToken = async (_pubData: SanitizedPubData, _initialData: InitialData) => { return { - firebaseToken, + firebaseToken: null, }; }; diff --git a/stubstub/firebase.ts b/stubstub/firebase.ts index 3dae305630..8fb8d3e89a 100644 --- a/stubstub/firebase.ts +++ b/stubstub/firebase.ts @@ -1,14 +1,11 @@ -import uuid from 'uuid/v4'; - -import { editFirebaseDraftByRef, getDatabaseRef, getPubDraftRef } from 'server/utils/firebaseAdmin'; +import { editDraft, getPubDraft } from 'server/utils/firebaseAdmin'; const stubstubClientId = 'stubstub-firebase'; -export const editFirebaseDraft = (refKey: string = uuid()) => { - return editFirebaseDraftByRef(getDatabaseRef(refKey)!, stubstubClientId); +export const editFirebaseDraft = (_refKey?: string) => { + throw new Error('editFirebaseDraft is deprecated. Use editPub instead.'); }; export const editPub = async (pubId: string) => { - const draftRef = await getPubDraftRef(pubId); - return editFirebaseDraftByRef(draftRef, stubstubClientId); + return editDraft(pubId, stubstubClientId); }; diff --git a/stubstub/global/setup.ts b/stubstub/global/setup.ts index 274017e134..4ab54f927c 100644 --- a/stubstub/global/setup.ts +++ b/stubstub/global/setup.ts @@ -27,7 +27,7 @@ export default async () => { const dotenv = require('dotenv'); dotenv.config({ path: path.join(__dirname, '..', '..', 'infra', '.env.test') }); - const { env, refreshEnv } = await import('server/env'); + const { env, refreshEnv } = await import('server/env.js'); if (!process.env.DATABASE_URL) { console.log('\nSit tight while a local test database is created...'); @@ -49,7 +49,7 @@ export default async () => { * create the tables in the test db, leading to "relation does not exist" errors when running * tests */ - const { sequelize } = await import('../../server/models'); + const { sequelize } = await import('../../server/models.js'); // Install pg_trgm before sync so the User model's GIN trigram indexes can be created await sequelize.query('CREATE EXTENSION IF NOT EXISTS pg_trgm;'); await sequelize.sync(); @@ -60,7 +60,7 @@ export default async () => { * * If this is not here, then the tests will fail. */ - const { FeatureFlag } = await import('../../server/models'); + const { FeatureFlag } = await import('../../server/models.js'); await FeatureFlag.findOrCreate({ where: { diff --git a/stubstub/stub.ts b/stubstub/stub.ts index 311460fe70..1ddcceeac8 100644 --- a/stubstub/stub.ts +++ b/stubstub/stub.ts @@ -89,10 +89,8 @@ export const stubFirebaseAdmin = () => { size: 0, }), ); - const getFirebaseTokenStub = sinon - .stub(firebaseAdmin, 'getFirebaseToken') - .returns(Promise.resolve('')); - stubs = [getPubDraftDocStub, getFirebaseTokenStub]; + + stubs = [getPubDraftDocStub]; }); afterAll(() => stubs.forEach((stub) => stub.restore())); diff --git a/tools/bootstrapCheckpoints.ts b/tools/bootstrapCheckpoints.ts index f272f2dec4..74519e2c62 100644 --- a/tools/bootstrapCheckpoints.ts +++ b/tools/bootstrapCheckpoints.ts @@ -23,8 +23,6 @@ * pnpm run tools bootstrapCheckpoints --execute --replaceErrors # Write fallback docs for corrupted drafts */ -import type firebase from 'firebase'; - import firebaseAdmin from 'firebase-admin'; import { Op, QueryTypes } from 'sequelize'; @@ -407,7 +405,7 @@ const deleteFirebasePath = async (path: string): Promise => { const normalizePath = async (draft: Draft): Promise => { const { id, firebasePath } = draft; - if (!isLegacyPath(firebasePath)) { + if (!firebasePath || !isLegacyPath(firebasePath)) { stats.pathsSkipped++; return; } @@ -445,6 +443,11 @@ const normalizePath = async (draft: Draft): Promise => { const extractCheckpoint = async (draft: Draft): Promise => { const { id: draftId, firebasePath } = draft; + if (!firebasePath) { + stats.checkpointsSkippedEmpty++; + return; + } + // Skip if a checkpoint already exists const existing = await DraftCheckpoint.findOne({ where: { draftId } }); if (existing) { @@ -735,7 +738,7 @@ const main = async () => { // Phase 1: Normalize legacy paths (rolling concurrency) if (!skipPathNormalization) { - const legacyDrafts = drafts.filter((d) => isLegacyPath(d.firebasePath)); + const legacyDrafts = drafts.filter((d) => d.firebasePath && isLegacyPath(d.firebasePath)); log(''); log( `Phase 1: Path normalization (${legacyDrafts.length} legacy paths, concurrency=${CONCURRENCY})`, diff --git a/tools/cleanupFirebase.ts b/tools/cleanupFirebase.ts index bb79c49ac2..ff593008ee 100644 --- a/tools/cleanupFirebase.ts +++ b/tools/cleanupFirebase.ts @@ -19,8 +19,6 @@ * pnpm run tools cleanupFirebase --pubId= --execute */ -import type firebase from 'firebase'; - import type { DiscussionInfo } from 'components/Editor/plugins/discussions/types'; import firebaseAdmin from 'firebase-admin'; @@ -204,7 +202,7 @@ const stats: CleanupStats = { /** * Get all checkpoint keys for a draft */ -const getCheckpointKeys = async (draftRef: firebase.database.Reference): Promise => { +const getCheckpointKeys = async (draftRef: any): Promise => { const checkpointMapSnapshot = await draftRef.child('checkpointMap').once('value'); const checkpointMap = checkpointMapSnapshot.val(); @@ -225,7 +223,7 @@ const getCheckpointKeys = async (draftRef: firebase.database.Reference): Promise * Get the highest checkpoint key for a draft */ const getLatestCheckpointKey = async ( - draftRef: firebase.database.Reference, + draftRef: any, ): Promise => { const keys = await getCheckpointKeys(draftRef); if (keys.length === 0) return null; @@ -236,7 +234,7 @@ const getLatestCheckpointKey = async ( * Get the highest checkpoint key at or before a threshold */ // const _getCheckpointKeyAtOrBefore = async ( -// draftRef: firebase.database.Reference, +// draftRef: any, // threshold: number, // ): Promise => { // const keys = await getCheckpointKeys(draftRef); @@ -292,7 +290,7 @@ const batchGetLatestReleaseKeys = async (pubIds: string[]): Promise> => { const discussionsSnapshot = await draftRef.child('discussions').once('value'); const discussionsData = discussionsSnapshot.val(); @@ -380,7 +378,7 @@ const getFirebaseDiscussions = async ( * Fast-forward all outdated discussions to the target key */ const fastForwardDiscussions = async ( - draftRef: firebase.database.Reference, + draftRef: any, targetKey: number, ): Promise => { const discussions = await getFirebaseDiscussions(draftRef); @@ -449,7 +447,7 @@ const fastForwardDiscussions = async ( * Falls back to individual deletes if batch update fails with WRITE_TOO_BIG. */ const pruneKeysBefore = async ( - parentRef: firebase.database.Reference, + parentRef: any, childName: string, thresholdKey: number, ): Promise => { @@ -776,7 +774,7 @@ const deleteOrphanedDraft = async (draft: Draft): Promise => { if (!isDryRun) { // Delete from Firebase first (using deleteFirebasePath to handle large drafts) - await deleteFirebasePath(firebasePath); + if (firebasePath) await deleteFirebasePath(firebasePath); // Then delete from Postgres await draft.destroy(); @@ -811,7 +809,7 @@ const getValidFirebasePaths = async (): Promise> => { const drafts = await Draft.findAll({ attributes: ['firebasePath'], }); - return new Set(drafts.map((d) => d.firebasePath)); + return new Set(drafts.map((d) => d.firebasePath).filter((p): p is string => p !== null)); }; /** @@ -1082,8 +1080,8 @@ const processPubDraft = async (pubId: string): Promise => { return; } - if (!pub.draft) { - log(`Pub ${pubId} has no draft`); + if (!pub.draft || !pub.draft.firebasePath) { + log(`Pub ${pubId} has no draft or no firebase path`); return; } @@ -1107,8 +1105,8 @@ const processPubDraft = async (pubId: string): Promise => { const processDraftById = async (draftId: string): Promise => { const draft = await Draft.findOne({ where: { id: draftId } }); - if (!draft) { - log(`Draft not found: ${draftId}`); + if (!draft || !draft.firebasePath) { + log(`Draft not found or has no firebase path: ${draftId}`); return; } @@ -1195,6 +1193,8 @@ const processAllDrafts = async (): Promise => { const pubQueue = pubsWithFirebaseData.filter((pub) => { if (!pub.draft) return false; const { firebasePath } = pub.draft; + if (!firebasePath) return false; + // Modern format: drafts/draft-{draftId} if (firebasePath.startsWith('drafts/draft-')) { return firebaseDraftIdSet.has(pub.draft.id); @@ -1238,14 +1238,14 @@ const processAllDrafts = async (): Promise => { const releaseKey = releaseKeyCache.get(pub.id) ?? null; // biome-ignore lint/performance/noAwaitInLoops: worker pool pattern requires sequential processing await pruneDraft( - pub.draft!.firebasePath, + pub.draft!.firebasePath!, pub.id, releaseKey, pubLabel, localStats, pub.draft!.id, ); - await cleanupOrphanedBranchesForPub(pub.id, pub.draft!.firebasePath, localStats); + await cleanupOrphanedBranchesForPub(pub.id, pub.draft!.firebasePath!, localStats); localStats.draftsProcessed++; totalProcessed++; diff --git a/tools/coldStorage.ts b/tools/coldStorage.ts index d0bd47336f..c1729aa4ff 100644 --- a/tools/coldStorage.ts +++ b/tools/coldStorage.ts @@ -22,8 +22,6 @@ * (Prod/dev is determined by env vars: DATABASE_URL, FIREBASE_SERVICE_ACCOUNT_BASE64) */ -import type firebase from 'firebase'; - import firebaseAdmin from 'firebase-admin'; import { uncompressSelectionJSON } from 'prosemirror-compress-pubpub'; import { Op, QueryTypes } from 'sequelize'; @@ -139,7 +137,7 @@ const firebaseRest = async ( * List child keys at a Firebase path using REST API with ?shallow=true. * Never downloads the actual content, so safe for huge nodes. */ -const getShallowKeys = async (ref: firebase.database.Reference): Promise => { +const getShallowKeys = async (ref: any): Promise => { const refPath = ref.toString().replace(/^https:\/\/[^/]+\//, ''); const data = await firebaseRest | null>('GET', refPath, undefined, { shallow: 'true', @@ -267,6 +265,12 @@ const freezeDraft = async (draft: Draft, pubId: string): Promise => { const { id: draftId, firebasePath } = draft; const prefix = `[${draftId.slice(0, 8)}]`; + if (!firebasePath) { + verbose(`${prefix} No firebase path, skipping`); + stats.draftsAlreadyCold++; + return; + } + try { const draftRef = getDatabaseRef(firebasePath); diff --git a/tools/index.js b/tools/index.js index 5a1cb2b885..280d24fab0 100644 --- a/tools/index.js +++ b/tools/index.js @@ -81,6 +81,7 @@ const commandFiles = { migration2020_06_24: "./migration2020_06_24", migrationCommunityTemplates: "./migrationCommunityTemplates", migrationsDeprecated: "./migrationsDeprecated", + migrateFirebaseToPostgres: "./migrateFirebaseToPostgres", movePubs: "./movePubs", pubCrawl: "./pubCrawl", purgeNotifications: "./purgeNotifications", diff --git a/tools/localdb.ts b/tools/localdb.ts index 32033cef6f..ce0ee7fac4 100644 --- a/tools/localdb.ts +++ b/tools/localdb.ts @@ -2,7 +2,7 @@ import { setupLocalDatabase } from '../localDatabase'; const main = async () => { await setupLocalDatabase(true); - const { modelize } = await import('stubstub'); + const { modelize } = await import('stubstub/index.js'); const models = modelize` Community { createFullCommunity: true diff --git a/tools/migrateFirebasePaths.ts b/tools/migrateFirebasePaths.ts index e444eec193..885fc0739c 100644 --- a/tools/migrateFirebasePaths.ts +++ b/tools/migrateFirebasePaths.ts @@ -113,6 +113,11 @@ const copyFirebaseData = async (sourcePath: string, destPath: string): Promise => { const { id, firebasePath } = draft; + + if (!firebasePath) { + return false; + } + const modernPath = getModernPath(id); log(` Migrating draft ${id}`); @@ -201,7 +206,7 @@ const main = async () => { // Process drafts for (const draft of drafts) { - if (!isLegacyPath(draft.firebasePath)) { + if (!draft.firebasePath || !isLegacyPath(draft.firebasePath)) { verbose(` Skipping ${draft.id}: not a legacy path (${draft.firebasePath})`); stats.skipped++; continue; diff --git a/tools/migrateFirebaseToPostgres.ts b/tools/migrateFirebaseToPostgres.ts new file mode 100644 index 0000000000..f1fffda33e --- /dev/null +++ b/tools/migrateFirebaseToPostgres.ts @@ -0,0 +1,306 @@ +/** + * Migration Tool: Firebase -> Postgres Collab + * + * Migrates all active drafts from Firebase to the new Pitter Patter Collab + * data model in Postgres. For each draft: + * + * 1. Ensures a DraftCheckpoint exists (runs cold storage if needed) + * 2. Extracts any Firebase changes after the checkpoint + * 3. Inserts those changes into the CollabCommits table + * 4. Sets Draft.version to the latest commit version + * + * This script is safe to run repeatedly -- drafts that already have commits + * in Postgres are skipped. + * + * Usage: + * pnpm run tools migrateFirebaseToPostgres # Dry run + * pnpm run tools migrateFirebaseToPostgres --execute # Actually migrate + * pnpm run tools migrateFirebaseToPostgres --pubId= # Single pub + * pnpm run tools migrateFirebaseToPostgres --batchSize=100 # Custom batch size + * pnpm run tools migrateFirebaseToPostgres --verbose # Verbose output + */ + +import firebaseAdmin from 'firebase-admin'; +import { uncompressStepJSON } from 'prosemirror-compress-pubpub'; +import { Op, QueryTypes } from 'sequelize'; +import { v4 as uuid } from 'uuid'; + +import { editorSchema, getFirebaseDoc } from 'components/Editor'; +import { getDraftCheckpoint, upsertDraftCheckpoint } from 'server/draftCheckpoint/queries'; +import { CollabCommit, Draft, Pub } from 'server/models'; +import { sequelize } from 'server/sequelize'; +import { getFirebaseConfig } from 'utils/editor/firebaseConfig'; + +const { + argv: { execute, pubId: specificPubId, batchSize: batchSizeArg = 50, verbose: verboseFlag }, +} = require('yargs'); + +const isDryRun = !execute; +const BATCH_SIZE = Number(batchSizeArg); + +const log = (msg: string) => console.log(`[migrate] ${new Date().toISOString()} ${msg}`); +const verbose = (msg: string) => verboseFlag && log(msg); + +const formatDuration = (ms: number) => { + if (ms < 1000) return `${ms}ms`; + if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s`; + return `${Math.floor(ms / 60_000)}m ${Math.round((ms % 60_000) / 1000)}s`; +}; + +const getFirebaseApp = () => { + if (firebaseAdmin.apps.length > 0) { + return firebaseAdmin.apps[0]!; + } + + const serviceAccount = JSON.parse( + Buffer.from(process.env.FIREBASE_SERVICE_ACCOUNT_BASE64 as string, 'base64').toString(), + ); + + return firebaseAdmin.initializeApp({ + credential: firebaseAdmin.credential.cert(serviceAccount), + databaseURL: getFirebaseConfig().databaseURL, + }); +}; + +const getTotalDraftCount = async (): Promise => { + if (specificPubId) { + return 1; + } + + const [result] = await sequelize.query<{ count: string }>( + `SELECT COUNT(*) as count FROM "Drafts" WHERE "firebasePath" IS NOT NULL`, + { type: QueryTypes.SELECT }, + ); + + return parseInt(result.count, 10); +}; + +const getDraftBatch = async (offset: number): Promise => { + if (specificPubId) { + const pub = await Pub.findOne({ + where: { id: specificPubId }, + include: [{ model: Draft, as: 'draft' }], + }); + + if (!pub?.draft) { + throw new Error(`Pub ${specificPubId} not found or has no draft`); + } + + return [pub.draft]; + } + + return Draft.findAll({ + where: { + firebasePath: { [Op.ne]: null }, + }, + order: [['createdAt', 'ASC']], + limit: BATCH_SIZE, + offset, + }); +}; + +const migrateDraft = async (draft: Draft, firebaseApp: firebaseAdmin.app.App) => { + const draftId = draft.id; + const firebasePath = draft.firebasePath; + + if (!firebasePath) { + return { skipped: true }; + } + + const existingCommitCount = await CollabCommit.count({ where: { draftId } }); + if (existingCommitCount > 0) { + verbose(` [${draftId.slice(0, 8)}] already has ${existingCommitCount} commits, skipping`); + return { skipped: true }; + } + + const database = firebaseApp.database(); + const ref = database.ref(firebasePath) as any; + + let checkpoint = await getDraftCheckpoint(draftId); + + if (!checkpoint) { + verbose(` [${draftId.slice(0, 8)}] building checkpoint from Firebase...`); + + try { + const { doc, key: currentKey } = await getFirebaseDoc(ref, editorSchema); + + if (currentKey < 0) { + verbose(` [${draftId.slice(0, 8)}] empty Firebase, skipping`); + return { skipped: true }; + } + + if (!isDryRun) { + checkpoint = await upsertDraftCheckpoint( + draftId, + currentKey, + doc.toJSON() as any, + Date.now(), + ); + } + } catch (err) { + log(` [${draftId.slice(0, 8)}] ERROR building checkpoint: ${err}`); + return { error: true }; + } + } + + if (!checkpoint && isDryRun) { + return { wouldMigrate: true }; + } + + if (!checkpoint) { + return { error: true }; + } + + const checkpointKey = checkpoint.historyKey; + + try { + const changesSnapshot = await ref + .child('changes') + .orderByKey() + .startAt(String(checkpointKey + 1)) + .once('value'); + + const mergesSnapshot = await ref + .child('merges') + .orderByKey() + .startAt(String(checkpointKey + 1)) + .once('value'); + + const allKeyables = { + ...(changesSnapshot.val() || {}), + ...(mergesSnapshot.val() || {}), + }; + + const keys = Object.keys(allKeyables).sort((a, b) => parseInt(a, 10) - parseInt(b, 10)); + + if (keys.length === 0) { + if (!isDryRun) { + await Draft.update({ version: checkpointKey }, { where: { id: draftId } }); + } + + return { migrated: true, commits: 0 }; + } + + if (isDryRun) { + log(` [${draftId.slice(0, 8)}] would migrate ${keys.length} change(s)`); + return { wouldMigrate: true, commits: keys.length }; + } + + await sequelize.transaction(async (txn) => { + for (const key of keys) { + const keyNum = parseInt(key, 10); + const changeData = allKeyables[key]; + const changes = Array.isArray(changeData) ? changeData : [changeData]; + + for (const change of changes) { + const stepsJson = change.s.map((compressed: any) => + uncompressStepJSON(compressed), + ); + + await CollabCommit.create( + { + draftId, + version: keyNum, + ref: uuid(), + steps: stepsJson, + }, + { transaction: txn }, + ); + } + } + + const latestVersion = parseInt(keys[keys.length - 1], 10); + await Draft.update( + { version: latestVersion }, + { where: { id: draftId }, transaction: txn }, + ); + }); + + verbose(` [${draftId.slice(0, 8)}] migrated ${keys.length} commit(s)`); + return { migrated: true, commits: keys.length }; + } catch (err) { + log(` [${draftId.slice(0, 8)}] ERROR: ${err}`); + return { error: true }; + } +}; + +const main = async () => { + const startTime = Date.now(); + + log(isDryRun ? 'DRY RUN (pass --execute to apply)' : 'EXECUTING migration'); + log(`Batch size: ${BATCH_SIZE}`); + + const firebaseApp = getFirebaseApp(); + const totalDrafts = await getTotalDraftCount(); + + log(`Total drafts with firebasePath: ${totalDrafts}`); + + let migrated = 0; + let skipped = 0; + let errors = 0; + let totalCommits = 0; + let processed = 0; + let offset = 0; + + while (true) { + const batch = await getDraftBatch(offset); + + if (batch.length === 0) { + break; + } + + const batchNum = Math.floor(offset / BATCH_SIZE) + 1; + const totalBatches = Math.ceil(totalDrafts / BATCH_SIZE); + log(`Processing batch ${batchNum}/${totalBatches} (${batch.length} drafts, offset ${offset})`); + + for (const draft of batch) { + const result = await migrateDraft(draft, firebaseApp); + + if (result.skipped) { + skipped++; + } else if (result.error) { + errors++; + } else { + migrated++; + totalCommits += (result as any).commits ?? 0; + } + + processed++; + + if (processed % 100 === 0) { + const elapsed = Date.now() - startTime; + const rate = processed / (elapsed / 1000); + const remaining = totalDrafts - processed; + const eta = remaining / rate; + + log( + ` Progress: ${processed}/${totalDrafts} (${Math.round((processed / totalDrafts) * 100)}%) ` + + `| migrated=${migrated} skipped=${skipped} errors=${errors} ` + + `| ${rate.toFixed(1)} drafts/sec, ETA ${formatDuration(eta * 1000)}`, + ); + } + } + + offset += batch.length; + + // for a single pub, one iteration is enough + if (specificPubId) { + break; + } + } + + const elapsed = Date.now() - startTime; + log(''); + log(`Finished in ${formatDuration(elapsed)}`); + log(` Processed: ${processed}`); + log(` Migrated: ${migrated} (${totalCommits} total commits)`); + log(` Skipped: ${skipped}`); + log(` Errors: ${errors}`); + + process.exit(errors > 0 ? 1 : 0); +}; + +main().catch((err) => { + console.error('Fatal error:', err); + process.exit(1); +}); diff --git a/tools/migrateRedshift.ts b/tools/migrateRedshift.ts index 2aabcea21d..c0a594805b 100644 --- a/tools/migrateRedshift.ts +++ b/tools/migrateRedshift.ts @@ -445,7 +445,7 @@ async function main() { log('[7/7] creating & refreshing summary materialized views...'); const { createSummaryViews, refreshSummaryViews } = await import( - 'server/analytics/summaryViews' + 'server/analytics/summaryViews.js' ); await createSummaryViews(); await refreshSummaryViews(); diff --git a/tools/migrations/2026_06_16_addCollabCommitTableAndDraftVersion.js b/tools/migrations/2026_06_16_addCollabCommitTableAndDraftVersion.js new file mode 100644 index 0000000000..8f5ad22e3c --- /dev/null +++ b/tools/migrations/2026_06_16_addCollabCommitTableAndDraftVersion.js @@ -0,0 +1,92 @@ +export const up = async ({ Sequelize, sequelize }) => { + const qi = sequelize.queryInterface; + + // add version column to Drafts + const draftsDesc = await qi.describeTable('Drafts'); + if (!draftsDesc.version) { + await qi.addColumn('Drafts', 'version', { + type: Sequelize.INTEGER, + allowNull: false, + defaultValue: 0, + }); + } + + // make firebasePath nullable (was NOT NULL before) + if (draftsDesc.firebasePath && !draftsDesc.firebasePath.allowNull) { + await qi.changeColumn('Drafts', 'firebasePath', { + type: Sequelize.STRING, + allowNull: true, + }); + } + + // create CollabCommits table + const tableExists = await qi.describeTable('CollabCommits').catch(() => null); + if (!tableExists) { + await qi.createTable('CollabCommits', { + id: { + type: Sequelize.UUID, + primaryKey: true, + defaultValue: Sequelize.UUIDV4, + }, + draftId: { + type: Sequelize.UUID, + allowNull: false, + references: { model: 'Drafts', key: 'id' }, + onDelete: 'CASCADE', + }, + version: { + type: Sequelize.INTEGER, + allowNull: false, + }, + ref: { + type: Sequelize.TEXT, + allowNull: false, + }, + steps: { + type: Sequelize.JSONB, + allowNull: false, + }, + createdAt: { + type: Sequelize.DATE, + allowNull: false, + }, + updatedAt: { + type: Sequelize.DATE, + allowNull: false, + }, + }); + + await qi.addConstraint('CollabCommits', { + fields: ['draftId', 'version'], + type: 'unique', + name: 'collab_commits_draft_version_unique', + }); + + await qi.addIndex('CollabCommits', ['draftId', 'ref'], { + name: 'collab_commits_draft_ref_idx', + }); + + await qi.addIndex('CollabCommits', ['draftId', 'version'], { + name: 'collab_commits_draft_version_idx', + }); + } +}; + +export const down = async ({ Sequelize, sequelize }) => { + const qi = sequelize.queryInterface; + + await qi.dropTable('CollabCommits').catch(() => {}); + + const draftsDesc = await qi.describeTable('Drafts'); + + if (draftsDesc.version) { + await qi.removeColumn('Drafts', 'version'); + } + + if (draftsDesc.firebasePath) { + await qi.changeColumn('Drafts', 'firebasePath', { + type: Sequelize.STRING, + allowNull: false, + }); + } +}; diff --git a/tools/migrations/2026_06_16_removeFirebasePath.js b/tools/migrations/2026_06_16_removeFirebasePath.js new file mode 100644 index 0000000000..77535cc80a --- /dev/null +++ b/tools/migrations/2026_06_16_removeFirebasePath.js @@ -0,0 +1,26 @@ +/** + * Final cleanup migration: removes the firebasePath column from Drafts. + * Run ONLY after all drafts have been fully migrated to Pitter Patter + * and the Firebase migration tool has been run successfully. + */ + +export const up = async ({ Sequelize, sequelize }) => { + const qi = sequelize.queryInterface; + const draftsDesc = await qi.describeTable('Drafts'); + + if (draftsDesc.firebasePath) { + await qi.removeColumn('Drafts', 'firebasePath'); + } +}; + +export const down = async ({ Sequelize, sequelize }) => { + const qi = sequelize.queryInterface; + const draftsDesc = await qi.describeTable('Drafts'); + + if (!draftsDesc.firebasePath) { + await qi.addColumn('Drafts', 'firebasePath', { + type: Sequelize.STRING, + allowNull: true, + }); + } +}; diff --git a/tsconfig.json b/tsconfig.json index 97b79e1da3..9439529f38 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -10,7 +10,7 @@ /* Visit https://aka.ms/tsconfig.json to read more about this file */ /* Basic Options */ "target": "es6" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */, - "module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */, + "module": "nodenext" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */, "lib": ["es2019", "dom"] /* Specify library files to be included in the compilation. */, "allowJs": true /* Allow javascript files to be compiled. */, // "checkJs": true, /* Report errors in .js files. */ @@ -43,7 +43,7 @@ // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ /* Module Resolution Options */ - // "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ + "moduleResolution": "nodenext" /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */, "baseUrl": "./" /* Base directory to resolve non-absolute module names. */, "paths": { "client": ["./client"], diff --git a/utils/api/schemas/draft.ts b/utils/api/schemas/draft.ts index 8031f48af7..15a346d14b 100644 --- a/utils/api/schemas/draft.ts +++ b/utils/api/schemas/draft.ts @@ -6,5 +6,5 @@ export const draftSchema = z.object({ .date() .transform((d) => d.toString()) .nullable() as z.ZodType, - firebasePath: z.string(), + firebasePath: z.string().nullable(), }); diff --git a/utils/caching/__tests__/purge.test.ts b/utils/caching/__tests__/purge.test.ts index 2f6bf6ddea..5d7a909455 100644 --- a/utils/caching/__tests__/purge.test.ts +++ b/utils/caching/__tests__/purge.test.ts @@ -149,7 +149,7 @@ let serviceId: string; setup(beforeAll, async () => { await models.resolve(); - const { env } = await import('server/env'); + const { env } = await import('server/env.js'); token = env.FASTLY_PURGE_TOKEN; serviceId = env.FASTLY_SERVICE_ID; @@ -165,7 +165,7 @@ setup(beforeAll, async () => { }); teardown(afterAll, async () => { - const { env } = await import('server/env'); + const { env } = await import('server/env.js'); env.TEST_FASTLY_PURGE = false; setEnvironment(false, false, false); diff --git a/workers/tasks/export/html.tsx b/workers/tasks/export/html.tsx index 42d5939367..8e97fd9e9e 100644 --- a/workers/tasks/export/html.tsx +++ b/workers/tasks/export/html.tsx @@ -8,7 +8,7 @@ import fs from 'fs'; import path from 'path'; import ReactDOMServer from 'react-dom/server'; -import { editorSchema, renderStatic } from 'components/Editor'; +import { editorSchema, renderStatic } from 'components/Editor/utils'; import { intersperse, unique } from 'utils/arrays'; import { renderNotesForListing } from '../../../utils/notes'; diff --git a/workers/tasks/export/notes.ts b/workers/tasks/export/notes.ts index e1fea916a7..6e851160b0 100644 --- a/workers/tasks/export/notes.ts +++ b/workers/tasks/export/notes.ts @@ -7,7 +7,7 @@ import type { NotesData } from './types'; import sanitizeHtml from 'sanitize-html'; import { NoteManager } from 'client/utils/notes'; -import { getNotesByKindFromDoc, jsonToNode } from 'components/Editor'; +import { getNotesByKindFromDoc, jsonToNode } from 'components/Editor/utils'; import { getStructuredCitations } from 'server/utils/citations'; export type PandocNote = { diff --git a/workers/tasks/export/pandoc.ts b/workers/tasks/export/pandoc.ts index 7db436fc63..bf79d6161c 100644 --- a/workers/tasks/export/pandoc.ts +++ b/workers/tasks/export/pandoc.ts @@ -12,7 +12,7 @@ import fs from 'fs'; import path from 'path'; import YAML from 'yaml'; -import { editorSchema, getReactedDocFromJson } from 'components/Editor'; +import { editorSchema, getReactedDocFromJson } from 'components/Editor/utils'; import { getPathToCslFileForCitationStyleKind } from 'server/utils/citations'; import { normalizeOrcid } from 'utils/orcid'; diff --git a/workers/tasks/export/styles/buildCss.mts b/workers/tasks/export/styles/buildCss.mts index 51be237ed3..1d2ce1be74 100644 --- a/workers/tasks/export/styles/buildCss.mts +++ b/workers/tasks/export/styles/buildCss.mts @@ -7,7 +7,6 @@ const katexCdnPrefix = 'https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.13.18/'; * this is run once per build to generate the export CSS */ export const buildExportCss = async () => { - // @ts-expect-error shh const stylesDir = path.join(new URL('.', import.meta.url).pathname); const entrypoint = path.join(stylesDir, 'printDocument.scss'); const cssPath = path.join(stylesDir, 'printDocument.css'); @@ -39,7 +38,6 @@ export const buildExportCss = async () => { return cssPath; }; -// @ts-expect-error shh if (import.meta.main) { buildExportCss(); } diff --git a/workers/tasks/export/types.ts b/workers/tasks/export/types.ts index f345cd60c7..e174a73e29 100644 --- a/workers/tasks/export/types.ts +++ b/workers/tasks/export/types.ts @@ -1,5 +1,5 @@ import type { NoteManager } from 'client/utils/notes'; -import type { NodeLabelMap } from 'components/Editor'; +import type { NodeLabelMap } from 'components/Editor/types'; import type { AttributionWithUser, Collection, Maybe, RenderedLicense } from 'types'; import type { CitationInlineStyleKind, CitationStyleKind } from 'utils/citations'; import type { Note, RenderedStructuredValues } from 'utils/notes'; diff --git a/workers/tasks/import/bulk/directives/pub.ts b/workers/tasks/import/bulk/directives/pub.ts index 1e0adc5c67..2ca3dd2890 100644 --- a/workers/tasks/import/bulk/directives/pub.ts +++ b/workers/tasks/import/bulk/directives/pub.ts @@ -8,13 +8,13 @@ import { Op } from 'sequelize'; import tmp from 'tmp-promise'; import YAML from 'yaml'; -import { buildSchema, createFirebaseChange } from 'components/Editor/utils'; +import { buildSchema } from 'components/Editor/utils'; import { createCollection } from 'server/collection/queries'; import { createCollectionPub } from 'server/collectionPub/queries'; import { Collection, PubAttribution } from 'server/models'; import { createPub as createPubQuery } from 'server/pub/queries'; import { setSummarizeParentScopesOnPubCreation } from 'server/scopeSummary'; -import { getPubDraftRef } from 'server/utils/firebaseAdmin'; +import { editDraft } from 'server/utils/firebaseAdmin'; import { bibliographyFormats, extensionToPandocFormat } from 'utils/import/formats'; import { getUrlForAssetKey, uploadFileToAssetStore } from '../../assetStore'; @@ -438,12 +438,15 @@ const addFileNodeToDocument = (document, fileNodeAttrs) => { }; const writeDocumentToPubDraft = async (pubId, document) => { - const draftRef = await getPubDraftRef(pubId); const hydratedDocument = Node.fromJSON(documentSchema, document); const documentSlice = new Slice(Fragment.from(hydratedDocument.content), 0, 0); const replaceStep = new ReplaceStep(0, 0, documentSlice); - const change = createFirebaseChange([replaceStep], 'bulk-importer'); - await draftRef.child('changes').child('0').set(change); + + const editor = await editDraft(pubId, 'bulk-importer'); + editor.transform((tr) => { + tr.step(replaceStep); + }); + await editor.writeChange(); }; export const resolvePubDirective = async ({ directive, targetPath, community, collection }) => { diff --git a/workers/tasks/import/rules.ts b/workers/tasks/import/rules.ts index 0f56f84904..b0bbca80d5 100644 --- a/workers/tasks/import/rules.ts +++ b/workers/tasks/import/rules.ts @@ -1,7 +1,7 @@ import { pandocUtils, RuleSet, transformers, transformUtils } from '@pubpub/prosemirror-pandoc'; import md5 from 'crypto-js/md5'; -import { editorSchema } from 'components/Editor'; +import { editorSchema } from 'components/Editor/utils'; import { renderToKatexString } from 'utils/katex'; const { createAttr, flatten, intersperse, textFromStrSpace, textToStrSpace } = transformUtils; From 1eea774c3b31ca5b79fe5ed19640481a66073ee6 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Wed, 17 Jun 2026 14:36:40 +0200 Subject: [PATCH 02/17] fix: some progress --- .../Editor/plugins/collaborative/index.ts | 9 +- .../PubHeaderThemeEditor/TextStyleChoice.tsx | 68 +-- .../FormattingBar/BlockTypeSelector.tsx | 116 +++--- .../components/FormattingBar/CommandMenu.tsx | 2 +- .../FormattingBar/FormattingBarButton.tsx | 3 +- .../FormattingBarMediaButton.tsx | 54 +-- client/components/Menu/Menu.tsx | 134 +++--- client/components/PubEdge/PubEdgeEditor.tsx | 4 +- .../Pub/PubHeader/LargeHeaderButton.tsx | 116 +++--- package.json | 24 +- pnpm-lock.yaml | 387 +++++++++++------- server/collab/api.ts | 2 + server/collab/authority.ts | 1 + server/server.ts | 4 + .../2024_01_11_addAnalyticsSettings.js | 10 - ...4_02_19_addMastodonAndInstagramSettings.js | 1 - .../2024_06_12_replaceLinkedinHandles.js | 3 +- .../2025_11_19_addDiscussionCreationAccess.js | 1 - tsconfig.json | 8 +- types/global.d.ts | 6 +- types/scss.d.ts | 3 + .../tasks/archive/siteDownloaderTransform.ts | 3 +- workers/tasks/import/bulk/paths.ts | 4 +- 23 files changed, 538 insertions(+), 425 deletions(-) create mode 100644 types/scss.d.ts diff --git a/client/components/Editor/plugins/collaborative/index.ts b/client/components/Editor/plugins/collaborative/index.ts index 330a55dcfc..8b6d1ba48e 100644 --- a/client/components/Editor/plugins/collaborative/index.ts +++ b/client/components/Editor/plugins/collaborative/index.ts @@ -1,5 +1,7 @@ +import type { CollabState } from '@stepwisehq/prosemirror-collab-commit/collab-commit'; + import { collab } from '@pitter-patter/collab-client'; -import { PluginKey } from 'prosemirror-state'; +import { type Plugin, PluginKey } from 'prosemirror-state'; import { generateHash } from 'utils/hashes'; @@ -16,7 +18,10 @@ export default (schema, props) => { const localClientId = `${props.collaborativeOptions.clientData.id}-${generateHash(6)}`; return [ - collab({ version: props.collaborativeOptions.initialDocKey }), + collab({ + version: props.collaborativeOptions.initialDocKey, + }), + //as unknown as Plugin, buildDocument(schema, props, collabDocPluginKey, localClientId), buildCursors(schema, props, collabDocPluginKey), ]; diff --git a/client/components/FacetEditor/definitions/PubHeaderThemeEditor/TextStyleChoice.tsx b/client/components/FacetEditor/definitions/PubHeaderThemeEditor/TextStyleChoice.tsx index 719ae49b5d..5658896253 100644 --- a/client/components/FacetEditor/definitions/PubHeaderThemeEditor/TextStyleChoice.tsx +++ b/client/components/FacetEditor/definitions/PubHeaderThemeEditor/TextStyleChoice.tsx @@ -11,44 +11,50 @@ import { PubHeaderBackground } from 'components'; type Props = { className?: string; communityData: Community; - label: React.ReactNode; + label: string; onClick: () => unknown; selected?: boolean; style?: React.CSSProperties; pubHeaderTheme: CascadedFacetType; }; -const TextStyleChoice = React.forwardRef((props: Props, ref) => { - const { - label, - className, - onClick, - selected = false, - style = {}, - communityData, - pubHeaderTheme, - } = props; +const TextStyleChoice = React.forwardRef( + (props: Props, ref: React.ForwardedRef) => { + const { + label, + className, + onClick, + selected = false, + style = {}, + communityData, + pubHeaderTheme, + } = props; - return ( - // @ts-expect-error ts-migrate(2769) FIXME: No overload matches this call. - - ); -}); + +
Aa
+ +
{label}
+ + ); + }, +); export default TextStyleChoice; diff --git a/client/components/FormattingBar/BlockTypeSelector.tsx b/client/components/FormattingBar/BlockTypeSelector.tsx index e109b8591f..f3c959abb3 100644 --- a/client/components/FormattingBar/BlockTypeSelector.tsx +++ b/client/components/FormattingBar/BlockTypeSelector.tsx @@ -83,65 +83,67 @@ const blockTypeDefinitions = [ }, ]; -const BlockTypeSelector = React.forwardRef((props: Props, ref) => { - const { editorChangeObject, isSmall, ...restProps } = props; - - const renderDisclosure = (disclosureProps: CommandMenuDisclosureProps) => { - const { commands, commandStates, disclosureElementProps } = disclosureProps; - const { ref: innerRef, ...restDisclosureElementProps } = disclosureElementProps; - const commandsFlat = commands.reduce((a, b) => [...a, ...b], []); - - const activeCommandEntry = commandsFlat.reduce( - (found, entry) => { - if (found) { - return found; - } - const commandState = commandStates[entry.key]; - if (commandState?.isActive && 'command' in entry) { - return entry; - } - return null; - }, - null as null | CommandDefinition, - ); - - const matchingBlockType = blockTypeDefinitions.find( - (def) => def.command === activeCommandEntry?.command, - ); - - const runnableCommandEntries = commandsFlat.filter((command) => { - const commandState = commandStates[command.key]; - return commandState?.canRun; - }); - - const effectiveBlockType = matchingBlockType || paragraphBlockType; +const BlockTypeSelector = React.forwardRef( + (props: Props, ref: React.ForwardedRef) => { + const { editorChangeObject, isSmall, ...restProps } = props; + + const renderDisclosure = (disclosureProps: CommandMenuDisclosureProps) => { + const { commands, commandStates, disclosureElementProps } = disclosureProps; + const { ref: innerRef, ...restDisclosureElementProps } = disclosureElementProps; + const commandsFlat = commands.reduce((a, b) => [...a, ...b], []); + + const activeCommandEntry = commandsFlat.reduce( + (found, entry) => { + if (found) { + return found; + } + const commandState = commandStates[entry.key]; + if (commandState?.isActive && 'command' in entry) { + return entry; + } + return null; + }, + null as null | CommandDefinition, + ); + + const matchingBlockType = blockTypeDefinitions.find( + (def) => def.command === activeCommandEntry?.command, + ); + + const runnableCommandEntries = commandsFlat.filter((command) => { + const commandState = commandStates[command.key]; + return commandState?.canRun; + }); + + const effectiveBlockType = matchingBlockType || paragraphBlockType; + + return ( + + ); + }; return ( - + ); - }; - - return ( - - ); -}); + }, +); export default BlockTypeSelector; diff --git a/client/components/FormattingBar/CommandMenu.tsx b/client/components/FormattingBar/CommandMenu.tsx index 229519c7b7..b5387b8db8 100644 --- a/client/components/FormattingBar/CommandMenu.tsx +++ b/client/components/FormattingBar/CommandMenu.tsx @@ -27,7 +27,7 @@ type Props = { markActiveItems?: boolean; }; -const CommandMenu = React.forwardRef((props: Props, ref) => { +const CommandMenu = React.forwardRef((props: Props, ref: React.ForwardedRef) => { const { editorChangeObject: { view }, className = '', diff --git a/client/components/FormattingBar/FormattingBarButton.tsx b/client/components/FormattingBar/FormattingBarButton.tsx index 71bf560927..6d4188e2e1 100644 --- a/client/components/FormattingBar/FormattingBarButton.tsx +++ b/client/components/FormattingBar/FormattingBarButton.tsx @@ -55,7 +55,7 @@ const getIndicatorStyle = (accentColor) => { const popoverModifiers = { preventOverflow: { enabled: false }, flip: { enabled: false } }; -const FormattingBarButton = React.forwardRef((props: FormattingBarButtonProps, ref) => { +const FormattingBarButton = React.forwardRef((props: FormattingBarButtonProps, ref: React.ForwardedRef) => { const { disabled = false, formattingItem, @@ -73,7 +73,6 @@ const FormattingBarButton = React.forwardRef((props: FormattingBarButtonProps, r } = props; let button = ( - // @ts-expect-error ts-migrate(2769) FIXME: No overload matches this call. - ); -}); + + ); + }, +); export default LargeHeaderButton; diff --git a/package.json b/package.json index 0889dc5225..c941ba647c 100644 --- a/package.json +++ b/package.json @@ -21,15 +21,15 @@ "build-dev-once": "rm -rf ./dist/client && pnpm run build-client-dev && pnpm run build-server", "build-prod": "rm -rf ./dist && webpack --config ./client/webpack/webpackConfig.prod.js && pnpm run build:export-css && pnpm run build-server", "heroku-postbuild": "pnpm run build-prod && [ -z $CI ] && pnpm run upload-sentry-sourcemaps && pnpm run write-commit-version || true", - "build-server": "tsc -p tsconfig.server.json", - "watch-server": "pnpm run build-server -w --preserveWatchOutput", + "build-server": "tsgo -p tsconfig.server.json", + "watch-server": "pnpm run build-server --watch --preserveWatchOutput", "postbuild-server": "cpy '**/*' '!**/*.ts' '!**/*.tsx' '!**/*.js' '!tsconfig.json' ../dist/server/server/ --cwd=server/ --parents && pnpm run copy-worker-assets", "copy-worker-assets": "cpy '**/*' '!**/*.ts' '!**/*.tsx' '!**/*.js' '!tsconfig.json' '!**/__tests__' ../dist/server/workers/ --cwd=workers/ --parents", "build:export-css": "NODE_PATH=./client:./ tsx ./workers/tasks/export/styles/buildCss.mts", "prod": "pnpm run build-prod && pnpm run api-prod", "lint": "biome check --max-diagnostics=10000 --diagnostic-level=error && pnpm run check", "lint:fix": "biome check --max-diagnostics=10000 --diagnostic-level=error --write", - "check": "tsc --noEmit", + "check": "tsgo --noEmit", "cron": "tsx tools/cron.ts", "install-git-hooks": "rm -f ./.git/hooks/* && cp ./.githooks/* ./.git/hooks && chmod +x ./.git/hooks/*", "continue": "concurrently --kill-others \"pnpm run api-dev\" \"pnpm run build-dev\"", @@ -97,10 +97,10 @@ "@lezer/rust": "^1.0.0", "@lezer/xml": "^1.0.0", "@monaco-editor/react": "4.1.1", - "@pitter-patter/collab-client": "^0.1.0", - "@pitter-patter/collab-server": "^0.1.0", - "@pitter-patter/presence-client": "^0.1.0", - "@pitter-patter/presence-server": "^0.1.0", + "@pitter-patter/collab-client": "^0.1.3", + "@pitter-patter/collab-server": "^0.1.3", + "@pitter-patter/presence-client": "^0.2.1", + "@pitter-patter/presence-server": "^0.1.3", "@popperjs/core": "^2.11.5", "@pubpub/deposit-utils": "^0.1.10", "@pubpub/prosemirror-pandoc": "^1.1.5", @@ -328,6 +328,9 @@ "@types/supertest": "^2.0.12", "@types/unidecode": "^0.1.1", "@types/uuid": "^3.4.10", + "@typescript/native-preview": "7.0.0-dev.20260616.1", + + "@typescript/native-preview-linux-arm64": "7.0.0-dev.20260616.1", "autoprefixer": "^9.5.0", "chalk": "^2.4.2", "chokidar": "^3.5.3", @@ -359,7 +362,7 @@ "terser-webpack-plugin": "^1.2.3", "ts-loader": "^8.0.11", "ts-morph": "^21.0.1", - "typescript": "^5.3.3", + "typescript": "^6.0.3", "vitest": "^4.0.10", "webpack": "^4.41.5", "webpack-bundle-analyzer": "^3.1.0", @@ -382,6 +385,11 @@ "protobufjs", "validate-with-xmllint" ], + "overrides": { + "prosemirror-model": "1.25.4", + "prosemirror-state": "1.4.4", + "prosemirror-view": "1.41.6" + }, "patchedDependencies": { "reakit": "patches/reakit.patch", "@pubpub/deposit-utils": "patches/@pubpub__deposit-utils.patch" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f03e44e2b1..36ab7b0522 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4,6 +4,11 @@ settings: autoInstallPeers: true excludeLinksFromLockfile: false +overrides: + prosemirror-model: 1.25.4 + prosemirror-state: 1.4.4 + prosemirror-view: 1.41.6 + patchedDependencies: '@pubpub/deposit-utils': hash: 6fef8933046b7751abda1bd2ed0422094c79d7828b07f7d5afba26a47c696d89 @@ -146,17 +151,17 @@ importers: specifier: 4.1.1 version: 4.1.1(monaco-editor@0.21.3)(react-dom@16.14.0(react@16.14.0))(react@16.14.0) '@pitter-patter/collab-client': - specifier: ^0.1.0 - version: 0.1.0(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0) + specifier: ^0.1.3 + version: 0.1.3(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0) '@pitter-patter/collab-server': - specifier: ^0.1.0 - version: 0.1.0(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0) + specifier: ^0.1.3 + version: 0.1.3(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0) '@pitter-patter/presence-client': - specifier: ^0.1.0 - version: 0.1.0(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0)(prosemirror-view@1.41.6) + specifier: ^0.2.1 + version: 0.2.1(@handlewithcare/react-prosemirror@3.1.6(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-view@1.41.6)(react-dom@16.14.0(react@16.14.0))(react-reconciler@0.32.0(react@16.14.0))(react@16.14.0))(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0)(prosemirror-view@1.41.6)(react-dom@16.14.0(react@16.14.0))(react-reconciler@0.32.0(react@16.14.0))(react@16.14.0) '@pitter-patter/presence-server': - specifier: ^0.1.0 - version: 0.1.0 + specifier: ^0.1.3 + version: 0.1.3 '@popperjs/core': specifier: ^2.11.5 version: 2.11.8 @@ -485,13 +490,13 @@ importers: specifier: ^1.1.3 version: 1.2.3 prosemirror-model: - specifier: ^1.18.3 + specifier: 1.25.4 version: 1.25.4 prosemirror-schema-list: specifier: ^1.2.2 version: 1.5.1 prosemirror-state: - specifier: ^1.4.2 + specifier: 1.4.4 version: 1.4.4 prosemirror-suggest: specifier: ^0.7.6 @@ -503,7 +508,7 @@ importers: specifier: ^1.7.0 version: 1.11.0 prosemirror-view: - specifier: ^1.29.2 + specifier: 1.41.6 version: 1.41.6 query-string: specifier: ^6.4.0 @@ -585,7 +590,7 @@ importers: version: 2.1.6(@types/node@24.11.0)(@types/validator@13.15.10)(reflect-metadata@0.1.14)(sequelize@6.37.7(pg@8.18.0)) sequelize-typescript-generator: specifier: ^10.1.2 - version: 10.1.2(@types/node@24.11.0)(@types/validator@13.15.10)(pg@8.18.0)(reflect-metadata@0.1.14)(typescript@5.9.3) + version: 10.1.2(@types/node@24.11.0)(@types/validator@13.15.10)(pg@8.18.0)(reflect-metadata@0.1.14)(typescript@6.0.3) sitemap: specifier: ^6.2.0 version: 6.4.0 @@ -609,7 +614,7 @@ importers: version: 2.1.1 ts-node: specifier: ^10.9.2 - version: 10.9.2(@types/node@24.11.0)(typescript@5.9.3) + version: 10.9.2(@types/node@24.11.0)(typescript@6.0.3) tsx: specifier: ^4.21.0 version: 4.21.0 @@ -633,7 +638,7 @@ importers: version: 1.0.9 vite-tsconfig-paths: specifier: ^5.1.4 - version: 5.1.4(typescript@5.9.3)(vite@7.3.1(@types/node@24.11.0)(jiti@2.6.1)(sass@1.67.0)(terser@5.46.0)(tsx@4.21.0)(yaml@1.10.2)) + version: 5.1.4(typescript@6.0.3)(vite@7.3.1(@types/node@24.11.0)(jiti@2.6.1)(sass@1.67.0)(terser@5.46.0)(tsx@4.21.0)(yaml@1.10.2)) xmlbuilder: specifier: ^13.0.2 version: 13.0.2 @@ -679,7 +684,7 @@ importers: version: 6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0) '@storybook/react': specifier: ^6.4.0 - version: 6.5.16(@babel/core@7.29.0)(@types/webpack@4.41.40)(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(require-from-string@2.0.2)(type-fest@0.21.3)(typescript@5.9.3)(webpack-cli@3.3.12)(webpack-dev-server@5.2.2(tslib@1.14.1)(webpack-cli@3.3.12)(webpack@4.47.0))(webpack-hot-middleware@2.26.1) + version: 6.5.16(@babel/core@7.29.0)(@types/webpack@4.41.40)(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(require-from-string@2.0.2)(type-fest@0.21.3)(typescript@6.0.3)(webpack-cli@3.3.12)(webpack-dev-server@5.2.2(tslib@1.14.1)(webpack-cli@3.3.12)(webpack@4.47.0))(webpack-hot-middleware@2.26.1) '@types/amqplib': specifier: ^0.5.3 version: 0.5.17 @@ -815,6 +820,12 @@ importers: '@types/uuid': specifier: ^3.4.10 version: 3.4.13 + '@typescript/native-preview': + specifier: 7.0.0-dev.20260616.1 + version: 7.0.0-dev.20260616.1 + '@typescript/native-preview-linux-arm64': + specifier: 7.0.0-dev.20260616.1 + version: 7.0.0-dev.20260616.1 autoprefixer: specifier: ^9.5.0 version: 9.8.8 @@ -904,13 +915,13 @@ importers: version: 1.4.6(webpack@4.47.0) ts-loader: specifier: ^8.0.11 - version: 8.4.0(typescript@5.9.3)(webpack@4.47.0) + version: 8.4.0(typescript@6.0.3)(webpack@4.47.0) ts-morph: specifier: ^21.0.1 version: 21.0.1 typescript: - specifier: ^5.3.3 - version: 5.9.3 + specifier: ^6.0.3 + version: 6.0.3 vitest: specifier: ^4.0.10 version: 4.0.18(@types/node@24.11.0)(jiti@2.6.1)(sass@1.67.0)(terser@5.46.0)(tsx@4.21.0)(yaml@1.10.2) @@ -1876,10 +1887,10 @@ packages: prosemirror-history: ^1.1.3 prosemirror-inputrules: '>=1.1.2' prosemirror-keymap: ^1.1.4 - prosemirror-model: ^1.13.3 - prosemirror-state: ^1.3.4 + prosemirror-model: 1.25.4 + prosemirror-state: 1.4.4 prosemirror-transform: ^1.2.12 - prosemirror-view: ^1.18.2 + prosemirror-view: 1.41.6 '@biomejs/biome@2.4.3': resolution: {integrity: sha512-cBrjf6PNF6yfL8+kcNl85AjiK2YHNsbU0EvDOwiZjBPbMbQ5QcgVGFpjD0O52p8nec5O8NYw7PKw3xUR7fPAkQ==} @@ -2406,9 +2417,9 @@ packages: '@tiptap/core': ^3.15.3 '@tiptap/pm': ^3.15.3 '@tiptap/react': ^3.15.3 - prosemirror-model: ^1.0.0 - prosemirror-state: ^1.0.0 - prosemirror-view: 1.41.7 + prosemirror-model: 1.25.4 + prosemirror-state: 1.4.4 + prosemirror-view: 1.41.6 react: '>=17 <20' react-dom: '>=17 <20' react-reconciler: '>=0.26.1 <=0.33.0' @@ -2751,33 +2762,46 @@ packages: '@pdf-lib/upng@1.0.1': resolution: {integrity: sha512-dQK2FUMQtowVP00mtIksrlZhdFXQZPC+taih1q4CvPZ5vqdxR/LKBaFg0oAfzd1GlHZXXSPdQfzQnt+ViGvEIQ==} - '@pitter-patter/collab-client@0.1.0': - resolution: {integrity: sha512-OszIplkYRS4ttck7MhPU/oCNrK2YDjnd6nZ6cuYUah7IfR19HRYkfa4d2vv4ys8PGSYsIp8251yscNivK5ORng==} + '@pitter-patter/collab-client@0.1.3': + resolution: {integrity: sha512-3KdbZ6G42pKzDJgJfKI0ZZqYU1h2ddkJSuUCBi1ClZk1ymf9RNTvsBV+0fvwJuPvXy2nSuTRo7LyXOEdhlHe+g==} peerDependencies: - prosemirror-model: ^1.0.0 - prosemirror-state: ^1.0.0 + prosemirror-model: 1.25.4 + prosemirror-state: 1.4.4 prosemirror-transform: ^1.0.0 - '@pitter-patter/collab-server@0.1.0': - resolution: {integrity: sha512-0esnVrkLyXRbByhn43hWR78ZwI1g4xKlL/Y6jx0cS1ACGQG2O+I2eKJqqcPXN3kmx1Jj2eB9RU2T1e2nOodWVA==} + '@pitter-patter/collab-server@0.1.3': + resolution: {integrity: sha512-61KRuqQJTIzLKPDVwjmBJgZXMaagOmVMGbXoAxU9ymCDGPlTXBHleKMwsWsYdQ3r3ezio0wwttddaqZRYhPs/A==} peerDependencies: - prosemirror-model: ^1.0.0 - prosemirror-state: ^1.0.0 + prosemirror-model: 1.25.4 + prosemirror-state: 1.4.4 prosemirror-transform: ^1.0.0 - '@pitter-patter/presence-client@0.1.0': - resolution: {integrity: sha512-B2b7nXf0j8p2kesB4+Z2dCP1+qGsqklBiT3AyKEIw5PhzyIjw16HVsb2P5b3/zfuITUcqsEN5/B080sPkKHzTQ==} + '@pitter-patter/presence-client@0.2.1': + resolution: {integrity: sha512-088LDBR1/OwdfGnWblrtYeeXXi1tNIZbTgv9WN/pUEX0GANIYt0uuKABJ1riwJxhAEwaRpFr4aEZhSc+8kcqnA==} peerDependencies: - prosemirror-model: ^1.0.0 - prosemirror-state: ^1.0.0 + '@handlewithcare/react-prosemirror': ^3.0.6 + prosemirror-model: 1.25.4 + prosemirror-state: 1.4.4 prosemirror-transform: ^1.0.0 - prosemirror-view: ^1.0.0 + prosemirror-view: 1.41.6 + react: ^19.2.0 + react-dom: ^19.2.0 + react-reconciler: ^0.32.0 + peerDependenciesMeta: + '@handlewithcare/react-prosemirror': + optional: true + react: + optional: true + react-dom: + optional: true + react-reconciler: + optional: true - '@pitter-patter/presence-server@0.1.0': - resolution: {integrity: sha512-0zQSikcCXKYFwUXG8sxmZjQ8rUgIIHpOOZFgrcTpCQtbxqaDeWskStxtMKvg8+bf0dLP4lF3ji8p2DcBmaf9fQ==} + '@pitter-patter/presence-server@0.1.3': + resolution: {integrity: sha512-zDEbiIMXrH9qsC6YufVo/S0mWkwDX4PWqrWT4AjEGkXUUuJQdSrfq+zqb9F2+geUlFHKry5xPEjb/yPWftnWWw==} - '@pitter-patter/refs@0.1.0': - resolution: {integrity: sha512-/zChXwSRr+IEOyCk+D2LHYXFvBnsAWyNKXR8B+wWPTW40A1sQF9lSoemeSyR4KaRtT56aa0jTw3+Rsj/687Bgg==} + '@pitter-patter/refs@0.1.3': + resolution: {integrity: sha512-uJDJpOgpcEtyapfhl7y7vXHIjdAd9pzSgX1rh4sLqEC33dnkamd5dH3I5MdiDzjek77S9Y1DhxP5tOvvNlCASA==} '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} @@ -2899,7 +2923,7 @@ packages: '@emotion/core': ^10 '@types/prosemirror-view': ^1.11.0 '@types/react': ^16.8.0 - prosemirror-view: ^1.11.3 + prosemirror-view: 1.41.6 react: ^16.8.0 '@remirror/core-utils@0.8.0': @@ -4371,6 +4395,52 @@ packages: resolution: {integrity: sha512-07ny+LHRzQXepkGg6w0mFY41fVUNBrL2Roj/++7V1txKugfjm/Ci/qSND03r2RhlJhJYMcTn9AhhSSqQp0Ysyw==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + '@typescript/native-preview-darwin-arm64@7.0.0-dev.20260616.1': + resolution: {integrity: sha512-9SwegwwE7fYutnZJjTi2PeUXhKGFg82MGjSpCFD2cj5v9YQcs5oO5QsmeeMit4fMG8Z83Jy8knOF97d9NaQ7Bg==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [darwin] + + '@typescript/native-preview-darwin-x64@7.0.0-dev.20260616.1': + resolution: {integrity: sha512-gl7R8OiwEBNxzs5wjbM9XOibTs5b6/gAgu0+En9pLpYuWR/EITs8Eh0mNQui4JYTp1SkoHvn09aRZKTQf21XTg==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [darwin] + + '@typescript/native-preview-linux-arm64@7.0.0-dev.20260616.1': + resolution: {integrity: sha512-CJjplWoE+EeYtbyNeP4fyuh0QcPiQBZJSLqPS07E3ugo3d9M/IG8WnL+3GFQ0g1p4c6QC/+OC0oTTQnGcNdd2Q==} + engines: {node: '>=16.20.0'} + os: [linux] + + '@typescript/native-preview-linux-arm@7.0.0-dev.20260616.1': + resolution: {integrity: sha512-QWXQS2CrhSpXbng7vBtCVDszFgwVBuJU8MCFhxZL0hH6s+XjQDSNNkGO1oHueBr+7HbSkkyqfNYVNXvgVMUJLQ==} + engines: {node: '>=16.20.0'} + cpu: [arm] + os: [linux] + + '@typescript/native-preview-linux-x64@7.0.0-dev.20260616.1': + resolution: {integrity: sha512-UVySBNnGTAnul2kO2/EhlVZl0CeTDcd6Lzi+WkvgyCgapTcU0BX1selQ4MEPtbVWKUbt9HBhxNku2dyaCcmKVw==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [linux] + + '@typescript/native-preview-win32-arm64@7.0.0-dev.20260616.1': + resolution: {integrity: sha512-kdX0QcDXiESH0o5DFdSWw15Hth0EtQobT9tX28ofqBUwGU1FFhwTKzA6qo7chaYUSW5MMd6XIME9rBwk+WizLw==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [win32] + + '@typescript/native-preview-win32-x64@7.0.0-dev.20260616.1': + resolution: {integrity: sha512-EB0Pj/0+nXifS3+wN0HdR1mKu7IieSpjMXoDjdXtJAdiPGlmKazBgHj97qn6CBvXisgLx0Eyb7tLCEQNDG53cA==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [win32] + + '@typescript/native-preview@7.0.0-dev.20260616.1': + resolution: {integrity: sha512-+AuZUl7nkLPXL1rwsyZZF7iasu0HkrL5O6aWwUC0cObD5EvNgxz3hXbBhsSvuJ8tb2JRpVwFsE0BHPcYVe5GkA==} + engines: {node: '>=16.20.0'} + hasBin: true + '@ungap/structured-clone@1.3.0': resolution: {integrity: sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==} @@ -9568,7 +9638,7 @@ packages: resolution: {integrity: sha512-byzffTO/uk8v0JRGIOO2H8h97u5fS01KcxmPXzuc0IDgxvcmrP55XsXerVORAoqJlJFlN0DGAPojQENkDLSXsA==} peerDependencies: '@types/prosemirror-view': ^1.11.0 - prosemirror-view: ^1.11.1 + prosemirror-view: 1.41.6 prosemirror-tables@1.8.5: resolution: {integrity: sha512-V/0cDCsHKHe/tfWkeCmthNUcEp1IVO3p6vwN8XtwE9PZQLAZJigbw3QoraAdfJPir4NKJtNvOB8oYGKRl+t0Dw==} @@ -9764,11 +9834,6 @@ packages: peerDependencies: react: ^16.14.0 - react-dom@19.2.7: - resolution: {integrity: sha512-t0BRVXvbiE/o20Hfw669rLbMCDWtYZLvmJigy2f0MxsXF+71pxhR3xOkspmsO8h3ZlNzyibAmtCa3l4lYKk6gQ==} - peerDependencies: - react: ^19.2.7 - react-dropzone@10.2.2: resolution: {integrity: sha512-U5EKckXVt6IrEyhMMsgmHQiWTGLudhajPPG77KFSvgsMqNEHSyGpqWvOMc5+DhEah/vH4E1n+J5weBNLd5VtyA==} engines: {node: '>= 8'} @@ -9913,10 +9978,6 @@ packages: resolution: {integrity: sha512-0X2CImDkJGApiAlcf0ODKIneSwBPhqJawOa5wCtKbu7ZECrmS26NvtSILynQ66cgkT/RJ4LidJOc3bUESwmU8g==} engines: {node: '>=0.10.0'} - react@19.2.7: - resolution: {integrity: sha512-HNe9WslTbXmFK8o8cmwgAeJFSBvt1bPdHCVKtaaV+WlAN36mpT4hcRpwbf3fY56ar2oIXzsBpOAiIRHAdY0OlQ==} - engines: {node: '>=0.10.0'} - reactcss@1.2.3: resolution: {integrity: sha512-KiwVUcFu1RErkI97ywr8nvx8dNOpT03rbnma0SSalTYjkrPYaEajR4a/MRt6DZ46K6arDRbWMNHF+xH7G7n/8A==} peerDependencies: @@ -10314,9 +10375,6 @@ packages: scheduler@0.26.0: resolution: {integrity: sha512-NlHwttCI/l5gCPR3D1nNXtWABUmBwvZpEQiD4IXSbIDq8BzLIK/7Ir5gTFSGZDUu37K5cMNp0hFtzO38sC7gWA==} - scheduler@0.27.0: - resolution: {integrity: sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==} - schema-utils@1.0.0: resolution: {integrity: sha512-i27Mic4KovM/lnGsy8whRCHhc7VicJajAjTrYg11K9zfZXnYIt4k5F+kZkwjnrhKzLic/HLU4j11mjsz2G/75g==} engines: {node: '>= 4'} @@ -11302,8 +11360,8 @@ packages: typedarray@0.0.6: resolution: {integrity: sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==} - typescript@5.9.3: - resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + typescript@6.0.3: + resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} engines: {node: '>=14.17'} hasBin: true @@ -14239,15 +14297,16 @@ snapshots: yargs: 17.7.2 optional: true - '@handlewithcare/react-prosemirror@3.1.6(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-view@1.41.6)(react-dom@19.2.7(react@19.2.7))(react-reconciler@0.32.0(react@19.2.7))(react@19.2.7)': + '@handlewithcare/react-prosemirror@3.1.6(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-view@1.41.6)(react-dom@16.14.0(react@16.14.0))(react-reconciler@0.32.0(react@16.14.0))(react@16.14.0)': dependencies: classnames: 2.5.1 prosemirror-model: 1.25.4 prosemirror-state: 1.4.4 prosemirror-view: 1.41.6 - react: 19.2.7 - react-dom: 19.2.7(react@19.2.7) - react-reconciler: 0.32.0(react@19.2.7) + react: 16.14.0 + react-dom: 16.14.0(react@16.14.0) + react-reconciler: 0.32.0(react@16.14.0) + optional: true '@humanwhocodes/config-array@0.13.0': dependencies: @@ -14685,14 +14744,14 @@ snapshots: dependencies: pako: 1.0.11 - '@pitter-patter/collab-client@0.1.0(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0)': + '@pitter-patter/collab-client@0.1.3(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0)': dependencies: '@stepwisehq/prosemirror-collab-commit': 1.0.5 prosemirror-model: 1.25.4 prosemirror-state: 1.4.4 prosemirror-transform: 1.11.0 - '@pitter-patter/collab-server@0.1.0(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0)': + '@pitter-patter/collab-server@0.1.3(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0)': dependencies: '@stepwisehq/prosemirror-collab-commit': 1.0.5 prosemirror-model: 1.25.4 @@ -14703,32 +14762,29 @@ snapshots: - '@node-rs/xxhash' - '@opentelemetry/api' - '@pitter-patter/presence-client@0.1.0(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0)(prosemirror-view@1.41.6)': + '@pitter-patter/presence-client@0.2.1(@handlewithcare/react-prosemirror@3.1.6(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-view@1.41.6)(react-dom@16.14.0(react@16.14.0))(react-reconciler@0.32.0(react@16.14.0))(react@16.14.0))(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0)(prosemirror-view@1.41.6)(react-dom@16.14.0(react@16.14.0))(react-reconciler@0.32.0(react@16.14.0))(react@16.14.0)': dependencies: - '@handlewithcare/react-prosemirror': 3.1.6(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-view@1.41.6)(react-dom@19.2.7(react@19.2.7))(react-reconciler@0.32.0(react@19.2.7))(react@19.2.7) - '@pitter-patter/collab-client': 0.1.0(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0) - '@pitter-patter/refs': 0.1.0 + '@pitter-patter/collab-client': 0.1.3(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0) + '@pitter-patter/refs': 0.1.3 classnames: 2.5.1 prosemirror-model: 1.25.4 prosemirror-state: 1.4.4 prosemirror-transform: 1.11.0 prosemirror-view: 1.41.6 - react: 19.2.7 - react-dom: 19.2.7(react@19.2.7) - react-reconciler: 0.32.0(react@19.2.7) - transitivePeerDependencies: - - '@tiptap/core' - - '@tiptap/pm' - - '@tiptap/react' + optionalDependencies: + '@handlewithcare/react-prosemirror': 3.1.6(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-view@1.41.6)(react-dom@16.14.0(react@16.14.0))(react-reconciler@0.32.0(react@16.14.0))(react@16.14.0) + react: 16.14.0 + react-dom: 16.14.0(react@16.14.0) + react-reconciler: 0.32.0(react@16.14.0) - '@pitter-patter/presence-server@0.1.0': + '@pitter-patter/presence-server@0.1.3': dependencies: redis: 5.12.1 transitivePeerDependencies: - '@node-rs/xxhash' - '@opentelemetry/api' - '@pitter-patter/refs@0.1.0': {} + '@pitter-patter/refs@0.1.3': {} '@pkgjs/parseargs@0.11.0': optional: true @@ -15789,7 +15845,7 @@ snapshots: ts-dedent: 2.2.0 util-deprecate: 1.0.2 - '@storybook/builder-webpack4@6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@5.9.3)(webpack-cli@3.3.12)': + '@storybook/builder-webpack4@6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@6.0.3)(webpack-cli@3.3.12)': dependencies: '@babel/core': 7.29.0 '@storybook/addons': 6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0) @@ -15799,7 +15855,7 @@ snapshots: '@storybook/client-api': 6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0) '@storybook/client-logger': 6.5.16 '@storybook/components': 6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0) - '@storybook/core-common': 6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@5.9.3)(webpack-cli@3.3.12) + '@storybook/core-common': 6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@6.0.3)(webpack-cli@3.3.12) '@storybook/core-events': 6.5.16 '@storybook/node-logger': 6.5.16 '@storybook/preview-web': 6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0) @@ -15817,12 +15873,12 @@ snapshots: css-loader: 3.6.0(webpack@4.47.0) file-loader: 6.2.0(webpack@4.47.0) find-up: 5.0.0 - fork-ts-checker-webpack-plugin: 4.1.6(eslint@8.57.1)(typescript@5.9.3)(webpack@4.47.0) + fork-ts-checker-webpack-plugin: 4.1.6(eslint@8.57.1)(typescript@6.0.3)(webpack@4.47.0) glob: 7.2.3 glob-promise: 3.4.0(glob@7.2.3) global: 4.4.0 html-webpack-plugin: 4.5.2(webpack@4.47.0) - pnp-webpack-plugin: 1.6.4(typescript@5.9.3) + pnp-webpack-plugin: 1.6.4(typescript@6.0.3) postcss: 7.0.39 postcss-flexbugs-fixes: 4.2.1 postcss-loader: 4.3.0(postcss@7.0.39)(webpack@4.47.0) @@ -15841,7 +15897,7 @@ snapshots: webpack-hot-middleware: 2.26.1 webpack-virtual-modules: 0.2.2 optionalDependencies: - typescript: 5.9.3 + typescript: 6.0.3 transitivePeerDependencies: - bluebird - eslint @@ -15917,7 +15973,7 @@ snapshots: regenerator-runtime: 0.13.11 util-deprecate: 1.0.2 - '@storybook/core-client@6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@5.9.3)(webpack@4.47.0)': + '@storybook/core-client@6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@6.0.3)(webpack@4.47.0)': dependencies: '@storybook/addons': 6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0) '@storybook/channel-postmessage': 6.5.16 @@ -15943,9 +15999,9 @@ snapshots: util-deprecate: 1.0.2 webpack: 4.47.0(webpack-cli@3.3.12) optionalDependencies: - typescript: 5.9.3 + typescript: 6.0.3 - '@storybook/core-common@6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@5.9.3)(webpack-cli@3.3.12)': + '@storybook/core-common@6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@6.0.3)(webpack-cli@3.3.12)': dependencies: '@babel/core': 7.29.0 '@babel/plugin-proposal-class-properties': 7.18.6(@babel/core@7.29.0) @@ -15981,7 +16037,7 @@ snapshots: express: 4.22.1 file-system-cache: 1.1.0 find-up: 5.0.0 - fork-ts-checker-webpack-plugin: 6.5.3(eslint@8.57.1)(typescript@5.9.3)(webpack@4.47.0) + fork-ts-checker-webpack-plugin: 6.5.3(eslint@8.57.1)(typescript@6.0.3)(webpack@4.47.0) fs-extra: 9.1.0 glob: 7.2.3 handlebars: 4.7.8 @@ -16000,7 +16056,7 @@ snapshots: util-deprecate: 1.0.2 webpack: 4.47.0(webpack-cli@3.3.12) optionalDependencies: - typescript: 5.9.3 + typescript: 6.0.3 transitivePeerDependencies: - eslint - supports-color @@ -16012,20 +16068,20 @@ snapshots: dependencies: core-js: 3.48.0 - '@storybook/core-server@6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@5.9.3)(webpack-cli@3.3.12)': + '@storybook/core-server@6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@6.0.3)(webpack-cli@3.3.12)': dependencies: '@discoveryjs/json-ext': 0.5.7 - '@storybook/builder-webpack4': 6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@5.9.3)(webpack-cli@3.3.12) - '@storybook/core-client': 6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@5.9.3)(webpack@4.47.0) - '@storybook/core-common': 6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@5.9.3)(webpack-cli@3.3.12) + '@storybook/builder-webpack4': 6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@6.0.3)(webpack-cli@3.3.12) + '@storybook/core-client': 6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@6.0.3)(webpack@4.47.0) + '@storybook/core-common': 6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@6.0.3)(webpack-cli@3.3.12) '@storybook/core-events': 6.5.16 '@storybook/csf': 0.0.2--canary.4566f4d.1 '@storybook/csf-tools': 6.5.16 - '@storybook/manager-webpack4': 6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@5.9.3)(webpack-cli@3.3.12) + '@storybook/manager-webpack4': 6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@6.0.3)(webpack-cli@3.3.12) '@storybook/node-logger': 6.5.16 '@storybook/semver': 7.3.2 '@storybook/store': 6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0) - '@storybook/telemetry': 6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@5.9.3)(webpack-cli@3.3.12) + '@storybook/telemetry': 6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@6.0.3)(webpack-cli@3.3.12) '@types/node': 16.18.126 '@types/node-fetch': 2.6.13 '@types/pretty-hrtime': 1.0.3 @@ -16062,7 +16118,7 @@ snapshots: ws: 8.19.0 x-default-browser: 0.4.0 optionalDependencies: - typescript: 5.9.3 + typescript: 6.0.3 transitivePeerDependencies: - '@storybook/mdx2-csf' - bluebird @@ -16075,15 +16131,15 @@ snapshots: - webpack-cli - webpack-command - '@storybook/core@6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@5.9.3)(webpack-cli@3.3.12)(webpack@4.47.0)': + '@storybook/core@6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@6.0.3)(webpack-cli@3.3.12)(webpack@4.47.0)': dependencies: - '@storybook/core-client': 6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@5.9.3)(webpack@4.47.0) - '@storybook/core-server': 6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@5.9.3)(webpack-cli@3.3.12) + '@storybook/core-client': 6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@6.0.3)(webpack@4.47.0) + '@storybook/core-server': 6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@6.0.3)(webpack-cli@3.3.12) react: 16.14.0 react-dom: 16.14.0(react@16.14.0) webpack: 4.47.0(webpack-cli@3.3.12) optionalDependencies: - typescript: 5.9.3 + typescript: 6.0.3 transitivePeerDependencies: - '@storybook/mdx2-csf' - bluebird @@ -16133,14 +16189,14 @@ snapshots: - react-dom - supports-color - '@storybook/manager-webpack4@6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@5.9.3)(webpack-cli@3.3.12)': + '@storybook/manager-webpack4@6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@6.0.3)(webpack-cli@3.3.12)': dependencies: '@babel/core': 7.29.0 '@babel/plugin-transform-template-literals': 7.27.1(@babel/core@7.29.0) '@babel/preset-react': 7.28.5(@babel/core@7.29.0) '@storybook/addons': 6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0) - '@storybook/core-client': 6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@5.9.3)(webpack@4.47.0) - '@storybook/core-common': 6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@5.9.3)(webpack-cli@3.3.12) + '@storybook/core-client': 6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@6.0.3)(webpack@4.47.0) + '@storybook/core-common': 6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@6.0.3)(webpack-cli@3.3.12) '@storybook/node-logger': 6.5.16 '@storybook/theming': 6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0) '@storybook/ui': 6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0) @@ -16157,7 +16213,7 @@ snapshots: fs-extra: 9.1.0 html-webpack-plugin: 4.5.2(webpack@4.47.0) node-fetch: 2.7.0 - pnp-webpack-plugin: 1.6.4(typescript@5.9.3) + pnp-webpack-plugin: 1.6.4(typescript@6.0.3) react: 16.14.0 react-dom: 16.14.0(react@16.14.0) read-pkg-up: 7.0.1 @@ -16173,7 +16229,7 @@ snapshots: webpack-dev-middleware: 3.7.3(webpack@4.47.0) webpack-virtual-modules: 0.2.2 optionalDependencies: - typescript: 5.9.3 + typescript: 6.0.3 transitivePeerDependencies: - bluebird - encoding @@ -16229,33 +16285,33 @@ snapshots: unfetch: 4.2.0 util-deprecate: 1.0.2 - '@storybook/react-docgen-typescript-plugin@1.0.2-canary.6.9d540b91e815f8fc2f8829189deb00553559ff63.0(typescript@5.9.3)(webpack@4.47.0)': + '@storybook/react-docgen-typescript-plugin@1.0.2-canary.6.9d540b91e815f8fc2f8829189deb00553559ff63.0(typescript@6.0.3)(webpack@4.47.0)': dependencies: debug: 4.4.3(supports-color@5.5.0) endent: 2.1.0 find-cache-dir: 3.3.2 flat-cache: 3.2.0 micromatch: 4.0.8 - react-docgen-typescript: 2.4.0(typescript@5.9.3) + react-docgen-typescript: 2.4.0(typescript@6.0.3) tslib: 2.8.1 - typescript: 5.9.3 + typescript: 6.0.3 webpack: 4.47.0(webpack-cli@3.3.12) transitivePeerDependencies: - supports-color - '@storybook/react@6.5.16(@babel/core@7.29.0)(@types/webpack@4.41.40)(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(require-from-string@2.0.2)(type-fest@0.21.3)(typescript@5.9.3)(webpack-cli@3.3.12)(webpack-dev-server@5.2.2(tslib@1.14.1)(webpack-cli@3.3.12)(webpack@4.47.0))(webpack-hot-middleware@2.26.1)': + '@storybook/react@6.5.16(@babel/core@7.29.0)(@types/webpack@4.41.40)(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(require-from-string@2.0.2)(type-fest@0.21.3)(typescript@6.0.3)(webpack-cli@3.3.12)(webpack-dev-server@5.2.2(tslib@1.14.1)(webpack-cli@3.3.12)(webpack@4.47.0))(webpack-hot-middleware@2.26.1)': dependencies: '@babel/preset-flow': 7.27.1(@babel/core@7.29.0) '@babel/preset-react': 7.28.5(@babel/core@7.29.0) '@pmmmwh/react-refresh-webpack-plugin': 0.5.17(@types/webpack@4.41.40)(react-refresh@0.11.0)(type-fest@0.21.3)(webpack-dev-server@5.2.2(tslib@1.14.1)(webpack-cli@3.3.12)(webpack@4.47.0))(webpack-hot-middleware@2.26.1)(webpack@4.47.0) '@storybook/addons': 6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0) '@storybook/client-logger': 6.5.16 - '@storybook/core': 6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@5.9.3)(webpack-cli@3.3.12)(webpack@4.47.0) - '@storybook/core-common': 6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@5.9.3)(webpack-cli@3.3.12) + '@storybook/core': 6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@6.0.3)(webpack-cli@3.3.12)(webpack@4.47.0) + '@storybook/core-common': 6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@6.0.3)(webpack-cli@3.3.12) '@storybook/csf': 0.0.2--canary.4566f4d.1 '@storybook/docs-tools': 6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0) '@storybook/node-logger': 6.5.16 - '@storybook/react-docgen-typescript-plugin': 1.0.2-canary.6.9d540b91e815f8fc2f8829189deb00553559ff63.0(typescript@5.9.3)(webpack@4.47.0) + '@storybook/react-docgen-typescript-plugin': 1.0.2-canary.6.9d540b91e815f8fc2f8829189deb00553559ff63.0(typescript@6.0.3)(webpack@4.47.0) '@storybook/semver': 7.3.2 '@storybook/store': 6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0) '@types/estree': 0.0.51 @@ -16285,7 +16341,7 @@ snapshots: webpack: 4.47.0(webpack-cli@3.3.12) optionalDependencies: '@babel/core': 7.29.0 - typescript: 5.9.3 + typescript: 6.0.3 transitivePeerDependencies: - '@storybook/mdx2-csf' - '@types/webpack' @@ -16339,10 +16395,10 @@ snapshots: ts-dedent: 2.2.0 util-deprecate: 1.0.2 - '@storybook/telemetry@6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@5.9.3)(webpack-cli@3.3.12)': + '@storybook/telemetry@6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@6.0.3)(webpack-cli@3.3.12)': dependencies: '@storybook/client-logger': 6.5.16 - '@storybook/core-common': 6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@5.9.3)(webpack-cli@3.3.12) + '@storybook/core-common': 6.5.16(eslint@8.57.1)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)(typescript@6.0.3)(webpack-cli@3.3.12) chalk: 4.1.2 core-js: 3.48.0 detect-package-manager: 2.0.1 @@ -17007,15 +17063,15 @@ snapshots: '@types/yoga-layout@1.9.2': {} - '@typescript-eslint/parser@5.62.0(eslint@8.57.1)(typescript@5.9.3)': + '@typescript-eslint/parser@5.62.0(eslint@8.57.1)(typescript@6.0.3)': dependencies: '@typescript-eslint/scope-manager': 5.62.0 '@typescript-eslint/types': 5.62.0 - '@typescript-eslint/typescript-estree': 5.62.0(typescript@5.9.3) + '@typescript-eslint/typescript-estree': 5.62.0(typescript@6.0.3) debug: 4.4.3(supports-color@5.5.0) eslint: 8.57.1 optionalDependencies: - typescript: 5.9.3 + typescript: 6.0.3 transitivePeerDependencies: - supports-color @@ -17026,7 +17082,7 @@ snapshots: '@typescript-eslint/types@5.62.0': {} - '@typescript-eslint/typescript-estree@5.62.0(typescript@5.9.3)': + '@typescript-eslint/typescript-estree@5.62.0(typescript@6.0.3)': dependencies: '@typescript-eslint/types': 5.62.0 '@typescript-eslint/visitor-keys': 5.62.0 @@ -17034,9 +17090,9 @@ snapshots: globby: 11.1.0 is-glob: 4.0.3 semver: 7.7.4 - tsutils: 3.21.0(typescript@5.9.3) + tsutils: 3.21.0(typescript@6.0.3) optionalDependencies: - typescript: 5.9.3 + typescript: 6.0.3 transitivePeerDependencies: - supports-color @@ -17045,6 +17101,36 @@ snapshots: '@typescript-eslint/types': 5.62.0 eslint-visitor-keys: 3.4.3 + '@typescript/native-preview-darwin-arm64@7.0.0-dev.20260616.1': + optional: true + + '@typescript/native-preview-darwin-x64@7.0.0-dev.20260616.1': + optional: true + + '@typescript/native-preview-linux-arm64@7.0.0-dev.20260616.1': {} + + '@typescript/native-preview-linux-arm@7.0.0-dev.20260616.1': + optional: true + + '@typescript/native-preview-linux-x64@7.0.0-dev.20260616.1': + optional: true + + '@typescript/native-preview-win32-arm64@7.0.0-dev.20260616.1': + optional: true + + '@typescript/native-preview-win32-x64@7.0.0-dev.20260616.1': + optional: true + + '@typescript/native-preview@7.0.0-dev.20260616.1': + optionalDependencies: + '@typescript/native-preview-darwin-arm64': 7.0.0-dev.20260616.1 + '@typescript/native-preview-darwin-x64': 7.0.0-dev.20260616.1 + '@typescript/native-preview-linux-arm': 7.0.0-dev.20260616.1 + '@typescript/native-preview-linux-arm64': 7.0.0-dev.20260616.1 + '@typescript/native-preview-linux-x64': 7.0.0-dev.20260616.1 + '@typescript/native-preview-win32-arm64': 7.0.0-dev.20260616.1 + '@typescript/native-preview-win32-x64': 7.0.0-dev.20260616.1 + '@ungap/structured-clone@1.3.0': {} '@vitest/expect@4.0.18': @@ -20143,7 +20229,7 @@ snapshots: forever-agent@0.6.1: {} - fork-ts-checker-webpack-plugin@4.1.6(eslint@8.57.1)(typescript@5.9.3)(webpack@4.47.0): + fork-ts-checker-webpack-plugin@4.1.6(eslint@8.57.1)(typescript@6.0.3)(webpack@4.47.0): dependencies: '@babel/code-frame': 7.29.0 chalk: 2.4.2 @@ -20151,7 +20237,7 @@ snapshots: minimatch: 3.1.2 semver: 5.7.2 tapable: 1.1.3 - typescript: 5.9.3 + typescript: 6.0.3 webpack: 4.47.0(webpack-cli@3.3.12) worker-rpc: 0.1.1 optionalDependencies: @@ -20159,7 +20245,7 @@ snapshots: transitivePeerDependencies: - supports-color - fork-ts-checker-webpack-plugin@6.5.3(eslint@8.57.1)(typescript@5.9.3)(webpack@4.47.0): + fork-ts-checker-webpack-plugin@6.5.3(eslint@8.57.1)(typescript@6.0.3)(webpack@4.47.0): dependencies: '@babel/code-frame': 7.29.0 '@types/json-schema': 7.0.15 @@ -20174,7 +20260,7 @@ snapshots: schema-utils: 2.7.0 semver: 7.7.4 tapable: 1.1.3 - typescript: 5.9.3 + typescript: 6.0.3 webpack: 4.47.0(webpack-cli@3.3.12) optionalDependencies: eslint: 8.57.1 @@ -23061,9 +23147,9 @@ snapshots: pluralize@8.0.0: {} - pnp-webpack-plugin@1.6.4(typescript@5.9.3): + pnp-webpack-plugin@1.6.4(typescript@6.0.3): dependencies: - ts-pnp: 1.2.0(typescript@5.9.3) + ts-pnp: 1.2.0(typescript@6.0.3) transitivePeerDependencies: - typescript @@ -23588,9 +23674,9 @@ snapshots: react: 16.14.0 react-dom: 16.14.0(react@16.14.0) - react-docgen-typescript@2.4.0(typescript@5.9.3): + react-docgen-typescript@2.4.0(typescript@6.0.3): dependencies: - typescript: 5.9.3 + typescript: 6.0.3 react-docgen@5.4.3: dependencies: @@ -23625,11 +23711,6 @@ snapshots: react: 16.14.0 scheduler: 0.19.1 - react-dom@19.2.7(react@19.2.7): - dependencies: - react: 19.2.7 - scheduler: 0.27.0 - react-dropzone@10.2.2(react@16.14.0): dependencies: attr-accept: 2.2.5 @@ -23699,10 +23780,11 @@ snapshots: react: 16.14.0 scheduler: 0.18.0 - react-reconciler@0.32.0(react@19.2.7): + react-reconciler@0.32.0(react@16.14.0): dependencies: - react: 19.2.7 + react: 16.14.0 scheduler: 0.26.0 + optional: true react-redux@7.2.9(react-dom@16.14.0(react@16.14.0))(react@16.14.0): dependencies: @@ -23819,8 +23901,6 @@ snapshots: object-assign: 4.1.1 prop-types: 15.8.1 - react@19.2.7: {} - reactcss@1.2.3(react@16.14.0): dependencies: lodash: 4.17.23 @@ -24376,9 +24456,8 @@ snapshots: loose-envify: 1.4.0 object-assign: 4.1.1 - scheduler@0.26.0: {} - - scheduler@0.27.0: {} + scheduler@0.26.0: + optional: true schema-utils@1.0.0: dependencies: @@ -24460,16 +24539,16 @@ snapshots: sequelize-pool@8.0.1: {} - sequelize-typescript-generator@10.1.2(@types/node@24.11.0)(@types/validator@13.15.10)(pg@8.18.0)(reflect-metadata@0.1.14)(typescript@5.9.3): + sequelize-typescript-generator@10.1.2(@types/node@24.11.0)(@types/validator@13.15.10)(pg@8.18.0)(reflect-metadata@0.1.14)(typescript@6.0.3): dependencies: '@types/eslint': 8.56.12 - '@typescript-eslint/parser': 5.62.0(eslint@8.57.1)(typescript@5.9.3) + '@typescript-eslint/parser': 5.62.0(eslint@8.57.1)(typescript@6.0.3) change-case: 4.1.2 eslint: 8.57.1 pluralize: 8.0.0 sequelize: 6.37.7(pg@8.18.0) sequelize-typescript: 2.1.6(@types/node@24.11.0)(@types/validator@13.15.10)(reflect-metadata@0.1.14)(sequelize@6.37.7(pg@8.18.0)) - typescript: 5.9.3 + typescript: 6.0.3 yargs: 17.7.2 transitivePeerDependencies: - '@types/node' @@ -25425,14 +25504,14 @@ snapshots: ts-easing@0.2.0: {} - ts-loader@8.4.0(typescript@5.9.3)(webpack@4.47.0): + ts-loader@8.4.0(typescript@6.0.3)(webpack@4.47.0): dependencies: chalk: 4.1.2 enhanced-resolve: 4.5.0 loader-utils: 2.0.4 micromatch: 4.0.8 semver: 7.7.4 - typescript: 5.9.3 + typescript: 6.0.3 webpack: 4.47.0(webpack-cli@3.3.12) ts-morph@21.0.1: @@ -25440,7 +25519,7 @@ snapshots: '@ts-morph/common': 0.22.0 code-block-writer: 12.0.0 - ts-node@10.9.2(@types/node@24.11.0)(typescript@5.9.3): + ts-node@10.9.2(@types/node@24.11.0)(typescript@6.0.3): dependencies: '@cspotcode/source-map-support': 0.8.1 '@tsconfig/node10': 1.0.12 @@ -25454,17 +25533,17 @@ snapshots: create-require: 1.1.1 diff: 4.0.4 make-error: 1.3.6 - typescript: 5.9.3 + typescript: 6.0.3 v8-compile-cache-lib: 3.0.1 yn: 3.1.1 - ts-pnp@1.2.0(typescript@5.9.3): + ts-pnp@1.2.0(typescript@6.0.3): optionalDependencies: - typescript: 5.9.3 + typescript: 6.0.3 - tsconfck@3.1.6(typescript@5.9.3): + tsconfck@3.1.6(typescript@6.0.3): optionalDependencies: - typescript: 5.9.3 + typescript: 6.0.3 tslib@1.10.0: {} @@ -25474,10 +25553,10 @@ snapshots: tslib@2.8.1: {} - tsutils@3.21.0(typescript@5.9.3): + tsutils@3.21.0(typescript@6.0.3): dependencies: tslib: 1.14.1 - typescript: 5.9.3 + typescript: 6.0.3 tsx@4.21.0: dependencies: @@ -25561,7 +25640,7 @@ snapshots: typedarray@0.0.6: {} - typescript@5.9.3: {} + typescript@6.0.3: {} uglify-js@3.19.3: {} @@ -25859,11 +25938,11 @@ snapshots: d3-time: 3.1.0 d3-timer: 3.0.1 - vite-tsconfig-paths@5.1.4(typescript@5.9.3)(vite@7.3.1(@types/node@24.11.0)(jiti@2.6.1)(sass@1.67.0)(terser@5.46.0)(tsx@4.21.0)(yaml@1.10.2)): + vite-tsconfig-paths@5.1.4(typescript@6.0.3)(vite@7.3.1(@types/node@24.11.0)(jiti@2.6.1)(sass@1.67.0)(terser@5.46.0)(tsx@4.21.0)(yaml@1.10.2)): dependencies: debug: 4.4.3(supports-color@5.5.0) globrex: 0.1.2 - tsconfck: 3.1.6(typescript@5.9.3) + tsconfck: 3.1.6(typescript@6.0.3) optionalDependencies: vite: 7.3.1(@types/node@24.11.0)(jiti@2.6.1)(sass@1.67.0)(terser@5.46.0)(tsx@4.21.0)(yaml@1.10.2) transitivePeerDependencies: diff --git a/server/collab/api.ts b/server/collab/api.ts index 1a17686cb8..5b9752a143 100644 --- a/server/collab/api.ts +++ b/server/collab/api.ts @@ -25,6 +25,7 @@ const getDraftIdForPub = async (pubId: string): Promise => { router.post( '/api/pubs/:pubId/commits', wrap(async (req, res) => { + console.log('receiveCommit', req.body); const draftId = await getDraftIdForPub(req.params.pubId); if (!draftId) { @@ -35,6 +36,7 @@ router.post( await collabAuthority.receiveCommit(draftId, req.body); } catch (e) { if (e instanceof TooMuchContentionError) { + console.log('TooMuchContentionError', e); return res.status(409).json(null); } throw e; diff --git a/server/collab/authority.ts b/server/collab/authority.ts index 1176517ebd..d724ec59d8 100644 --- a/server/collab/authority.ts +++ b/server/collab/authority.ts @@ -83,6 +83,7 @@ export const collabAuthority = new CollabAuthority({ }, saveCommit: async (tr, docId, commitRef, commitVersion, commitSteps) => { + console.log('saveCommit', docId, commitRef, commitVersion, commitSteps); await CollabCommit.create( { draftId: docId, diff --git a/server/server.ts b/server/server.ts index ad672c7e79..62d8b52c9f 100755 --- a/server/server.ts +++ b/server/server.ts @@ -308,6 +308,10 @@ appRouter.use((req, res, next) => { const userId = req.user?.id ?? 'anon'; const ip = req.headers['x-forwarded-for'] ?? req.socket.remoteAddress ?? 'unknown'; + if (req.path.includes('/discussions/positions') || req.path.includes('/commits')) { + return; + } + console.log( `${req.method} ${res.statusCode} ${req.path} ${durationMs}ms | host=${host} | user=${userId} | ip=${ip} | size=${contentLength} | ua=${userAgent} | origin=${req.headers.origin}`, ); diff --git a/tools/migrations/2024_01_11_addAnalyticsSettings.js b/tools/migrations/2024_01_11_addAnalyticsSettings.js index 4b055d8184..833ae0d540 100644 --- a/tools/migrations/2024_01_11_addAnalyticsSettings.js +++ b/tools/migrations/2024_01_11_addAnalyticsSettings.js @@ -1,22 +1,12 @@ // @ts-check -/** - * @param {object} options - * @param {import('sequelize').Sequelize} options.Sequelize - * @param {import('server/sequelize').sequelize} options.sequelize - */ export const up = async ({ Sequelize, sequelize }) => { await sequelize.getQueryInterface().addColumn('Communities', 'analyticsSettings', { - // @ts-expect-error type: Sequelize.JSONB, defaultValue: null, }); }; -/** - * @param {object} options - * @param {import('server/sequelize').sequelize} options.sequelize - */ export const down = async ({ sequelize }) => { await sequelize.getQueryInterface().removeColumn('Communities', 'analyticsSettings'); }; diff --git a/tools/migrations/2024_02_19_addMastodonAndInstagramSettings.js b/tools/migrations/2024_02_19_addMastodonAndInstagramSettings.js index 949575261b..1282f84589 100644 --- a/tools/migrations/2024_02_19_addMastodonAndInstagramSettings.js +++ b/tools/migrations/2024_02_19_addMastodonAndInstagramSettings.js @@ -1,4 +1,3 @@ -// @ts-check const newCommunityColumns = ['instagram', 'mastodon', 'linkedin', 'bluesky', 'github']; const newUserColumns = ['mastodon', 'instagram', 'linkedin', 'bluesky']; diff --git a/tools/migrations/2024_06_12_replaceLinkedinHandles.js b/tools/migrations/2024_06_12_replaceLinkedinHandles.js index 9ebf61b7ab..64e9ace75e 100644 --- a/tools/migrations/2024_06_12_replaceLinkedinHandles.js +++ b/tools/migrations/2024_06_12_replaceLinkedinHandles.js @@ -1,4 +1,3 @@ -// @ts-check const { asyncMap } = require('utils/async'); const { Op } = require('sequelize'); @@ -6,7 +5,7 @@ const { Op } = require('sequelize'); /** * @param {object} options * @param {import('sequelize').Sequelize} options.Sequelize - * @param {import('server/sequelize').sequelize} options.sequelize + * @param {import('server/sequelize')} options.sequelize */ export const up = async ({ sequelize }) => { const communitiesWithLinkedin = await sequelize.models.Community.findAll({ diff --git a/tools/migrations/2025_11_19_addDiscussionCreationAccess.js b/tools/migrations/2025_11_19_addDiscussionCreationAccess.js index a8c093d61f..31005a2e2b 100644 --- a/tools/migrations/2025_11_19_addDiscussionCreationAccess.js +++ b/tools/migrations/2025_11_19_addDiscussionCreationAccess.js @@ -1,4 +1,3 @@ -// @ts-check const { Sequelize } = require('sequelize'); diff --git a/tsconfig.json b/tsconfig.json index 9439529f38..4bdc04a04b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -26,7 +26,7 @@ // "removeComments": true, /* Do not emit comments to output. */ "noEmit": true /* Do not emit outputs. */, // "importHelpers": true, /* Import emit helpers from 'tslib'. */ - "downlevelIteration": true /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */, + // "downlevelIteration": true /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */, // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ /* Strict Type-Checking Options */ "strict": true /* Enable all strict type-checking options. */, @@ -44,7 +44,7 @@ // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ /* Module Resolution Options */ "moduleResolution": "nodenext" /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */, - "baseUrl": "./" /* Base directory to resolve non-absolute module names. */, + // "baseUrl": "./" /* Base directory to resolve non-absolute module names. */, "paths": { "client": ["./client"], "client/*": ["./client/*"], @@ -54,6 +54,10 @@ "containers/*": ["./client/containers/*"], "server": ["./server"], "server/*": ["./server/*"], + "deposit": ["./deposit"], + "deposit/*": ["./deposit/*"], + "workers": ["./workers"], + "workers/*": ["./workers/*"], "stubstub": ["./stubstub"], "stubstub/*": ["./stubstub/*"], "types": ["./types"], diff --git a/types/global.d.ts b/types/global.d.ts index 6527e96067..b078ae34d8 100644 --- a/types/global.d.ts +++ b/types/global.d.ts @@ -1,4 +1,4 @@ -import { UserWithPrivateFields } from './user'; +import type { UserWithPrivateFields } from './user'; export {}; @@ -9,3 +9,7 @@ declare global { } } } + +declare module '*.scss' { + const content: string; +} diff --git a/types/scss.d.ts b/types/scss.d.ts new file mode 100644 index 0000000000..8355fa2ce4 --- /dev/null +++ b/types/scss.d.ts @@ -0,0 +1,3 @@ +declare module '*.scss' { + const content: string; +} diff --git a/workers/tasks/archive/siteDownloaderTransform.ts b/workers/tasks/archive/siteDownloaderTransform.ts index 4aadd3a325..71154f0f3f 100644 --- a/workers/tasks/archive/siteDownloaderTransform.ts +++ b/workers/tasks/archive/siteDownloaderTransform.ts @@ -11,8 +11,9 @@ import { getAssetUrlFromResizedUrl } from 'utils/images'; // https://stackoverflow.com/questions/76958222/how-to-pipe-response-from-nodejs-fetch-response-to-an-express-response#comment139165202_77589444 declare global { + // @ts-ignore response type incompatibility with node 24 / ts 7 interface Response { - readonly body: streamWeb.ReadableStream | null; + readonly body: streamWeb.ReadableStream> | null; } } diff --git a/workers/tasks/import/bulk/paths.ts b/workers/tasks/import/bulk/paths.ts index 8d46c4e28e..f8e39fd548 100644 --- a/workers/tasks/import/bulk/paths.ts +++ b/workers/tasks/import/bulk/paths.ts @@ -1,4 +1,4 @@ -const zipArrays = (first, second) => { +const zipArrays = (first, second): unknown[] => { const length = Math.max(first.length, second.length); const firstFilled = [...first, ...new Array(length - first.length).fill(null)]; const secondFilled = [...second, ...new Array(length - second.length).fill(null)]; @@ -18,9 +18,7 @@ export const pathMatchesPattern = (filePath, pattern) => { if (pathPart === null || patternPart === null) { return false; } - // @ts-expect-error ts-migrate(2339) FIXME: Property 'split' does not exist on type 'never'. const pathDotSegments = pathPart.split('.'); - // @ts-expect-error ts-migrate(2339) FIXME: Property 'split' does not exist on type 'never'. const patternDotSegments = patternPart.split('.'); return zipArrays(patternDotSegments, pathDotSegments).every( // @ts-expect-error ts-migrate(2488) FIXME: Type 'never' must have a '[Symbol.iterator]()' met... Remove this comment to see the full error message From 8b5e7b201d398d5221b7dc65e376b68226cadb77 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Wed, 17 Jun 2026 15:20:45 +0200 Subject: [PATCH 03/17] fix: patch prosemirror st pitterpatter doesnt try to get the esm version --- .../Editor/plugins/collaborative/index.ts | 11 +- package.json | 15 +- patches/prosemirror-model.patch | 12 ++ patches/prosemirror-state.patch | 12 ++ patches/prosemirror-transform.patch | 12 ++ patches/prosemirror-view.patch | 12 ++ pnpm-lock.yaml | 201 ++++++++++-------- server/collab/authority.ts | 3 +- 8 files changed, 172 insertions(+), 106 deletions(-) create mode 100644 patches/prosemirror-model.patch create mode 100644 patches/prosemirror-state.patch create mode 100644 patches/prosemirror-transform.patch create mode 100644 patches/prosemirror-view.patch diff --git a/client/components/Editor/plugins/collaborative/index.ts b/client/components/Editor/plugins/collaborative/index.ts index 8b6d1ba48e..a283c8a489 100644 --- a/client/components/Editor/plugins/collaborative/index.ts +++ b/client/components/Editor/plugins/collaborative/index.ts @@ -1,7 +1,5 @@ -import type { CollabState } from '@stepwisehq/prosemirror-collab-commit/collab-commit'; - import { collab } from '@pitter-patter/collab-client'; -import { type Plugin, PluginKey } from 'prosemirror-state'; +import { PluginKey } from 'prosemirror-state'; import { generateHash } from 'utils/hashes'; @@ -10,7 +8,7 @@ import buildDocument from './document'; export const collabDocPluginKey = new PluginKey('collaborative'); -export default (schema, props) => { +export default (schema: any, props: any) => { if (!props.collaborativeOptions) { return []; } @@ -18,10 +16,7 @@ export default (schema, props) => { const localClientId = `${props.collaborativeOptions.clientData.id}-${generateHash(6)}`; return [ - collab({ - version: props.collaborativeOptions.initialDocKey, - }), - //as unknown as Plugin, + collab({ version: props.collaborativeOptions.initialDocKey }), buildDocument(schema, props, collabDocPluginKey, localClientId), buildCursors(schema, props, collabDocPluginKey), ]; diff --git a/package.json b/package.json index c941ba647c..fe7523e932 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "build-dev-once": "rm -rf ./dist/client && pnpm run build-client-dev && pnpm run build-server", "build-prod": "rm -rf ./dist && webpack --config ./client/webpack/webpackConfig.prod.js && pnpm run build:export-css && pnpm run build-server", "heroku-postbuild": "pnpm run build-prod && [ -z $CI ] && pnpm run upload-sentry-sourcemaps && pnpm run write-commit-version || true", - "build-server": "tsgo -p tsconfig.server.json", + "build-server": "tsc -p tsconfig.server.json", "watch-server": "pnpm run build-server --watch --preserveWatchOutput", "postbuild-server": "cpy '**/*' '!**/*.ts' '!**/*.tsx' '!**/*.js' '!tsconfig.json' ../dist/server/server/ --cwd=server/ --parents && pnpm run copy-worker-assets", "copy-worker-assets": "cpy '**/*' '!**/*.ts' '!**/*.tsx' '!**/*.js' '!tsconfig.json' '!**/__tests__' ../dist/server/workers/ --cwd=workers/ --parents", @@ -101,13 +101,13 @@ "@pitter-patter/collab-server": "^0.1.3", "@pitter-patter/presence-client": "^0.2.1", "@pitter-patter/presence-server": "^0.1.3", + "@stepwisehq/prosemirror-collab-commit": "^1.0.0", "@popperjs/core": "^2.11.5", "@pubpub/deposit-utils": "^0.1.10", "@pubpub/prosemirror-pandoc": "^1.1.5", "@pubpub/prosemirror-reactive": "^0.2.0", "@sentry/node": "^7.77.0", "@sentry/react": "^7.77.0", - "@stepwisehq/prosemirror-collab-commit": "^1.0.5", "@ts-rest/core": "^3.30.5", "@ts-rest/express": "^3.30.5", "@ts-rest/open-api": "^3.30.5", @@ -388,11 +388,18 @@ "overrides": { "prosemirror-model": "1.25.4", "prosemirror-state": "1.4.4", - "prosemirror-view": "1.41.6" + "prosemirror-view": "1.41.6", + "@pitter-patter/collab-server@0>prosemirror-state": "1.4.4", + "@pitter-patter/collab-server@0>prosemirror-view": "1.41.6", + "@pitter-patter/collab-server@0>prosemirror-model": "1.25.4" }, "patchedDependencies": { "reakit": "patches/reakit.patch", - "@pubpub/deposit-utils": "patches/@pubpub__deposit-utils.patch" + "@pubpub/deposit-utils": "patches/@pubpub__deposit-utils.patch", + "prosemirror-model": "patches/prosemirror-model.patch", + "prosemirror-transform": "patches/prosemirror-transform.patch", + "prosemirror-state": "patches/prosemirror-state.patch", + "prosemirror-view": "patches/prosemirror-view.patch" } } } diff --git a/patches/prosemirror-model.patch b/patches/prosemirror-model.patch new file mode 100644 index 0000000000..ca145fd1b7 --- /dev/null +++ b/patches/prosemirror-model.patch @@ -0,0 +1,12 @@ +diff --git a/package.json b/package.json +--- a/package.json ++++ b/package.json +@@ -7,7 +7,7 @@ + "module": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { +- "import": "./dist/index.js", ++ "import": "./dist/index.cjs", + "require": "./dist/index.cjs" + }, + "sideEffects": false, diff --git a/patches/prosemirror-state.patch b/patches/prosemirror-state.patch new file mode 100644 index 0000000000..ca145fd1b7 --- /dev/null +++ b/patches/prosemirror-state.patch @@ -0,0 +1,12 @@ +diff --git a/package.json b/package.json +--- a/package.json ++++ b/package.json +@@ -7,7 +7,7 @@ + "module": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { +- "import": "./dist/index.js", ++ "import": "./dist/index.cjs", + "require": "./dist/index.cjs" + }, + "sideEffects": false, diff --git a/patches/prosemirror-transform.patch b/patches/prosemirror-transform.patch new file mode 100644 index 0000000000..ca145fd1b7 --- /dev/null +++ b/patches/prosemirror-transform.patch @@ -0,0 +1,12 @@ +diff --git a/package.json b/package.json +--- a/package.json ++++ b/package.json +@@ -7,7 +7,7 @@ + "module": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { +- "import": "./dist/index.js", ++ "import": "./dist/index.cjs", + "require": "./dist/index.cjs" + }, + "sideEffects": false, diff --git a/patches/prosemirror-view.patch b/patches/prosemirror-view.patch new file mode 100644 index 0000000000..2e238ff4fe --- /dev/null +++ b/patches/prosemirror-view.patch @@ -0,0 +1,12 @@ +diff --git a/package.json b/package.json +--- a/package.json ++++ b/package.json +@@ -8,7 +8,7 @@ + "types": "dist/index.d.ts", + "exports": { + ".": { +- "import": "./dist/index.js", ++ "import": "./dist/index.cjs", + "require": "./dist/index.cjs" + }, + "./style/prosemirror.css": "./style/prosemirror.css" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 36ab7b0522..5990768cb1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,11 +8,26 @@ overrides: prosemirror-model: 1.25.4 prosemirror-state: 1.4.4 prosemirror-view: 1.41.6 + '@pitter-patter/collab-server@0>prosemirror-state': 1.4.4 + '@pitter-patter/collab-server@0>prosemirror-view': 1.41.6 + '@pitter-patter/collab-server@0>prosemirror-model': 1.25.4 patchedDependencies: '@pubpub/deposit-utils': hash: 6fef8933046b7751abda1bd2ed0422094c79d7828b07f7d5afba26a47c696d89 path: patches/@pubpub__deposit-utils.patch + prosemirror-model: + hash: 8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8 + path: patches/prosemirror-model.patch + prosemirror-state: + hash: 8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8 + path: patches/prosemirror-state.patch + prosemirror-transform: + hash: 8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8 + path: patches/prosemirror-transform.patch + prosemirror-view: + hash: ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5 + path: patches/prosemirror-view.patch reakit: hash: 31488077229fe3d19a71c66458b888c58eab18010632a29e3b5c485df77242a8 path: patches/reakit.patch @@ -44,7 +59,7 @@ importers: version: 3.1032.0 '@benrbray/prosemirror-math': specifier: git+https://github.com/pubpub/prosemirror-math#9e6722987690bfad58444d8edbd73294e255de17 - version: https://codeload.github.com/pubpub/prosemirror-math/tar.gz/9e6722987690bfad58444d8edbd73294e255de17(katex@0.13.24)(prosemirror-commands@1.7.1)(prosemirror-gapcursor@1.4.0)(prosemirror-history@1.5.0)(prosemirror-inputrules@1.5.1)(prosemirror-keymap@1.2.3)(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0)(prosemirror-view@1.41.6) + version: https://codeload.github.com/pubpub/prosemirror-math/tar.gz/9e6722987690bfad58444d8edbd73294e255de17(katex@0.13.24)(prosemirror-commands@1.7.1)(prosemirror-gapcursor@1.4.0)(prosemirror-history@1.5.0)(prosemirror-inputrules@1.5.1)(prosemirror-keymap@1.2.3)(prosemirror-model@1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-state@1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-transform@1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-view@1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5)) '@blueprintjs/core': specifier: 3.26.0 version: 3.26.0(react-dom@16.14.0(react@16.14.0))(react@16.14.0) @@ -152,13 +167,13 @@ importers: version: 4.1.1(monaco-editor@0.21.3)(react-dom@16.14.0(react@16.14.0))(react@16.14.0) '@pitter-patter/collab-client': specifier: ^0.1.3 - version: 0.1.3(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0) + version: 0.1.3(prosemirror-model@1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-state@1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-transform@1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8)) '@pitter-patter/collab-server': specifier: ^0.1.3 - version: 0.1.3(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0) + version: 0.1.3(prosemirror-model@1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-state@1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-transform@1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8)) '@pitter-patter/presence-client': specifier: ^0.2.1 - version: 0.2.1(@handlewithcare/react-prosemirror@3.1.6(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-view@1.41.6)(react-dom@16.14.0(react@16.14.0))(react-reconciler@0.32.0(react@16.14.0))(react@16.14.0))(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0)(prosemirror-view@1.41.6)(react-dom@16.14.0(react@16.14.0))(react-reconciler@0.32.0(react@16.14.0))(react@16.14.0) + version: 0.2.1(@handlewithcare/react-prosemirror@3.1.6(prosemirror-model@1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-state@1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-view@1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5))(react-dom@16.14.0(react@16.14.0))(react-reconciler@0.32.0(react@16.14.0))(react@16.14.0))(prosemirror-model@1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-state@1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-transform@1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-view@1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5))(react-dom@16.14.0(react@16.14.0))(react-reconciler@0.32.0(react@16.14.0))(react@16.14.0) '@pitter-patter/presence-server': specifier: ^0.1.3 version: 0.1.3 @@ -181,7 +196,7 @@ importers: specifier: ^7.77.0 version: 7.120.4(react@16.14.0) '@stepwisehq/prosemirror-collab-commit': - specifier: ^1.0.5 + specifier: ^1.0.0 version: 1.0.5 '@ts-rest/core': specifier: ^3.30.5 @@ -491,25 +506,25 @@ importers: version: 1.2.3 prosemirror-model: specifier: 1.25.4 - version: 1.25.4 + version: 1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) prosemirror-schema-list: specifier: ^1.2.2 version: 1.5.1 prosemirror-state: specifier: 1.4.4 - version: 1.4.4 + version: 1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) prosemirror-suggest: specifier: ^0.7.6 - version: 0.7.6(@emotion/core@10.3.1(react@16.14.0))(@types/prosemirror-view@1.24.0)(@types/react-dom@17.0.26(@types/react@16.14.69))(@types/react@16.14.69)(prosemirror-view@1.41.6)(react-dom@16.14.0(react@16.14.0))(react@16.14.0) + version: 0.7.6(@emotion/core@10.3.1(react@16.14.0))(@types/prosemirror-view@1.24.0)(@types/react-dom@17.0.26(@types/react@16.14.69))(@types/react@16.14.69)(prosemirror-view@1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5))(react-dom@16.14.0(react@16.14.0))(react@16.14.0) prosemirror-tables: specifier: ^1.1.1 version: 1.8.5 prosemirror-transform: specifier: ^1.7.0 - version: 1.11.0 + version: 1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) prosemirror-view: specifier: 1.41.6 - version: 1.41.6 + version: 1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5) query-string: specifier: ^6.4.0 version: 6.14.1 @@ -13673,7 +13688,7 @@ snapshots: '@bcoe/v8-coverage@0.2.3': {} - '@benrbray/prosemirror-math@https://codeload.github.com/pubpub/prosemirror-math/tar.gz/9e6722987690bfad58444d8edbd73294e255de17(katex@0.13.24)(prosemirror-commands@1.7.1)(prosemirror-gapcursor@1.4.0)(prosemirror-history@1.5.0)(prosemirror-inputrules@1.5.1)(prosemirror-keymap@1.2.3)(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0)(prosemirror-view@1.41.6)': + '@benrbray/prosemirror-math@https://codeload.github.com/pubpub/prosemirror-math/tar.gz/9e6722987690bfad58444d8edbd73294e255de17(katex@0.13.24)(prosemirror-commands@1.7.1)(prosemirror-gapcursor@1.4.0)(prosemirror-history@1.5.0)(prosemirror-inputrules@1.5.1)(prosemirror-keymap@1.2.3)(prosemirror-model@1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-state@1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-transform@1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-view@1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5))': dependencies: katex: 0.13.24 prosemirror-commands: 1.7.1 @@ -13681,10 +13696,10 @@ snapshots: prosemirror-history: 1.5.0 prosemirror-inputrules: 1.5.1 prosemirror-keymap: 1.2.3 - prosemirror-model: 1.25.4 - prosemirror-state: 1.4.4 - prosemirror-transform: 1.11.0 - prosemirror-view: 1.41.6 + prosemirror-model: 1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-state: 1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-transform: 1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-view: 1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5) '@biomejs/biome@2.4.3': optionalDependencies: @@ -14297,12 +14312,12 @@ snapshots: yargs: 17.7.2 optional: true - '@handlewithcare/react-prosemirror@3.1.6(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-view@1.41.6)(react-dom@16.14.0(react@16.14.0))(react-reconciler@0.32.0(react@16.14.0))(react@16.14.0)': + '@handlewithcare/react-prosemirror@3.1.6(prosemirror-model@1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-state@1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-view@1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5))(react-dom@16.14.0(react@16.14.0))(react-reconciler@0.32.0(react@16.14.0))(react@16.14.0)': dependencies: classnames: 2.5.1 - prosemirror-model: 1.25.4 - prosemirror-state: 1.4.4 - prosemirror-view: 1.41.6 + prosemirror-model: 1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-state: 1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-view: 1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5) react: 16.14.0 react-dom: 16.14.0(react@16.14.0) react-reconciler: 0.32.0(react@16.14.0) @@ -14744,35 +14759,35 @@ snapshots: dependencies: pako: 1.0.11 - '@pitter-patter/collab-client@0.1.3(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0)': + '@pitter-patter/collab-client@0.1.3(prosemirror-model@1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-state@1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-transform@1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))': dependencies: '@stepwisehq/prosemirror-collab-commit': 1.0.5 - prosemirror-model: 1.25.4 - prosemirror-state: 1.4.4 - prosemirror-transform: 1.11.0 + prosemirror-model: 1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-state: 1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-transform: 1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) - '@pitter-patter/collab-server@0.1.3(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0)': + '@pitter-patter/collab-server@0.1.3(prosemirror-model@1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-state@1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-transform@1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))': dependencies: '@stepwisehq/prosemirror-collab-commit': 1.0.5 - prosemirror-model: 1.25.4 - prosemirror-state: 1.4.4 - prosemirror-transform: 1.11.0 + prosemirror-model: 1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-state: 1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-transform: 1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) redis: 5.12.1 transitivePeerDependencies: - '@node-rs/xxhash' - '@opentelemetry/api' - '@pitter-patter/presence-client@0.2.1(@handlewithcare/react-prosemirror@3.1.6(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-view@1.41.6)(react-dom@16.14.0(react@16.14.0))(react-reconciler@0.32.0(react@16.14.0))(react@16.14.0))(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0)(prosemirror-view@1.41.6)(react-dom@16.14.0(react@16.14.0))(react-reconciler@0.32.0(react@16.14.0))(react@16.14.0)': + '@pitter-patter/presence-client@0.2.1(@handlewithcare/react-prosemirror@3.1.6(prosemirror-model@1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-state@1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-view@1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5))(react-dom@16.14.0(react@16.14.0))(react-reconciler@0.32.0(react@16.14.0))(react@16.14.0))(prosemirror-model@1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-state@1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-transform@1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-view@1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5))(react-dom@16.14.0(react@16.14.0))(react-reconciler@0.32.0(react@16.14.0))(react@16.14.0)': dependencies: - '@pitter-patter/collab-client': 0.1.3(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-transform@1.11.0) + '@pitter-patter/collab-client': 0.1.3(prosemirror-model@1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-state@1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-transform@1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8)) '@pitter-patter/refs': 0.1.3 classnames: 2.5.1 - prosemirror-model: 1.25.4 - prosemirror-state: 1.4.4 - prosemirror-transform: 1.11.0 - prosemirror-view: 1.41.6 + prosemirror-model: 1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-state: 1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-transform: 1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-view: 1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5) optionalDependencies: - '@handlewithcare/react-prosemirror': 3.1.6(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-view@1.41.6)(react-dom@16.14.0(react@16.14.0))(react-reconciler@0.32.0(react@16.14.0))(react@16.14.0) + '@handlewithcare/react-prosemirror': 3.1.6(prosemirror-model@1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-state@1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-view@1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5))(react-dom@16.14.0(react@16.14.0))(react-reconciler@0.32.0(react@16.14.0))(react@16.14.0) react: 16.14.0 react-dom: 16.14.0(react@16.14.0) react-reconciler: 0.32.0(react@16.14.0) @@ -14849,9 +14864,9 @@ snapshots: '@pubpub/prosemirror-reactive@0.2.0': dependencies: - prosemirror-model: 1.25.4 - prosemirror-state: 1.4.4 - prosemirror-view: 1.41.6 + prosemirror-model: 1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-state: 1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-view: 1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5) '@redis/bloom@5.12.1(@redis/client@5.12.1)': dependencies: @@ -14877,11 +14892,11 @@ snapshots: dependencies: '@babel/runtime': 7.28.6 - '@remirror/core-helpers@0.7.6(@emotion/core@10.3.1(react@16.14.0))(@types/prosemirror-view@1.24.0)(@types/react-dom@17.0.26(@types/react@16.14.69))(@types/react@16.14.69)(prosemirror-view@1.41.6)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)': + '@remirror/core-helpers@0.7.6(@emotion/core@10.3.1(react@16.14.0))(@types/prosemirror-view@1.24.0)(@types/react-dom@17.0.26(@types/react@16.14.69))(@types/react@16.14.69)(prosemirror-view@1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5))(react-dom@16.14.0(react@16.14.0))(react@16.14.0)': dependencies: '@babel/runtime': 7.28.6 '@remirror/core-constants': 0.7.4 - '@remirror/core-types': 0.9.0(@emotion/core@10.3.1(react@16.14.0))(@types/prosemirror-view@1.24.0)(@types/react-dom@17.0.26(@types/react@16.14.69))(@types/react@16.14.69)(prosemirror-view@1.41.6)(react-dom@16.14.0(react@16.14.0))(react@16.14.0) + '@remirror/core-types': 0.9.0(@emotion/core@10.3.1(react@16.14.0))(@types/prosemirror-view@1.24.0)(@types/react-dom@17.0.26(@types/react@16.14.69))(@types/react@16.14.69)(prosemirror-view@1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5))(react-dom@16.14.0(react@16.14.0))(react@16.14.0) '@types/object.omit': 3.0.3 '@types/object.pick': 1.3.4 '@types/throttle-debounce': 2.1.0 @@ -14901,7 +14916,7 @@ snapshots: - react - react-dom - '@remirror/core-types@0.9.0(@emotion/core@10.3.1(react@16.14.0))(@types/prosemirror-view@1.24.0)(@types/react-dom@17.0.26(@types/react@16.14.69))(@types/react@16.14.69)(prosemirror-view@1.41.6)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)': + '@remirror/core-types@0.9.0(@emotion/core@10.3.1(react@16.14.0))(@types/prosemirror-view@1.24.0)(@types/react-dom@17.0.26(@types/react@16.14.69))(@types/react@16.14.69)(prosemirror-view@1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5))(react-dom@16.14.0(react@16.14.0))(react@16.14.0)': dependencies: '@babel/runtime': 7.28.6 '@emotion/core': 10.3.1(react@16.14.0) @@ -14922,23 +14937,23 @@ snapshots: prosemirror-commands: 1.7.1 prosemirror-inputrules: 1.5.1 prosemirror-keymap: 1.2.3 - prosemirror-model: 1.25.4 + prosemirror-model: 1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) prosemirror-schema-list: 1.5.1 - prosemirror-state: 1.4.4 - prosemirror-transform: 1.11.0 - prosemirror-view: 1.41.6 + prosemirror-state: 1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-transform: 1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-view: 1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5) react: 16.14.0 type-fest: 0.11.0 transitivePeerDependencies: - '@types/react-dom' - react-dom - '@remirror/core-utils@0.8.0(@emotion/core@10.3.1(react@16.14.0))(@types/prosemirror-view@1.24.0)(@types/react-dom@17.0.26(@types/react@16.14.69))(@types/react@16.14.69)(prosemirror-view@1.41.6)(react-dom@16.14.0(react@16.14.0))(react@16.14.0)': + '@remirror/core-utils@0.8.0(@emotion/core@10.3.1(react@16.14.0))(@types/prosemirror-view@1.24.0)(@types/react-dom@17.0.26(@types/react@16.14.69))(@types/react@16.14.69)(prosemirror-view@1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5))(react-dom@16.14.0(react@16.14.0))(react@16.14.0)': dependencies: '@babel/runtime': 7.28.6 '@remirror/core-constants': 0.7.4 - '@remirror/core-helpers': 0.7.6(@emotion/core@10.3.1(react@16.14.0))(@types/prosemirror-view@1.24.0)(@types/react-dom@17.0.26(@types/react@16.14.69))(@types/react@16.14.69)(prosemirror-view@1.41.6)(react-dom@16.14.0(react@16.14.0))(react@16.14.0) - '@remirror/core-types': 0.9.0(@emotion/core@10.3.1(react@16.14.0))(@types/prosemirror-view@1.24.0)(@types/react-dom@17.0.26(@types/react@16.14.69))(@types/react@16.14.69)(prosemirror-view@1.41.6)(react-dom@16.14.0(react@16.14.0))(react@16.14.0) + '@remirror/core-helpers': 0.7.6(@emotion/core@10.3.1(react@16.14.0))(@types/prosemirror-view@1.24.0)(@types/react-dom@17.0.26(@types/react@16.14.69))(@types/react@16.14.69)(prosemirror-view@1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5))(react-dom@16.14.0(react@16.14.0))(react@16.14.0) + '@remirror/core-types': 0.9.0(@emotion/core@10.3.1(react@16.14.0))(@types/prosemirror-view@1.24.0)(@types/react-dom@17.0.26(@types/react@16.14.69))(@types/react@16.14.69)(prosemirror-view@1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5))(react-dom@16.14.0(react@16.14.0))(react@16.14.0) '@types/min-document': 2.19.2 '@types/node': 24.11.0 '@types/prosemirror-commands': 1.3.0 @@ -14949,9 +14964,9 @@ snapshots: min-document: 2.19.0 prosemirror-commands: 1.7.1 prosemirror-inputrules: 1.5.1 - prosemirror-model: 1.25.4 + prosemirror-model: 1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) prosemirror-schema-list: 1.5.1 - prosemirror-state: 1.4.4 + prosemirror-state: 1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) transitivePeerDependencies: - '@emotion/core' - '@types/prosemirror-view' @@ -15764,7 +15779,7 @@ snapshots: '@stepwisehq/prosemirror-collab-commit@1.0.5': dependencies: - prosemirror-state: 1.4.4 + prosemirror-state: 1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) '@storybook/addon-knobs@6.4.0(@storybook/addons@6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0))(@storybook/api@6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0))(@storybook/components@6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0))(@storybook/core-events@6.5.16)(@storybook/theming@6.5.16(react-dom@16.14.0(react@16.14.0))(react@16.14.0))(react-dom@16.14.0(react@16.14.0))(react@16.14.0)': dependencies: @@ -16887,7 +16902,7 @@ snapshots: '@types/prosemirror-model@1.17.0': dependencies: - prosemirror-model: 1.25.4 + prosemirror-model: 1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) '@types/prosemirror-schema-list@1.2.0': dependencies: @@ -16895,15 +16910,15 @@ snapshots: '@types/prosemirror-state@1.4.0': dependencies: - prosemirror-state: 1.4.4 + prosemirror-state: 1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) '@types/prosemirror-transform@1.5.0': dependencies: - prosemirror-transform: 1.11.0 + prosemirror-transform: 1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) '@types/prosemirror-view@1.24.0': dependencies: - prosemirror-view: 1.41.6 + prosemirror-view: 1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5) '@types/qs@6.14.0': {} @@ -23318,13 +23333,13 @@ snapshots: prosemirror-changeset@2.4.0: dependencies: - prosemirror-transform: 1.11.0 + prosemirror-transform: 1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) prosemirror-commands@1.7.1: dependencies: - prosemirror-model: 1.25.4 - prosemirror-state: 1.4.4 - prosemirror-transform: 1.11.0 + prosemirror-model: 1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-state: 1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-transform: 1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) prosemirror-compress-pubpub@0.0.3: {} @@ -23336,8 +23351,8 @@ snapshots: jotai: 1.13.1(@babel/core@7.29.0)(@babel/template@7.28.6)(react@16.14.0) jsondiffpatch: 0.4.1 nanoid: 2.1.11 - prosemirror-model: 1.25.4 - prosemirror-state: 1.4.4 + prosemirror-model: 1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-state: 1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) react: 16.14.0 react-dock: 0.6.0(@types/react@16.14.69)(react@16.14.0) react-dom: 16.14.0(react@16.14.0) @@ -23359,57 +23374,57 @@ snapshots: prosemirror-gapcursor@1.4.0: dependencies: prosemirror-keymap: 1.2.3 - prosemirror-model: 1.25.4 - prosemirror-state: 1.4.4 - prosemirror-view: 1.41.6 + prosemirror-model: 1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-state: 1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-view: 1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5) prosemirror-history@1.5.0: dependencies: - prosemirror-state: 1.4.4 - prosemirror-transform: 1.11.0 - prosemirror-view: 1.41.6 + prosemirror-state: 1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-transform: 1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-view: 1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5) rope-sequence: 1.3.4 prosemirror-inputrules@1.5.1: dependencies: - prosemirror-state: 1.4.4 - prosemirror-transform: 1.11.0 + prosemirror-state: 1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-transform: 1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) prosemirror-keymap@1.2.3: dependencies: - prosemirror-state: 1.4.4 + prosemirror-state: 1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) w3c-keyname: 2.2.8 - prosemirror-model@1.25.4: + prosemirror-model@1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8): dependencies: orderedmap: 2.1.1 prosemirror-schema-list@1.5.1: dependencies: - prosemirror-model: 1.25.4 - prosemirror-state: 1.4.4 - prosemirror-transform: 1.11.0 + prosemirror-model: 1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-state: 1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-transform: 1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) - prosemirror-state@1.4.4: + prosemirror-state@1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8): dependencies: - prosemirror-model: 1.25.4 - prosemirror-transform: 1.11.0 - prosemirror-view: 1.41.6 + prosemirror-model: 1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-transform: 1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-view: 1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5) - prosemirror-suggest@0.7.6(@emotion/core@10.3.1(react@16.14.0))(@types/prosemirror-view@1.24.0)(@types/react-dom@17.0.26(@types/react@16.14.69))(@types/react@16.14.69)(prosemirror-view@1.41.6)(react-dom@16.14.0(react@16.14.0))(react@16.14.0): + prosemirror-suggest@0.7.6(@emotion/core@10.3.1(react@16.14.0))(@types/prosemirror-view@1.24.0)(@types/react-dom@17.0.26(@types/react@16.14.69))(@types/react@16.14.69)(prosemirror-view@1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5))(react-dom@16.14.0(react@16.14.0))(react@16.14.0): dependencies: '@babel/runtime': 7.28.6 '@remirror/core-constants': 0.7.4 - '@remirror/core-helpers': 0.7.6(@emotion/core@10.3.1(react@16.14.0))(@types/prosemirror-view@1.24.0)(@types/react-dom@17.0.26(@types/react@16.14.69))(@types/react@16.14.69)(prosemirror-view@1.41.6)(react-dom@16.14.0(react@16.14.0))(react@16.14.0) - '@remirror/core-types': 0.9.0(@emotion/core@10.3.1(react@16.14.0))(@types/prosemirror-view@1.24.0)(@types/react-dom@17.0.26(@types/react@16.14.69))(@types/react@16.14.69)(prosemirror-view@1.41.6)(react-dom@16.14.0(react@16.14.0))(react@16.14.0) - '@remirror/core-utils': 0.8.0(@emotion/core@10.3.1(react@16.14.0))(@types/prosemirror-view@1.24.0)(@types/react-dom@17.0.26(@types/react@16.14.69))(@types/react@16.14.69)(prosemirror-view@1.41.6)(react-dom@16.14.0(react@16.14.0))(react@16.14.0) + '@remirror/core-helpers': 0.7.6(@emotion/core@10.3.1(react@16.14.0))(@types/prosemirror-view@1.24.0)(@types/react-dom@17.0.26(@types/react@16.14.69))(@types/react@16.14.69)(prosemirror-view@1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5))(react-dom@16.14.0(react@16.14.0))(react@16.14.0) + '@remirror/core-types': 0.9.0(@emotion/core@10.3.1(react@16.14.0))(@types/prosemirror-view@1.24.0)(@types/react-dom@17.0.26(@types/react@16.14.69))(@types/react@16.14.69)(prosemirror-view@1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5))(react-dom@16.14.0(react@16.14.0))(react@16.14.0) + '@remirror/core-utils': 0.8.0(@emotion/core@10.3.1(react@16.14.0))(@types/prosemirror-view@1.24.0)(@types/react-dom@17.0.26(@types/react@16.14.69))(@types/react@16.14.69)(prosemirror-view@1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5))(react-dom@16.14.0(react@16.14.0))(react@16.14.0) '@types/prosemirror-keymap': 1.2.0 '@types/prosemirror-state': 1.4.0 '@types/prosemirror-view': 1.24.0 escape-string-regexp: 2.0.0 prosemirror-keymap: 1.2.3 - prosemirror-state: 1.4.4 - prosemirror-view: 1.41.6 + prosemirror-state: 1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-view: 1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5) transitivePeerDependencies: - '@emotion/core' - '@types/react' @@ -23420,20 +23435,20 @@ snapshots: prosemirror-tables@1.8.5: dependencies: prosemirror-keymap: 1.2.3 - prosemirror-model: 1.25.4 - prosemirror-state: 1.4.4 - prosemirror-transform: 1.11.0 - prosemirror-view: 1.41.6 + prosemirror-model: 1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-state: 1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-transform: 1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-view: 1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5) - prosemirror-transform@1.11.0: + prosemirror-transform@1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8): dependencies: - prosemirror-model: 1.25.4 + prosemirror-model: 1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) - prosemirror-view@1.41.6: + prosemirror-view@1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5): dependencies: - prosemirror-model: 1.25.4 - prosemirror-state: 1.4.4 - prosemirror-transform: 1.11.0 + prosemirror-model: 1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-state: 1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) + prosemirror-transform: 1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) proto-list@1.2.4: {} diff --git a/server/collab/authority.ts b/server/collab/authority.ts index d724ec59d8..db7b0025fa 100644 --- a/server/collab/authority.ts +++ b/server/collab/authority.ts @@ -3,7 +3,7 @@ import type { Transaction } from 'sequelize'; import { CollabAuthority, RedisBroadcastManager } from '@pitter-patter/collab-server'; import { Op } from 'sequelize'; -import { editorSchema } from 'client/components/Editor/utils'; +import { editorSchema } from 'client/components/Editor/utils/schema'; import { env } from 'server/env'; import { CollabCommit, Draft, DraftCheckpoint } from 'server/models'; import { sequelize } from 'server/sequelize'; @@ -25,6 +25,7 @@ export const collabAuthority = new CollabAuthority({ return sequelize.transaction((tr) => callback(tr)); }, + // getDoc: async (tr, docId) => { const draft = await Draft.findOne({ where: { id: docId }, From b669704b8467014c67a3c64e0f2c29f5a732c8a7 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Mon, 22 Jun 2026 15:16:53 +0200 Subject: [PATCH 04/17] fix: correct comment syncing, but at what cost --- .../Editor/plugins/collaborative/cursors.ts | 42 +++- .../Editor/plugins/collaborative/document.ts | 13 +- .../Editor/plugins/discussions/decorations.ts | 7 + .../plugins/discussions/discussionsState.ts | 45 +++- .../plugins/discussions/historyState.ts | 5 +- .../Editor/plugins/discussions/plugin.ts | 56 ++++- .../discussions/{firebase.ts => polling.ts} | 0 client/components/Editor/types.ts | 2 + client/components/Editor/utils/view.ts | 16 +- client/containers/Pub/PubDocument/PubBody.tsx | 52 +++- server/collab/api.ts | 46 +++- server/collab/authority.ts | 237 +++++++++--------- server/collab/discussionPositions.ts | 68 ++++- server/collab/presence.ts | 24 +- server/utils/firebaseAdmin.ts | 4 +- 15 files changed, 446 insertions(+), 171 deletions(-) rename client/components/Editor/plugins/discussions/{firebase.ts => polling.ts} (100%) diff --git a/client/components/Editor/plugins/collaborative/cursors.ts b/client/components/Editor/plugins/collaborative/cursors.ts index 96c1fe2950..b79d71250d 100644 --- a/client/components/Editor/plugins/collaborative/cursors.ts +++ b/client/components/Editor/plugins/collaborative/cursors.ts @@ -5,7 +5,7 @@ import { Decoration, DecorationSet, type EditorView } from 'prosemirror-view'; export const cursorsPluginKey = new PluginKey('cursors'); const generateCursorDecorations = (cursorData: any, editorState: any, localClientId: string) => { - if (cursorData.id === localClientId) { + if (cursorData.clientId === localClientId) { return []; } @@ -21,7 +21,7 @@ const generateCursorDecorations = (cursorData: any, editorState: any, localClien return []; } - const formattedDataId = `c-${cursorData.id}`; + const formattedDataId = `c-${cursorData.clientId}`; const elem = document.createElement('span'); elem.className = `collab-cursor ${formattedDataId}`; @@ -191,8 +191,10 @@ export default (schema: any, props: any, collabDocPluginKey: PluginKey) => { const pollPresence = async () => { let refs: Record = {}; + const MIN_POLL_INTERVAL = 1000; while (polling && !abortController!.signal.aborted) { + const pollStart = Date.now(); try { const response = await fetch(`/api/pubs/${pubId}/presence`, { method: 'POST', @@ -209,17 +211,45 @@ export default (schema: any, props: any, collabDocPluginKey: PluginKey) => { const indicators = await response.json(); if (indicators && typeof indicators === 'object') { - for (const [id, indicator] of Object.entries(indicators) as any) { - if (id !== localClientId && indicator) { - currentIndicators.set(id, { id, ...indicator }); - refs[id] = indicator.ref ?? ''; + const localUserId = localClientData?.id; + for (const [clientId, indicator] of Object.entries(indicators) as any) { + if (clientId === localClientId) continue; + if (indicator) { + const userId = indicator.id; + if (localUserId && userId === localUserId) { + currentIndicators.delete(clientId); + delete refs[clientId]; + continue; + } + if (userId) { + for (const [existingClientId, existing] of currentIndicators) { + if (existing.id === userId && existingClientId !== clientId) { + currentIndicators.delete(existingClientId); + delete refs[existingClientId]; + } + } + } + currentIndicators.set(clientId, indicator); + refs[clientId] = indicator.ref ?? ''; + } else { + currentIndicators.delete(clientId); + delete refs[clientId]; } } + props.collaborativeOptions?.onPresenceChange?.( + Array.from(currentIndicators.values()), + ); + const { tr } = view.state; tr.setMeta('presenceIndicators', true); view.dispatch(tr); } + + const elapsed = Date.now() - pollStart; + if (elapsed < MIN_POLL_INTERVAL) { + await new Promise((r) => setTimeout(r, MIN_POLL_INTERVAL - elapsed)); + } } catch (e: any) { if (e.name === 'AbortError') break; await new Promise((r) => setTimeout(r, 3000)); diff --git a/client/components/Editor/plugins/collaborative/document.ts b/client/components/Editor/plugins/collaborative/document.ts index 06be74a23a..fc4622b427 100644 --- a/client/components/Editor/plugins/collaborative/document.ts +++ b/client/components/Editor/plugins/collaborative/document.ts @@ -37,10 +37,15 @@ export default ( } onStatusChange('saving'); - collabClient.send(newState).catch((e) => { - console.error('Error sending collab commit:', e); - onError(e); - }); + collabClient + .send(newState) + .then(() => { + onStatusChange('saved'); + }) + .catch((e) => { + console.error('Error sending collab commit:', e); + onError(e); + }); }; const startCollab = (initialState: any) => { diff --git a/client/components/Editor/plugins/discussions/decorations.ts b/client/components/Editor/plugins/discussions/decorations.ts index 0b9292b44a..3ff8aa3e11 100644 --- a/client/components/Editor/plugins/discussions/decorations.ts +++ b/client/components/Editor/plugins/discussions/decorations.ts @@ -48,9 +48,16 @@ const getDecorationsForDiscussion = (discussionId: string, discussion: Discussio }; const getNewDecorations = (updateResult: DiscussionsUpdateResult): DiscussionDecoration[] => { + const docSize = updateResult.doc.content.size; return flattenOnce( [...updateResult.addedDiscussionIds].map((id) => { const discussion = updateResult.discussions[id]; + if (discussion.selection) { + const { anchor, head } = discussion.selection; + if (anchor > docSize || head > docSize || anchor < 0 || head < 0) { + return []; + } + } return getDecorationsForDiscussion(id, discussion); }), ); diff --git a/client/components/Editor/plugins/discussions/discussionsState.ts b/client/components/Editor/plugins/discussions/discussionsState.ts index d066f5a59d..800598c991 100644 --- a/client/components/Editor/plugins/discussions/discussionsState.ts +++ b/client/components/Editor/plugins/discussions/discussionsState.ts @@ -2,6 +2,7 @@ import type { Node } from 'prosemirror-model'; import type { EditorState, Transaction } from 'prosemirror-state'; import type { + DiscussionInfo, DiscussionSelection, Discussions, DiscussionsFastForwardFn, @@ -21,6 +22,7 @@ type Options = { initialHistoryKey: number; fastForwardDiscussions: null | DiscussionsFastForwardFn; remoteDiscussions: null | RemoteDiscussions; + onNewDiscussionIds?: (ids: string[]) => void; onUpdateDiscussions: (result: DiscussionsUpdateResult) => unknown; }; @@ -58,15 +60,18 @@ const filterDiscussionsUpdate = ( const addedDiscussionIds: Set = new Set(); Object.entries(update).forEach(([id, next]) => { if (next) { - if (next.currentKey === currentKey) { + if (next.currentKey <= currentKey) { + const adjusted = next.currentKey < currentKey + ? { ...next, currentKey } + : next; const previous = discussions[id]; - const hasKeyAdvanced = !previous || previous.currentKey < next.currentKey; - const isKeyMonotonic = !previous || previous.currentKey <= next.currentKey; + const hasKeyAdvanced = !previous || previous.currentKey < adjusted.currentKey; + const isKeyMonotonic = !previous || previous.currentKey <= adjusted.currentKey; if (hasKeyAdvanced) { - sendableDiscussions[id] = next; + sendableDiscussions[id] = adjusted; } if (isKeyMonotonic) { - updatableDiscussions[id] = next; + updatableDiscussions[id] = adjusted; } if (!previous) { addedDiscussionIds.add(id); @@ -84,9 +89,22 @@ const filterDiscussionsUpdate = ( }; }; +const isValidDiscussionInfo = (d: any): d is DiscussionInfo => + d && typeof d.currentKey === 'number' && typeof d.initKey === 'number' && d.selection; + +const sanitizeRemoteDiscussions = (raw: NullableDiscussions): NullableDiscussions => { + const result: NullableDiscussions = {}; + for (const [id, d] of Object.entries(raw)) { + if (d === null || isValidDiscussionInfo(d)) { + result[id] = d; + } + } + return result; +}; + const getHighestCurrentKeyFromDiscussions = (discussions: NullableDiscussions) => { return Object.values(discussions).reduce((max, discussion) => { - if (discussion) { + if (discussion && typeof discussion.currentKey === 'number') { return Math.max(max, discussion.currentKey); } return max; @@ -100,6 +118,7 @@ export const createDiscussionsState = (options: Options) => { initialDoc, fastForwardDiscussions, onUpdateDiscussions, + onNewDiscussionIds, remoteDiscussions, } = options; const history = createHistoryState(initialDoc, initialHistoryKey); @@ -183,8 +202,20 @@ export const createDiscussionsState = (options: Options) => { }); }; - remoteDiscussions?.receiveDiscussions((update: NullableDiscussions) => { + remoteDiscussions?.receiveDiscussions((rawUpdate: NullableDiscussions) => { + const update = sanitizeRemoteDiscussions(rawUpdate); + if (Object.keys(update).length === 0) return; + + const newIds = Object.keys(update).filter( + (id) => update[id] && !discussions[id], + ); + if (newIds.length > 0) { + onNewDiscussionIds?.(newIds); + } + const remoteKey = getHighestCurrentKeyFromDiscussions(update); + if (remoteKey < 0) return; + history.onReachesKey(remoteKey, () => { const { currentDoc, currentHistoryKey } = history.getState(); asynchronouslyUpdateDiscussions(update); diff --git a/client/components/Editor/plugins/discussions/historyState.ts b/client/components/Editor/plugins/discussions/historyState.ts index 9d2422560c..b08f2e5b3e 100644 --- a/client/components/Editor/plugins/discussions/historyState.ts +++ b/client/components/Editor/plugins/discussions/historyState.ts @@ -1,7 +1,7 @@ import type { Node } from 'prosemirror-model'; import type { EditorState, Transaction } from 'prosemirror-state'; -import { collabDocPluginKey } from '../collaborative'; +import { getVersion } from '@pitter-patter/collab-client'; type Callback = () => unknown; @@ -49,8 +49,7 @@ export const createHistoryState = (initialDoc: Node, initialHistoryKey: number) const previousHistoryKey = historyKey; const previousDoc = doc; - const collabState = collabDocPluginKey.getState(nextState); - const nextHistoryKey: number = collabState?.mostRecentRemoteKey ?? -1; + const nextHistoryKey: number = getVersion(nextState) ?? historyKey; const nextDoc = tr.doc; if (nextHistoryKey >= historyKey) { diff --git a/client/components/Editor/plugins/discussions/plugin.ts b/client/components/Editor/plugins/discussions/plugin.ts index 1a754abb53..aa6daaccf5 100644 --- a/client/components/Editor/plugins/discussions/plugin.ts +++ b/client/components/Editor/plugins/discussions/plugin.ts @@ -1,15 +1,23 @@ import type { Node } from 'prosemirror-model'; import type { DiscussionsOptions, PluginsOptions } from '../../types'; -import type { DiscussionDecoration, DiscussionSelection, DiscussionsUpdateResult } from './types'; +import type { + DiscussionDecoration, + DiscussionSelection, + DiscussionsFastForwardFn, + DiscussionsUpdateResult, + NullableDiscussions, +} from './types'; import { type EditorState, Plugin, PluginKey, type Transaction } from 'prosemirror-state'; +import { Step } from 'prosemirror-transform'; import { DecorationSet, type EditorView } from 'prosemirror-view'; import { getDiscussionsFromAnchors } from './anchors'; import { getDecorationsForDiscussions, getDecorationsForUpdateResult } from './decorations'; import { createDiscussionsState } from './discussionsState'; -import { connectToRemoteDiscussions } from './firebase'; +import { connectToRemoteDiscussions } from './polling'; +import { mapDiscussionThroughSteps } from './util'; export const discussionsPluginKey = new PluginKey('discussions'); @@ -20,8 +28,43 @@ type PluginState = { addDiscussion: SyncDraftDiscussions['addDiscussion']; }; +const createFastForward = (pubId: string, schema: any): DiscussionsFastForwardFn => { + return async (discussions: NullableDiscussions, _fromDoc: Node, toKey: number) => { + const lowestKey = Object.values(discussions).reduce((min, d) => { + if (d && d.currentKey < min) return d.currentKey; + return min; + }, toKey); + + if (lowestKey >= toKey) return {}; + + try { + const response = await fetch( + `/api/pubs/${pubId}/commits/steps?from=${lowestKey}&to=${toKey}`, + ); + if (!response.ok) return {}; + const commits = (await response.json()) as { version: number; steps: any[] }[]; + + const result: Record = {}; + for (const [id, discussion] of Object.entries(discussions)) { + if (!discussion) continue; + const relevantCommits = commits.filter((c) => c.version > discussion.currentKey); + const steps = relevantCommits.flatMap((c) => + c.steps.map((s: any) => Step.fromJSON(schema, s)), + ); + if (steps.length > 0) { + const mapped = mapDiscussionThroughSteps(discussion, steps); + result[id] = { ...mapped, currentKey: toKey }; + } + } + return result; + } catch { + return {}; + } + }; +}; + const createPlugin = (discussionsOptions: DiscussionsOptions, initialDoc: Node) => { - const { discussionAnchors, pubId, initialHistoryKey } = discussionsOptions; + const { discussionAnchors, pubId, initialHistoryKey, onNewDiscussionIds } = discussionsOptions; const remote = pubId ? connectToRemoteDiscussions(pubId) : null; const initialDiscussions = getDiscussionsFromAnchors(discussionAnchors); @@ -32,7 +75,8 @@ const createPlugin = (discussionsOptions: DiscussionsOptions, initialDoc: Node) initialHistoryKey, initialDoc, remoteDiscussions: remote || null, - fastForwardDiscussions: null, + fastForwardDiscussions: pubId ? createFastForward(pubId, initialDoc.type.schema) : null, + onNewDiscussionIds, onUpdateDiscussions: (updateResult: DiscussionsUpdateResult) => { if (editorView) { const { tr } = editorView.state; @@ -61,8 +105,8 @@ const createPlugin = (discussionsOptions: DiscussionsOptions, initialDoc: Node) }; }; - const apply = (tr: Transaction, pluginState: PluginState, editorState: EditorState) => { - const updateResult = getUpdateResult(tr, editorState); + const apply = (tr: Transaction, pluginState: PluginState, _oldState: EditorState, newState: EditorState) => { + const updateResult = getUpdateResult(tr, newState); if (updateResult) { return { ...pluginState, diff --git a/client/components/Editor/plugins/discussions/firebase.ts b/client/components/Editor/plugins/discussions/polling.ts similarity index 100% rename from client/components/Editor/plugins/discussions/firebase.ts rename to client/components/Editor/plugins/discussions/polling.ts diff --git a/client/components/Editor/types.ts b/client/components/Editor/types.ts index bb41c3c272..d10e2338df 100644 --- a/client/components/Editor/types.ts +++ b/client/components/Editor/types.ts @@ -45,12 +45,14 @@ export type CollaborativeOptions = { initialDocKey: number; onStatusChange?: (status: CollaborativeEditorStatus) => unknown; onUpdateLatestKey?: (key: number) => unknown; + onPresenceChange?: (users: any[]) => unknown; }; export type DiscussionsOptions = { pubId?: string | null; initialHistoryKey: number; discussionAnchors: DiscussionAnchor[]; + onNewDiscussionIds?: (ids: string[]) => void; }; export type MediaUploadInstance = { diff --git a/client/components/Editor/utils/view.ts b/client/components/Editor/utils/view.ts index 7c13f78aa2..ac45f25353 100644 --- a/client/components/Editor/utils/view.ts +++ b/client/components/Editor/utils/view.ts @@ -5,6 +5,8 @@ import type { DocJson } from 'types'; import { Node, Slice } from 'prosemirror-model'; import { type EditorState, Selection } from 'prosemirror-state'; +import { getVersion } from '@pitter-patter/collab-client'; + import { addDiscussionToView } from '../plugins/discussions'; import { editorHasPasteDecorations } from '../plugins/paste/plugin'; import { isEmptyDocNode } from './doc'; @@ -152,11 +154,8 @@ export const getLocalHighlightText = (editorView, highlightId) => { }; export const reanchorDiscussion = (editorView, pubId: string, discussionId: string) => { - const collabPlugin = editorView.state.collaborative$ || {}; - const newCurrentKey = collabPlugin.mostRecentRemoteKey; - const selection = editorView.state.selection; - const newAnchor = selection.anchor; - const newHead = selection.head; + const currentKey = getVersion(editorView.state) ?? 0; + const { anchor, head } = editorView.state.selection; const transaction = editorView.state.tr; transaction.setMeta('removeDiscussion', { id: discussionId }); @@ -167,8 +166,11 @@ export const reanchorDiscussion = (editorView, pubId: string, discussionId: stri headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ [discussionId]: { - currentKey: newCurrentKey, - selection: { type: 'text', anchor: newAnchor, head: newHead }, + initKey: currentKey, + currentKey, + initAnchor: anchor, + initHead: head, + selection: { type: 'text', anchor, head }, }, }), }).catch((err) => { diff --git a/client/containers/Pub/PubDocument/PubBody.tsx b/client/containers/Pub/PubDocument/PubBody.tsx index d93ad34830..5441f0a788 100644 --- a/client/containers/Pub/PubDocument/PubBody.tsx +++ b/client/containers/Pub/PubDocument/PubBody.tsx @@ -1,11 +1,12 @@ import type { CollaborativeEditorStatus, EditorChangeObject } from 'client/components/Editor'; -import React, { useCallback, useContext, useState } from 'react'; +import React, { useCallback, useContext, useEffect, useRef, useState } from 'react'; import * as Sentry from '@sentry/react'; import { useBeforeUnload } from 'react-use'; import { useDebouncedCallback } from 'use-debounce/lib'; +import { apiFetch } from 'client/utils/apiFetch'; import malformedDocPlugin from 'client/components/Editor/plugins/malformedDoc'; import buildSuggestedEdits from 'client/components/Editor/plugins/suggestedEdits'; import { useFacetsQuery } from 'client/utils/useFacets'; @@ -36,6 +37,7 @@ const PubBody = (props: Props) => { pubData, noteManager, updateCollabData, + updateLocalData, historyData: { setLatestHistoryKey }, collabData: { status, localCollabUser }, pubBodyState: { @@ -83,18 +85,64 @@ const PubBody = (props: Props) => { [updateCollabData], ); + const handlePresenceChange = useCallback( + (users: any[]) => { + updateCollabData({ remoteCollabUsers: users }); + }, + [updateCollabData], + ); + + const fetchedDiscussionIds = useRef(new Set()); + + const handleNewDiscussionIds = useCallback( + (ids: string[]) => { + const unfetched = ids.filter((id) => !fetchedDiscussionIds.current.has(id)); + if (unfetched.length === 0) return; + unfetched.forEach((id) => fetchedDiscussionIds.current.add(id)); + + apiFetch(`/api/pubs/${pubData.id}/discussions?ids=${unfetched.join(',')}`) + .then((newDiscussions: any[]) => { + if (newDiscussions.length > 0) { + updateLocalData('pub', { + discussions: [...pubData.discussions, ...newDiscussions], + }); + } + }) + .catch(() => {}); + }, + [pubData.id, pubData.discussions, updateLocalData], + ); + + const updateLocalDataRef = useRef(updateLocalData); + updateLocalDataRef.current = updateLocalData; + + useEffect(() => { + if (!includeCollabPlugin || !includeDiscussionsPlugin) return undefined; + const pubId = pubData.id; + const interval = setInterval(() => { + apiFetch(`/api/pubs/${pubId}/discussions`) + .then((discussions: any[]) => { + updateLocalDataRef.current('pub', { discussions }); + }) + .catch(() => {}); + }, 5000); + return () => clearInterval(interval); + }, [pubData.id, includeCollabPlugin, includeDiscussionsPlugin]); + const collaborativeOptions = includeCollabPlugin && { pubId: pubData.id, initialDocKey: initialHistoryKey, clientData: localCollabUser, onStatusChange: handleStatusChange, onUpdateLatestKey: setLatestHistoryKey, + onPresenceChange: handlePresenceChange, }; const discussionOptions = includeDiscussionsPlugin && { - pubId: pubData.id, + pubId: includeCollabPlugin ? pubData.id : null, initialHistoryKey, discussionAnchors: discussionAnchors || [], + onNewDiscussionIds: handleNewDiscussionIds, }; return ( diff --git a/server/collab/api.ts b/server/collab/api.ts index 5b9752a143..f04a0df6bb 100644 --- a/server/collab/api.ts +++ b/server/collab/api.ts @@ -2,12 +2,13 @@ import type { PresenceIndicator } from '@pitter-patter/presence-server'; import { TooMuchContentionError } from '@pitter-patter/collab-server'; import { Router } from 'express'; +import { Op } from 'sequelize'; -import { Draft, Pub } from 'server/models'; +import { CollabCommit, Draft, Pub } from 'server/models'; import { wrap } from 'server/wrap'; -import { collabAuthority } from './authority'; -import { presenceAuthority } from './presence'; +import { getCollabAuthority } from './authority'; +import { getPresenceAuthority } from './presence'; export const router = Router(); @@ -33,7 +34,7 @@ router.post( } try { - await collabAuthority.receiveCommit(draftId, req.body); + await getCollabAuthority().receiveCommit(draftId, req.body); } catch (e) { if (e instanceof TooMuchContentionError) { console.log('TooMuchContentionError', e); @@ -62,17 +63,48 @@ router.get( return res.status(400).json({ error: 'Missing or invalid version query parameter' }); } - const commits = await collabAuthority.listenForCommit(draftId, version); + const commits = await getCollabAuthority().listenForCommit(draftId, version); return res.status(200).json(commits); }), ); +// get steps between two versions (non-blocking, for discussion fast-forwarding) +router.get( + '/api/pubs/:pubId/commits/steps', + wrap(async (req, res) => { + const draftId = await getDraftIdForPub(req.params.pubId); + + if (!draftId) { + return res.status(404).json({ error: 'Pub or draft not found' }); + } + + const from = parseInt(req.query.from as string, 10); + const to = parseInt(req.query.to as string, 10); + + if (Number.isNaN(from) || Number.isNaN(to)) { + return res.status(400).json({ error: 'Missing or invalid from/to' }); + } + + const commits = await CollabCommit.findAll({ + where: { + draftId, + version: { [Op.gt]: from, [Op.lte]: to }, + }, + order: [['version', 'ASC']], + }); + + return res.status(200).json( + commits.map((c: any) => ({ version: c.version, steps: c.steps })), + ); + }), +); + // update presence indicator router.post( '/api/pubs/:pubId/presence/:clientId', wrap(async (req, res) => { const indicator = req.body as PresenceIndicator; - await presenceAuthority.updatePresence(req.params.pubId, indicator); + await getPresenceAuthority().updatePresence(req.params.pubId, indicator); return res.status(204).send(null); }), ); @@ -86,7 +118,7 @@ router.post( clientId: string; }; - const presence = await presenceAuthority.listenForPresence( + const presence = await getPresenceAuthority().listenForPresence( req.params.pubId, clientId, refs, diff --git a/server/collab/authority.ts b/server/collab/authority.ts index db7b0025fa..16ccd33299 100644 --- a/server/collab/authority.ts +++ b/server/collab/authority.ts @@ -8,125 +8,134 @@ import { env } from 'server/env'; import { CollabCommit, Draft, DraftCheckpoint } from 'server/models'; import { sequelize } from 'server/sequelize'; -const broadcastManager = new RedisBroadcastManager({ - redisUrl: env.VALKEY_URL ?? 'redis://localhost:6379', -}); +let authority: CollabAuthority | null = null; + +const createAuthority = (bm: RedisBroadcastManager) => + new CollabAuthority({ + schema: editorSchema, + broadcastManager: bm, + + runWithTransaction: async (callback) => { + return sequelize.transaction((tr) => callback(tr)); + }, + + getDoc: async (tr, docId) => { + const draft = await Draft.findOne({ + where: { id: docId }, + ...(tr && { lock: tr.LOCK.UPDATE }), + transaction: tr ?? undefined, + }); + + if (!draft) { + throw new Error(`Draft not found: ${docId}`); + } + + const checkpoint = await DraftCheckpoint.findOne({ + where: { draftId: docId }, + transaction: tr ?? undefined, + }); + + if (!checkpoint) { + const emptyDoc = editorSchema.topNodeType.createAndFill()!; + + return { + docJSON: emptyDoc.toJSON(), + version: 0, + lastUpdatedTimestamp: Date.now(), + }; + } -export const connectCollabRedis = async () => { - await broadcastManager.connect(); - console.log('[collab] collab broadcast redis connected'); -}; - -export const collabAuthority = new CollabAuthority({ - schema: editorSchema, - broadcastManager, - - runWithTransaction: async (callback) => { - return sequelize.transaction((tr) => callback(tr)); - }, + return { + docJSON: checkpoint.doc, + version: draft.version, + lastUpdatedTimestamp: draft.latestKeyAt?.valueOf() ?? Date.now(), + }; + }, - // - getDoc: async (tr, docId) => { - const draft = await Draft.findOne({ - where: { id: docId }, - ...(tr && { lock: tr.LOCK.UPDATE }), - transaction: tr ?? undefined, - }); + saveDoc: async (tr, docId, docJSON, version) => { + await Draft.update( + { version, latestKeyAt: new Date() }, + { where: { id: docId }, transaction: tr }, + ); - if (!draft) { - throw new Error(`Draft not found: ${docId}`); - } + const existing = await DraftCheckpoint.findOne({ + where: { draftId: docId }, + transaction: tr, + }); + + if (existing) { + await existing.update( + { doc: docJSON, historyKey: version, timestamp: Date.now() }, + { transaction: tr }, + ); + } else { + await DraftCheckpoint.create( + { draftId: docId, doc: docJSON, historyKey: version, timestamp: Date.now() }, + { transaction: tr }, + ); + } + }, + + saveCommit: async (tr, docId, commitRef, commitVersion, commitSteps) => { + console.log('saveCommit', docId, commitRef, commitVersion, commitSteps); + await CollabCommit.create( + { + draftId: docId, + ref: commitRef, + version: commitVersion, + steps: commitSteps, + }, + { transaction: tr }, + ); + }, - const checkpoint = await DraftCheckpoint.findOne({ - where: { draftId: docId }, - transaction: tr ?? undefined, - }); + getCommit: async (tr, docId, commitRef) => { + const commit = await CollabCommit.findOne({ + where: { draftId: docId, ref: commitRef }, + transaction: tr ?? undefined, + }); - if (!checkpoint) { - const emptyDoc = editorSchema.topNodeType.createAndFill()!; + if (!commit) { + return null; + } return { - docJSON: emptyDoc.toJSON(), - version: 0, - lastUpdatedTimestamp: Date.now(), + ref: commit.ref, + version: commit.version, + steps: commit.steps, }; - } - - return { - docJSON: checkpoint.doc, - version: draft.version, - lastUpdatedTimestamp: draft.latestKeyAt?.valueOf() ?? Date.now(), - }; - }, - - saveDoc: async (tr, docId, docJSON, version) => { - await Draft.update( - { version, latestKeyAt: new Date() }, - { where: { id: docId }, transaction: tr }, - ); - - const existing = await DraftCheckpoint.findOne({ - where: { draftId: docId }, - transaction: tr, - }); - - if (existing) { - await existing.update( - { doc: docJSON, historyKey: version, timestamp: Date.now() }, - { transaction: tr }, - ); - } else { - await DraftCheckpoint.create( - { draftId: docId, doc: docJSON, historyKey: version, timestamp: Date.now() }, - { transaction: tr }, - ); - } - }, - - saveCommit: async (tr, docId, commitRef, commitVersion, commitSteps) => { - console.log('saveCommit', docId, commitRef, commitVersion, commitSteps); - await CollabCommit.create( - { - draftId: docId, - ref: commitRef, - version: commitVersion, - steps: commitSteps, - }, - { transaction: tr }, - ); - }, - - getCommit: async (tr, docId, commitRef) => { - const commit = await CollabCommit.findOne({ - where: { draftId: docId, ref: commitRef }, - transaction: tr ?? undefined, - }); - - if (!commit) { - return null; - } - - return { - ref: commit.ref, - version: commit.version, - steps: commit.steps, - }; - }, - - getCommits: async (tr, docId, version) => { - const commits = await CollabCommit.findAll({ - where: { - draftId: docId, - version: { [Op.gt]: version }, - }, - order: [['version', 'ASC']], - transaction: tr ?? undefined, - }); - - return commits.map((c) => ({ - ref: c.ref, - version: c.version, - steps: c.steps, - })); - }, -}); + }, + + getCommits: async (tr, docId, version) => { + const commits = await CollabCommit.findAll({ + where: { + draftId: docId, + version: { [Op.gt]: version }, + }, + order: [['version', 'ASC']], + transaction: tr ?? undefined, + }); + + return commits.map((c) => ({ + ref: c.ref, + version: c.version, + steps: c.steps, + })); + }, + }); + +export const getCollabAuthority = () => { + if (!authority) { + throw new Error('[collab] Collab Redis not connected. Call connectCollabRedis() first.'); + } + return authority; +}; + +export const connectCollabRedis = async () => { + const broadcastManager = new RedisBroadcastManager({ + redisUrl: env.VALKEY_URL ?? 'redis://localhost:6379', + }); + await broadcastManager.connect(); + authority = createAuthority(broadcastManager); + console.log('[collab] collab broadcast redis connected'); +}; diff --git a/server/collab/discussionPositions.ts b/server/collab/discussionPositions.ts index 58065f3dda..e1932cf022 100644 --- a/server/collab/discussionPositions.ts +++ b/server/collab/discussionPositions.ts @@ -1,10 +1,19 @@ import { Router } from 'express'; +import { Op } from 'sequelize'; -import { Draft, DraftCheckpoint, Pub } from 'server/models'; +import { Commenter, Discussion, DiscussionAnchor, Draft, DraftCheckpoint, Pub } from 'server/models'; import { wrap } from 'server/wrap'; +import { authorIncludes, baseVisibility, threadIncludes } from 'server/utils/queryHelpers/util'; export const router = Router(); +const isValidPositionEntry = (entry: any) => + entry && + typeof entry === 'object' && + typeof entry.currentKey === 'number' && + typeof entry.initKey === 'number' && + entry.selection; + // get current discussion positions for a pub's draft router.get( '/api/pubs/:pubId/discussions/positions', @@ -23,7 +32,53 @@ router.get( where: { draftId: pub.draft.id }, }); - return res.status(200).json(checkpoint?.discussions ?? {}); + if (!checkpoint?.discussions) { + return res.status(200).json({}); + } + + // filter out corrupted entries and auto-clean the checkpoint + const raw = checkpoint.discussions as Record; + const clean: Record = {}; + let needsCleanup = false; + for (const [id, entry] of Object.entries(raw)) { + if (isValidPositionEntry(entry)) { + clean[id] = entry; + } else { + needsCleanup = true; + } + } + + if (needsCleanup) { + await checkpoint.update({ discussions: clean }); + } + + return res.status(200).json(clean); + }), +); + +// fetch full discussion data (all or by IDs) for collab sync +router.get( + '/api/pubs/:pubId/discussions', + wrap(async (req, res) => { + const ids = (req.query.ids as string)?.split(',').filter(Boolean); + + const where: any = { pubId: req.params.pubId }; + if (ids && ids.length > 0) { + where.id = { [Op.in]: ids }; + } + + const discussions = await Discussion.findAll({ + where, + include: [ + ...authorIncludes(), + { model: DiscussionAnchor, as: 'anchors' }, + ...baseVisibility, + ...threadIncludes(), + { model: Commenter, as: 'commenter' }, + ], + }); + + return res.status(200).json(discussions); }), ); @@ -52,9 +107,14 @@ router.post( }); if (checkpoint) { - // merge incoming discussion positions with existing ones const existing = checkpoint.discussions ?? {}; - const merged = { ...existing, ...discussions }; + const validated: Record = {}; + for (const [id, entry] of Object.entries(discussions)) { + if (isValidPositionEntry(entry)) { + validated[id] = entry; + } + } + const merged = { ...existing, ...validated }; await checkpoint.update({ discussions: merged }); } diff --git a/server/collab/presence.ts b/server/collab/presence.ts index 2fae57e4d2..940b8b3ff0 100644 --- a/server/collab/presence.ts +++ b/server/collab/presence.ts @@ -6,17 +6,23 @@ import { import { env } from 'server/env'; -const redisUrl = env.VALKEY_URL ?? 'redis://localhost:6379'; +let presenceAuthority: PresenceAuthority | null = null; -const presenceBroadcaster = new RedisPresenceBroadcastManager({ redisUrl }); -const presencePersister = new RedisPresencePersistenceManager({ redisUrl }); - -export const presenceAuthority = new PresenceAuthority({ - persistenceManager: presencePersister, - broadcastManager: presenceBroadcaster, -}); +export const getPresenceAuthority = () => { + if (!presenceAuthority) { + throw new Error('[collab] Presence Redis not connected. Call connectPresenceRedis() first.'); + } + return presenceAuthority; +}; export const connectPresenceRedis = async () => { - await Promise.all([presenceBroadcaster.connect(), presencePersister.connect()]); + const redisUrl = env.VALKEY_URL ?? 'redis://localhost:6379'; + const broadcaster = new RedisPresenceBroadcastManager({ redisUrl }); + const persister = new RedisPresencePersistenceManager({ redisUrl }); + await Promise.all([broadcaster.connect(), persister.connect()]); + presenceAuthority = new PresenceAuthority({ + persistenceManager: persister, + broadcastManager: broadcaster, + }); console.log('[collab] presence redis connected'); }; diff --git a/server/utils/firebaseAdmin.ts b/server/utils/firebaseAdmin.ts index 6c8f1d72df..3f4abd636c 100644 --- a/server/utils/firebaseAdmin.ts +++ b/server/utils/firebaseAdmin.ts @@ -195,7 +195,7 @@ export const editDraft = async (pubId: string, clientId: string, schema: Schema return true; } - const { collabAuthority } = await import('server/collab/authority.js'); + const { getCollabAuthority } = await import('server/collab/authority.js'); try { const commitData = { @@ -205,7 +205,7 @@ export const editDraft = async (pubId: string, clientId: string, schema: Schema ref: `server-${Date.now()}-${Math.random().toString(36).slice(2)}`, }; - await collabAuthority.receiveCommit(draft.id, commitData); + await getCollabAuthority().receiveCommit(draft.id, commitData); currentVersion++; pendingSteps = []; return true; From 37b1ff1bb6793d46170dfb646ea4248ed9a510ab Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Mon, 22 Jun 2026 18:02:59 +0200 Subject: [PATCH 05/17] fix: dont think you are smarter than the robot --- .../Editor/plugins/discussions/polling.ts | 4 + server/collab/api.ts | 16 +- server/collab/authority.ts | 172 +++++++++++++----- server/collab/presence.ts | 6 +- server/draft/model.ts | 14 +- server/draftCheckpoint/queries.ts | 1 + server/utils/firebaseAdmin.ts | 32 +--- 7 files changed, 157 insertions(+), 88 deletions(-) diff --git a/client/components/Editor/plugins/discussions/polling.ts b/client/components/Editor/plugins/discussions/polling.ts index 8abbb2cc6d..da34e44ca6 100644 --- a/client/components/Editor/plugins/discussions/polling.ts +++ b/client/components/Editor/plugins/discussions/polling.ts @@ -51,6 +51,10 @@ export const connectToRemoteDiscussions = (pubId: string): RemoteDiscussions => }; const sendDiscussions = (discussions: Discussions) => { + if (Object.keys(discussions).length === 0) { + return; + } + fetch(`/api/pubs/${pubId}/discussions/positions`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, diff --git a/server/collab/api.ts b/server/collab/api.ts index f04a0df6bb..aab7630f39 100644 --- a/server/collab/api.ts +++ b/server/collab/api.ts @@ -26,7 +26,6 @@ const getDraftIdForPub = async (pubId: string): Promise => { router.post( '/api/pubs/:pubId/commits', wrap(async (req, res) => { - console.log('receiveCommit', req.body); const draftId = await getDraftIdForPub(req.params.pubId); if (!draftId) { @@ -34,7 +33,7 @@ router.post( } try { - await getCollabAuthority().receiveCommit(draftId, req.body); + await (await getCollabAuthority()).receiveCommit(draftId, req.body); } catch (e) { if (e instanceof TooMuchContentionError) { console.log('TooMuchContentionError', e); @@ -63,7 +62,7 @@ router.get( return res.status(400).json({ error: 'Missing or invalid version query parameter' }); } - const commits = await getCollabAuthority().listenForCommit(draftId, version); + const commits = await (await getCollabAuthority()).listenForCommit(draftId, version); return res.status(200).json(commits); }), ); @@ -93,9 +92,9 @@ router.get( order: [['version', 'ASC']], }); - return res.status(200).json( - commits.map((c: any) => ({ version: c.version, steps: c.steps })), - ); + return res + .status(200) + .json(commits.map((c: any) => ({ version: c.version, steps: c.steps }))); }), ); @@ -104,7 +103,8 @@ router.post( '/api/pubs/:pubId/presence/:clientId', wrap(async (req, res) => { const indicator = req.body as PresenceIndicator; - await getPresenceAuthority().updatePresence(req.params.pubId, indicator); + await (await getPresenceAuthority()).updatePresence(req.params.pubId, indicator); + return res.status(204).send(null); }), ); @@ -118,7 +118,7 @@ router.post( clientId: string; }; - const presence = await getPresenceAuthority().listenForPresence( + const presence = await (await getPresenceAuthority()).listenForPresence( req.params.pubId, clientId, refs, diff --git a/server/collab/authority.ts b/server/collab/authority.ts index 16ccd33299..56564bab8f 100644 --- a/server/collab/authority.ts +++ b/server/collab/authority.ts @@ -1,15 +1,42 @@ import type { Transaction } from 'sequelize'; +import type { DocJson } from 'types'; import { CollabAuthority, RedisBroadcastManager } from '@pitter-patter/collab-server'; +import { Node } from 'prosemirror-model'; +import { Step } from 'prosemirror-transform'; import { Op } from 'sequelize'; import { editorSchema } from 'client/components/Editor/utils/schema'; +import { upsertDraftCheckpoint } from 'server/draftCheckpoint/queries'; import { env } from 'server/env'; import { CollabCommit, Draft, DraftCheckpoint } from 'server/models'; import { sequelize } from 'server/sequelize'; +import { createLogger } from 'server/utils/queryHelpers/communityGet'; let authority: CollabAuthority | null = null; +const CHECKPOINT_INTERVAL = 50; + +export const replayCommitsOntoDoc = ( + docJSON: Record, + commits: { steps: Record[] }[], +): Node => { + let doc = Node.fromJSON(editorSchema, docJSON); + + for (const commit of commits) { + for (const stepJSON of commit.steps) { + const step = Step.fromJSON(editorSchema, stepJSON); + const result = step.apply(doc); + + if (result.doc) { + doc = result.doc; + } + } + } + + return doc; +}; + const createAuthority = (bm: RedisBroadcastManager) => new CollabAuthority({ schema: editorSchema, @@ -20,21 +47,28 @@ const createAuthority = (bm: RedisBroadcastManager) => }, getDoc: async (tr, docId) => { - const draft = await Draft.findOne({ - where: { id: docId }, - ...(tr && { lock: tr.LOCK.UPDATE }), - transaction: tr ?? undefined, - }); + const logger = createLogger('getDoc'); + + const [draft, checkpoint] = await logger.log( + 'getDocAndCheckpoint', + Promise.all([ + Draft.findOne({ + where: { id: docId }, + ...(tr && { lock: tr.LOCK.NO_KEY_UPDATE }), + transaction: tr ?? undefined, + }), + DraftCheckpoint.findOne({ + where: { draftId: docId }, + order: [['historyKey', 'DESC']], + transaction: tr ?? undefined, + }), + ]), + ); if (!draft) { throw new Error(`Draft not found: ${docId}`); } - const checkpoint = await DraftCheckpoint.findOne({ - where: { draftId: docId }, - transaction: tr ?? undefined, - }); - if (!checkpoint) { const emptyDoc = editorSchema.topNodeType.createAndFill()!; @@ -45,6 +79,33 @@ const createAuthority = (bm: RedisBroadcastManager) => }; } + const checkpointVersion = checkpoint.historyKey ?? 0; + + if (checkpointVersion < draft.version) { + const missedCommits = await logger.log( + 'getMissedCommits', + CollabCommit.findAll({ + where: { + draftId: docId, + version: { [Op.gt]: checkpointVersion, [Op.lte]: draft.version }, + }, + order: [['version', 'ASC']], + transaction: tr ?? undefined, + }), + ); + + const reconstructedDoc = replayCommitsOntoDoc(checkpoint.doc, missedCommits); + logger.end(); + + return { + docJSON: reconstructedDoc.toJSON(), + version: draft.version, + lastUpdatedTimestamp: draft.latestKeyAt?.valueOf() ?? Date.now(), + }; + } + + logger.end(); + return { docJSON: checkpoint.doc, version: draft.version, @@ -53,46 +114,59 @@ const createAuthority = (bm: RedisBroadcastManager) => }, saveDoc: async (tr, docId, docJSON, version) => { - await Draft.update( - { version, latestKeyAt: new Date() }, - { where: { id: docId }, transaction: tr }, - ); + try { + await Draft.update( + { version, latestKeyAt: new Date() }, + { where: { id: docId }, transaction: tr }, + ); - const existing = await DraftCheckpoint.findOne({ - where: { draftId: docId }, - transaction: tr, - }); + const shouldCheckpoint = version % CHECKPOINT_INTERVAL === 0 || version <= 1; - if (existing) { - await existing.update( - { doc: docJSON, historyKey: version, timestamp: Date.now() }, - { transaction: tr }, - ); - } else { - await DraftCheckpoint.create( - { draftId: docId, doc: docJSON, historyKey: version, timestamp: Date.now() }, - { transaction: tr }, - ); + if (!shouldCheckpoint) { + return; + } + + const truncateBelow = version - CHECKPOINT_INTERVAL; + + await upsertDraftCheckpoint(docId, version, docJSON as DocJson, Date.now(), tr); + + if (truncateBelow > 0) { + await CollabCommit.destroy({ + where: { + draftId: docId, + version: { [Op.lt]: truncateBelow }, + }, + transaction: tr, + }); + } + } catch (error) { + console.error('Error saving doc', error); + throw error; } }, saveCommit: async (tr, docId, commitRef, commitVersion, commitSteps) => { - console.log('saveCommit', docId, commitRef, commitVersion, commitSteps); - await CollabCommit.create( - { - draftId: docId, - ref: commitRef, - version: commitVersion, - steps: commitSteps, - }, - { transaction: tr }, - ); + try { + await CollabCommit.create( + { + draftId: docId, + ref: commitRef, + version: commitVersion, + steps: commitSteps, + }, + { transaction: tr }, + ); + } catch (error) { + console.error('Error saving commit', error); + throw error; + } }, getCommit: async (tr, docId, commitRef) => { const commit = await CollabCommit.findOne({ where: { draftId: docId, ref: commitRef }, transaction: tr ?? undefined, + plain: true, }); if (!commit) { @@ -107,14 +181,15 @@ const createAuthority = (bm: RedisBroadcastManager) => }, getCommits: async (tr, docId, version) => { - const commits = await CollabCommit.findAll({ - where: { - draftId: docId, - version: { [Op.gt]: version }, - }, - order: [['version', 'ASC']], - transaction: tr ?? undefined, - }); + const commits = + (await CollabCommit.findAll({ + where: { + draftId: docId, + version: { [Op.gt]: version }, + }, + order: [['version', 'ASC']], + transaction: tr ?? undefined, + })) ?? []; return commits.map((c) => ({ ref: c.ref, @@ -124,9 +199,9 @@ const createAuthority = (bm: RedisBroadcastManager) => }, }); -export const getCollabAuthority = () => { +export const getCollabAuthority = async () => { if (!authority) { - throw new Error('[collab] Collab Redis not connected. Call connectCollabRedis() first.'); + return await connectCollabRedis(); } return authority; }; @@ -138,4 +213,5 @@ export const connectCollabRedis = async () => { await broadcastManager.connect(); authority = createAuthority(broadcastManager); console.log('[collab] collab broadcast redis connected'); + return authority; }; diff --git a/server/collab/presence.ts b/server/collab/presence.ts index 940b8b3ff0..87c9db1e7b 100644 --- a/server/collab/presence.ts +++ b/server/collab/presence.ts @@ -8,9 +8,10 @@ import { env } from 'server/env'; let presenceAuthority: PresenceAuthority | null = null; -export const getPresenceAuthority = () => { +export const getPresenceAuthority = async () => { if (!presenceAuthority) { - throw new Error('[collab] Presence Redis not connected. Call connectPresenceRedis() first.'); + // throw new Error('[collab] Presence Redis not connected. Call connectPresenceRedis() first.'); + return await connectPresenceRedis(); } return presenceAuthority; }; @@ -25,4 +26,5 @@ export const connectPresenceRedis = async () => { broadcastManager: broadcaster, }); console.log('[collab] presence redis connected'); + return presenceAuthority; }; diff --git a/server/draft/model.ts b/server/draft/model.ts index d389b6d5f9..a57498f120 100644 --- a/server/draft/model.ts +++ b/server/draft/model.ts @@ -2,14 +2,9 @@ import type { CreationOptional, InferAttributes, InferCreationAttributes } from import type { SerializedModel } from 'types'; -import { - Column, - DataType, - Default, - Model, - PrimaryKey, - Table, -} from 'sequelize-typescript'; +import { Column, DataType, Default, HasOne, Model, PrimaryKey, Table } from 'sequelize-typescript'; + +import { DraftCheckpoint } from 'server/draftCheckpoint/model'; @Table export class Draft extends Model, InferCreationAttributes> { @@ -30,4 +25,7 @@ export class Draft extends Model, InferCreationAttributes @Default(0) @Column(DataType.INTEGER) declare version: CreationOptional; + + @HasOne(() => DraftCheckpoint, { as: 'checkpoint', foreignKey: 'draftId' }) + declare checkpoint?: DraftCheckpoint; } diff --git a/server/draftCheckpoint/queries.ts b/server/draftCheckpoint/queries.ts index 989e29c1ca..494c33ac7b 100644 --- a/server/draftCheckpoint/queries.ts +++ b/server/draftCheckpoint/queries.ts @@ -56,6 +56,7 @@ export const upsertDraftCheckpoint = async ( export const getDraftCheckpoint = async (draftId: string, sequelizeTransaction: any = null) => { return DraftCheckpoint.findOne({ where: { draftId }, + order: [['historyKey', 'DESC']], transaction: sequelizeTransaction, }); }; diff --git a/server/utils/firebaseAdmin.ts b/server/utils/firebaseAdmin.ts index 3f4abd636c..3e773abbf2 100644 --- a/server/utils/firebaseAdmin.ts +++ b/server/utils/firebaseAdmin.ts @@ -7,6 +7,7 @@ import { Step, Transform } from 'prosemirror-transform'; import { Op } from 'sequelize'; import { editorSchema } from 'components/Editor/utils'; +import { replayCommitsOntoDoc } from 'server/collab/authority'; import { getDraftCheckpoint } from 'server/draftCheckpoint/queries'; import { CollabCommit, Draft, Pub } from 'server/models'; import { expect } from 'utils/assert'; @@ -56,8 +57,6 @@ const applyCommitsOnDoc = async ( order: [['version', 'ASC']], }); - const allStepsJson = commits.flatMap((commit) => commit.steps); - const currentKey = commits.length > 0 ? commits[commits.length - 1].version : checkpointKey; const currentTimestamp = @@ -65,18 +64,7 @@ const applyCommitsOnDoc = async ( ? (commits[commits.length - 1].createdAt?.valueOf() ?? checkpointTimestamp) : checkpointTimestamp; - let doc = Node.fromJSON(editorSchema, checkpointDoc); - - for (const stepJson of allStepsJson) { - const step = Step.fromJSON(editorSchema, stepJson); - const { failed, doc: nextDoc } = step.apply(doc); - - if (failed) { - console.error(`Failed with: ${failed}`); - } else if (nextDoc) { - doc = nextDoc; - } - } + const doc = replayCommitsOntoDoc(checkpointDoc, commits); return { doc, @@ -198,14 +186,14 @@ export const editDraft = async (pubId: string, clientId: string, schema: Schema const { getCollabAuthority } = await import('server/collab/authority.js'); try { - const commitData = { - steps: pendingSteps.map((s) => s.toJSON()), - version: currentVersion, - clientId, - ref: `server-${Date.now()}-${Math.random().toString(36).slice(2)}`, - }; - - await getCollabAuthority().receiveCommit(draft.id, commitData); + const commitData = { + steps: pendingSteps.map((s) => s.toJSON()), + version: currentVersion, + clientId, + ref: `server-${Date.now()}-${Math.random().toString(36).slice(2)}`, + }; + + await (await getCollabAuthority()).receiveCommit(draft.id, commitData); currentVersion++; pendingSteps = []; return true; From 322474fd69ae985f61938039bfe597cfcb20a595 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Mon, 22 Jun 2026 18:19:25 +0200 Subject: [PATCH 06/17] fix: removee firebse thingies --- server/collab/authority.ts | 26 +- server/collab/replay.ts | 24 + tools/cleanupCollab.ts | 239 +++++++ tools/cleanupFirebase.ts | 1393 ------------------------------------ tools/coldStorage.ts | 620 ---------------- tools/cron.ts | 6 +- tools/index.js | 3 +- 7 files changed, 269 insertions(+), 2042 deletions(-) create mode 100644 server/collab/replay.ts create mode 100644 tools/cleanupCollab.ts delete mode 100644 tools/cleanupFirebase.ts delete mode 100644 tools/coldStorage.ts diff --git a/server/collab/authority.ts b/server/collab/authority.ts index 56564bab8f..a94b928c02 100644 --- a/server/collab/authority.ts +++ b/server/collab/authority.ts @@ -2,8 +2,6 @@ import type { Transaction } from 'sequelize'; import type { DocJson } from 'types'; import { CollabAuthority, RedisBroadcastManager } from '@pitter-patter/collab-server'; -import { Node } from 'prosemirror-model'; -import { Step } from 'prosemirror-transform'; import { Op } from 'sequelize'; import { editorSchema } from 'client/components/Editor/utils/schema'; @@ -13,29 +11,13 @@ import { CollabCommit, Draft, DraftCheckpoint } from 'server/models'; import { sequelize } from 'server/sequelize'; import { createLogger } from 'server/utils/queryHelpers/communityGet'; -let authority: CollabAuthority | null = null; - -const CHECKPOINT_INTERVAL = 50; +import { replayCommitsOntoDoc } from './replay'; -export const replayCommitsOntoDoc = ( - docJSON: Record, - commits: { steps: Record[] }[], -): Node => { - let doc = Node.fromJSON(editorSchema, docJSON); +export { replayCommitsOntoDoc }; - for (const commit of commits) { - for (const stepJSON of commit.steps) { - const step = Step.fromJSON(editorSchema, stepJSON); - const result = step.apply(doc); - - if (result.doc) { - doc = result.doc; - } - } - } +let authority: CollabAuthority | null = null; - return doc; -}; +const CHECKPOINT_INTERVAL = 50; const createAuthority = (bm: RedisBroadcastManager) => new CollabAuthority({ diff --git a/server/collab/replay.ts b/server/collab/replay.ts new file mode 100644 index 0000000000..8039d2f90d --- /dev/null +++ b/server/collab/replay.ts @@ -0,0 +1,24 @@ +import { Node } from 'prosemirror-model'; +import { Step } from 'prosemirror-transform'; + +import { editorSchema } from 'client/components/Editor/utils/schema'; + +export const replayCommitsOntoDoc = ( + docJSON: Record, + commits: { steps: Record[] }[], +): Node => { + let doc = Node.fromJSON(editorSchema, docJSON); + + for (const commit of commits) { + for (const stepJSON of commit.steps) { + const step = Step.fromJSON(editorSchema, stepJSON); + const result = step.apply(doc); + + if (result.doc) { + doc = result.doc; + } + } + } + + return doc; +}; diff --git a/tools/cleanupCollab.ts b/tools/cleanupCollab.ts new file mode 100644 index 0000000000..6d5c86db04 --- /dev/null +++ b/tools/cleanupCollab.ts @@ -0,0 +1,239 @@ +/** + * Collab Cleanup Tool + * + * Maintains the CollabCommit and DraftCheckpoint tables: + * 1. Checkpoints stale drafts (not edited within threshold) and wipes their commits + * 2. Truncates old commits for active drafts as a safety net + * 3. Deletes orphaned drafts (no associated Pub) + * + * Usage: + * pnpm run tools cleanupCollab # dry run, all drafts + * pnpm run tools cleanupCollab --execute # actually delete data + * pnpm run tools cleanupCollab --pubId= # test on single pub + * pnpm run tools cleanupCollab --daysOld=60 # custom staleness threshold + */ + +import { Op, QueryTypes } from 'sequelize'; + +import { replayCommitsOntoDoc } from 'server/collab/replay'; +import { upsertDraftCheckpoint } from 'server/draftCheckpoint/queries'; +import { CollabCommit, Draft, DraftCheckpoint } from 'server/models'; +import { sequelize } from 'server/sequelize'; + +const { + argv: { execute, pubId: specificPubId, daysOld: daysOldArg = 30, verbose: verboseFlag }, +} = require('yargs'); + +const isDryRun = !execute; +const DAYS_OLD = Number(daysOldArg); +const CHECKPOINT_BUFFER = 50; + +// biome-ignore lint/suspicious/noConsole: CLI tool output +const log = (msg: string) => console.log(`[collab-cleanup] ${new Date().toISOString()} ${msg}`); +const verbose = (msg: string) => verboseFlag && log(msg); + +interface Stats { + staleDraftsCheckpointed: number; + staleCommitsDeleted: number; + activeCommitsTruncated: number; + orphanedDraftsDeleted: number; + errorsEncountered: number; +} + +const stats: Stats = { + staleDraftsCheckpointed: 0, + staleCommitsDeleted: 0, + activeCommitsTruncated: 0, + orphanedDraftsDeleted: 0, + errorsEncountered: 0, +}; + +/** + * For stale drafts whose checkpoint is behind Draft.version, replay outstanding + * commits onto the checkpoint doc to bring it up to date, then delete all commits. + */ +const checkpointStaleDrafts = async () => { + const cutoff = new Date(); + cutoff.setDate(cutoff.getDate() - DAYS_OLD); + + log(`Checkpointing stale drafts (not edited since ${cutoff.toISOString()})...`); + + type StaleDraft = { draftId: string; version: number; checkpointKey: number | null }; + + const where = specificPubId + ? `AND p.id = :pubId` + : `AND (d."latestKeyAt" IS NULL OR d."latestKeyAt" < :cutoff)`; + + const staleDrafts = await sequelize.query( + ` + SELECT d.id AS "draftId", d.version, dc."historyKey" AS "checkpointKey" + FROM "Drafts" d + INNER JOIN "Pubs" p ON p."draftId" = d.id + LEFT JOIN "DraftCheckpoints" dc ON dc."draftId" = d.id + WHERE d.version > COALESCE(dc."historyKey", 0) + ${where} + ORDER BY d."latestKeyAt" ASC NULLS FIRST + `, + { + replacements: { cutoff: cutoff.toISOString(), pubId: specificPubId }, + type: QueryTypes.SELECT, + }, + ); + + log(` Found ${staleDrafts.length} stale drafts needing checkpoint`); + + for (const { draftId, version, checkpointKey } of staleDrafts) { + try { + const fromVersion = checkpointKey ?? 0; + + // biome-ignore lint/performance/noAwaitInLoops: sequential to avoid overwhelming the DB + const [checkpoint, commits] = await Promise.all([ + DraftCheckpoint.findOne({ where: { draftId } }), + CollabCommit.findAll({ + where: { draftId, version: { [Op.gt]: fromVersion, [Op.lte]: version } }, + order: [['version', 'ASC']], + }), + ]); + + if (commits.length === 0) { + verbose(` [${draftId.slice(0, 8)}] no commits to replay, skipping`); + continue; + } + + const baseDoc = checkpoint?.doc ?? { type: 'doc', content: [{ type: 'paragraph' }] }; + const reconstructed = replayCommitsOntoDoc(baseDoc, commits); + + verbose( + ` [${draftId.slice(0, 8)}] replaying ${commits.length} commits (${fromVersion} -> ${version})`, + ); + + if (!isDryRun) { + await sequelize.transaction(async (tr) => { + await upsertDraftCheckpoint(draftId, version, reconstructed.toJSON() as any, Date.now(), tr); + + await CollabCommit.destroy({ + where: { draftId }, + transaction: tr, + }); + }); + } + + stats.staleDraftsCheckpointed++; + stats.staleCommitsDeleted += commits.length; + } catch (err) { + log(` Error checkpointing draft ${draftId}: ${(err as Error).message}`); + stats.errorsEncountered++; + } + } +}; + +/** + * For active drafts, delete commits well below the checkpoint version. + * This catches any leftovers the real-time truncation in authority.ts missed. + */ +const truncateActiveCommits = async () => { + log('Truncating old commits for active drafts...'); + + // single bulk query: delete commits where version < (checkpoint.historyKey - buffer) + const [results] = await sequelize.query<{ deleted: string }>( + isDryRun + ? ` + SELECT COUNT(*) AS deleted + FROM "CollabCommits" cc + INNER JOIN "DraftCheckpoints" dc ON dc."draftId" = cc."draftId" + WHERE cc.version < dc."historyKey" - :buffer + AND dc."historyKey" > :buffer + ` + : ` + DELETE FROM "CollabCommits" cc + USING "DraftCheckpoints" dc + WHERE dc."draftId" = cc."draftId" + AND cc.version < dc."historyKey" - :buffer + AND dc."historyKey" > :buffer + RETURNING cc.id + `, + { + replacements: { buffer: CHECKPOINT_BUFFER }, + type: QueryTypes.SELECT, + }, + ); + + const totalDeleted = isDryRun + ? parseInt((results as any)?.deleted ?? '0', 10) + : (results as any[]).length; + + stats.activeCommitsTruncated = totalDeleted; + log(` ${isDryRun ? 'Would truncate' : 'Truncated'} ${totalDeleted} old commits`); +}; + +/** + * Find drafts with no associated Pub and delete them (cascades to commits and checkpoints). + */ +const deleteOrphanedDrafts = async () => { + log('Looking for orphaned drafts...'); + + const orphaned = await sequelize.query<{ id: string }>( + ` + SELECT d.id + FROM "Drafts" d + LEFT JOIN "Pubs" p ON p."draftId" = d.id + WHERE p.id IS NULL + `, + { type: QueryTypes.SELECT }, + ); + + log(` Found ${orphaned.length} orphaned drafts`); + + if (orphaned.length === 0) { + return; + } + + if (!isDryRun) { + const ids = orphaned.map((d) => d.id); + + await CollabCommit.destroy({ where: { draftId: { [Op.in]: ids } } }); + await DraftCheckpoint.destroy({ where: { draftId: { [Op.in]: ids } } }); + await Draft.destroy({ where: { id: { [Op.in]: ids } } }); + } + + stats.orphanedDraftsDeleted = orphaned.length; +}; + +const printSummary = () => { + log('=== Cleanup Summary ==='); + log(`Mode: ${isDryRun ? 'DRY RUN (no data changed)' : 'EXECUTE'}`); + log(`Stale drafts checkpointed: ${stats.staleDraftsCheckpointed}`); + log(`Stale commits deleted: ${stats.staleCommitsDeleted}`); + log(`Active commits truncated: ${stats.activeCommitsTruncated}`); + log(`Orphaned drafts deleted: ${stats.orphanedDraftsDeleted}`); + log(`Errors encountered: ${stats.errorsEncountered}`); +}; + +const main = async () => { + log('Collab Cleanup Tool'); + log(`Mode: ${isDryRun ? 'DRY RUN' : 'EXECUTE'}`); + log(`Staleness threshold: ${DAYS_OLD} days`); + + if (specificPubId) { + log(`Target: single pub ${specificPubId}`); + } + + log(''); + + try { + await checkpointStaleDrafts(); + await truncateActiveCommits(); + + if (!specificPubId) { + await deleteOrphanedDrafts(); + } + + printSummary(); + } catch (err) { + log(`Fatal error: ${(err as Error).message}`); + console.error(err); + process.exit(1); + } +}; + +main().finally(() => process.exit(0)); diff --git a/tools/cleanupFirebase.ts b/tools/cleanupFirebase.ts deleted file mode 100644 index ff593008ee..0000000000 --- a/tools/cleanupFirebase.ts +++ /dev/null @@ -1,1393 +0,0 @@ -/** - * Firebase Cleanup Tool - * - * Reduces Firebase storage costs by: - * 1. Fast-forwarding outdated discussions to the safe prune threshold - * 2. Pruning changes/merges/checkpoints before the safe prune threshold - * 3. Removing orphaned drafts (drafts without an associated Pub) - * 4. Removing orphaned Firebase paths (paths not referenced in Postgres) - * 5. Removing v5→v6 migration stubs (drafts containing only {"lastMergeKey":-1}) - * - * IMPORTANT: The safe prune threshold is min(latestCheckpointKey, latestReleaseKey). - * - We must preserve changes from latestCheckpointKey onwards to reconstruct the doc - * - We must preserve changes from latestReleaseKey onwards for discussion migration during release - * - * Usage: - * pnpm run tools cleanupFirebase # Dry run, all drafts - * pnpm run tools cleanupFirebase --execute # Actually delete data - * pnpm run tools cleanupFirebase --pubId= # Test on single pub - * pnpm run tools cleanupFirebase --pubId= --execute - */ - -import type { DiscussionInfo } from 'components/Editor/plugins/discussions/types'; - -import firebaseAdmin from 'firebase-admin'; -import { - compressSelectionJSON, - compressStateJSON, - uncompressSelectionJSON, -} from 'prosemirror-compress-pubpub'; -import { QueryTypes } from 'sequelize'; - -import { editorSchema, getFirebaseDoc, getStepsInChangeRange } from 'components/Editor'; -import { createFastForwarder } from 'components/Editor/plugins/discussions/fastForward'; -import { getDraftCheckpoint } from 'server/draftCheckpoint/queries'; -import { Doc, Draft, Pub, Release } from 'server/models'; -import { sequelize } from 'server/sequelize'; -import { getDatabaseRef } from 'server/utils/firebaseAdmin'; -import { postToSlack } from 'server/utils/slack'; -import { getFirebaseConfig } from 'utils/editor/firebaseConfig'; - -const { - argv: { execute, pubId, draftId, batchSize = 100, verbose: verboseFlag, scanFirebase = true }, -} = require('yargs'); - -const isDryRun = !execute; -const isVerbose = verboseFlag; - -// biome-ignore lint/suspicious/noConsole: CLI tool output -const log = (msg: string) => console.log(`[cleanup] ${new Date().toISOString()} ${msg}`); -const verbose = (msg: string) => isVerbose && log(msg); - -/** - * Fetch only the keys at a Firebase path using REST API with shallow=true - * This avoids loading potentially huge amounts of data - */ -let cachedAccessToken: { token: string; expiresAt: number } | null = null; - -const getAccessToken = async (): Promise => { - const now = Date.now(); - if (cachedAccessToken && cachedAccessToken.expiresAt > now + 60000) { - return cachedAccessToken.token; - } - - const credential = firebaseAdmin.credential.cert( - JSON.parse( - Buffer.from(process.env.FIREBASE_SERVICE_ACCOUNT_BASE64 as string, 'base64').toString(), - ), - ); - const tokenResult = await credential.getAccessToken(); - cachedAccessToken = { - token: tokenResult.access_token, - expiresAt: now + (tokenResult.expires_in ?? 3600) * 1000, - }; - return cachedAccessToken.token; -}; - -const getShallowKeys = async (path: string, retries = 3): Promise => { - const databaseURL = getFirebaseConfig().databaseURL; - const accessToken = await getAccessToken(); - - const url = `${databaseURL}/${path}.json?shallow=true&access_token=${accessToken}`; - - for (let attempt = 1; attempt <= retries; attempt++) { - try { - // biome-ignore lint/performance/noAwaitInLoops: intentionally sequential - const response = await fetch(url); - - if (!response.ok) { - throw new Error( - `Firebase REST API error: ${response.status} ${response.statusText}`, - ); - } - - const data = await response.json(); - if (!data || typeof data !== 'object') { - return []; - } - return Object.keys(data); - } catch (error: any) { - if (attempt === retries) { - throw error; - } - const delay = Math.min(1000 * 2 ** attempt, 10000); - verbose(` Retry ${attempt}/${retries} for ${path} after ${delay}ms: ${error.message}`); - await new Promise((r) => setTimeout(r, delay)); - } - } - return []; // unreachable -}; - -/** - * Recursively delete a Firebase path, handling WRITE_TOO_BIG errors by - * deleting children first. This is necessary for large subtrees. - */ -const deleteFirebasePath = async (path: string, depth = 0): Promise => { - const indent = ' '.repeat(depth); - try { - await getDatabaseRef(path).remove(); - verbose(`${indent}Deleted: ${path}`); - } catch (error: any) { - const errorCode = error?.code || error?.message || String(error); - if (errorCode.includes('WRITE_TOO_BIG') || errorCode.includes('write_too_big')) { - verbose(`${indent}Path too large for single delete, deleting children first: ${path}`); - // Get children and delete them with concurrency - const childKeys = await getShallowKeys(path); - await runWithConcurrency( - childKeys.map( - (childKey) => () => deleteFirebasePath(`${path}/${childKey}`, depth + 1), - ), - 30, - ); - // Now delete the (empty) parent - await getDatabaseRef(path).remove(); - verbose(`${indent}Deleted (after children): ${path}`); - } else { - throw error; - } - } -}; - -/** - * Run async tasks with limited concurrency (worker pool pattern) - */ -const runWithConcurrency = async ( - tasks: (() => Promise)[], - concurrency: number, -): Promise => { - const results: T[] = []; - let index = 0; - - const worker = async (): Promise => { - while (index < tasks.length) { - const currentIndex = index++; - // biome-ignore lint/performance/noAwaitInLoops: worker pool pattern requires sequential processing - results[currentIndex] = await tasks[currentIndex](); - } - }; - - const workers = Array.from({ length: Math.min(concurrency, tasks.length) }, () => worker()); - await Promise.all(workers); - return results; -}; - -interface CleanupStats { - draftsProcessed: number; - draftsSkipped: number; - draftsRepairedFromRelease: number; - orphanedDraftsFound: number; - orphanedDraftsDeleted: number; - orphanedFirebasePathsFound: number; - orphanedFirebasePathsDeleted: number; - migrationStubsFound: number; - migrationStubsDeleted: number; - discussionsUpdated: number; - changesDeleted: number; - mergesDeleted: number; - checkpointsDeleted: number; - checkpointsCreated: number; - metadataDeleted: number; - errorsEncountered: number; -} - -const stats: CleanupStats = { - draftsProcessed: 0, - draftsSkipped: 0, - draftsRepairedFromRelease: 0, - orphanedDraftsFound: 0, - orphanedDraftsDeleted: 0, - orphanedFirebasePathsFound: 0, - orphanedFirebasePathsDeleted: 0, - migrationStubsFound: 0, - migrationStubsDeleted: 0, - discussionsUpdated: 0, - changesDeleted: 0, - mergesDeleted: 0, - checkpointsDeleted: 0, - checkpointsCreated: 0, - metadataDeleted: 0, - errorsEncountered: 0, -}; - -/** - * Get all checkpoint keys for a draft - */ -const getCheckpointKeys = async (draftRef: any): Promise => { - const checkpointMapSnapshot = await draftRef.child('checkpointMap').once('value'); - const checkpointMap = checkpointMapSnapshot.val(); - - if (!checkpointMap) { - // Try the deprecated singular checkpoint - const checkpointSnapshot = await draftRef.child('checkpoint').once('value'); - const checkpoint = checkpointSnapshot.val(); - if (checkpoint?.k) { - return [parseInt(checkpoint.k, 10)]; - } - return []; - } - - return Object.keys(checkpointMap).map((k) => parseInt(k, 10)); -}; - -/** - * Get the highest checkpoint key for a draft - */ -const getLatestCheckpointKey = async ( - draftRef: any, -): Promise => { - const keys = await getCheckpointKeys(draftRef); - if (keys.length === 0) return null; - return Math.max(...keys); -}; - -/** - * Get the highest checkpoint key at or before a threshold - */ -// const _getCheckpointKeyAtOrBefore = async ( -// draftRef: any, -// threshold: number, -// ): Promise => { -// const keys = await getCheckpointKeys(draftRef); -// const validKeys = keys.filter((k) => k <= threshold); -// if (validKeys.length === 0) return null; -// return Math.max(...validKeys); -// }; - -/** - * Get the latest release history key for a pub - */ -const getLatestReleaseHistoryKey = async (pubId: string): Promise => { - const release = await Release.findOne({ - where: { pubId }, - attributes: ['historyKey'], - order: [['historyKey', 'DESC']], - }); - return release?.historyKey ?? null; -}; - -/** - * Batch fetch latest release history keys for multiple pubs - * More efficient than individual queries - */ -const batchGetLatestReleaseKeys = async (pubIds: string[]): Promise> => { - if (pubIds.length === 0) return new Map(); - - const releases = await Release.findAll({ - where: { pubId: pubIds }, - attributes: ['pubId', 'historyKey'], - order: [['historyKey', 'DESC']], - }); - - // Group by pubId and take the max historyKey for each - const result = new Map(); - for (const release of releases) { - const existing = result.get(release.pubId); - if (existing === undefined || release.historyKey > existing) { - result.set(release.pubId, release.historyKey); - } - } - return result; -}; - -/** - * Try to repair a corrupted draft by creating a checkpoint from a Release's doc - * - * When getFirebaseDoc fails with "Position X out of range", the draft's history - * is corrupted. But if there's a Release at or before the target key, we can - * use the Release's doc to create a checkpoint, making the draft recoverable. - * - * @returns the historyKey where checkpoint was created, or null if repair failed - */ -const tryRepairFromRelease = async ( - pubId: string, - draftRef: any, - targetKey: number, - prefix: string = '', - localStats: CleanupStats = stats, -): Promise => { - // Find a release at or before the target key - const release = await Release.findOne({ - where: { pubId }, - include: [{ model: Doc, as: 'doc' }], - order: [['historyKey', 'DESC']], - }); - - if (!release) { - log(`${prefix}No releases found for repair`); - return null; - } - - if (release.historyKey > targetKey) { - // The release is newer than where we need the checkpoint - we could use it - // but the doc state would be from a later point. This is still valid for - // pruning old data, but let's warn about it. - log( - `${prefix}Warning: Using release at key ${release.historyKey} (newer than target ${targetKey})`, - ); - } - - const docContent = release.doc?.content; - if (!docContent) { - log(`${prefix}Release ${release.id} has no doc content`); - return null; - } - - // Create checkpoint at the release's historyKey using the release's doc - const historyKey = release.historyKey; - const compressedDoc = compressStateJSON({ doc: docContent }).d; - const timestamp = release.createdAt.getTime(); - - const checkpoint = { - d: compressedDoc, - k: historyKey, - t: timestamp, - }; - - log(`${prefix}Repairing: creating checkpoint at key ${historyKey} from release ${release.id}`); - - if (!isDryRun) { - await draftRef.update({ - [`checkpoints/${historyKey}`]: checkpoint, - checkpoint, - [`checkpointMap/${historyKey}`]: timestamp, - }); - localStats.draftsRepairedFromRelease++; - localStats.checkpointsCreated++; - } - - return historyKey; -}; - -/** - * Get discussions from Firebase and uncompress them - */ -const getFirebaseDiscussions = async ( - draftRef: any, -): Promise> => { - const discussionsSnapshot = await draftRef.child('discussions').once('value'); - const discussionsData = discussionsSnapshot.val(); - - if (!discussionsData) { - return {}; - } - - const discussions: Record = {}; - for (const [id, compressed] of Object.entries(discussionsData as Record)) { - const selection = compressed.selection - ? uncompressSelectionJSON(compressed.selection) - : null; - discussions[id] = { - ...compressed, - selection, - }; - } - return discussions; -}; - -/** - * Fast-forward all outdated discussions to the target key - */ -const fastForwardDiscussions = async ( - draftRef: any, - targetKey: number, -): Promise => { - const discussions = await getFirebaseDiscussions(draftRef); - const outdatedDiscussions = Object.entries(discussions).filter( - ([_, d]) => d && d.currentKey < targetKey, - ); - - if (outdatedDiscussions.length === 0) { - return 0; - } - - verbose(` Found ${outdatedDiscussions.length} outdated discussions`); - - // Get the current doc at targetKey to use for fast-forwarding - const { doc } = await getFirebaseDoc(draftRef, editorSchema, targetKey); - const fastForward = createFastForwarder(draftRef); - - const discussionsById: Record = {}; - for (const [id, discussion] of outdatedDiscussions) { - discussionsById[id] = discussion; - } - - const fastForwardedDiscussions = await fastForward(discussionsById, doc, targetKey); - - // Collect all updates for batch write - const updates: Record = {}; - let updatedCount = 0; - for (const [id, updatedDiscussion] of Object.entries(fastForwardedDiscussions)) { - if (updatedDiscussion) { - const compressed = { - ...updatedDiscussion, - selection: updatedDiscussion.selection - ? compressSelectionJSON(updatedDiscussion.selection) - : null, - }; - // Skip discussions with invalid selections (NaN positions from failed mapping) - if ( - compressed.selection && - (Number.isNaN(compressed.selection.a) || Number.isNaN(compressed.selection.h)) - ) { - verbose( - ` Skipping discussion ${id} - invalid selection after fast-forward (NaN position)`, - ); - continue; - } - verbose( - ` Fast-forwarding discussion ${id} from ${discussions[id].currentKey} to ${targetKey}`, - ); - updates[id] = compressed; - updatedCount++; - } - } - - // Write all discussion updates in a single batch - if (!isDryRun && updatedCount > 0) { - await draftRef.child('discussions').update(updates); - } - - return updatedCount; -}; - -/** - * Count and optionally delete keys before a threshold in a Firebase child. - * Uses shallow key listing to avoid loading content into memory. - * Uses batch multi-path updates for much faster deletion. - * Falls back to individual deletes if batch update fails with WRITE_TOO_BIG. - */ -const pruneKeysBefore = async ( - parentRef: any, - childName: string, - thresholdKey: number, -): Promise => { - // Get parent path and use shallow key listing - const parentPath = parentRef.toString().replace(/^https:\/\/[^/]+\//, ''); - const childPath = parentPath ? `${parentPath}/${childName}` : childName; - - const allKeys = await getShallowKeys(childPath); - const keysToDelete = allKeys - .map((k) => parseInt(k, 10)) - .filter((k) => !Number.isNaN(k) && k < thresholdKey) - .map((k) => String(k)); - - if (keysToDelete.length === 0) { - return 0; - } - - verbose(` Found ${keysToDelete.length} ${childName} entries before key ${thresholdKey}`); - - if (!isDryRun) { - // Use batch multi-path updates for much faster deletion - // Firebase limits multi-path updates to ~1000 paths or 16MB payload - const BATCH_SIZE = 500; - const childRef = parentRef.child(childName); - - for (let i = 0; i < keysToDelete.length; i += BATCH_SIZE) { - const batch = keysToDelete.slice(i, i + BATCH_SIZE); - const updates: Record = {}; - for (const key of batch) { - updates[key] = null; - } - try { - // biome-ignore lint/performance/noAwaitInLoops: batched processing requires sequential batches - await childRef.update(updates); - } catch (error: any) { - const errorCode = error?.code || error?.message || String(error); - if (errorCode.includes('WRITE_TOO_BIG') || errorCode.includes('write_too_big')) { - // Fall back to individual deletes for this batch - verbose(` Batch update too large, falling back to individual deletes`); - for (const key of batch) { - // biome-ignore lint/performance/noAwaitInLoops: sequential fallback for large items - await deleteFirebasePath(`${childPath}/${key}`); - } - } else { - throw error; - } - } - } - verbose(` Deleted ${keysToDelete.length} ${childName} entries`); - } - - return keysToDelete.length; -}; - -/** - * Prune old data from a single draft - * @param firebasePath - Firebase path to the draft - * @param pubId - Optional pub ID for release key lookup - * @param preloadedReleaseKey - Pre-fetched release key (for batch processing efficiency) - * @param label - Label for verbose logging (helps track logs during concurrent processing) - * @param localStats - Optional local stats object to update (for thread-safe concurrent processing) - */ -const pruneDraft = async ( - firebasePath: string, - pubId: string | null = null, - preloadedReleaseKey: number | null = null, - label: string = '', - localStats: CleanupStats = stats, - draftId: string | null = null, -): Promise => { - const prefix = label ? `[${label}] ` : ' '; - const draftRef = getDatabaseRef(firebasePath); - - // --- PG-checkpoint-aware fast path --- - // If a PG checkpoint exists, it is the source of truth for the doc. - // We can safely prune all Firebase changes/checkpoints before the PG key. - // Before pruning, capture stepMaps for the range we're about to delete. - if (draftId) { - const pgCheckpoint = await getDraftCheckpoint(draftId); - if (pgCheckpoint) { - const pgKey = pgCheckpoint.historyKey; - verbose(`${prefix}PG checkpoint at key ${pgKey}, using PG-aware prune`); - - // Before pruning, ensure stepMaps cover the range we're about to delete. - // If there's a release, we need stepMaps from release→pgKey. - const latestReleaseKey = - preloadedReleaseKey ?? (pubId ? await getLatestReleaseHistoryKey(pubId) : null); - - if (latestReleaseKey !== null && latestReleaseKey < pgKey) { - const existingToKey = pgCheckpoint.stepMapToKey; - // Only capture new stepMaps if there's a gap - const captureFrom = - existingToKey != null ? existingToKey + 1 : latestReleaseKey + 1; - - if (captureFrom <= pgKey) { - try { - const stepsByChange = await getStepsInChangeRange( - draftRef, - editorSchema, - captureFrom, - pgKey, - ); - const allSteps = stepsByChange.reduce((a, b) => [...a, ...b], []); - if (allSteps.length > 0) { - const newMaps = allSteps.map((step) => - Array.from((step.getMap() as any).ranges as number[]), - ); - const composedMaps = [...(pgCheckpoint.stepMaps ?? []), ...newMaps]; - verbose( - `${prefix}Capturing ${newMaps.length} stepMaps (${captureFrom}→${pgKey}) before prune`, - ); - - if (!isDryRun) { - await pgCheckpoint.update({ - stepMaps: composedMaps, - stepMapToKey: pgKey, - }); - } - } - } catch (err: any) { - log( - `${prefix}Warning: could not capture stepMaps before prune: ${err.message}`, - ); - } - } - } - - // Prune changes, merges, and old Firebase checkpoints before the PG key - const [changesDeleted, mergesDeleted, checkpointsDeleted] = await Promise.all([ - pruneKeysBefore(draftRef, 'changes', pgKey), - pruneKeysBefore(draftRef, 'merges', pgKey), - pruneKeysBefore(draftRef, 'checkpoints', pgKey + 1), // remove ALL Firebase checkpoints at or below pgKey - ]); - localStats.changesDeleted += changesDeleted; - localStats.mergesDeleted += mergesDeleted; - localStats.checkpointsDeleted += checkpointsDeleted; - - // Clean up Firebase checkpointMap entries and deprecated singular checkpoint - if (!isDryRun) { - const checkpointKeys = await getCheckpointKeys(draftRef); - if (checkpointKeys.length > 0) { - const updates: Record = {}; - for (const k of checkpointKeys) { - updates[String(k)] = null; - } - await draftRef.child('checkpointMap').update(updates); - } - await draftRef.child('checkpoint').remove(); - } - - verbose( - `${prefix}PG-aware prune: deleted ${changesDeleted} changes, ${mergesDeleted} merges, ${checkpointsDeleted} checkpoints`, - ); - return; - } - } - - // --- Legacy path: no PG checkpoint, use Firebase checkpoints --- - const latestCheckpointKey = await getLatestCheckpointKey(draftRef); - - if (latestCheckpointKey === null) { - verbose(`${prefix}No checkpoint found, skipping pruning`); - localStats.draftsSkipped++; - return; - } - - verbose(`${prefix}Latest Firebase checkpoint key: ${latestCheckpointKey}`); - - // Determine safe prune threshold: min(latestCheckpointKey, latestReleaseHistoryKey) - // - We need changes from latestCheckpointKey onwards to reconstruct the doc - // - We need changes from latestReleaseKey onwards to migrate discussions during release - let pruneThreshold = latestCheckpointKey; - - // Use preloaded release key if available, otherwise fetch it - const latestReleaseKey = - preloadedReleaseKey ?? (pubId ? await getLatestReleaseHistoryKey(pubId) : null); - if (latestReleaseKey !== null && latestReleaseKey < pruneThreshold) { - verbose( - `${prefix}Using release history key ${latestReleaseKey} (lower than checkpoint ${latestCheckpointKey})`, - ); - pruneThreshold = latestReleaseKey; - } - - verbose(`${prefix}Safe prune threshold: ${pruneThreshold}`); - - // Ensure there's a checkpoint at EXACTLY the prune threshold. - // We must create one if it doesn't exist, even if there's an older checkpoint, - // because we'll delete all checkpoints before the threshold. - const checkpointKeys = await getCheckpointKeys(draftRef); - const hasCheckpointAtThreshold = checkpointKeys.includes(pruneThreshold); - if (!hasCheckpointAtThreshold) { - verbose(`${prefix}No checkpoint at ${pruneThreshold}, creating one...`); - if (!isDryRun) { - try { - const { doc } = await getFirebaseDoc(draftRef, editorSchema, pruneThreshold); - verbose(`${prefix}Doc size at key ${pruneThreshold}: ${doc.content.size} nodes`); - // Get the timestamp from the change at this key - const changeSnapshot = await draftRef - .child(`changes/${pruneThreshold}`) - .once('value'); - const change = changeSnapshot.val(); - const timestamp = change?.t ?? Date.now(); - // Store checkpoint with the original change's timestamp - const compressedDoc = compressStateJSON({ doc: doc.toJSON() }).d; - const checkpointSize = JSON.stringify(compressedDoc).length; - verbose( - `${prefix}Compressed checkpoint size: ${(checkpointSize / 1024).toFixed(1)} KB`, - ); - const checkpoint = { - d: compressedDoc, - k: pruneThreshold, - t: timestamp, - }; - // Write all checkpoint data in a single multi-path update - verbose(`${prefix}Writing checkpoint at key ${pruneThreshold}...`); - await draftRef.update({ - [`checkpoints/${pruneThreshold}`]: checkpoint, - checkpoint, - [`checkpointMap/${pruneThreshold}`]: timestamp, - }); - localStats.checkpointsCreated++; - verbose( - `${prefix}Created checkpoint at key ${pruneThreshold} with timestamp ${timestamp}`, - ); - } catch (err) { - const error = err as Error & { code?: string }; - const errorStr = `${error.code || ''} ${error.message || ''}`; - log(`${prefix}Error during checkpoint creation: ${errorStr}`); - - // Check if this is a corrupted history error - // - "Position X out of range" = missing/corrupted steps - // - "Invalid content for node doc" = reconstructed doc is empty/malformed - // - "Inconsistent open depths" = malformed ProseMirror steps - // - "Cannot read properties of null" = null doc from failed reconstruction - const isCorruptedHistory = - (errorStr.includes('Position') && errorStr.includes('out of range')) || - errorStr.includes('Invalid content for node') || - errorStr.includes('Inconsistent open depths') || - errorStr.includes('Cannot read properties of null'); - - if (isCorruptedHistory && pubId) { - // Try to repair by creating a checkpoint from a Release - log(`${prefix}Attempting repair from Release...`); - const repairedAtKey = await tryRepairFromRelease( - pubId, - draftRef, - pruneThreshold, - prefix, - localStats, - ); - if (repairedAtKey !== null) { - log( - `${prefix}Repair successful at key ${repairedAtKey}, continuing with prune`, - ); - // Update pruneThreshold to the repaired checkpoint's key - // so subsequent operations (fast-forward, prune) work correctly - pruneThreshold = repairedAtKey; - } else { - log(`${prefix}Repair failed, skipping prune for this draft`); - localStats.draftsSkipped++; - return; - } - } else if ( - errorStr.includes('WRITE_TOO_BIG') || - errorStr.includes('write_too_big') - ) { - // Doc is too large to write as a single checkpoint. - // We can't safely prune without a checkpoint at the threshold, - // so skip pruning this draft entirely. - log( - `${prefix}Warning: Doc too large to create checkpoint at ${pruneThreshold}, skipping prune for this draft`, - ); - localStats.draftsSkipped++; - return; - } else { - throw err; - } - } - } else { - verbose(`${prefix}Would create checkpoint at key ${pruneThreshold}`); - } - } - - // First, fast-forward any outdated discussions to the prune threshold - const discussionsUpdated = await fastForwardDiscussions(draftRef, pruneThreshold); - localStats.discussionsUpdated += discussionsUpdated; - - // Prune changes, merges, and checkpoints in parallel (they're independent) - const [changesDeleted, mergesDeleted, checkpointsDeleted] = await Promise.all([ - pruneKeysBefore(draftRef, 'changes', pruneThreshold), - pruneKeysBefore(draftRef, 'merges', pruneThreshold), - pruneKeysBefore(draftRef, 'checkpoints', pruneThreshold), - ]); - localStats.changesDeleted += changesDeleted; - localStats.mergesDeleted += mergesDeleted; - localStats.checkpointsDeleted += checkpointsDeleted; - - // Clean up checkpointMap entries for deleted checkpoints - // Reuse checkpointKeys we already fetched earlier - const oldMapKeys = checkpointKeys.filter((k) => k < pruneThreshold).map(String); - if (oldMapKeys.length > 0 && !isDryRun) { - const updates: Record = {}; - for (const key of oldMapKeys) { - updates[key] = null; - } - await draftRef.child('checkpointMap').update(updates); - verbose(`${prefix}Cleaned up ${oldMapKeys.length} checkpointMap entries`); - } - - // Remove deprecated singular checkpoint if we deleted any checkpoints - // (avoiding extra read - if we pruned checkpoints, the checkpoints/ path exists) - if (!isDryRun && checkpointsDeleted > 0) { - await draftRef.child('checkpoint').remove(); - } -}; - -/** - * Delete an orphaned draft from both Firebase and Postgres - */ -const deleteOrphanedDraft = async (draft: Draft): Promise => { - const { id, firebasePath } = draft; - - log(` Deleting orphaned draft: ${id} (${firebasePath})`); - - if (!isDryRun) { - // Delete from Firebase first (using deleteFirebasePath to handle large drafts) - if (firebasePath) await deleteFirebasePath(firebasePath); - - // Then delete from Postgres - await draft.destroy(); - stats.orphanedDraftsDeleted++; - } -}; - -/** - * Find all orphaned drafts (drafts without an associated Pub) - */ -const findOrphanedDrafts = async (): Promise => { - const orphanedDrafts = await sequelize.query( - ` - SELECT d.* - FROM "Drafts" d - LEFT JOIN "Pubs" p ON p."draftId" = d.id - WHERE p.id IS NULL - `, - { - model: Draft, - mapToModel: true, - type: QueryTypes.SELECT, - }, - ); - return orphanedDrafts; -}; - -/** - * Get all valid firebase paths from the database - */ -const getValidFirebasePaths = async (): Promise> => { - const drafts = await Draft.findAll({ - attributes: ['firebasePath'], - }); - return new Set(drafts.map((d) => d.firebasePath).filter((p): p is string => p !== null)); -}; - -/** - * Shared Firebase key discovery — called once, results shared across - * orphan cleanup, stub cleanup, and prune pre-filtering. - */ -interface FirebaseKeyDiscovery { - /** All keys under drafts/ (e.g. 'draft-abc-123') */ - draftKeys: string[]; - /** All pub-* keys at root */ - legacyPubKeys: string[]; - /** Map of pub-* key → child keys (branches, metadata, etc.) */ - legacyPubChildren: Map; -} - -const discoverFirebaseKeys = async (): Promise => { - log('Discovering Firebase keys...'); - - const [draftKeys, rootKeys] = await Promise.all([getShallowKeys('drafts'), getShallowKeys('')]); - log(` Found ${draftKeys.length} keys under drafts/`); - - const legacyPubKeys = rootKeys.filter((key) => key.startsWith('pub-')); - const legacyPubChildren = new Map(); - - if (legacyPubKeys.length > 0) { - log(` Found ${legacyPubKeys.length} legacy pub-* paths, scanning children...`); - let fetched = 0; - await runWithConcurrency( - legacyPubKeys.map((pubKey) => async () => { - const childKeys = await getShallowKeys(pubKey); - legacyPubChildren.set(pubKey, childKeys); - fetched++; - if (fetched % 1000 === 0) { - log(` Scanned ${fetched}/${legacyPubKeys.length} legacy pubs...`); - } - }), - 50, - ); - } - - return { draftKeys, legacyPubKeys, legacyPubChildren }; -}; - -/** - * Scan Firebase for orphaned top-level paths and delete them - */ -const cleanupOrphanedFirebasePaths = async (discovery: FirebaseKeyDiscovery): Promise => { - log('Scanning Firebase for orphaned paths...'); - - const validPaths = await getValidFirebasePaths(); - verbose(` Found ${validPaths.size} valid paths in database`); - - const { draftKeys, legacyPubKeys, legacyPubChildren } = discovery; - verbose(` ${draftKeys.length} keys under drafts/`); - - // Identify orphaned drafts - const orphanedDraftKeys = draftKeys.filter((key) => !validPaths.has(`drafts/${key}`)); - stats.orphanedFirebasePathsFound += orphanedDraftKeys.length; - if (orphanedDraftKeys.length > 0) { - log(` Found ${orphanedDraftKeys.length} orphaned draft paths`); - } - - // Delete orphaned drafts with concurrency - if (!isDryRun && orphanedDraftKeys.length > 0) { - let deleted = 0; - await runWithConcurrency( - orphanedDraftKeys.map((draftKey) => async () => { - await deleteFirebasePath(`drafts/${draftKey}`); - deleted++; - if (deleted % 100 === 0) { - log(` Deleted ${deleted}/${orphanedDraftKeys.length} orphaned drafts...`); - } - }), - 30, - ); - stats.orphanedFirebasePathsDeleted += orphanedDraftKeys.length; - log(` Deleted ${orphanedDraftKeys.length} orphaned draft paths`); - } - - // Process legacy pubs using pre-fetched children - log(` ${legacyPubKeys.length} legacy pub-* paths at root`); - - interface LegacyPubInfo { - pubKey: string; - childKeys: string[]; - orphanedBranches: string[]; - hasValidBranch: boolean; - } - - const legacyPubInfos: LegacyPubInfo[] = legacyPubKeys.map((pubKey) => { - const childKeys = legacyPubChildren.get(pubKey) ?? []; - const branchKeys = childKeys.filter((k) => k.startsWith('branch-')); - const orphanedBranches: string[] = []; - let hasValidBranch = false; - - for (const branchKey of branchKeys) { - const firebasePath = `${pubKey}/${branchKey}`; - if (validPaths.has(firebasePath)) { - hasValidBranch = true; - } else { - orphanedBranches.push(firebasePath); - } - } - - return { pubKey, childKeys, orphanedBranches, hasValidBranch }; - }); - - // Collect all paths to delete - const pathsToDelete: string[] = []; - let metadataCount = 0; - - for (const info of legacyPubInfos) { - if (!info.hasValidBranch) { - // Delete entire pub - pathsToDelete.push(info.pubKey); - if (info.childKeys.includes('metadata')) { - metadataCount++; - } - } else { - // Delete only orphaned branches - pathsToDelete.push(...info.orphanedBranches); - } - } - - stats.orphanedFirebasePathsFound += pathsToDelete.length; - if (pathsToDelete.length > 0) { - log(` Found ${pathsToDelete.length} orphaned legacy paths to delete`); - } - - // Delete all orphaned paths with concurrency - if (!isDryRun && pathsToDelete.length > 0) { - let deleted = 0; - await runWithConcurrency( - pathsToDelete.map((path) => async () => { - await deleteFirebasePath(path); - deleted++; - if (deleted % 100 === 0) { - log(` Deleted ${deleted}/${pathsToDelete.length} legacy paths...`); - } - }), - 30, - ); - stats.orphanedFirebasePathsDeleted += pathsToDelete.length; - stats.metadataDeleted += metadataCount; - log(` Deleted ${pathsToDelete.length} orphaned legacy paths`); - } - - log(`Found ${stats.orphanedFirebasePathsFound} orphaned Firebase paths`); -}; - -/** - * Remove v5→v6 migration stubs from Firebase. - * - * During the v5→v6 migration, every branch was written with a `lastMergeKey` field - * (the highest merge index, defaulting to -1 when no merges existed). Many drafts - * never received any edits after migration, so they still contain only - * `{"lastMergeKey": -1}` — 18 bytes of dead weight per path. - * - * Nothing reads `lastMergeKey` at runtime (client, server, or collab-service). - * These stubs are safe to delete. - */ -const cleanupMigrationStubs = async (discovery: FirebaseKeyDiscovery): Promise => { - log('Scanning for v5→v6 migration stubs...'); - - const { draftKeys } = discovery; - - // For each draft, check if its only child key is 'lastMergeKey' - let scanned = 0; - const stubPaths: string[] = []; - - await runWithConcurrency( - draftKeys.map((draftKey) => async () => { - const childKeys = await getShallowKeys(`drafts/${draftKey}`); - scanned++; - if (scanned % 500 === 0) { - verbose(` Scanned ${scanned}/${draftKeys.length} drafts for stubs...`); - } - if (childKeys.length === 1 && childKeys[0] === 'lastMergeKey') { - stubPaths.push(`drafts/${draftKey}`); - } - }), - 50, - ); - - stats.migrationStubsFound = stubPaths.length; - log(` Found ${stubPaths.length} migration stubs`); - - if (!isDryRun && stubPaths.length > 0) { - let deleted = 0; - await runWithConcurrency( - stubPaths.map((path) => async () => { - await getDatabaseRef(path).child('lastMergeKey').remove(); - deleted++; - if (deleted % 100 === 0) { - log(` Deleted ${deleted}/${stubPaths.length} stubs...`); - } - }), - 30, - ); - stats.migrationStubsDeleted = stubPaths.length; - log(` Deleted ${stubPaths.length} migration stubs`); - } -}; - -/** - * Clean up orphaned branches and legacy metadata for a specific pub (legacy format only) - * If the draft uses pub-{pubId}/branch-{branchId} format, delete any other branches - * and the legacy metadata field at pub-{pubId}/metadata - * @param localStats - Optional local stats object to update (for thread-safe concurrent processing) - */ -const cleanupOrphanedBranchesForPub = async ( - pubId: string, - activePath: string, - localStats: CleanupStats = stats, -): Promise => { - // Check if this is a legacy format path - const legacyMatch = activePath.match(/^(pub-[^/]+)\/(branch-[^/]+)$/); - if (!legacyMatch) { - // New format (drafts/draft-xxx), no sibling branches to clean - return; - } - - const [, pubKey, activeBranchKey] = legacyMatch; - verbose(` Checking for orphaned branches under ${pubKey}`); - - const branchKeys = await getShallowKeys(pubKey); - const orphanedBranches = branchKeys.filter( - (key) => key.startsWith('branch-') && key !== activeBranchKey, - ); - - if (orphanedBranches.length > 0) { - localStats.orphanedFirebasePathsFound += orphanedBranches.length; - log(` Found ${orphanedBranches.length} orphaned branches under ${pubKey}`); - if (!isDryRun) { - await runWithConcurrency( - orphanedBranches.map((branchKey) => async () => { - await deleteFirebasePath(`${pubKey}/${branchKey}`); - }), - 30, - ); - localStats.orphanedFirebasePathsDeleted += orphanedBranches.length; - } - } - - // Delete legacy metadata field at pub-{pubId}/metadata (no longer used in v6) - const pubRef = getDatabaseRef(pubKey); - const metadataSnapshot = await pubRef.child('metadata').once('value'); - if (metadataSnapshot.exists()) { - verbose(` Removing legacy metadata field at ${pubKey}/metadata`); - if (!isDryRun) { - await pubRef.child('metadata').remove(); - localStats.metadataDeleted++; - } - } -}; - -/** - * Process a single pub's draft - */ -const processPubDraft = async (pubId: string): Promise => { - const pub = await Pub.findOne({ - where: { id: pubId }, - include: [{ model: Draft, as: 'draft' }], - }); - - if (!pub) { - log(`Pub not found: ${pubId}`); - return; - } - - if (!pub.draft || !pub.draft.firebasePath) { - log(`Pub ${pubId} has no draft or no firebase path`); - return; - } - - log(`Processing pub ${pub.slug || pub.id}`); - const pubLabel = pub.slug || pub.id.slice(0, 8); - verbose(`[${pubLabel}] Draft path: ${pub.draft.firebasePath}`); - - try { - await pruneDraft(pub.draft.firebasePath, pubId, null, pubLabel, stats, pub.draft.id); - await cleanupOrphanedBranchesForPub(pubId, pub.draft.firebasePath); - stats.draftsProcessed++; - } catch (err) { - log(` Error processing draft: ${(err as Error).message}`); - stats.errorsEncountered++; - } -}; - -/** - * Process a single draft by ID - */ -const processDraftById = async (draftId: string): Promise => { - const draft = await Draft.findOne({ where: { id: draftId } }); - - if (!draft || !draft.firebasePath) { - log(`Draft not found or has no firebase path: ${draftId}`); - return; - } - - // Try to find associated pub for release key lookup - const pub = await Pub.findOne({ where: { draftId } }); - - log(`Processing draft ${draftId}`); - const draftLabel = draftId.slice(0, 8); - verbose(`[${draftLabel}] Draft path: ${draft.firebasePath}`); - - try { - await pruneDraft(draft.firebasePath, pub?.id ?? null, null, draftLabel, stats, draft.id); - stats.draftsProcessed++; - } catch (err) { - log(` Error processing draft: ${(err as Error).message}`); - stats.errorsEncountered++; - } -}; - -/** - * Process all drafts in batches - */ -const processAllDrafts = async (): Promise => { - // First, handle orphaned drafts in Postgres - log('Looking for orphaned drafts in Postgres...'); - const orphanedDrafts = await findOrphanedDrafts(); - stats.orphanedDraftsFound = orphanedDrafts.length; - - if (orphanedDrafts.length > 0) { - log(`Found ${orphanedDrafts.length} orphaned drafts`); - for (const draft of orphanedDrafts) { - try { - // biome-ignore lint/performance/noAwaitInLoops: intentionally sequential - await deleteOrphanedDraft(draft); - } catch (err) { - log(` Error deleting orphaned draft ${draft.id}: ${(err as Error).message}`); - stats.errorsEncountered++; - } - } - } else { - log('No orphaned drafts found in Postgres'); - } - - // Single Firebase key discovery pass — shared across orphan cleanup, - // stub cleanup, and prune pre-filtering. - const discovery = await discoverFirebaseKeys(); - - // Scan Firebase for orphaned paths and migration stubs - if (scanFirebase) { - await cleanupOrphanedFirebasePaths(discovery); - await cleanupMigrationStubs(discovery); - } - - // Pre-filter: only process pubs whose drafts actually have Firebase data. - // After orphan/stub cleanup, some paths may have been deleted — re-read - // draft keys if we ran cleanup, otherwise use the discovery data as-is. - const currentDraftKeys = - scanFirebase && !isDryRun ? await getShallowKeys('drafts') : discovery.draftKeys; - const firebaseDraftIdSet = new Set(currentDraftKeys.map((k) => k.replace('draft-', ''))); - - // Build legacy path set from pre-fetched children (no extra Firebase calls) - const legacyFirebasePaths = new Set(); - for (const [pubKey, children] of discovery.legacyPubChildren) { - for (const child of children) { - if (child.startsWith('branch-')) { - legacyFirebasePaths.add(`${pubKey}/${child}`); - } - } - } - - log( - `Found ${firebaseDraftIdSet.size} modern drafts and ${legacyFirebasePaths.size} legacy paths with Firebase data`, - ); - - // Fetch only pubs whose drafts have Firebase data - log('Fetching pubs with active Firebase data...'); - const pubsWithFirebaseData = await Pub.findAll({ - attributes: ['id', 'slug'], - include: [{ model: Draft, as: 'draft', attributes: ['id', 'firebasePath'] }], - order: [['id', 'ASC']], - }); - - // Filter to only pubs whose draft has data in Firebase - const pubQueue = pubsWithFirebaseData.filter((pub) => { - if (!pub.draft) return false; - const { firebasePath } = pub.draft; - if (!firebasePath) return false; - - // Modern format: drafts/draft-{draftId} - if (firebasePath.startsWith('drafts/draft-')) { - return firebaseDraftIdSet.has(pub.draft.id); - } - // Legacy format: pub-{pubId}/branch-{branchId} - return legacyFirebasePaths.has(firebasePath); - }); - - const totalPubs = pubQueue.length; - log( - `Found ${totalPubs} pubs with Firebase data to process (skipped ${pubsWithFirebaseData.length - totalPubs} empty)`, - ); - - // Prefetch all release keys in a single batch - const pubIdsToProcess = pubQueue.map((p) => p.id); - const releaseKeyCache = new Map(); - for (let i = 0; i < pubIdsToProcess.length; i += batchSize) { - const batch = pubIdsToProcess.slice(i, i + batchSize); - // biome-ignore lint/performance/noAwaitInLoops: sequential batch fetching - const releaseKeys = await batchGetLatestReleaseKeys(batch); - for (const [pid, key] of releaseKeys) { - releaseKeyCache.set(pid, key); - } - } - - const WORKER_COUNT = 50; - - // Track total processed across all workers for progress logging - let totalProcessed = 0; - let queueIndex = 0; - - // Worker function that continuously processes pubs from the pre-built queue - const workerWithStats = async (workerId: number, localStats: CleanupStats) => { - while (queueIndex < pubQueue.length) { - const currentIndex = queueIndex++; - const pub = pubQueue[currentIndex]; - - try { - const pubLabel = pub.slug || pub.id.slice(0, 8); - verbose(`[W${workerId}:${pubLabel}] Processing...`); - const releaseKey = releaseKeyCache.get(pub.id) ?? null; - // biome-ignore lint/performance/noAwaitInLoops: worker pool pattern requires sequential processing - await pruneDraft( - pub.draft!.firebasePath!, - pub.id, - releaseKey, - pubLabel, - localStats, - pub.draft!.id, - ); - await cleanupOrphanedBranchesForPub(pub.id, pub.draft!.firebasePath!, localStats); - localStats.draftsProcessed++; - totalProcessed++; - - // Clear from cache to prevent memory buildup - releaseKeyCache.delete(pub.id); - - // Log progress every 500 processed (approximate due to concurrency) - if (totalProcessed % 500 === 0) { - log(` Processed ~${totalProcessed}/${totalPubs} drafts...`); - } - } catch (err) { - log(` Error processing pub ${pub.id}: ${(err as Error).message}`); - localStats.errorsEncountered++; - releaseKeyCache.delete(pub.id); - } - } - }; - - // Create per-worker stats to avoid race conditions - const createLocalStats = (): CleanupStats => ({ - draftsProcessed: 0, - draftsSkipped: 0, - draftsRepairedFromRelease: 0, - orphanedDraftsFound: 0, - orphanedDraftsDeleted: 0, - orphanedFirebasePathsFound: 0, - orphanedFirebasePathsDeleted: 0, - migrationStubsFound: 0, - migrationStubsDeleted: 0, - discussionsUpdated: 0, - changesDeleted: 0, - mergesDeleted: 0, - checkpointsDeleted: 0, - checkpointsCreated: 0, - metadataDeleted: 0, - errorsEncountered: 0, - }); - - const workerStats: CleanupStats[] = []; - - // Start workers with local stats - const workers = Array.from({ length: WORKER_COUNT }, (_, i) => { - const localStats = createLocalStats(); - workerStats.push(localStats); - return workerWithStats(i, localStats); - }); - await Promise.all(workers); - - // Aggregate worker stats into global stats - for (const ws of workerStats) { - stats.draftsProcessed += ws.draftsProcessed; - stats.draftsSkipped += ws.draftsSkipped; - stats.draftsRepairedFromRelease += ws.draftsRepairedFromRelease; - stats.discussionsUpdated += ws.discussionsUpdated; - stats.changesDeleted += ws.changesDeleted; - stats.mergesDeleted += ws.mergesDeleted; - stats.checkpointsDeleted += ws.checkpointsDeleted; - stats.checkpointsCreated += ws.checkpointsCreated; - stats.errorsEncountered += ws.errorsEncountered; - stats.orphanedFirebasePathsFound += ws.orphanedFirebasePathsFound; - stats.orphanedFirebasePathsDeleted += ws.orphanedFirebasePathsDeleted; - stats.metadataDeleted += ws.metadataDeleted; - } - - log( - ` Processed ${stats.draftsProcessed} drafts total (${stats.changesDeleted} changes deleted)`, - ); -}; - -const printSummary = () => { - log('=== Cleanup Summary ==='); - log(`Mode: ${isDryRun ? 'DRY RUN (no data deleted)' : 'EXECUTE'}`); - log(`Drafts processed: ${stats.draftsProcessed}`); - log(`Drafts skipped (no checkpoint): ${stats.draftsSkipped}`); - log(`Drafts repaired from release: ${stats.draftsRepairedFromRelease}`); - log(`Orphaned drafts found (Postgres): ${stats.orphanedDraftsFound}`); - log(`Orphaned drafts deleted: ${stats.orphanedDraftsDeleted}`); - log(`Orphaned Firebase paths found: ${stats.orphanedFirebasePathsFound}`); - log(`Orphaned Firebase paths deleted: ${stats.orphanedFirebasePathsDeleted}`); - log(`Migration stubs found: ${stats.migrationStubsFound}`); - log(`Migration stubs deleted: ${stats.migrationStubsDeleted}`); - log(`Discussions fast-forwarded: ${stats.discussionsUpdated}`); - log(`Changes deleted: ${stats.changesDeleted}`); - log(`Merges deleted: ${stats.mergesDeleted}`); - log(`Checkpoints created: ${stats.checkpointsCreated}`); - log(`Checkpoints deleted: ${stats.checkpointsDeleted}`); - log(`Metadata fields deleted: ${stats.metadataDeleted}`); - log(`Errors encountered: ${stats.errorsEncountered}`); -}; - -const _postSummaryToSlack = async () => { - if (isDryRun) return; - - const text = [ - '*Firebase Cleanup Completed*', - `• Drafts processed: ${stats.draftsProcessed}`, - stats.draftsRepairedFromRelease > 0 - ? `• Drafts repaired from release: ${stats.draftsRepairedFromRelease}` - : '', - `• Orphaned drafts deleted: ${stats.orphanedDraftsDeleted}`, - `• Orphaned Firebase paths deleted: ${stats.orphanedFirebasePathsDeleted}`, - stats.migrationStubsDeleted > 0 - ? `• Migration stubs deleted: ${stats.migrationStubsDeleted}` - : '', - `• Discussions fast-forwarded: ${stats.discussionsUpdated}`, - `• Changes pruned: ${stats.changesDeleted}`, - `• Merges pruned: ${stats.mergesDeleted}`, - `• Checkpoints created: ${stats.checkpointsCreated}`, - `• Checkpoints pruned: ${stats.checkpointsDeleted}`, - `• Metadata fields deleted: ${stats.metadataDeleted}`, - stats.errorsEncountered > 0 ? `• ⚠️ Errors: ${stats.errorsEncountered}` : '', - ] - .filter(Boolean) - .join('\n'); - - try { - await postToSlack({ text, icon_emoji: ':broom:' }); - } catch (err) { - log(`Failed to post to Slack: ${(err as Error).message}`); - } -}; - -const main = async () => { - log('Starting Firebase cleanup'); - log(`Mode: ${isDryRun ? 'DRY RUN' : 'EXECUTE'}`); - - try { - if (pubId) { - await processPubDraft(pubId); - } else if (draftId) { - await processDraftById(draftId); - } else { - await processAllDrafts(); - } - - printSummary(); - // await postSummaryToSlack(); - } catch (err) { - log(`Fatal error: ${(err as Error).message}`); - console.error(err); - process.exit(1); - } -}; - -main().finally(() => process.exit(0)); diff --git a/tools/coldStorage.ts b/tools/coldStorage.ts deleted file mode 100644 index c1729aa4ff..0000000000 --- a/tools/coldStorage.ts +++ /dev/null @@ -1,620 +0,0 @@ -/** - * Cold Storage Tool - * - * Moves inactive drafts from Firebase to Postgres by: - * 1. Finding drafts not edited within a threshold (default: 30 days) - * 2. Building a checkpoint from the current Firebase state (checkpoint + changes) - * 3. Storing that checkpoint in the DraftCheckpoints Postgres table - * 4. Wiping all data from the Firebase path (changes, checkpoints, merges, etc.) - * - * When a user next opens a cold-stored draft, the server loads the checkpoint - * from Postgres and the client connects to an empty Firebase ref — ready for - * new edits. A future run of this tool will re-checkpoint those new edits. - * - * This tool is safe to run repeatedly. Drafts already cold-stored (with an - * empty Firebase ref) are skipped. - * - * Usage: - * pnpm run tools coldStorage # Dry run, all stale drafts - * pnpm run tools coldStorage --execute # Actually migrate + wipe - * pnpm run tools coldStorage --daysOld=60 # Custom threshold - * pnpm run tools coldStorage --pubId= # Single pub - * (Prod/dev is determined by env vars: DATABASE_URL, FIREBASE_SERVICE_ACCOUNT_BASE64) - */ - -import firebaseAdmin from 'firebase-admin'; -import { uncompressSelectionJSON } from 'prosemirror-compress-pubpub'; -import { Op, QueryTypes } from 'sequelize'; - -import { editorSchema, getFirebaseDoc, getStepsInChangeRange } from 'components/Editor'; -import { getDraftCheckpoint } from 'server/draftCheckpoint/queries'; -import { Draft, DraftCheckpoint, Pub, Release } from 'server/models'; -import { sequelize } from 'server/sequelize'; -import { getDatabaseRef, getPubDraftDoc } from 'server/utils/firebaseAdmin'; -import { getFirebaseConfig } from 'utils/editor/firebaseConfig'; - -const { - argv: { - execute, - pubId: specificPubId, - daysOld: daysOldArg = 30, - batchSize: batchSizeArg = 100, - concurrency: concurrencyArg = 10, - verbose: verboseFlag, - }, -} = require('yargs'); - -const isDryRun = !execute; -const DAYS_OLD = Number(daysOldArg); -const BATCH_SIZE = Number(batchSizeArg); -const CONCURRENCY = Number(concurrencyArg); - -// biome-ignore lint/suspicious/noConsole: CLI tool output -const log = (msg: string) => console.log(`[cold-storage] ${new Date().toISOString()} ${msg}`); -const verbose = (msg: string) => verboseFlag && log(msg); - -// --- Firebase REST helpers (avoid SDK WebSocket throttling) --- - -let cachedAccessToken: { token: string; expiresAt: number } | null = null; - -const getAccessToken = async (): Promise => { - const now = Date.now(); - if (cachedAccessToken && cachedAccessToken.expiresAt > now + 60_000) { - return cachedAccessToken.token; - } - const credential = firebaseAdmin.credential.cert( - JSON.parse( - Buffer.from(process.env.FIREBASE_SERVICE_ACCOUNT_BASE64 as string, 'base64').toString(), - ), - ); - const tokenResult = await credential.getAccessToken(); - cachedAccessToken = { - token: tokenResult.access_token, - expiresAt: now + (tokenResult.expires_in ?? 3600) * 1000, - }; - return cachedAccessToken.token; -}; - -const REST_TIMEOUT_MS = 60_000; -const REST_MAX_RETRIES = 5; - -/** - * General-purpose Firebase REST API helper. - * Each call is an independent HTTP request — no shared WebSocket. - */ -const firebaseRest = async ( - method: 'GET' | 'PUT' | 'PATCH' | 'DELETE', - path: string, - body?: any, - queryParams?: Record, -): Promise => { - const databaseURL = getFirebaseConfig().databaseURL; - - for (let attempt = 1; attempt <= REST_MAX_RETRIES; attempt++) { - // biome-ignore lint/performance/noAwaitInLoops: retry loop - const accessToken = await getAccessToken(); - const params = new URLSearchParams({ access_token: accessToken, ...queryParams }); - const url = `${databaseURL}/${path}.json?${params}`; - - try { - const options: RequestInit = { - method, - signal: AbortSignal.timeout(REST_TIMEOUT_MS), - }; - if (body !== undefined) { - options.headers = { 'Content-Type': 'application/json' }; - options.body = JSON.stringify(body); - } - const response = await fetch(url, options); - if (!response.ok) { - const text = await response.text(); - throw new Error(`Firebase REST ${method} ${response.status}: ${text}`); - } - if (method === 'DELETE') return null as T; - return (await response.json()) as T; - } catch (error: any) { - // Don't retry deterministic errors like WRITE_TOO_BIG - const errMsg = error?.message || String(error); - if ( - errMsg.includes('Data to write exceeds') || - errMsg.includes('WRITE_TOO_BIG') || - errMsg.includes('write_too_big') - ) { - throw error; - } - if (attempt === REST_MAX_RETRIES) throw error; - const delay = Math.min(2000 * 2 ** attempt, 30_000); - log( - ` [rest] ${method} ${path}: attempt ${attempt} failed, retrying in ${delay / 1000}s (${errMsg})`, - ); - await new Promise((r) => setTimeout(r, delay)); - } - } - throw new Error('unreachable'); -}; - -/** - * List child keys at a Firebase path using REST API with ?shallow=true. - * Never downloads the actual content, so safe for huge nodes. - */ -const getShallowKeys = async (ref: any): Promise => { - const refPath = ref.toString().replace(/^https:\/\/[^/]+\//, ''); - const data = await firebaseRest | null>('GET', refPath, undefined, { - shallow: 'true', - }); - if (!data || typeof data !== 'object') return []; - return Object.keys(data); -}; - -/** - * Recursively delete a Firebase path via REST, handling WRITE_TOO_BIG errors - * by listing children shallowly and batch-deleting with multi-path PATCH. - */ -const DELETE_BATCH_SIZE = 2500; - -const deleteFirebasePath = async (path: string): Promise => { - try { - await firebaseRest('DELETE', path); - } catch (error: any) { - const msg = error?.message || String(error); - if ( - !msg.includes('Data to write exceeds') && - !msg.includes('WRITE_TOO_BIG') && - !msg.includes('write_too_big') - ) - throw error; - - verbose(` ${path} too large, deleting in batches`); - const childKeys = await firebaseRest | null>('GET', path, undefined, { - shallow: 'true', - }); - if (!childKeys || typeof childKeys !== 'object') return; - const keys = Object.keys(childKeys); - - for (let i = 0; i < keys.length; i += DELETE_BATCH_SIZE) { - const batch = keys.slice(i, i + DELETE_BATCH_SIZE); - const updates: Record = {}; - for (const key of batch) { - updates[key] = null; - } - try { - // biome-ignore lint/performance/noAwaitInLoops: batched deletion - await firebaseRest('PATCH', path, updates); - } catch (batchErr: any) { - const batchMsg = batchErr?.message || String(batchErr); - if ( - batchMsg.includes('Data to write exceeds') || - batchMsg.includes('WRITE_TOO_BIG') || - batchMsg.includes('write_too_big') - ) { - for (const key of batch) { - // biome-ignore lint/performance/noAwaitInLoops: sequential fallback - await deleteFirebasePath(`${path}/${key}`); - } - } else { - throw batchErr; - } - } - } - - // Delete the now-empty parent - await firebaseRest('DELETE', path); - } -}; - -// --- Concurrency helper --- - -const runWithConcurrency = async ( - tasks: (() => Promise)[], - concurrency: number, -): Promise => { - const results: T[] = []; - let index = 0; - const worker = async (): Promise => { - while (index < tasks.length) { - const currentIndex = index++; - // biome-ignore lint/performance/noAwaitInLoops: worker pool pattern - results[currentIndex] = await tasks[currentIndex](); - } - }; - await Promise.all(Array.from({ length: Math.min(concurrency, tasks.length) }, () => worker())); - return results; -}; - -// --- Stats --- - -interface ColdStorageStats { - draftsScanned: number; - draftsAlreadyCold: number; - draftsEmpty: number; - draftsFrozen: number; - draftsSkippedError: number; - bytesFreed: number; -} - -const stats: ColdStorageStats = { - draftsScanned: 0, - draftsAlreadyCold: 0, - draftsEmpty: 0, - draftsFrozen: 0, - draftsSkippedError: 0, - bytesFreed: 0, -}; - -const formatBytes = (bytes: number): string => { - if (bytes < 1024) return `${bytes} B`; - if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; - if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; - return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`; -}; - -// --- Core logic --- - -/** - * Freeze a single draft: build checkpoint from Firebase, store in Postgres, wipe Firebase. - * - * Steps: - * 1. Build the current doc from Firebase (checkpoint + changes) - * 2. Collect Firebase discussions and fast-forward them to the current key - * 3. Compute cumulative StepMaps from latest release to current key - * 4. Store checkpoint, discussions, and stepMaps in Postgres - * 5. Update latestKeyAt on the Draft - * 6. Wipe all Firebase data for this draft - */ -const freezeDraft = async (draft: Draft, pubId: string): Promise => { - const { id: draftId, firebasePath } = draft; - const prefix = `[${draftId.slice(0, 8)}]`; - - if (!firebasePath) { - verbose(`${prefix} No firebase path, skipping`); - stats.draftsAlreadyCold++; - return; - } - - try { - const draftRef = getDatabaseRef(firebasePath); - - // Check if Firebase has any data at all - const topLevelKeys = await getShallowKeys(draftRef); - if (topLevelKeys.length === 0) { - verbose(`${prefix} Firebase path empty, marking as already cold`); - stats.draftsAlreadyCold++; - return; - } - - // Build the current doc using PG-first logic (handles previously cold-stored drafts - // where Firebase only has changes since the last thaw, not the full history). - const draftDocInfo = await getPubDraftDoc(pubId); - const currentKey = draftDocInfo.mostRecentRemoteKey; - const currentTimestamp = draftDocInfo.latestTimestamp; - - if (currentKey < 0) { - verbose(`${prefix} No history found (empty doc, no changes)`); - stats.draftsEmpty++; - return; - } - - const docJson = draftDocInfo.doc; - const docSize = JSON.stringify(docJson).length; - - verbose(`${prefix} Doc at key ${currentKey}, size ${formatBytes(docSize)}`); - - // Collect and fast-forward Firebase discussions (via REST to avoid WebSocket throttling) - let frozenDiscussions: Record | null = null; - const rawDiscussions = await firebaseRest | null>( - 'GET', - `${firebasePath}/discussions`, - ); - if (rawDiscussions && typeof rawDiscussions === 'object') { - // Fast-forward outdated discussions to currentKey using step maps. - // Discussions whose currentKey matches are already at the latest position. - const outdatedIds = Object.entries(rawDiscussions).filter( - ([_, d]: [string, any]) => - d && typeof d.currentKey === 'number' && d.currentKey < currentKey, - ); - - if (outdatedIds.length > 0) { - // Only fetch steps that are actually available in Firebase. - // After a previous cold storage, early changes are gone — the PG - // checkpoint key is the earliest we can fetch from. - const existingCheckpoint = await getDraftCheckpoint(draftId); - const earliestFirebaseKey = existingCheckpoint ? existingCheckpoint.historyKey : 0; - const mostOutdatedKey = Math.max( - earliestFirebaseKey, - Math.min(...outdatedIds.map(([_, d]: [string, any]) => d.currentKey)), - ); - verbose( - `${prefix} Fast-forwarding ${outdatedIds.length} discussions from key ${mostOutdatedKey}`, - ); - - try { - const stepsByChange = await getStepsInChangeRange( - draftRef, - editorSchema, - mostOutdatedKey + 1, - currentKey, - ); - const allSteps = stepsByChange.reduce((a, b) => [...a, ...b], []); - - for (const [id, discussion] of outdatedIds) { - const disc = discussion as any; - if (disc.selection) { - const sel = - disc.selection.a !== undefined - ? uncompressSelectionJSON(disc.selection) - : disc.selection; - - let from = Math.min(sel.anchor, sel.head); - let to = Math.max(sel.anchor, sel.head); - const stepsToApply = allSteps.slice(disc.currentKey - mostOutdatedKey); - - for (const step of stepsToApply) { - const map = step.getMap(); - from = map.map(from, 1); - to = map.map(to, -1); - } - - if (from < to && from > 0) { - rawDiscussions[id] = { - ...disc, - selection: { type: 'text', a: from, h: to }, - currentKey, - }; - } else { - rawDiscussions[id] = { ...disc, selection: null, currentKey }; - } - } else { - rawDiscussions[id] = { ...disc, currentKey }; - } - } - } catch (err: any) { - verbose( - `${prefix} Warning: could not fast-forward discussions: ${err.message}`, - ); - } - } - - frozenDiscussions = rawDiscussions; - verbose(`${prefix} Freezing ${Object.keys(rawDiscussions).length} discussions`); - } - - // Compute stepMaps from latest release to this checkpoint for discussion anchor mapping. - // If we already have stored stepMaps (from a previous cold storage), compose them - // with the new Firebase-only changes rather than trying to fetch wiped history. - let stepMaps: number[][] | null = null; - let stepMapToKey: number | null = null; - const existingPgCheckpoint = await getDraftCheckpoint(draftId); - const latestRelease = await Release.findOne({ - where: { pubId }, - attributes: ['historyKey'], - order: [['historyKey', 'DESC']], - }); - - if (latestRelease && latestRelease.historyKey < currentKey) { - try { - // Start from existing stored stepMaps if available (covers wiped range) - const existingMaps = existingPgCheckpoint?.stepMaps ?? []; - const existingToKey = existingPgCheckpoint?.stepMapToKey ?? null; - - // Determine what range of new Firebase steps we need - const newStepsStartKey = - existingToKey != null ? existingToKey + 1 : latestRelease.historyKey + 1; - - let newMaps: number[][] = []; - if (newStepsStartKey <= currentKey) { - const stepsByChange = await getStepsInChangeRange( - draftRef, - editorSchema, - newStepsStartKey, - currentKey, - ); - const allSteps = stepsByChange.reduce((a, b) => [...a, ...b], []); - newMaps = allSteps.map((step) => - Array.from((step.getMap() as any).ranges as number[]), - ); - } - - stepMaps = [...existingMaps, ...newMaps]; - stepMapToKey = currentKey; - verbose( - `${prefix} Stored ${stepMaps.length} stepMaps (${existingMaps.length} existing + ${newMaps.length} new, up to key ${currentKey}) from release key ${latestRelease.historyKey}`, - ); - } catch (err: any) { - verbose(`${prefix} Warning: could not compute stepMaps: ${err.message}`); - } - } - - if (isDryRun) { - log( - `${prefix} Would freeze: key=${currentKey} docSize=${formatBytes(docSize)} discussions=${frozenDiscussions ? Object.keys(frozenDiscussions).length : 0} stepMaps=${stepMaps?.length ?? 0} stepMapToKey=${stepMapToKey}`, - ); - stats.draftsFrozen++; - stats.bytesFreed += docSize; - return; - } - - // Store checkpoint in Postgres (upsert) - await sequelize.transaction(async (txn) => { - const existing = await DraftCheckpoint.findOne({ - where: { draftId }, - transaction: txn, - }); - - if (existing) { - await existing.update( - { - historyKey: currentKey, - doc: docJson, - timestamp: currentTimestamp, - discussions: frozenDiscussions, - stepMaps, - stepMapToKey, - }, - { transaction: txn }, - ); - } else { - await DraftCheckpoint.create( - { - draftId, - historyKey: currentKey, - doc: docJson, - timestamp: currentTimestamp, - discussions: frozenDiscussions, - stepMaps, - stepMapToKey, - }, - { transaction: txn }, - ); - } - - // Update latestKeyAt on the Draft - if (currentTimestamp) { - await Draft.update( - { latestKeyAt: new Date(currentTimestamp) }, - { where: { id: draftId }, transaction: txn }, - ); - } - }); - - // Wipe Firebase data - await deleteFirebasePath(firebasePath); - - verbose(`${prefix} Frozen successfully`); - stats.draftsFrozen++; - } catch (err: any) { - log(`${prefix} Error: ${err.message}`); - stats.draftsSkippedError++; - } - - stats.draftsScanned++; -}; - -// --- Main --- - -const main = async () => { - log('Firebase Cold Storage Tool'); - log(`Mode: ${isDryRun ? 'DRY RUN' : 'EXECUTE'}`); - log(`Threshold: ${DAYS_OLD} days old`); - log(`Batch size: ${BATCH_SIZE}`); - log(`Concurrency: ${CONCURRENCY}`); - log(''); - - const cutoffDate = new Date(); - cutoffDate.setDate(cutoffDate.getDate() - DAYS_OLD); - log(`Cutoff date: ${cutoffDate.toISOString()}`); - log(''); - - // Find stale drafts - let draftsWithPubs: { draft: Draft; pubId: string }[]; - - if (specificPubId) { - const pub = await Pub.findOne({ - where: { id: specificPubId }, - include: [{ model: Draft, as: 'draft' }], - }); - if (!pub?.draft) { - log(`No draft found for pub ${specificPubId}`); - process.exit(1); - } - draftsWithPubs = [{ draft: pub.draft, pubId: pub.id }]; - log(`Processing single pub: ${specificPubId}`); - } else { - // First, list what actually exists in Firebase (one shallow REST call). - // This avoids doing tens of thousands of per-draft Firebase checks for - // drafts that have no data (already cold-stored or never had any). - log('Listing Firebase paths with data...'); - const firebaseDraftKeys = await firebaseRest | null>( - 'GET', - 'drafts', - undefined, - { shallow: 'true' }, - ); - const firebaseDraftIds = new Set( - firebaseDraftKeys - ? Object.keys(firebaseDraftKeys).map((k) => k.replace('draft-', '')) - : [], - ); - log(`Found ${firebaseDraftIds.size} drafts with Firebase data`); - - // Find stale drafts that ALSO have Firebase data - const results = await sequelize.query<{ draftId: string; pubId: string }>( - ` - SELECT d.id as "draftId", p.id as "pubId" - FROM "Drafts" d - INNER JOIN "Pubs" p ON p."draftId" = d.id - WHERE (d."latestKeyAt" IS NULL OR d."latestKeyAt" < :cutoff) - ORDER BY d."latestKeyAt" ASC NULLS FIRST - `, - { - replacements: { cutoff: cutoffDate.toISOString() }, - type: QueryTypes.SELECT, - }, - ); - - // Intersect: only process drafts that are stale AND have Firebase data - const filteredResults = results.filter((r) => firebaseDraftIds.has(r.draftId)); - log( - `${results.length} stale drafts total, ${filteredResults.length} with Firebase data to freeze`, - ); - - // Load draft models - const draftIds = filteredResults.map((r) => r.draftId); - const pubIdByDraftId = new Map(filteredResults.map((r) => [r.draftId, r.pubId])); - - const drafts = await Draft.findAll({ - where: { id: { [Op.in]: draftIds } }, - }); - - draftsWithPubs = drafts.map((d) => ({ - draft: d, - pubId: pubIdByDraftId.get(d.id)!, - })); - - log(`Found ${draftsWithPubs.length} stale drafts (older than ${DAYS_OLD} days)`); - } - - log(''); - - // Process in batches - for (let i = 0; i < draftsWithPubs.length; i += BATCH_SIZE) { - const batch = draftsWithPubs.slice(i, i + BATCH_SIZE); - const batchNum = Math.floor(i / BATCH_SIZE) + 1; - const totalBatches = Math.ceil(draftsWithPubs.length / BATCH_SIZE); - - log(`Batch ${batchNum}/${totalBatches} (${batch.length} drafts)`); - - // biome-ignore lint/performance/noAwaitInLoops: batched processing - await runWithConcurrency( - batch.map( - ({ draft, pubId }) => - () => - freezeDraft(draft, pubId), - ), - CONCURRENCY, - ); - - log(` Frozen so far: ${stats.draftsFrozen}, Errors: ${stats.draftsSkippedError}`); - } - - log(''); - log('=== RESULTS ==='); - log(`Drafts scanned: ${stats.draftsScanned}`); - log(`Drafts frozen: ${stats.draftsFrozen}`); - log(`Already cold/empty: ${stats.draftsAlreadyCold + stats.draftsEmpty}`); - log(`Errors: ${stats.draftsSkippedError}`); - if (isDryRun) { - log(`Est. data to free: ${formatBytes(stats.bytesFreed)}`); - } - log(''); - - if (isDryRun) { - log('This was a DRY RUN. Re-run with --execute to apply changes.'); - } - - process.exit(0); -}; - -main().catch((err) => { - console.error(err); - process.exit(1); -}); diff --git a/tools/cron.ts b/tools/cron.ts index 6ed53ebc24..47c8e35341 100644 --- a/tools/cron.ts +++ b/tools/cron.ts @@ -30,10 +30,6 @@ if (process.env.PUBPUB_PRODUCTION === 'true') { timezone: 'UTC', }); - cron.schedule('0 5 * * 6', () => run('Cold Storage', 'tools-prod coldStorage --execute'), { - timezone: 'UTC', - }); // Weekly on Saturday at 5 AM UTC (day before cleanup) - cron.schedule( '0 2 * * 0', () => run('Purge Notifications', 'tools-prod purgeNotifications --execute'), @@ -48,7 +44,7 @@ if (process.env.PUBPUB_PRODUCTION === 'true') { cron.schedule( '0 5 * * 0', - () => run('Firebase Cleanup', 'tools-prod cleanupFirebase --execute'), + () => run('Collab Cleanup', 'tools-prod cleanupCollab --execute'), { timezone: 'UTC', }, diff --git a/tools/index.js b/tools/index.js index 280d24fab0..ea7a7ef32e 100644 --- a/tools/index.js +++ b/tools/index.js @@ -52,8 +52,7 @@ const commandFiles = { branchMaintenance: "./branchMaintenance", bulkimport: "../workers/tasks/import/bulk/cli", checkpointBackfill: "./dashboardMigrations/backfillCheckpoints", - cleanupFirebase: "./cleanupFirebase", - coldStorage: "./coldStorage", + cleanupCollab: "./cleanupCollab", measureFirebaseBreakdown: "./measureFirebaseBreakdown", measureNonCheckpointSize: "./measureNonCheckpointSize", clone: "./clone", From 2c94b4b316936c87e38a31ce528fb6303438685e Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Mon, 22 Jun 2026 18:20:18 +0200 Subject: [PATCH 07/17] chore: lint --- client/components/Editor/Editor.stories.tsx | 38 ++--- .../Editor/plugins/collaborative/cursors.ts | 29 +++- .../Editor/plugins/collaborative/document.ts | 16 +- .../plugins/discussions/discussionsState.ts | 8 +- .../Editor/plugins/discussions/plugin.ts | 7 +- client/components/Editor/utils/index.ts | 7 +- client/components/Editor/utils/view.ts | 3 +- .../FormattingBar/FormattingBarButton.tsx | 140 +++++++++--------- client/containers/Pub/PubContextProvider.tsx | 2 +- client/containers/Pub/PubDocument/PubBody.tsx | 2 +- client/containers/Pub/usePubCollabState.ts | 11 +- server/collab/authority.ts | 1 + server/collab/discussionPositions.ts | 11 +- server/collab/featureFlag.ts | 4 +- server/discussion/utils.ts | 8 +- server/models.ts | 2 +- server/release/queries.ts | 1 + server/utils/queryHelpers/pubEnrich.ts | 5 +- tools/cleanupCollab.ts | 8 +- tools/cron.ts | 10 +- tools/migrateFirebaseToPostgres.ts | 7 +- 21 files changed, 172 insertions(+), 148 deletions(-) diff --git a/client/components/Editor/Editor.stories.tsx b/client/components/Editor/Editor.stories.tsx index 51295c331e..305d85a9b3 100644 --- a/client/components/Editor/Editor.stories.tsx +++ b/client/components/Editor/Editor.stories.tsx @@ -145,19 +145,19 @@ storiesOf('Editor', module) onChange={(evt) => { updatechangeObject(evt); }} - collaborativeOptions={{ - pubId: 'storybook-pub-id', - clientData, - initialDocKey: -1, - onStatusChange: (status) => console.info('collab status is', status), - }} - /> - - ); - }; - return ; -}) -.add('collaborative2', () => { + collaborativeOptions={{ + pubId: 'storybook-pub-id', + clientData, + initialDocKey: -1, + onStatusChange: (status) => console.info('collab status is', status), + }} + /> + + ); + }; + return ; + }) + .add('collaborative2', () => { const Thing = () => { const [changeObject, _updatechangeObject] = useState({}); return ( @@ -219,12 +219,12 @@ storiesOf('Editor', module) }, 1000); } }} - collaborativeOptions={{ - pubId: 'storybook-pub-id', - clientData, - initialDocKey: -1, - onStatusChange: (status) => console.info('collab status is', status), - }} + collaborativeOptions={{ + pubId: 'storybook-pub-id', + clientData, + initialDocKey: -1, + onStatusChange: (status) => console.info('collab status is', status), + }} /> ); diff --git a/client/components/Editor/plugins/collaborative/cursors.ts b/client/components/Editor/plugins/collaborative/cursors.ts index b79d71250d..c88f5a7343 100644 --- a/client/components/Editor/plugins/collaborative/cursors.ts +++ b/client/components/Editor/plugins/collaborative/cursors.ts @@ -12,9 +12,10 @@ const generateCursorDecorations = (cursorData: any, editorState: any, localClien let selection: Selection; try { // handle both compressed (legacy) and uncompressed selection formats - const selJSON = cursorData.selection?.a !== undefined - ? uncompressSelectionJSON(cursorData.selection) - : cursorData.selection; + const selJSON = + cursorData.selection?.a !== undefined + ? uncompressSelectionJSON(cursorData.selection) + : cursorData.selection; selection = Selection.fromJSON(editorState.doc, selJSON); } catch (_err) { @@ -109,7 +110,7 @@ const generateCursorDecorations = (cursorData: any, editorState: any, localClien export default (schema: any, props: any, collabDocPluginKey: PluginKey) => { let abortController: AbortController | null = null; - let currentIndicators: Map = new Map(); + const currentIndicators: Map = new Map(); return new Plugin({ key: cursorsPluginKey, @@ -119,7 +120,12 @@ export default (schema: any, props: any, collabDocPluginKey: PluginKey) => { cursorDecorations: DecorationSet.create(editorState.doc, []), }; }, - apply: (transaction: any, pluginState: any, _prevEditorState: any, editorState: any) => { + apply: ( + transaction: any, + pluginState: any, + _prevEditorState: any, + editorState: any, + ) => { if (props.isReadOnly) { return pluginState; } @@ -190,12 +196,13 @@ export default (schema: any, props: any, collabDocPluginKey: PluginKey) => { }; const pollPresence = async () => { - let refs: Record = {}; + const refs: Record = {}; const MIN_POLL_INTERVAL = 1000; while (polling && !abortController!.signal.aborted) { const pollStart = Date.now(); try { + // biome-ignore lint/performance/noAwaitInLoops: shh const response = await fetch(`/api/pubs/${pubId}/presence`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, @@ -222,8 +229,14 @@ export default (schema: any, props: any, collabDocPluginKey: PluginKey) => { continue; } if (userId) { - for (const [existingClientId, existing] of currentIndicators) { - if (existing.id === userId && existingClientId !== clientId) { + for (const [ + existingClientId, + existing, + ] of currentIndicators) { + if ( + existing.id === userId && + existingClientId !== clientId + ) { currentIndicators.delete(existingClientId); delete refs[existingClientId]; } diff --git a/client/components/Editor/plugins/collaborative/document.ts b/client/components/Editor/plugins/collaborative/document.ts index fc4622b427..e39b43bd3a 100644 --- a/client/components/Editor/plugins/collaborative/document.ts +++ b/client/components/Editor/plugins/collaborative/document.ts @@ -6,8 +6,8 @@ import type { PluginsOptions } from '../../types'; import { CollabClient, - LongPollListener, collab, + LongPollListener, receiveCommitTransaction, } from '@pitter-patter/collab-client'; import { Plugin, type PluginKey } from 'prosemirror-state'; @@ -94,14 +94,12 @@ export default ( collabClient = new CollabClient(collabConfig); abortController = new AbortController(); - collabClient - .listen(initialState, abortController.signal) - .catch((e) => { - if (e.name !== 'AbortError') { - console.error('Collab listener error:', e); - onError(e); - } - }); + collabClient.listen(initialState, abortController.signal).catch((e) => { + if (e.name !== 'AbortError') { + console.error('Collab listener error:', e); + onError(e); + } + }); onStatusChange('connected'); }; diff --git a/client/components/Editor/plugins/discussions/discussionsState.ts b/client/components/Editor/plugins/discussions/discussionsState.ts index 800598c991..eda36e15d1 100644 --- a/client/components/Editor/plugins/discussions/discussionsState.ts +++ b/client/components/Editor/plugins/discussions/discussionsState.ts @@ -61,9 +61,7 @@ const filterDiscussionsUpdate = ( Object.entries(update).forEach(([id, next]) => { if (next) { if (next.currentKey <= currentKey) { - const adjusted = next.currentKey < currentKey - ? { ...next, currentKey } - : next; + const adjusted = next.currentKey < currentKey ? { ...next, currentKey } : next; const previous = discussions[id]; const hasKeyAdvanced = !previous || previous.currentKey < adjusted.currentKey; const isKeyMonotonic = !previous || previous.currentKey <= adjusted.currentKey; @@ -206,9 +204,7 @@ export const createDiscussionsState = (options: Options) => { const update = sanitizeRemoteDiscussions(rawUpdate); if (Object.keys(update).length === 0) return; - const newIds = Object.keys(update).filter( - (id) => update[id] && !discussions[id], - ); + const newIds = Object.keys(update).filter((id) => update[id] && !discussions[id]); if (newIds.length > 0) { onNewDiscussionIds?.(newIds); } diff --git a/client/components/Editor/plugins/discussions/plugin.ts b/client/components/Editor/plugins/discussions/plugin.ts index aa6daaccf5..84073ad03b 100644 --- a/client/components/Editor/plugins/discussions/plugin.ts +++ b/client/components/Editor/plugins/discussions/plugin.ts @@ -105,7 +105,12 @@ const createPlugin = (discussionsOptions: DiscussionsOptions, initialDoc: Node) }; }; - const apply = (tr: Transaction, pluginState: PluginState, _oldState: EditorState, newState: EditorState) => { + const apply = ( + tr: Transaction, + pluginState: PluginState, + _oldState: EditorState, + newState: EditorState, + ) => { const updateResult = getUpdateResult(tr, newState); if (updateResult) { return { diff --git a/client/components/Editor/utils/index.ts b/client/components/Editor/utils/index.ts index 41b317193e..34cf0794ec 100644 --- a/client/components/Editor/utils/index.ts +++ b/client/components/Editor/utils/index.ts @@ -1,5 +1,8 @@ export * from './changes'; export * from './doc'; +// legacy firebase exports -- only used by migration tools, not the app bundle +export { createFirebaseChange, flattenKeyables, storeCheckpoint } from './firebase'; +export { getFirebaseDoc, getFirstKeyAndTimestamp, getLatestKeyAndTimestamp } from './firebaseDoc'; export * from './media'; export * from './misc'; export * from './nodes'; @@ -10,7 +13,3 @@ export * from './renderStatic'; export * from './schema'; export * from './selection'; export * from './view'; - -// legacy firebase exports -- only used by migration tools, not the app bundle -export { storeCheckpoint, flattenKeyables, createFirebaseChange } from './firebase'; -export { getFirebaseDoc, getFirstKeyAndTimestamp, getLatestKeyAndTimestamp } from './firebaseDoc'; diff --git a/client/components/Editor/utils/view.ts b/client/components/Editor/utils/view.ts index ac45f25353..5a88337864 100644 --- a/client/components/Editor/utils/view.ts +++ b/client/components/Editor/utils/view.ts @@ -2,11 +2,10 @@ import type { EditorView } from 'prosemirror-view'; import type { DocJson } from 'types'; +import { getVersion } from '@pitter-patter/collab-client'; import { Node, Slice } from 'prosemirror-model'; import { type EditorState, Selection } from 'prosemirror-state'; -import { getVersion } from '@pitter-patter/collab-client'; - import { addDiscussionToView } from '../plugins/discussions'; import { editorHasPasteDecorations } from '../plugins/paste/plugin'; import { isEmptyDocNode } from './doc'; diff --git a/client/components/FormattingBar/FormattingBarButton.tsx b/client/components/FormattingBar/FormattingBarButton.tsx index 6d4188e2e1..b91335fd8f 100644 --- a/client/components/FormattingBar/FormattingBarButton.tsx +++ b/client/components/FormattingBar/FormattingBarButton.tsx @@ -55,78 +55,82 @@ const getIndicatorStyle = (accentColor) => { const popoverModifiers = { preventOverflow: { enabled: false }, flip: { enabled: false } }; -const FormattingBarButton = React.forwardRef((props: FormattingBarButtonProps, ref: React.ForwardedRef) => { - const { - disabled = false, - formattingItem, - isActive = false, - isIndicated = false, - isDetached = false, - isOpen = false, - isSmall = false, - label = null, - onClick, - accentColor = 'white', - outerRef, - popoverContent, - ...restProps - } = props; +const FormattingBarButton = React.forwardRef( + (props: FormattingBarButtonProps, ref: React.ForwardedRef) => { + const { + disabled = false, + formattingItem, + isActive = false, + isIndicated = false, + isDetached = false, + isOpen = false, + isSmall = false, + label = null, + onClick, + accentColor = 'white', + outerRef, + popoverContent, + ...restProps + } = props; - let button = ( - - ); + let button = ( + + ); + + if (popoverContent) { + button = ( + + {button} + + ); + } - if (popoverContent) { - button = ( - {button} - + {isIndicated && ( +
+ )} + ); - } - - return ( - - {button} - {isIndicated &&
} - - ); -}); + }, +); export default FormattingBarButton; diff --git a/client/containers/Pub/PubContextProvider.tsx b/client/containers/Pub/PubContextProvider.tsx index bc831213ba..c667e7561d 100644 --- a/client/containers/Pub/PubContextProvider.tsx +++ b/client/containers/Pub/PubContextProvider.tsx @@ -61,7 +61,7 @@ export const ImmediatePubContext = React.createContext(shimPubCo export const PubContextProvider = (props: Props) => { const { children, pubData: initialPubData } = props; const [pubData, updatePubData] = useIdlyUpdatedState(initialPubData); - const [collabData, updateCollabData] = usePubCollabState({ pubData }); + const [collabData, updateCollabData] = usePubCollabState(); const historyData = usePubHistoryState({ pubData, editorView: collabData.editorChangeObject?.view ?? null, diff --git a/client/containers/Pub/PubDocument/PubBody.tsx b/client/containers/Pub/PubDocument/PubBody.tsx index 5441f0a788..2017ae5c3c 100644 --- a/client/containers/Pub/PubDocument/PubBody.tsx +++ b/client/containers/Pub/PubDocument/PubBody.tsx @@ -6,9 +6,9 @@ import * as Sentry from '@sentry/react'; import { useBeforeUnload } from 'react-use'; import { useDebouncedCallback } from 'use-debounce/lib'; -import { apiFetch } from 'client/utils/apiFetch'; import malformedDocPlugin from 'client/components/Editor/plugins/malformedDoc'; import buildSuggestedEdits from 'client/components/Editor/plugins/suggestedEdits'; +import { apiFetch } from 'client/utils/apiFetch'; import { useFacetsQuery } from 'client/utils/useFacets'; import { Editor } from 'components'; import discussionSchema from 'components/Editor/schemas/discussion'; diff --git a/client/containers/Pub/usePubCollabState.ts b/client/containers/Pub/usePubCollabState.ts index 91c19dbcd8..fd3c919672 100644 --- a/client/containers/Pub/usePubCollabState.ts +++ b/client/containers/Pub/usePubCollabState.ts @@ -1,7 +1,5 @@ import type { EditorChangeObject } from 'components/Editor'; -import type { LoginData, Maybe, PubPageData } from 'types'; - -import { useCallback } from 'react'; +import type { LoginData, Maybe } from 'types'; import { useIdlyUpdatedState } from 'client/utils/useIdlyUpdatedState'; import { getRandomColor } from 'utils/colors'; @@ -26,10 +24,6 @@ export type PubCollabState = { remoteCollabUsers: CollabUser[]; }; -type Options = { - pubData: PubPageData; -}; - const getLocalCollabUser = (canEdit: boolean, loginData: LoginData) => { const userColor = getRandomColor(loginData.id); return { @@ -43,8 +37,7 @@ const getLocalCollabUser = (canEdit: boolean, loginData: LoginData) => { }; }; -export const usePubCollabState = (options: Options) => { - const { pubData } = options; +export const usePubCollabState = () => { const { loginData, scopeData: { diff --git a/server/collab/authority.ts b/server/collab/authority.ts index a94b928c02..c9b6dad52f 100644 --- a/server/collab/authority.ts +++ b/server/collab/authority.ts @@ -1,4 +1,5 @@ import type { Transaction } from 'sequelize'; + import type { DocJson } from 'types'; import { CollabAuthority, RedisBroadcastManager } from '@pitter-patter/collab-server'; diff --git a/server/collab/discussionPositions.ts b/server/collab/discussionPositions.ts index e1932cf022..d1957d79a4 100644 --- a/server/collab/discussionPositions.ts +++ b/server/collab/discussionPositions.ts @@ -1,9 +1,16 @@ import { Router } from 'express'; import { Op } from 'sequelize'; -import { Commenter, Discussion, DiscussionAnchor, Draft, DraftCheckpoint, Pub } from 'server/models'; -import { wrap } from 'server/wrap'; +import { + Commenter, + Discussion, + DiscussionAnchor, + Draft, + DraftCheckpoint, + Pub, +} from 'server/models'; import { authorIncludes, baseVisibility, threadIncludes } from 'server/utils/queryHelpers/util'; +import { wrap } from 'server/wrap'; export const router = Router(); diff --git a/server/collab/featureFlag.ts b/server/collab/featureFlag.ts index 227511ca99..919cfae022 100644 --- a/server/collab/featureFlag.ts +++ b/server/collab/featureFlag.ts @@ -26,9 +26,7 @@ export const isPitterPatterEnabled = async (communityId: string): Promise fc.communityId === communityId, - ); + const communityOptedIn = flag.communities?.some((fc) => fc.communityId === communityId); if (communityOptedIn) { return true; diff --git a/server/discussion/utils.ts b/server/discussion/utils.ts index 2a6e19e7c1..54ef38087b 100644 --- a/server/discussion/utils.ts +++ b/server/discussion/utils.ts @@ -1,10 +1,10 @@ +import type { Step } from 'prosemirror-transform'; + import type { DiscussionInfo } from 'components/Editor/plugins/discussions/types'; import type * as types from 'types'; -import { Step } from 'prosemirror-transform'; - -import { editorSchema, jsonToNode } from 'client/components/Editor/utils'; import { mapDiscussionThroughSteps } from 'client/components/Editor/plugins/discussions/util'; +import { editorSchema, jsonToNode } from 'client/components/Editor/utils'; import { createDiscussionAnchor } from 'server/discussionAnchor/queries'; import { Discussion, DiscussionAnchor, Doc, Release } from 'server/models'; import { getPubDraft, getStepsBetweenVersions } from 'server/utils/firebaseAdmin'; @@ -76,7 +76,7 @@ export const createDiscussionAnchorsForLatestRelease = async ( pubId: string, discussionIds: string[], ) => { - const { doc, historyKey } = await getLatestReleaseInfo(pubId); + const { historyKey } = await getLatestReleaseInfo(pubId); const { draft } = await getPubDraft(pubId); const discussions = await getDiscussions(discussionIds, pubId); diff --git a/server/models.ts b/server/models.ts index 8b7eb3d9f8..dc657222a2 100644 --- a/server/models.ts +++ b/server/models.ts @@ -6,6 +6,7 @@ import { ActivityItem } from './activityItem/model'; import { AnalyticsEvent } from './analytics/model'; import { AnalyticsCloudflareCache } from './analyticsCloudflareCache/model'; import { AuthToken } from './authToken/model'; +import { CollabCommit } from './collabCommit/model'; import { Collection } from './collection/model'; import { CollectionAttribution } from './collectionAttribution/model'; import { CollectionPub } from './collectionPub/model'; @@ -19,7 +20,6 @@ import { DepositTarget } from './depositTarget/model'; import { Discussion } from './discussion/model'; import { DiscussionAnchor } from './discussionAnchor/model'; import { Doc } from './doc/model'; -import { CollabCommit } from './collabCommit/model'; import { Draft } from './draft/model'; import { DraftCheckpoint } from './draftCheckpoint/model'; import { EmailChangeToken } from './emailChangeToken/model'; diff --git a/server/release/queries.ts b/server/release/queries.ts index f76e6ef707..83eafde878 100644 --- a/server/release/queries.ts +++ b/server/release/queries.ts @@ -16,6 +16,7 @@ import { getPubDraft, getPubDraftDoc, getStepsBetweenVersions } from 'server/uti type ReleaseErrorReason = 'merge-failed' | 'duplicate-release'; export class ReleaseQueryError extends Error { + // biome-ignore lint/complexity/noUselessConstructor: types constructor(reason: ReleaseErrorReason) { super(reason); } diff --git a/server/utils/queryHelpers/pubEnrich.ts b/server/utils/queryHelpers/pubEnrich.ts index f0d64c7a82..77d50dc752 100644 --- a/server/utils/queryHelpers/pubEnrich.ts +++ b/server/utils/queryHelpers/pubEnrich.ts @@ -67,7 +67,10 @@ export const getPubRelease = async ( * @deprecated Firebase token is no longer needed. Collab uses Pitter Patter with Postgres. * Kept as a stub for any callers that haven't been updated yet. */ -export const getPubFirebaseToken = async (_pubData: SanitizedPubData, _initialData: InitialData) => { +export const getPubFirebaseToken = async ( + _pubData: SanitizedPubData, + _initialData: InitialData, +) => { return { firebaseToken: null, }; diff --git a/tools/cleanupCollab.ts b/tools/cleanupCollab.ts index 6d5c86db04..f8a91223a2 100644 --- a/tools/cleanupCollab.ts +++ b/tools/cleanupCollab.ts @@ -109,7 +109,13 @@ const checkpointStaleDrafts = async () => { if (!isDryRun) { await sequelize.transaction(async (tr) => { - await upsertDraftCheckpoint(draftId, version, reconstructed.toJSON() as any, Date.now(), tr); + await upsertDraftCheckpoint( + draftId, + version, + reconstructed.toJSON() as any, + Date.now(), + tr, + ); await CollabCommit.destroy({ where: { draftId }, diff --git a/tools/cron.ts b/tools/cron.ts index 47c8e35341..92e6ea9eff 100644 --- a/tools/cron.ts +++ b/tools/cron.ts @@ -42,13 +42,9 @@ if (process.env.PUBPUB_PRODUCTION === 'true') { timezone: 'UTC', }); // Weekly on Sunday at 3 AM UTC - cron.schedule( - '0 5 * * 0', - () => run('Collab Cleanup', 'tools-prod cleanupCollab --execute'), - { - timezone: 'UTC', - }, - ); // Weekly on Sunday at 5 AM UTC + cron.schedule('0 5 * * 0', () => run('Collab Cleanup', 'tools-prod cleanupCollab --execute'), { + timezone: 'UTC', + }); // Weekly on Sunday at 5 AM UTC cron.schedule( '30 3 * * *', diff --git a/tools/migrateFirebaseToPostgres.ts b/tools/migrateFirebaseToPostgres.ts index f1fffda33e..c4e46b05b4 100644 --- a/tools/migrateFirebaseToPostgres.ts +++ b/tools/migrateFirebaseToPostgres.ts @@ -197,6 +197,7 @@ const migrateDraft = async (draft: Draft, firebaseApp: firebaseAdmin.app.App) => uncompressStepJSON(compressed), ); + // biome-ignore lint/performance/noAwaitInLoops: shh await CollabCommit.create( { draftId, @@ -243,6 +244,7 @@ const main = async () => { let offset = 0; while (true) { + // biome-ignore lint/performance/noAwaitInLoops: shh const batch = await getDraftBatch(offset); if (batch.length === 0) { @@ -251,9 +253,12 @@ const main = async () => { const batchNum = Math.floor(offset / BATCH_SIZE) + 1; const totalBatches = Math.ceil(totalDrafts / BATCH_SIZE); - log(`Processing batch ${batchNum}/${totalBatches} (${batch.length} drafts, offset ${offset})`); + log( + `Processing batch ${batchNum}/${totalBatches} (${batch.length} drafts, offset ${offset})`, + ); for (const draft of batch) { + // biome-ignore lint/performance/noAwaitInLoops: shh const result = await migrateDraft(draft, firebaseApp); if (result.skipped) { From 833093e5bba8a3881ef9ff90a8d785a5c6bb8d21 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Tue, 23 Jun 2026 13:56:09 +0200 Subject: [PATCH 08/17] fix: optimize migration --- server/collab/featureFlag.ts | 41 -------- tools/migrateFirebaseToPostgres.ts | 161 +++++++++++++++++------------ utils/caching/purgeMiddleware.ts | 5 +- 3 files changed, 98 insertions(+), 109 deletions(-) delete mode 100644 server/collab/featureFlag.ts diff --git a/server/collab/featureFlag.ts b/server/collab/featureFlag.ts deleted file mode 100644 index 919cfae022..0000000000 --- a/server/collab/featureFlag.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { FeatureFlag, FeatureFlagCommunity } from 'server/models'; - -const PITTER_PATTER_FLAG_NAME = 'pitterPatterCollab'; - -/** - * Check if Pitter Patter collab is enabled for a given community. - * Uses the existing FeatureFlag system for gradual rollout. - * - * During migration: - * - Create a FeatureFlag named "pitterPatterCollab" - * - Add specific communities via FeatureFlagCommunity to opt them in - * - Or set enabledCommunitiesFraction to 1.0 to enable globally - * - * When the flag does not exist, Pitter Patter is assumed to be enabled - * (post-migration default). - */ -export const isPitterPatterEnabled = async (communityId: string): Promise => { - const flag = await FeatureFlag.findOne({ - where: { name: PITTER_PATTER_FLAG_NAME }, - include: [{ model: FeatureFlagCommunity, as: 'communities' }], - }); - - // if no flag exists, the migration is complete and everyone uses Pitter Patter - if (!flag) { - return true; - } - - // check if the community is explicitly opted in - const communityOptedIn = flag.communities?.some((fc) => fc.communityId === communityId); - - if (communityOptedIn) { - return true; - } - - // check fraction-based rollout - if (flag.enabledCommunitiesFraction && flag.enabledCommunitiesFraction >= 1.0) { - return true; - } - - return false; -}; diff --git a/tools/migrateFirebaseToPostgres.ts b/tools/migrateFirebaseToPostgres.ts index c4e46b05b4..ecfd4460f2 100644 --- a/tools/migrateFirebaseToPostgres.ts +++ b/tools/migrateFirebaseToPostgres.ts @@ -17,6 +17,7 @@ * pnpm run tools migrateFirebaseToPostgres --execute # Actually migrate * pnpm run tools migrateFirebaseToPostgres --pubId= # Single pub * pnpm run tools migrateFirebaseToPostgres --batchSize=100 # Custom batch size + * pnpm run tools migrateFirebaseToPostgres --concurrency=20 # Parallel drafts * pnpm run tools migrateFirebaseToPostgres --verbose # Verbose output */ @@ -32,11 +33,33 @@ import { sequelize } from 'server/sequelize'; import { getFirebaseConfig } from 'utils/editor/firebaseConfig'; const { - argv: { execute, pubId: specificPubId, batchSize: batchSizeArg = 50, verbose: verboseFlag }, + argv: { + execute, + pubId: specificPubId, + batchSize: batchSizeArg = 50, + concurrency: concurrencyArg = 20, + verbose: verboseFlag, + }, } = require('yargs'); const isDryRun = !execute; const BATCH_SIZE = Number(batchSizeArg); +const CONCURRENCY = Number(concurrencyArg); + +const runWithConcurrency = async (tasks: (() => Promise)[], limit: number) => { + let i = 0; + + const run = async () => { + while (i < tasks.length) { + const idx = i++; + // biome-ignore lint/performance/noAwaitInLoops: concurrency pool worker + await tasks[idx](); + } + }; + + const workers = Array.from({ length: Math.min(limit, tasks.length) }, () => run()); + await Promise.all(workers); +}; const log = (msg: string) => console.log(`[migrate] ${new Date().toISOString()} ${msg}`); const verbose = (msg: string) => verboseFlag && log(msg); @@ -107,12 +130,6 @@ const migrateDraft = async (draft: Draft, firebaseApp: firebaseAdmin.app.App) => return { skipped: true }; } - const existingCommitCount = await CollabCommit.count({ where: { draftId } }); - if (existingCommitCount > 0) { - verbose(` [${draftId.slice(0, 8)}] already has ${existingCommitCount} commits, skipping`); - return { skipped: true }; - } - const database = firebaseApp.database(); const ref = database.ref(firebasePath) as any; @@ -154,17 +171,10 @@ const migrateDraft = async (draft: Draft, firebaseApp: firebaseAdmin.app.App) => const checkpointKey = checkpoint.historyKey; try { - const changesSnapshot = await ref - .child('changes') - .orderByKey() - .startAt(String(checkpointKey + 1)) - .once('value'); - - const mergesSnapshot = await ref - .child('merges') - .orderByKey() - .startAt(String(checkpointKey + 1)) - .once('value'); + const [changesSnapshot, mergesSnapshot] = await Promise.all([ + ref.child('changes').orderByKey().startAt(String(checkpointKey + 1)).once('value'), + ref.child('merges').orderByKey().startAt(String(checkpointKey + 1)).once('value'), + ]); const allKeyables = { ...(changesSnapshot.val() || {}), @@ -186,29 +196,24 @@ const migrateDraft = async (draft: Draft, firebaseApp: firebaseAdmin.app.App) => return { wouldMigrate: true, commits: keys.length }; } - await sequelize.transaction(async (txn) => { - for (const key of keys) { - const keyNum = parseInt(key, 10); - const changeData = allKeyables[key]; - const changes = Array.isArray(changeData) ? changeData : [changeData]; - - for (const change of changes) { - const stepsJson = change.s.map((compressed: any) => - uncompressStepJSON(compressed), - ); - - // biome-ignore lint/performance/noAwaitInLoops: shh - await CollabCommit.create( - { - draftId, - version: keyNum, - ref: uuid(), - steps: stepsJson, - }, - { transaction: txn }, - ); - } + const rows: { draftId: string; version: number; ref: string; steps: any[] }[] = []; + + for (const key of keys) { + const keyNum = parseInt(key, 10); + const changeData = allKeyables[key]; + const changes = Array.isArray(changeData) ? changeData : [changeData]; + + for (const change of changes) { + const stepsJson = change.s.map((compressed: any) => + uncompressStepJSON(compressed), + ); + + rows.push({ draftId, version: keyNum, ref: uuid(), steps: stepsJson }); } + } + + await sequelize.transaction(async (txn) => { + await CollabCommit.bulkCreate(rows, { transaction: txn }); const latestVersion = parseInt(keys[keys.length - 1], 10); await Draft.update( @@ -229,7 +234,7 @@ const main = async () => { const startTime = Date.now(); log(isDryRun ? 'DRY RUN (pass --execute to apply)' : 'EXECUTING migration'); - log(`Batch size: ${BATCH_SIZE}`); + log(`Batch size: ${BATCH_SIZE}, concurrency: ${CONCURRENCY}`); const firebaseApp = getFirebaseApp(); const totalDrafts = await getTotalDraftCount(); @@ -244,7 +249,7 @@ const main = async () => { let offset = 0; while (true) { - // biome-ignore lint/performance/noAwaitInLoops: shh + // biome-ignore lint/performance/noAwaitInLoops: outer batch loop const batch = await getDraftBatch(offset); if (batch.length === 0) { @@ -257,38 +262,60 @@ const main = async () => { `Processing batch ${batchNum}/${totalBatches} (${batch.length} drafts, offset ${offset})`, ); - for (const draft of batch) { - // biome-ignore lint/performance/noAwaitInLoops: shh - const result = await migrateDraft(draft, firebaseApp); - - if (result.skipped) { - skipped++; - } else if (result.error) { - errors++; - } else { - migrated++; - totalCommits += (result as any).commits ?? 0; - } + // pre-filter: find which drafts in this batch already have commits + const batchIds = batch.map((d) => d.id); - processed++; + const alreadyMigrated = await sequelize.query<{ draftId: string }>( + `SELECT DISTINCT "draftId" FROM "CollabCommits" WHERE "draftId" IN (:ids)`, + { replacements: { ids: batchIds }, type: QueryTypes.SELECT }, + ); - if (processed % 100 === 0) { - const elapsed = Date.now() - startTime; - const rate = processed / (elapsed / 1000); - const remaining = totalDrafts - processed; - const eta = remaining / rate; + const migratedSet = new Set(alreadyMigrated.map((r) => r.draftId)); + const toMigrate = batch.filter((d) => !migratedSet.has(d.id)); + const batchSkipped = batch.length - toMigrate.length; - log( - ` Progress: ${processed}/${totalDrafts} (${Math.round((processed / totalDrafts) * 100)}%) ` + - `| migrated=${migrated} skipped=${skipped} errors=${errors} ` + - `| ${rate.toFixed(1)} drafts/sec, ETA ${formatDuration(eta * 1000)}`, - ); - } + skipped += batchSkipped; + processed += batchSkipped; + + if (batchSkipped > 0) { + verbose(` Skipped ${batchSkipped} already-migrated drafts`); } + await runWithConcurrency( + toMigrate.map( + (draft) => async () => { + const result = await migrateDraft(draft, firebaseApp); + + if (result.skipped) { + skipped++; + } else if (result.error) { + errors++; + } else { + migrated++; + totalCommits += (result as any).commits ?? 0; + } + + processed++; + + if (processed % 100 === 0) { + const elapsed = Date.now() - startTime; + const rate = processed / (elapsed / 1000); + const remaining = totalDrafts - processed; + const eta = remaining / rate; + + log( + ` Progress: ${processed}/${totalDrafts} (${Math.round((processed / totalDrafts) * 100)}%) ` + + `| migrated=${migrated} skipped=${skipped} errors=${errors} ` + + `| ${rate.toFixed(1)} drafts/sec, ETA ${formatDuration(eta * 1000)}`, + ); + } + }, + ), + CONCURRENCY, + ); + offset += batch.length; - // for a single pub, one iteration is enough if (specificPubId) { break; } diff --git a/utils/caching/purgeMiddleware.ts b/utils/caching/purgeMiddleware.ts index dfc16e22e3..97b871fa17 100644 --- a/utils/caching/purgeMiddleware.ts +++ b/utils/caching/purgeMiddleware.ts @@ -31,6 +31,8 @@ const nonPurgeNonGetRoutes = { '/api/spamTags/user': ['DELETE', 'PUT'], '/api/spamTags/queryCommunitiesForSpam': ['POST'], '/api/spamTags/userRecentDiscussions': ['POST'], + '/api/ev': ['POST'], + '/api/collabCommits': ['POST'], } satisfies { [Path in `/api/${string}`]: AllowedMethods[]; }; @@ -47,7 +49,8 @@ const userPaths = [ * These are routes with path parameters that we don't want to purge They aren't easily caught in * the map above */ -const otherNonPurgeRoutes = /\/api\/(pubs|collections)\/[^/]+\/doi\/preview/; +const otherNonPurgeRoutes = + /\/api\/(pubs|collections)\/[^/]+\/doi\/preview|\/api\/pubs\/[^/]+\/(presence|commits)/; async function getSurrogateTag(req: Request) { /** We don't want to purge GET/CORS/HEAD etc requests, that's the whole point! */ From 956b116ff485e8e430519be2cf2ec84f098d54ce Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Tue, 23 Jun 2026 14:11:40 +0200 Subject: [PATCH 09/17] fix: optimize endpoints --- server/collab/api.ts | 18 +++++++++++++++-- server/collab/authority.ts | 30 +++++++++++++++++++++++----- server/collab/discussionPositions.ts | 24 ++++++++-------------- 3 files changed, 49 insertions(+), 23 deletions(-) diff --git a/server/collab/api.ts b/server/collab/api.ts index aab7630f39..b2726988ed 100644 --- a/server/collab/api.ts +++ b/server/collab/api.ts @@ -12,14 +12,28 @@ import { getPresenceAuthority } from './presence'; export const router = Router(); -const getDraftIdForPub = async (pubId: string): Promise => { +const draftIdCache = new Map(); + +export const getDraftIdForPub = async (pubId: string): Promise => { + const cached = draftIdCache.get(pubId); + + if (cached) { + return cached; + } + const pub = await Pub.findOne({ where: { id: pubId }, include: [{ model: Draft, as: 'draft' }], attributes: ['id'], }); - return pub?.draft?.id ?? null; + const draftId = pub?.draft?.id ?? null; + + if (draftId) { + draftIdCache.set(pubId, draftId); + } + + return draftId; }; // receive a commit from a client diff --git a/server/collab/authority.ts b/server/collab/authority.ts index c9b6dad52f..e0a93591e7 100644 --- a/server/collab/authority.ts +++ b/server/collab/authority.ts @@ -20,6 +20,13 @@ let authority: CollabAuthority | null = null; const CHECKPOINT_INTERVAL = 50; +interface CachedCheckpoint { + historyKey: number; + doc: Record; +} + +const checkpointCache = new Map(); + const createAuthority = (bm: RedisBroadcastManager) => new CollabAuthority({ schema: editorSchema, @@ -32,6 +39,8 @@ const createAuthority = (bm: RedisBroadcastManager) => getDoc: async (tr, docId) => { const logger = createLogger('getDoc'); + const cached = checkpointCache.get(docId); + const [draft, checkpoint] = await logger.log( 'getDocAndCheckpoint', Promise.all([ @@ -40,11 +49,21 @@ const createAuthority = (bm: RedisBroadcastManager) => ...(tr && { lock: tr.LOCK.NO_KEY_UPDATE }), transaction: tr ?? undefined, }), - DraftCheckpoint.findOne({ - where: { draftId: docId }, - order: [['historyKey', 'DESC']], - transaction: tr ?? undefined, - }), + cached + ? Promise.resolve(cached) + : DraftCheckpoint.findOne({ + where: { draftId: docId }, + order: [['historyKey', 'DESC']], + transaction: tr ?? undefined, + }).then((cp) => { + if (cp) { + const entry = { historyKey: cp.historyKey, doc: cp.doc }; + checkpointCache.set(docId, entry); + return entry; + } + + return null; + }), ]), ); @@ -112,6 +131,7 @@ const createAuthority = (bm: RedisBroadcastManager) => const truncateBelow = version - CHECKPOINT_INTERVAL; await upsertDraftCheckpoint(docId, version, docJSON as DocJson, Date.now(), tr); + checkpointCache.set(docId, { historyKey: version, doc: docJSON as Record }); if (truncateBelow > 0) { await CollabCommit.destroy({ diff --git a/server/collab/discussionPositions.ts b/server/collab/discussionPositions.ts index d1957d79a4..4be2c77444 100644 --- a/server/collab/discussionPositions.ts +++ b/server/collab/discussionPositions.ts @@ -5,13 +5,13 @@ import { Commenter, Discussion, DiscussionAnchor, - Draft, DraftCheckpoint, - Pub, } from 'server/models'; import { authorIncludes, baseVisibility, threadIncludes } from 'server/utils/queryHelpers/util'; import { wrap } from 'server/wrap'; +import { getDraftIdForPub } from './api'; + export const router = Router(); const isValidPositionEntry = (entry: any) => @@ -25,18 +25,14 @@ const isValidPositionEntry = (entry: any) => router.get( '/api/pubs/:pubId/discussions/positions', wrap(async (req, res) => { - const pub = await Pub.findOne({ - where: { id: req.params.pubId }, - include: [{ model: Draft, as: 'draft' }], - attributes: ['id'], - }); + const draftId = await getDraftIdForPub(req.params.pubId); - if (!pub?.draft) { + if (!draftId) { return res.status(404).json({}); } const checkpoint = await DraftCheckpoint.findOne({ - where: { draftId: pub.draft.id }, + where: { draftId }, }); if (!checkpoint?.discussions) { @@ -93,13 +89,9 @@ router.get( router.post( '/api/pubs/:pubId/discussions/positions', wrap(async (req, res) => { - const pub = await Pub.findOne({ - where: { id: req.params.pubId }, - include: [{ model: Draft, as: 'draft' }], - attributes: ['id'], - }); + const draftId = await getDraftIdForPub(req.params.pubId); - if (!pub?.draft) { + if (!draftId) { return res.status(404).json({}); } @@ -110,7 +102,7 @@ router.post( } const checkpoint = await DraftCheckpoint.findOne({ - where: { draftId: pub.draft.id }, + where: { draftId }, }); if (checkpoint) { From a14baca803f170291474fe65d006426149423e5e Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Tue, 23 Jun 2026 15:26:54 +0200 Subject: [PATCH 10/17] fix: patch pitterpatter to debug --- package.json | 6 +++--- patches/@pitter-patter__collab-server.patch | 14 ++++++++++++++ pnpm-lock.yaml | 7 +++++-- 3 files changed, 22 insertions(+), 5 deletions(-) create mode 100644 patches/@pitter-patter__collab-server.patch diff --git a/package.json b/package.json index fe7523e932..d220d9022d 100644 --- a/package.json +++ b/package.json @@ -101,13 +101,13 @@ "@pitter-patter/collab-server": "^0.1.3", "@pitter-patter/presence-client": "^0.2.1", "@pitter-patter/presence-server": "^0.1.3", - "@stepwisehq/prosemirror-collab-commit": "^1.0.0", "@popperjs/core": "^2.11.5", "@pubpub/deposit-utils": "^0.1.10", "@pubpub/prosemirror-pandoc": "^1.1.5", "@pubpub/prosemirror-reactive": "^0.2.0", "@sentry/node": "^7.77.0", "@sentry/react": "^7.77.0", + "@stepwisehq/prosemirror-collab-commit": "^1.0.0", "@ts-rest/core": "^3.30.5", "@ts-rest/express": "^3.30.5", "@ts-rest/open-api": "^3.30.5", @@ -329,7 +329,6 @@ "@types/unidecode": "^0.1.1", "@types/uuid": "^3.4.10", "@typescript/native-preview": "7.0.0-dev.20260616.1", - "@typescript/native-preview-linux-arm64": "7.0.0-dev.20260616.1", "autoprefixer": "^9.5.0", "chalk": "^2.4.2", @@ -399,7 +398,8 @@ "prosemirror-model": "patches/prosemirror-model.patch", "prosemirror-transform": "patches/prosemirror-transform.patch", "prosemirror-state": "patches/prosemirror-state.patch", - "prosemirror-view": "patches/prosemirror-view.patch" + "prosemirror-view": "patches/prosemirror-view.patch", + "@pitter-patter/collab-server": "patches/@pitter-patter__collab-server.patch" } } } diff --git a/patches/@pitter-patter__collab-server.patch b/patches/@pitter-patter__collab-server.patch new file mode 100644 index 0000000000..4e2e438844 --- /dev/null +++ b/patches/@pitter-patter__collab-server.patch @@ -0,0 +1,14 @@ +diff --git a/dist/index.js b/dist/index.js +index 3647c143c60bf5cd28c34ef5687e42856aa65dab..05b206e11102a655e94f941c7b728fa834d01fc8 100644 +--- a/dist/index.js ++++ b/dist/index.js +@@ -41,7 +41,8 @@ export class CollabAuthority { + try { + return await this.runWithTransaction(callback); + } +- catch { ++ catch (e) { ++ console.error("Error in runWithTransactionRetries", e); + retries--; + } + } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5990768cb1..5398dcf3cd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,6 +13,9 @@ overrides: '@pitter-patter/collab-server@0>prosemirror-model': 1.25.4 patchedDependencies: + '@pitter-patter/collab-server': + hash: b1549b8792ceb2d79f51c93e26da54424e3ea417c727e40f326afca76d89cf5c + path: patches/@pitter-patter__collab-server.patch '@pubpub/deposit-utils': hash: 6fef8933046b7751abda1bd2ed0422094c79d7828b07f7d5afba26a47c696d89 path: patches/@pubpub__deposit-utils.patch @@ -170,7 +173,7 @@ importers: version: 0.1.3(prosemirror-model@1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-state@1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-transform@1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8)) '@pitter-patter/collab-server': specifier: ^0.1.3 - version: 0.1.3(prosemirror-model@1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-state@1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-transform@1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8)) + version: 0.1.3(patch_hash=b1549b8792ceb2d79f51c93e26da54424e3ea417c727e40f326afca76d89cf5c)(prosemirror-model@1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-state@1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-transform@1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8)) '@pitter-patter/presence-client': specifier: ^0.2.1 version: 0.2.1(@handlewithcare/react-prosemirror@3.1.6(prosemirror-model@1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-state@1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-view@1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5))(react-dom@16.14.0(react@16.14.0))(react-reconciler@0.32.0(react@16.14.0))(react@16.14.0))(prosemirror-model@1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-state@1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-transform@1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-view@1.41.6(patch_hash=ba6c92b133cd1be07cc7afe47f420e9f46e5da1f0516a281aeae92fd0f98bbf5))(react-dom@16.14.0(react@16.14.0))(react-reconciler@0.32.0(react@16.14.0))(react@16.14.0) @@ -14766,7 +14769,7 @@ snapshots: prosemirror-state: 1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) prosemirror-transform: 1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) - '@pitter-patter/collab-server@0.1.3(prosemirror-model@1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-state@1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-transform@1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))': + '@pitter-patter/collab-server@0.1.3(patch_hash=b1549b8792ceb2d79f51c93e26da54424e3ea417c727e40f326afca76d89cf5c)(prosemirror-model@1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-state@1.4.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))(prosemirror-transform@1.11.0(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8))': dependencies: '@stepwisehq/prosemirror-collab-commit': 1.0.5 prosemirror-model: 1.25.4(patch_hash=8bb57c40eb270df7d5c07dca69a7563c1df164b1c8f92991dafcd88513cbaaf8) From ec0012aacfb087f4cf116f9c8912a5ad9e1f00d9 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Tue, 23 Jun 2026 15:46:49 +0200 Subject: [PATCH 11/17] fix: fix in mem cache issues --- server/collab/authority.ts | 89 ++++++++++++++++++++++++++++---------- 1 file changed, 66 insertions(+), 23 deletions(-) diff --git a/server/collab/authority.ts b/server/collab/authority.ts index e0a93591e7..f73af817b9 100644 --- a/server/collab/authority.ts +++ b/server/collab/authority.ts @@ -41,6 +41,21 @@ const createAuthority = (bm: RedisBroadcastManager) => const cached = checkpointCache.get(docId); + const fetchCheckpointFromDb = () => + DraftCheckpoint.findOne({ + where: { draftId: docId }, + order: [['historyKey', 'DESC']], + transaction: tr ?? undefined, + }).then((cp) => { + if (!cp) { + return null; + } + + const entry = { historyKey: cp.historyKey, doc: cp.doc }; + checkpointCache.set(docId, entry); + return entry; + }); + const [draft, checkpoint] = await logger.log( 'getDocAndCheckpoint', Promise.all([ @@ -49,21 +64,7 @@ const createAuthority = (bm: RedisBroadcastManager) => ...(tr && { lock: tr.LOCK.NO_KEY_UPDATE }), transaction: tr ?? undefined, }), - cached - ? Promise.resolve(cached) - : DraftCheckpoint.findOne({ - where: { draftId: docId }, - order: [['historyKey', 'DESC']], - transaction: tr ?? undefined, - }).then((cp) => { - if (cp) { - const entry = { historyKey: cp.historyKey, doc: cp.doc }; - checkpointCache.set(docId, entry); - return entry; - } - - return null; - }), + cached ? Promise.resolve(cached) : fetchCheckpointFromDb(), ]), ); @@ -96,14 +97,48 @@ const createAuthority = (bm: RedisBroadcastManager) => }), ); - const reconstructedDoc = replayCommitsOntoDoc(checkpoint.doc, missedCommits); - logger.end(); + try { + const reconstructedDoc = replayCommitsOntoDoc(checkpoint.doc, missedCommits); + logger.end(); + + return { + docJSON: reconstructedDoc.toJSON(), + version: draft.version, + lastUpdatedTimestamp: draft.latestKeyAt?.valueOf() ?? Date.now(), + }; + } catch (err) { + // stale cache: checkpoint doc doesn't match the commits in the db. + // refetch the checkpoint from postgres and retry. + if (!cached) { + throw err; + } + + checkpointCache.delete(docId); + const freshCheckpoint = await fetchCheckpointFromDb(); + + if (!freshCheckpoint) { + throw err; + } + + const freshVersion = freshCheckpoint.historyKey ?? 0; + const freshCommits = await CollabCommit.findAll({ + where: { + draftId: docId, + version: { [Op.gt]: freshVersion, [Op.lte]: draft.version }, + }, + order: [['version', 'ASC']], + transaction: tr ?? undefined, + }); + + const reconstructedDoc = replayCommitsOntoDoc(freshCheckpoint.doc, freshCommits); + logger.end(); - return { - docJSON: reconstructedDoc.toJSON(), - version: draft.version, - lastUpdatedTimestamp: draft.latestKeyAt?.valueOf() ?? Date.now(), - }; + return { + docJSON: reconstructedDoc.toJSON(), + version: draft.version, + lastUpdatedTimestamp: draft.latestKeyAt?.valueOf() ?? Date.now(), + }; + } } logger.end(); @@ -131,7 +166,15 @@ const createAuthority = (bm: RedisBroadcastManager) => const truncateBelow = version - CHECKPOINT_INTERVAL; await upsertDraftCheckpoint(docId, version, docJSON as DocJson, Date.now(), tr); - checkpointCache.set(docId, { historyKey: version, doc: docJSON as Record }); + + if (tr) { + tr.afterCommit(() => { + checkpointCache.set(docId, { + historyKey: version, + doc: docJSON as Record, + }); + }); + } if (truncateBelow > 0) { await CollabCommit.destroy({ From 994f4aa4b66cce126ca2b01f4f8f084dae41c8e9 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Tue, 23 Jun 2026 15:47:10 +0200 Subject: [PATCH 12/17] fix: lint --- server/collab/authority.ts | 5 +- server/collab/discussionPositions.ts | 7 +-- tools/migrateFirebaseToPostgres.ts | 72 +++++++++++++++------------- 3 files changed, 43 insertions(+), 41 deletions(-) diff --git a/server/collab/authority.ts b/server/collab/authority.ts index f73af817b9..9deff064f8 100644 --- a/server/collab/authority.ts +++ b/server/collab/authority.ts @@ -130,7 +130,10 @@ const createAuthority = (bm: RedisBroadcastManager) => transaction: tr ?? undefined, }); - const reconstructedDoc = replayCommitsOntoDoc(freshCheckpoint.doc, freshCommits); + const reconstructedDoc = replayCommitsOntoDoc( + freshCheckpoint.doc, + freshCommits, + ); logger.end(); return { diff --git a/server/collab/discussionPositions.ts b/server/collab/discussionPositions.ts index 4be2c77444..35c4336fcf 100644 --- a/server/collab/discussionPositions.ts +++ b/server/collab/discussionPositions.ts @@ -1,12 +1,7 @@ import { Router } from 'express'; import { Op } from 'sequelize'; -import { - Commenter, - Discussion, - DiscussionAnchor, - DraftCheckpoint, -} from 'server/models'; +import { Commenter, Discussion, DiscussionAnchor, DraftCheckpoint } from 'server/models'; import { authorIncludes, baseVisibility, threadIncludes } from 'server/utils/queryHelpers/util'; import { wrap } from 'server/wrap'; diff --git a/tools/migrateFirebaseToPostgres.ts b/tools/migrateFirebaseToPostgres.ts index ecfd4460f2..8b7931a3f5 100644 --- a/tools/migrateFirebaseToPostgres.ts +++ b/tools/migrateFirebaseToPostgres.ts @@ -172,8 +172,16 @@ const migrateDraft = async (draft: Draft, firebaseApp: firebaseAdmin.app.App) => try { const [changesSnapshot, mergesSnapshot] = await Promise.all([ - ref.child('changes').orderByKey().startAt(String(checkpointKey + 1)).once('value'), - ref.child('merges').orderByKey().startAt(String(checkpointKey + 1)).once('value'), + ref + .child('changes') + .orderByKey() + .startAt(String(checkpointKey + 1)) + .once('value'), + ref + .child('merges') + .orderByKey() + .startAt(String(checkpointKey + 1)) + .once('value'), ]); const allKeyables = { @@ -204,9 +212,7 @@ const migrateDraft = async (draft: Draft, firebaseApp: firebaseAdmin.app.App) => const changes = Array.isArray(changeData) ? changeData : [changeData]; for (const change of changes) { - const stepsJson = change.s.map((compressed: any) => - uncompressStepJSON(compressed), - ); + const stepsJson = change.s.map((compressed: any) => uncompressStepJSON(compressed)); rows.push({ draftId, version: keyNum, ref: uuid(), steps: stepsJson }); } @@ -282,35 +288,33 @@ const main = async () => { } await runWithConcurrency( - toMigrate.map( - (draft) => async () => { - const result = await migrateDraft(draft, firebaseApp); - - if (result.skipped) { - skipped++; - } else if (result.error) { - errors++; - } else { - migrated++; - totalCommits += (result as any).commits ?? 0; - } - - processed++; - - if (processed % 100 === 0) { - const elapsed = Date.now() - startTime; - const rate = processed / (elapsed / 1000); - const remaining = totalDrafts - processed; - const eta = remaining / rate; - - log( - ` Progress: ${processed}/${totalDrafts} (${Math.round((processed / totalDrafts) * 100)}%) ` + - `| migrated=${migrated} skipped=${skipped} errors=${errors} ` + - `| ${rate.toFixed(1)} drafts/sec, ETA ${formatDuration(eta * 1000)}`, - ); - } - }, - ), + toMigrate.map((draft) => async () => { + const result = await migrateDraft(draft, firebaseApp); + + if (result.skipped) { + skipped++; + } else if (result.error) { + errors++; + } else { + migrated++; + totalCommits += (result as any).commits ?? 0; + } + + processed++; + + if (processed % 100 === 0) { + const elapsed = Date.now() - startTime; + const rate = processed / (elapsed / 1000); + const remaining = totalDrafts - processed; + const eta = remaining / rate; + + log( + ` Progress: ${processed}/${totalDrafts} (${Math.round((processed / totalDrafts) * 100)}%) ` + + `| migrated=${migrated} skipped=${skipped} errors=${errors} ` + + `| ${rate.toFixed(1)} drafts/sec, ETA ${formatDuration(eta * 1000)}`, + ); + } + }), CONCURRENCY, ); From 278f55ad9d36528d0c2d950c5425be0dc954efa6 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Tue, 23 Jun 2026 15:48:41 +0200 Subject: [PATCH 13/17] fix: types --- tools/cleanupCollab.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/cleanupCollab.ts b/tools/cleanupCollab.ts index f8a91223a2..a41d224140 100644 --- a/tools/cleanupCollab.ts +++ b/tools/cleanupCollab.ts @@ -141,7 +141,7 @@ const truncateActiveCommits = async () => { log('Truncating old commits for active drafts...'); // single bulk query: delete commits where version < (checkpoint.historyKey - buffer) - const [results] = await sequelize.query<{ deleted: string }>( + const [results] = await sequelize.query<{ deleted: string } | { id: string }[]>( isDryRun ? ` SELECT COUNT(*) AS deleted From 745a0092da4e938e5969d72544890ec53af187ea Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Tue, 23 Jun 2026 15:55:40 +0200 Subject: [PATCH 14/17] fix: set some valkey url --- infra/.env.test | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/infra/.env.test b/infra/.env.test index 03b5a99c13..7390be693d 100644 --- a/infra/.env.test +++ b/infra/.env.test @@ -60,4 +60,5 @@ SESSION_SECRET=shhhhhh # FASTLY_PURGE_TOKEN: Required # SLACK_WEBHOOK_URL: Required # SENTRY_AUTH_TOKEN: Required -# SENTRY_ORG: Required \ No newline at end of file +# SENTRY_ORG: Required +VALKEY_URL='' \ No newline at end of file From d7120f9dac34a8d78a19208ccd06bf8b1ffdd29a Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Tue, 23 Jun 2026 15:56:27 +0200 Subject: [PATCH 15/17] fix: set some valkey url --- infra/.env.test | 2 +- server/server.ts | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/infra/.env.test b/infra/.env.test index 7390be693d..d9a85539cf 100644 --- a/infra/.env.test +++ b/infra/.env.test @@ -61,4 +61,4 @@ SESSION_SECRET=shhhhhh # SLACK_WEBHOOK_URL: Required # SENTRY_AUTH_TOKEN: Required # SENTRY_ORG: Required -VALKEY_URL='' \ No newline at end of file +VALKEY_URL=redis://localhost:6379 \ No newline at end of file diff --git a/server/server.ts b/server/server.ts index 91cc590c2a..0ef602fa31 100755 --- a/server/server.ts +++ b/server/server.ts @@ -387,14 +387,16 @@ export const startServer = async () => { console.warn('[OIDC] Discovery failed at startup (will retry on demand):', err.message); }); - // connect collab redis clients - const { connectCollabRedis } = await import('./collab/authority.js'); - const { connectPresenceRedis } = await import('./collab/presence.js'); - - await Promise.all([connectCollabRedis(), connectPresenceRedis()]).catch((err) => { - console.error('[collab] Failed to connect to Redis/Valkey:', err.message); - console.error('[collab] Collaborative editing will not work until Redis is available.'); - }); + if (env.NODE_ENV !== 'test') { + // connect collab redis clients + const { connectCollabRedis } = await import('./collab/authority.js'); + const { connectPresenceRedis } = await import('./collab/presence.js'); + + await Promise.all([connectCollabRedis(), connectPresenceRedis()]).catch((err) => { + console.error('[collab] Failed to connect to Redis/Valkey:', err.message); + console.error('[collab] Collaborative editing will not work until Redis is available.'); + }); + } await sequelizeSyncPromise; return app.listen( From 6196e9b76ed6a0142d06e5a9fbd1209d869bffe9 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Tue, 23 Jun 2026 18:16:14 +0200 Subject: [PATCH 16/17] fix: try to fix tests --- server/collab/api.ts | 4 +- server/collab/authority.ts | 395 +++++++++++++++++----------------- server/utils/firebaseAdmin.ts | 2 +- 3 files changed, 196 insertions(+), 205 deletions(-) diff --git a/server/collab/api.ts b/server/collab/api.ts index b2726988ed..12bb1f4a54 100644 --- a/server/collab/api.ts +++ b/server/collab/api.ts @@ -47,7 +47,7 @@ router.post( } try { - await (await getCollabAuthority()).receiveCommit(draftId, req.body); + await getCollabAuthority().receiveCommit(draftId, req.body); } catch (e) { if (e instanceof TooMuchContentionError) { console.log('TooMuchContentionError', e); @@ -76,7 +76,7 @@ router.get( return res.status(400).json({ error: 'Missing or invalid version query parameter' }); } - const commits = await (await getCollabAuthority()).listenForCommit(draftId, version); + const commits = await getCollabAuthority().listenForCommit(draftId, version); return res.status(200).json(commits); }), ); diff --git a/server/collab/authority.ts b/server/collab/authority.ts index 9deff064f8..84ac2c975b 100644 --- a/server/collab/authority.ts +++ b/server/collab/authority.ts @@ -16,8 +16,6 @@ import { replayCommitsOntoDoc } from './replay'; export { replayCommitsOntoDoc }; -let authority: CollabAuthority | null = null; - const CHECKPOINT_INTERVAL = 50; interface CachedCheckpoint { @@ -27,240 +25,233 @@ interface CachedCheckpoint { const checkpointCache = new Map(); -const createAuthority = (bm: RedisBroadcastManager) => - new CollabAuthority({ - schema: editorSchema, - broadcastManager: bm, +const broadcastManager = new RedisBroadcastManager({ + redisUrl: env.VALKEY_URL ?? 'redis://localhost:6379', +}); - runWithTransaction: async (callback) => { - return sequelize.transaction((tr) => callback(tr)); - }, +const authority = new CollabAuthority({ + schema: editorSchema, + broadcastManager, - getDoc: async (tr, docId) => { - const logger = createLogger('getDoc'); + runWithTransaction: async (callback) => { + return sequelize.transaction((tr) => callback(tr)); + }, - const cached = checkpointCache.get(docId); + getDoc: async (tr, docId) => { + const logger = createLogger('getDoc'); - const fetchCheckpointFromDb = () => - DraftCheckpoint.findOne({ - where: { draftId: docId }, - order: [['historyKey', 'DESC']], - transaction: tr ?? undefined, - }).then((cp) => { - if (!cp) { - return null; - } - - const entry = { historyKey: cp.historyKey, doc: cp.doc }; - checkpointCache.set(docId, entry); - return entry; - }); - - const [draft, checkpoint] = await logger.log( - 'getDocAndCheckpoint', - Promise.all([ - Draft.findOne({ - where: { id: docId }, - ...(tr && { lock: tr.LOCK.NO_KEY_UPDATE }), - transaction: tr ?? undefined, - }), - cached ? Promise.resolve(cached) : fetchCheckpointFromDb(), - ]), - ); + const cached = checkpointCache.get(docId); - if (!draft) { - throw new Error(`Draft not found: ${docId}`); - } + const fetchCheckpointFromDb = () => + DraftCheckpoint.findOne({ + where: { draftId: docId }, + order: [['historyKey', 'DESC']], + transaction: tr ?? undefined, + }).then((cp) => { + if (!cp) { + return null; + } - if (!checkpoint) { - const emptyDoc = editorSchema.topNodeType.createAndFill()!; + const entry = { historyKey: cp.historyKey, doc: cp.doc }; + checkpointCache.set(docId, entry); + return entry; + }); - return { - docJSON: emptyDoc.toJSON(), - version: 0, - lastUpdatedTimestamp: Date.now(), - }; - } + const [draft, checkpoint] = await logger.log( + 'getDocAndCheckpoint', + Promise.all([ + Draft.findOne({ + where: { id: docId }, + ...(tr && { lock: tr.LOCK.NO_KEY_UPDATE }), + transaction: tr ?? undefined, + }), + cached ? Promise.resolve(cached) : fetchCheckpointFromDb(), + ]), + ); - const checkpointVersion = checkpoint.historyKey ?? 0; - - if (checkpointVersion < draft.version) { - const missedCommits = await logger.log( - 'getMissedCommits', - CollabCommit.findAll({ - where: { - draftId: docId, - version: { [Op.gt]: checkpointVersion, [Op.lte]: draft.version }, - }, - order: [['version', 'ASC']], - transaction: tr ?? undefined, - }), - ); + if (!draft) { + throw new Error(`Draft not found: ${docId}`); + } - try { - const reconstructedDoc = replayCommitsOntoDoc(checkpoint.doc, missedCommits); - logger.end(); - - return { - docJSON: reconstructedDoc.toJSON(), - version: draft.version, - lastUpdatedTimestamp: draft.latestKeyAt?.valueOf() ?? Date.now(), - }; - } catch (err) { - // stale cache: checkpoint doc doesn't match the commits in the db. - // refetch the checkpoint from postgres and retry. - if (!cached) { - throw err; - } - - checkpointCache.delete(docId); - const freshCheckpoint = await fetchCheckpointFromDb(); - - if (!freshCheckpoint) { - throw err; - } - - const freshVersion = freshCheckpoint.historyKey ?? 0; - const freshCommits = await CollabCommit.findAll({ - where: { - draftId: docId, - version: { [Op.gt]: freshVersion, [Op.lte]: draft.version }, - }, - order: [['version', 'ASC']], - transaction: tr ?? undefined, - }); - - const reconstructedDoc = replayCommitsOntoDoc( - freshCheckpoint.doc, - freshCommits, - ); - logger.end(); - - return { - docJSON: reconstructedDoc.toJSON(), - version: draft.version, - lastUpdatedTimestamp: draft.latestKeyAt?.valueOf() ?? Date.now(), - }; - } - } - - logger.end(); + if (!checkpoint) { + const emptyDoc = editorSchema.topNodeType.createAndFill()!; return { - docJSON: checkpoint.doc, - version: draft.version, - lastUpdatedTimestamp: draft.latestKeyAt?.valueOf() ?? Date.now(), + docJSON: emptyDoc.toJSON(), + version: 0, + lastUpdatedTimestamp: Date.now(), }; - }, - - saveDoc: async (tr, docId, docJSON, version) => { - try { - await Draft.update( - { version, latestKeyAt: new Date() }, - { where: { id: docId }, transaction: tr }, - ); - - const shouldCheckpoint = version % CHECKPOINT_INTERVAL === 0 || version <= 1; + } - if (!shouldCheckpoint) { - return; - } + const checkpointVersion = checkpoint.historyKey ?? 0; - const truncateBelow = version - CHECKPOINT_INTERVAL; + if (checkpointVersion < draft.version) { + const missedCommits = await logger.log( + 'getMissedCommits', + CollabCommit.findAll({ + where: { + draftId: docId, + version: { [Op.gt]: checkpointVersion, [Op.lte]: draft.version }, + }, + order: [['version', 'ASC']], + transaction: tr ?? undefined, + }), + ); - await upsertDraftCheckpoint(docId, version, docJSON as DocJson, Date.now(), tr); + try { + const reconstructedDoc = replayCommitsOntoDoc(checkpoint.doc, missedCommits); + logger.end(); - if (tr) { - tr.afterCommit(() => { - checkpointCache.set(docId, { - historyKey: version, - doc: docJSON as Record, - }); - }); + return { + docJSON: reconstructedDoc.toJSON(), + version: draft.version, + lastUpdatedTimestamp: draft.latestKeyAt?.valueOf() ?? Date.now(), + }; + } catch (err) { + // stale cache: checkpoint doc doesn't match the commits in the db. + // refetch the checkpoint from postgres and retry. + if (!cached) { + throw err; } - if (truncateBelow > 0) { - await CollabCommit.destroy({ - where: { - draftId: docId, - version: { [Op.lt]: truncateBelow }, - }, - transaction: tr, - }); + checkpointCache.delete(docId); + const freshCheckpoint = await fetchCheckpointFromDb(); + + if (!freshCheckpoint) { + throw err; } - } catch (error) { - console.error('Error saving doc', error); - throw error; - } - }, - saveCommit: async (tr, docId, commitRef, commitVersion, commitSteps) => { - try { - await CollabCommit.create( - { + const freshVersion = freshCheckpoint.historyKey ?? 0; + const freshCommits = await CollabCommit.findAll({ + where: { draftId: docId, - ref: commitRef, - version: commitVersion, - steps: commitSteps, + version: { [Op.gt]: freshVersion, [Op.lte]: draft.version }, }, - { transaction: tr }, + order: [['version', 'ASC']], + transaction: tr ?? undefined, + }); + + const reconstructedDoc = replayCommitsOntoDoc( + freshCheckpoint.doc, + freshCommits, ); - } catch (error) { - console.error('Error saving commit', error); - throw error; + logger.end(); + + return { + docJSON: reconstructedDoc.toJSON(), + version: draft.version, + lastUpdatedTimestamp: draft.latestKeyAt?.valueOf() ?? Date.now(), + }; } - }, + } + + logger.end(); + + return { + docJSON: checkpoint.doc, + version: draft.version, + lastUpdatedTimestamp: draft.latestKeyAt?.valueOf() ?? Date.now(), + }; + }, + + saveDoc: async (tr, docId, docJSON, version) => { + try { + await Draft.update( + { version, latestKeyAt: new Date() }, + { where: { id: docId }, transaction: tr }, + ); - getCommit: async (tr, docId, commitRef) => { - const commit = await CollabCommit.findOne({ - where: { draftId: docId, ref: commitRef }, - transaction: tr ?? undefined, - plain: true, - }); + const shouldCheckpoint = version % CHECKPOINT_INTERVAL === 0 || version <= 1; - if (!commit) { - return null; + if (!shouldCheckpoint) { + return; } - return { - ref: commit.ref, - version: commit.version, - steps: commit.steps, - }; - }, + const truncateBelow = version - CHECKPOINT_INTERVAL; + + await upsertDraftCheckpoint(docId, version, docJSON as DocJson, Date.now(), tr); + + if (tr) { + tr.afterCommit(() => { + checkpointCache.set(docId, { + historyKey: version, + doc: docJSON as Record, + }); + }); + } - getCommits: async (tr, docId, version) => { - const commits = - (await CollabCommit.findAll({ + if (truncateBelow > 0) { + await CollabCommit.destroy({ where: { draftId: docId, - version: { [Op.gt]: version }, + version: { [Op.lt]: truncateBelow }, }, - order: [['version', 'ASC']], - transaction: tr ?? undefined, - })) ?? []; - - return commits.map((c) => ({ - ref: c.ref, - version: c.version, - steps: c.steps, - })); - }, - }); - -export const getCollabAuthority = async () => { - if (!authority) { - return await connectCollabRedis(); - } - return authority; -}; + transaction: tr, + }); + } + } catch (error) { + console.error('Error saving doc', error); + throw error; + } + }, + + saveCommit: async (tr, docId, commitRef, commitVersion, commitSteps) => { + try { + await CollabCommit.create( + { + draftId: docId, + ref: commitRef, + version: commitVersion, + steps: commitSteps, + }, + { transaction: tr }, + ); + } catch (error) { + console.error('Error saving commit', error); + throw error; + } + }, + + getCommit: async (tr, docId, commitRef) => { + const commit = await CollabCommit.findOne({ + where: { draftId: docId, ref: commitRef }, + transaction: tr ?? undefined, + plain: true, + }); + + if (!commit) { + return null; + } + + return { + ref: commit.ref, + version: commit.version, + steps: commit.steps, + }; + }, + + getCommits: async (tr, docId, version) => { + const commits = + (await CollabCommit.findAll({ + where: { + draftId: docId, + version: { [Op.gt]: version }, + }, + order: [['version', 'ASC']], + transaction: tr ?? undefined, + })) ?? []; + + return commits.map((c) => ({ + ref: c.ref, + version: c.version, + steps: c.steps, + })); + }, +}); + +export const getCollabAuthority = () => authority; export const connectCollabRedis = async () => { - const broadcastManager = new RedisBroadcastManager({ - redisUrl: env.VALKEY_URL ?? 'redis://localhost:6379', - }); await broadcastManager.connect(); - authority = createAuthority(broadcastManager); console.log('[collab] collab broadcast redis connected'); - return authority; }; diff --git a/server/utils/firebaseAdmin.ts b/server/utils/firebaseAdmin.ts index 3e773abbf2..9a31f61a78 100644 --- a/server/utils/firebaseAdmin.ts +++ b/server/utils/firebaseAdmin.ts @@ -193,7 +193,7 @@ export const editDraft = async (pubId: string, clientId: string, schema: Schema ref: `server-${Date.now()}-${Math.random().toString(36).slice(2)}`, }; - await (await getCollabAuthority()).receiveCommit(draft.id, commitData); + await getCollabAuthority().receiveCommit(draft.id, commitData); currentVersion++; pendingSteps = []; return true; From f9649c86c19df9ef069529ec745e185348286190 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Wed, 24 Jun 2026 15:29:10 +0200 Subject: [PATCH 17/17] fix: dont cache corrupted docs --- server/collab/replay.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/collab/replay.ts b/server/collab/replay.ts index 8039d2f90d..b4e209eca0 100644 --- a/server/collab/replay.ts +++ b/server/collab/replay.ts @@ -14,9 +14,11 @@ export const replayCommitsOntoDoc = ( const step = Step.fromJSON(editorSchema, stepJSON); const result = step.apply(doc); - if (result.doc) { - doc = result.doc; + if (result.failed) { + throw new Error(`Step replay failed: ${result.failed}`); } + + doc = result.doc!; } }