Add collab and presence overviews and fix presence flickers

Author: smoores-dev

.envrc

@@ -27,17 +27,25 @@ export function collab(config: Parameters<typeof base>[0]) {
         baseState.stepMaps = [];
         return baseState;
       },
-      apply(tr, ...args) {
-        const baseState = baseSpec.state!.apply(tr, ...args) as CollabState;
-        baseState.stepMaps ??= [];
-        const meta = tr.getMeta(collabKey) as BaseCollabState | undefined;
-        if (meta?.commit) {
-          baseState.stepMaps.push({
-            version: meta.commit.version,
-            mappables: meta.commit.steps.map((step) => step.getMap()),
+      apply(tr, value, ...args) {
+        const next = baseSpec.state!.apply(tr, value, ...args) as CollabState;
+        next.stepMaps ??= [...value.stepMaps];
+        if (!tr.docChanged && value.version === next.version) return next;
+
+        const lastMap = next.stepMaps[next.stepMaps.length - 1];
+
+        if (lastMap?.version !== next.version) {
+          next.stepMaps.push({
+            version: next.version,
+            mappables: tr.steps.map((step) => step.getMap()),
           });
+
+          return next;
         }
-        return baseState;
+
+        lastMap.mappables.push(...tr.steps.map((step) => step.getMap()));
+
+        return next;
       },
     },
     historyPreserveItems: true,

packages/collab-client/src/plugin.ts

@@ -0,0 +1,61 @@
+---
+title: Overview
+description: A ProseMirror plugin and backend implementation for fast, simple collaborative editing.
+---
+
+Collaborative editing in rich text is, in a word, hard. But ProseMirror was designed for it from the
+ground up, and even has a first-party collaboration plugin (`prosemirror-collab`).
+
+Unfortunately, `prosemirror-collab` only handles the _client_ collaboration implementation. It can
+be challenging to get the server-side constraints right, and even managing the client-side state and
+effects correctly isn’t trivial.
+
+**Pitter Patter Collab takes care of both the client and server implementation, and guides you
+through the relevant database and API decisions as well.**
+
+## How it works
+
+Pitter Patter Collab is based on
+[`prosemirror-collab-commit`](https://github.com/stepwisehq/prosemirror-collab-commit), an
+alternative `prosemirror-collab` implementation that provides improved throughput and fairness over
+the default implementation. This means that it uses ProseMirror steps as the unit of change, just
+like `prosemirror-collab` (and just like non-collaborative ProseMirror), which makes it
+straightforward to inspect, debug, and analyze.
+
+You can add Pitter Patter Collab to your existing stack. You provide the database and API, and it
+manages the collaboration state and synchronization.
+
+### Simple client configuration
+
+Just provide your client with the endpoints for sending and listening for changes. Collab handles
+the rest!
+
+### Long-polling clients
+
+Long-polling provides instant updates, is resilient to intermittent connections, and trivially
+supports horizontal autoscaling on the server side. No need to worry about sticky sessions or
+reconnecting websockets!
+
+### Redis/Valkey (or whatever you like!) for pub/sub
+
+Out of the box, we support Redis/Valkey for server-side pub/sub. This allows your horizontally
+scaled backend servers to notify each other of changes to documents that clients are subscribed to.
+
+Already using another pub/sub provider? No problem! The broadcast manager is fully configurable —
+you can build your own. It only needs to implement three methods: broadcast, listen, and abort!
+
+### BYODB
+
+Collab doesn’t have any opinions about your persistent datastore. We have guidance for various SQL
+databases in our step-by-step [guide](/docs/guides/collab), but NoSQL databases that support
+transactions (or thoughtfully crafted primary keys!) can be used as well. If you’re looking for
+guidance for a specific datastore, start a
+[Discussion](https://github.com/handlewithcarecollective/pitter-patter/discussions)!
+
+<Cards>
+  <Card href="/docs/guides/collab" title="Guide">
+    Set up Collab full-stack, step-by-step.
+  </Card>
+  <Card href="/docs/collab/reference/collab-client/README" title="Client API Reference" />
+  <Card href="/docs/collab/reference/collab-server/README" title="Server API Reference" />
+</Cards>

packages/docs/content/docs/collab/overview.mdx

@@ -137,8 +137,7 @@ const runWithTransaction = async (callback) => {
 
 Saves a document along with its docId, version, and lastUpdatedTimestamp to your database.
 
-If a transaction `tr` is provided, it must be used for all database operations. If `tr` is null, all
-database operations in this function should still be performed as an atomic unit.
+If a transaction `tr` is provided, it must be used for all database operations.
 
 ```ts
 const saveDoc = async (tr, docId, docJSON, version) => {
@@ -167,8 +166,7 @@ If you are using Postgres or MySql, getDoc should select the row holding the doc
 `SELECT FOR UPDATE`. This ensures that conflicting commits do not overwrite each other. We also
 recommend putting a unique constraint on the commit table for the fields docId and commit version.
 
-If a transaction `tr` is provided, it must be used for all database operations. If `tr` is null, all
-database operations in this function should still be performed as an atomic unit.
+If a transaction `tr` is provided, it must be used for all database operations.
 
 ```ts
 const getDoc = async (tr, docId) => {
@@ -190,8 +188,7 @@ export async function getDocument(tr: Transaction<DB> | null, id: string) {
 
 Saves a commit along with its version and ref to your database.
 
-If a transaction `tr` is provided, it must be used for all database operations. If `tr` is null, all
-database operations in this function should still be performed as an atomic unit.
+If a transaction `tr` is provided, it must be used for all database operations.
 
 ```ts
 const saveCommit = async (tr, docId, commitRef, commitVersion, commitSteps) => {
@@ -215,8 +212,7 @@ export async function createCommit(tr: Transaction<DB> | null, commit: Insertabl
 Given a docId and commitRef, retrieves the associated commit's steps and version from your database
 and returns a joined CommitJSON object.
 
-If a transaction `tr` is provided, it must be used for all database operations. If `tr` is null, all
-database operations in this function should still be performed as an atomic unit.
+If a transaction `tr` is provided, it must be used for all database operations.
 
 ```ts
 const getCommit = async (tr, docId, commitRef) => {
@@ -270,7 +266,7 @@ const broadcastManager = new RedisBroadcastManager({
 
 ### [Schema](https://pitter-patter.dev/docs/collab/reference/collab-server/interfaces/CollabAuthorityConfig#schema)
 
-This is just the schema for your Prosemirror document.
+This is just the schema for your ProseMirror document.
 
 ### All together
 
@@ -392,10 +388,7 @@ Next we need to create a listener for commit changes. The built-in
 import { LongPollListener as CollabLongPollListener } from "@pitter-patter/collab-client";
 
 const commitListener = new CollabLongPollListener(
-  new URL(
-    `/api/docs/${doc.id}/commits`,
-    typeof window !== "undefined" ? window.location.href : "http://localhost:3000",
-  ),
+  new URL(`/api/docs/${doc.id}/commits`, "http://localhost:3000"),
 );
 ```
 
@@ -429,7 +422,8 @@ const collabConfig = {
   },
   receiveCommits: (commits) => {
     // merge a new set of commits into your editor state for example:
-    // const newEditorState = commits.reduce((acc, commit) => acc.apply(receiveCommitTransaction(acc, commit)), oldEditorState),
+    // const newEditorState = commits.reduce((acc, commit) => acc.apply(receiveCommitTransaction(acc, commit)), oldEditorState)
+    // view.setState(newEditorState)
   },
   listener: commitListener,
 };
@@ -451,4 +445,4 @@ When the local editor state changes, you need to tell collabClient to create and
 collabClient.send(state).catch((e) => console.error(e));
 ```
 
-That's it! Your document can now be edited collaboratively by mulitiple simultaneous users.
+That's it! Your document can now be edited collaboratively by multiple simultaneous users.

packages/docs/content/docs/guides/collab.md

@@ -107,22 +107,16 @@ You can now connect your frontend client to the presence backend using PitterPat
 Like with the
 [CollabClient](https://pitter-patter.dev/docs/collab/reference/collab-client/classes/CollabClient),
 you create a listener for presence state updates with Presence's built-in
-[LongPollListner](https://pitter-patter.dev/docs/presence/reference/presence-client/classes/LongPollListener).
+[LongPollListener](https://pitter-patter.dev/docs/presence/reference/presence-client/classes/LongPollListener).
 
 ```ts
 const [presenceListener] = useState(
-  () =>
-    new PresenceListener(
-      new URL(
-        `/api/docs/${doc.id}/presence`,
-        typeof window !== "undefined" ? window.location.href : "http://localhost:3000",
-      ),
-    ),
+  () => new LongPollListener(new URL(`/api/docs/${doc.id}/presence`, "http://localhost:3000")),
 );
 ```
 
 The
-[LongPollListner](https://pitter-patter.dev/docs/presence/reference/presence-client/classes/LongPollListener)
+[LongPollListener](https://pitter-patter.dev/docs/presence/reference/presence-client/classes/LongPollListener)
 takes the url of the getPresence endpoint, as well as optional headers for requests to that
 endpoint. These headers can be updated with the listener's
 [update](https://pitter-patter.dev/docs/presence/reference/presence-client/classes/LongPollListener#update)

packages/docs/content/docs/guides/presence.md

@@ -9,4 +9,9 @@ description: ""
 
 ## Rich text editing
 
-- [`@pitter-patter/shuffle`](/docs/shuffle/overview): A 12-column, grid-based drag and drop library.
+- [`@pitter-patter/shuffle`](/docs/shuffle/overview): 12-column, grid-based drag and drop
+
+## Collaboration
+
+- [`@pitter-patter/collab`](/docs/collab/overview): Fast, simple, plug-and-play collaboration
+- [`@pitter-patter/presence`](/docs/presence/overview): Live presence

packages/docs/content/docs/introduction/getting-started.md

@@ -0,0 +1,51 @@
+---
+title: Overview
+description:
+  A ProseMirror plugin and backend implementation for fast, simple live presence indicators.
+---
+
+[Simple collaborative editing](/docs/collab/overview) is great, but without presence indicators it
+quickly gets confusing.
+
+**Pitter Patter Presence takes care of both the client and server implementation, and guides you
+through the relevant API decisions as well.**
+
+## How it works
+
+You can add Pitter Patter Presence to your existing stack. You provide the API, and it manages the
+presence state and synchronization.
+
+### Simple client configuration
+
+Just provide your client with the endpoints for sending and listening for indicators. Presence
+handles the rest!
+
+### Long-polling clients
+
+Long-polling provides instant updates, is resilient to intermittent connections, and trivially
+supports horizontal autoscaling on the server side. No need to worry about sticky sessions or
+reconnecting websockets!
+
+### Redis/Valkey (or whatever you like!) for pub/sub
+
+Out of the box, we support Redis/Valkey for server-side pub/sub. This allows your horizontally
+scaled backend servers to notify each other of changes to documents that clients are subscribed to.
+
+Already using another pub/sub provider? No problem! The broadcast manager is fully configurable —
+you can build your own. It only needs to implement three methods: broadcast, listen, and abort!
+
+### BYODB
+
+Presence doesn’t have any opinions about your persistent datastore. Out of the box, we support
+Redis/Valkey for server-side persistence, because presence state generally doesn’t need to be truly
+persistent. But you can use any datastore you like by implementing your own persistence manager. If
+you’re looking for guidance for a specific datastore, start a
+[Discussion](https://github.com/handlewithcarecollective/pitter-patter/discussions)!
+
+<Cards>
+  <Card href="/docs/guides/presence" title="Guide">
+    Set up Presence full-stack, step-by-step.
+  </Card>
+  <Card href="/docs/presence/reference/presence-client/README" title="Client API Reference" />
+  <Card href="/docs/presence/reference/presence-server/README" title="Server API Reference" />
+</Cards>

packages/docs/content/docs/presence/overview.mdx

@@ -89,17 +89,17 @@ export function presence(
             continue;
           }
 
-          const confirmedMappables = stepMaps
-            .filter(({ version }) => version > indicator.version)
-            .flatMap(({ mappables }) => mappables);
+          const confirmedMaps = stepMaps.filter(({ version }) => version > indicator.version);
 
           // This indicator may predate when we started tracking stepMaps.
           // If so, we have to ignore it, because we can't map it forward.
-          if (confirmedMappables.length < version - indicator.version) {
+          if (confirmedMaps.length < version - indicator.version) {
             continue;
           }
 
-          const mappables = confirmedMappables.concat(unconfirmed.map(({ step }) => step.getMap()));
+          const mappables = confirmedMaps
+            .flatMap(({ mappables }) => mappables)
+            .concat(unconfirmed.map(({ step }) => step.getMap()));
 
           const anchor = mappables.reduce((acc, mappable) => mappable.map(acc), indicator.anchor);
           const head = mappables.reduce((acc, mappable) => mappable.map(acc), indicator.head);
@@ -125,9 +125,11 @@ export function presence(
           );
         }
 
+        console.log(nextDecorations);
+
         return {
           decorations: DecorationSet.create(editorState.doc, nextDecorations),
-          indicators,
+          indicators: nextIndicators,
           stepMaps,
         };
       },