Add attachment replication support to Microsoft OneDrive plugin (#8592)

* feat(replication-microsoft-onedrive): implement attachment replication with base64 encoding

Mirrors the attachment replication support that was added to the
google-drive plugin: attachment binary data is serialised as base64
inside the document JSON file stored on OneDrive while `_attachments`
remains as clean stubs. Enabled automatically when the collection
schema declares `attachments: {}` and can be disabled via
`attachments: false`.

* refactor: share attachment serialization helpers between google-drive and onedrive plugins

Move serializeDocAttachments, deserializeDocAttachments and the
inline-stripForComparison logic (now stripAllAttachmentDataForComparison)
from each plugin's document-handling.ts / upstream.ts into
src/replication-protocol/helper.ts so both replication backends share
a single implementation. The plugins re-export the same names so the
public API is unchanged.

---------

Co-authored-by: Claude <noreply@anthropic.com>

Author: pubkey

orga/changelog/replication-microsoft-onedrive-attachments.md

@@ -0,0 +1,2 @@
+- ADD `replication-microsoft-onedrive` plugin now supports attachment replication. Attachment binary data is stored as base64 in a separate `_attachments_data` field of the document JSON file on OneDrive, keeping `_attachments` as clean stubs. Attachment replication is enabled automatically when the collection schema has `attachments: {}` defined and can be disabled by passing `attachments: false` to `replicateMicrosoftOneDrive()`.
+- REFACTOR move the attachment serialisation helpers (`serializeDocAttachments`, `deserializeDocAttachments`, `stripAllAttachmentDataForComparison`) to `src/replication-protocol/helper.ts` so they can be shared between the google-drive and microsoft-onedrive replication plugins.

src/plugins/replication-google-drive/document-handling.ts

@@ -1,15 +1,18 @@
 import { newRxError, newRxFetchError } from '../../rx-error.ts';
-import { stripAttachmentsDataFromDocument } from '../../rx-storage-helper.ts';
 import { ById } from '../../types/util';
 import { ensureNotFalsy } from '../utils/index.ts';
-import { blobToBase64String, createBlobFromBase64 } from '../utils/index.ts';
 import { applyDriveSpace, insertMultipartFile } from './google-drive-helper.ts';
 import type {
     DriveFileListResponse,
     GoogleDriveOptionsWithDefaults
 } from './google-drive-types.ts';
 import { DriveStructure } from './init.ts';
 
+export {
+    serializeDocAttachments,
+    deserializeDocAttachments
+} from '../../replication-protocol/helper.ts';
+
 const MAX_DRIVE_PAGE_SIZE = 1000;
 
 
@@ -217,69 +220,3 @@ export async function fetchDocumentContents<DocType>(
     return { byId, ordered };
 }
 
-/**
- * Serialises attachment data in a document clone so it can safely be stored
- * as JSON in Google Drive.
- *
- * - When `serializeData` is true: Blob values are extracted and stored as
- *   base64 strings in the top-level `_attachments_data` field, while
- *   `_attachments` is stripped to clean stubs via `stripAttachmentsDataFromDocument`.
- * - When `serializeData` is false: `_attachments` is set to `{}` so that
- *   attachment stubs (without binary data) are never stored in Drive.  This
- *   prevents the downstream replication protocol from trying to write attachment
- *   data it does not have when a peer pulls the document.
- *
- * The function returns a NEW document object; the original is not mutated.
- */
-export async function serializeDocAttachments<T>(doc: T, serializeData: boolean): Promise<T> {
-    const d = doc as any;
-    if (!d?._attachments) {
-        return doc;
-    }
-
-    if (!serializeData) {
-        return { ...d, _attachments: {} } as any;
-    }
-
-    const attachmentData: Record<string, string> = {};
-    await Promise.all(
-        Object.entries(d._attachments as Record<string, any>).map(async ([id, att]) => {
-            if (att.data instanceof Blob) {
-                attachmentData[id] = await blobToBase64String(att.data);
-            }
-        })
-    );
-
-    // Strip binary data from _attachments, leaving clean stubs {digest, length, type}.
-    const stripped = stripAttachmentsDataFromDocument(d) as any;
-
-    if (Object.keys(attachmentData).length > 0) {
-        return { ...stripped, _attachments_data: attachmentData } as any;
-    }
-    return stripped as any;
-}
-
-/**
- * Converts attachment data stored in `_attachments_data` back to Blobs in the
- * document, so that the downstream replication protocol can write them to the
- * fork storage instance correctly.
- *
- * Mutates the document in place and removes the `_attachments_data` field.
- */
-export async function deserializeDocAttachments(doc: any): Promise<void> {
-    const attachmentData: Record<string, string> | undefined = doc._attachments_data;
-    if (!attachmentData || !doc._attachments) {
-        return;
-    }
-    await Promise.all(
-        Object.entries(attachmentData).map(async ([id, base64]) => {
-            if (doc._attachments[id]) {
-                doc._attachments[id] = {
-                    ...doc._attachments[id],
-                    data: await createBlobFromBase64(base64, doc._attachments[id].type)
-                };
-            }
-        })
-    );
-    delete doc._attachments_data;
-}

src/plugins/replication-google-drive/upstream.ts

@@ -1,6 +1,6 @@
 import { RxReplicationWriteToMasterRow, WithDeletedAndAttachments } from '../../index.ts';
 import { newRxError, newRxFetchError } from '../../rx-error.ts';
-import { stripAttachmentsDataFromDocument } from '../../rx-storage-helper.ts';
+import { stripAllAttachmentDataForComparison } from '../../replication-protocol/helper.ts';
 import { deepEqual, ensureNotFalsy } from '../utils/index.ts';
 import { fetchDocumentContents, getDocumentFiles, insertDocumentFiles, updateDocumentFiles } from './document-handling.ts';
 import { DRIVE_MAX_BULK_SIZE, fillFileIfEtagMatches, getFileEtag } from './google-drive-helper.ts';
@@ -55,20 +55,10 @@ export async function fetchConflicts<RxDocType>(
             fileContent = contentsByFileId.byId[fileId];
         }
         if (row.assumedMasterState) {
-            /**
-             * Strip binary/serialised attachment data from both sides before
-             * comparing. `assumedMasterState` comes from the replication protocol
-             * which already stores clean stubs (no `data`). `fileContent` comes
-             * from Google Drive and stores base64 data in the separate
-             * `_attachments_data` field — strip that too so the structural
-             * metadata (digest / length / type) is all that is compared.
-             */
-            const stripForComparison = (d: any) => {
-                const s = stripAttachmentsDataFromDocument(d) as any;
-                delete s._attachments_data;
-                return s;
-            };
-            if (!deepEqual(stripForComparison(row.assumedMasterState), stripForComparison(fileContent))) {
+            if (!deepEqual(
+                stripAllAttachmentDataForComparison(row.assumedMasterState),
+                stripAllAttachmentDataForComparison(fileContent)
+            )) {
                 conflicts.push(ensureNotFalsy(fileContent));
             } else {
                 nonConflicts.push(row);

src/plugins/replication-microsoft-onedrive/document-handling.ts

@@ -9,6 +9,11 @@ import type {
 import { DriveStructure } from './init.ts';
 import { getDriveBaseUrl } from './microsoft-onedrive-helper.ts';
 
+export {
+    serializeDocAttachments,
+    deserializeDocAttachments
+} from '../../replication-protocol/helper.ts';
+
 const MAX_DRIVE_PAGE_SIZE = 999;
 
 export async function getDocumentFiles(
@@ -214,3 +219,4 @@ export async function fetchDocumentContents<DocType>(
     await Promise.all(Array.from({ length: concurrency }, () => worker()));
     return { byId, ordered };
 }
+

src/plugins/replication-microsoft-onedrive/index.ts

@@ -30,6 +30,10 @@ import {
     ensureProcessNextTickIsSet
 } from '../replication-webrtc/connection-handler-simple-peer.ts';
 import { SignalingOptions, SignalingState } from './signaling.ts';
+import {
+    deserializeDocAttachments,
+    serializeDocAttachments
+} from './document-handling.ts';
 
 export * from './microsoft-onedrive-types.ts';
 export * from './microsoft-onedrive-helper.ts';
@@ -105,6 +109,14 @@ export async function replicateMicrosoftOneDrive<RxDocType>(
     );
     const driveStructure = await initDriveStructure(oneDriveState);
 
+    /**
+     * When true, attachment binary data is serialised as base64 inside the
+     * document JSON file stored on OneDrive.  Defaults to true only when
+     * the collection schema has `attachments: {}` defined.  Can be disabled
+     * explicitly by passing `attachments: false` in the options.
+     */
+    const replicateAttachments = options.attachments !== false && !!collection.schema.jsonSchema.attachments;
+
 
     let replicationState: RxOneDriveReplicationState<RxDocType>;
 
@@ -131,6 +143,17 @@ export async function replicateMicrosoftOneDrive<RxDocType>(
                             lastPulledCheckpoint,
                             batchSize
                         );
+                        /**
+                         * Convert base64 attachment data that was stored in the
+                         * OneDrive JSON file back to Blobs so that the
+                         * downstream replication protocol can write them to the
+                         * fork storage instance correctly.
+                         */
+                        if (replicateAttachments) {
+                            await Promise.all(
+                                changes.documents.map(doc => deserializeDocAttachments(doc as any))
+                            );
+                        }
                         return changes as any;
                     }
                 );
@@ -148,6 +171,27 @@ export async function replicateMicrosoftOneDrive<RxDocType>(
             async handler(
                 rows: RxReplicationWriteToMasterRow<RxDocType>[]
             ) {
+                /**
+                 * Convert Blob attachment data to base64 strings before the
+                 * rows are written to the WAL file or stored as document JSON
+                 * on OneDrive.  We create new row objects so the originals
+                 * (which the replication protocol still uses for meta updates)
+                 * are not mutated.
+                 * When attachments are disabled we still run the conversion so
+                 * that Blob values are stripped rather than serialised as `{}`
+                 * by JSON.stringify.
+                 */
+                const rowsForDrive: RxReplicationWriteToMasterRow<RxDocType>[] =
+                    await Promise.all(
+                        rows.map(async row => {
+                            const newState = await serializeDocAttachments(
+                                row.newDocumentState,
+                                replicateAttachments
+                            );
+                            return { ...row, newDocumentState: newState };
+                        })
+                    );
+
                 return runInTransaction(
                     oneDriveState,
                     driveStructure,
@@ -157,8 +201,19 @@ export async function replicateMicrosoftOneDrive<RxDocType>(
                             oneDriveState,
                             driveStructure,
                             options.collection.schema.primaryPath as any,
-                            rows
+                            rowsForDrive
                         );
+                        /**
+                         * When attachment replication is enabled, deserialise the
+                         * base64 attachment data that OneDrive stored in the
+                         * `_attachments_data` field back into Blobs on each conflict
+                         * document before returning them to the replication protocol.
+                         */
+                        if (replicateAttachments && conflicts.length > 0) {
+                            await Promise.all(
+                                conflicts.map(c => deserializeDocAttachments(c as any))
+                            );
+                        }
                         return conflicts;
                     },
                     () => replicationState.notifyPeers().catch(() => { })

src/plugins/replication-microsoft-onedrive/microsoft-onedrive-types.ts

@@ -66,6 +66,14 @@ export type SyncOptionsOneDrive<RxDocType> = Omit<
     signalingOptions?: SignalingOptions;
     pull?: OneDriveSyncPullOptions<RxDocType>;
     push?: OneDriveSyncPushOptions<RxDocType>;
+    /**
+     * Set to false to disable attachment replication.
+     * When enabled (default), attachment binary data is stored as base64
+     * directly inside the document JSON file on OneDrive.
+     * Attachment replication only has an effect when the collection schema
+     * has `attachments: {}` defined.
+     */
+    attachments?: false;
 };
 
 export type OneDriveCheckpointType = {

src/plugins/replication-microsoft-onedrive/upstream.ts

@@ -1,5 +1,6 @@
 import { RxReplicationWriteToMasterRow, WithDeletedAndAttachments } from '../../index.ts';
 import { newRxError, newRxFetchError } from '../../rx-error.ts';
+import { stripAllAttachmentDataForComparison } from '../../replication-protocol/helper.ts';
 import { deepEqual, ensureNotFalsy } from '../utils/index.ts';
 import { fetchDocumentContents, getDocumentFiles, insertDocumentFiles, updateDocumentFiles } from './document-handling.ts';
 import { fillFileIfEtagMatches, getDriveBaseUrl } from './microsoft-onedrive-helper.ts';
@@ -57,7 +58,10 @@ export async function fetchConflicts<RxDocType>(
             fileContent = contentsByFileId.byId[fileId];
         }
         if (row.assumedMasterState) {
-            if (!deepEqual(row.assumedMasterState, fileContent)) {
+            if (!deepEqual(
+                stripAllAttachmentDataForComparison(row.assumedMasterState),
+                stripAllAttachmentDataForComparison(fileContent)
+            )) {
                 conflicts.push(ensureNotFalsy(fileContent));
             } else {
                 nonConflicts.push(row);

src/replication-protocol/helper.ts

@@ -8,7 +8,9 @@ import type {
     WithDeletedAndAttachments
 } from '../types/index.d.ts';
 import {
+    blobToBase64String,
     clone,
+    createBlobFromBase64,
     createRevision,
     flatClone,
     getDefaultRevision,
@@ -83,6 +85,86 @@ export function stripAttachmentsDataFromMetaWriteRows<RxDocType>(
     });
 }
 
+/**
+ * Serialises attachment data in a document clone so it can safely be stored
+ * as JSON in a remote storage that does not natively support attachments
+ * (e.g. Google Drive, OneDrive).
+ *
+ * - When `serializeData` is true: Blob values are extracted and stored as
+ *   base64 strings in the top-level `_attachments_data` field, while
+ *   `_attachments` is stripped to clean stubs via `stripAttachmentsDataFromDocument`.
+ * - When `serializeData` is false: `_attachments` is set to `{}` so that
+ *   attachment stubs (without binary data) are never persisted. This
+ *   prevents the downstream replication protocol from trying to write
+ *   attachment data it does not have when a peer pulls the document.
+ *
+ * The function returns a NEW document object; the original is not mutated.
+ */
+export async function serializeDocAttachments<T>(doc: T, serializeData: boolean): Promise<T> {
+    const d = doc as any;
+    if (!d?._attachments) {
+        return doc;
+    }
+
+    if (!serializeData) {
+        return { ...d, _attachments: {} } as any;
+    }
+
+    const attachmentData: Record<string, string> = {};
+    await Promise.all(
+        Object.entries(d._attachments as Record<string, any>).map(async ([id, att]) => {
+            if (att.data instanceof Blob) {
+                attachmentData[id] = await blobToBase64String(att.data);
+            }
+        })
+    );
+
+    const stripped = stripAttachmentsDataFromDocument(d) as any;
+
+    if (Object.keys(attachmentData).length > 0) {
+        return { ...stripped, _attachments_data: attachmentData } as any;
+    }
+    return stripped as any;
+}
+
+/**
+ * Strips both `_attachments` data and the serialised `_attachments_data` field
+ * from a document, leaving only the structural attachment metadata
+ * (digest / length / type). Used to compare two document states without
+ * being affected by attachment binary data that lives in different shapes on
+ * each side (Blobs in memory vs base64 in the serialised JSON file).
+ */
+export function stripAllAttachmentDataForComparison<T>(doc: T): T {
+    const stripped = stripAttachmentsDataFromDocument(doc as any) as any;
+    delete stripped._attachments_data;
+    return stripped;
+}
+
+/**
+ * Converts attachment data stored in `_attachments_data` back to Blobs in the
+ * document, so that the downstream replication protocol can write them to the
+ * fork storage instance correctly.
+ *
+ * Mutates the document in place and removes the `_attachments_data` field.
+ */
+export async function deserializeDocAttachments(doc: any): Promise<void> {
+    const attachmentData: Record<string, string> | undefined = doc._attachments_data;
+    if (!attachmentData || !doc._attachments) {
+        return;
+    }
+    await Promise.all(
+        Object.entries(attachmentData).map(async ([id, base64]) => {
+            if (doc._attachments[id]) {
+                doc._attachments[id] = {
+                    ...doc._attachments[id],
+                    data: await createBlobFromBase64(base64, doc._attachments[id].type)
+                };
+            }
+        })
+    );
+    delete doc._attachments_data;
+}
+
 export function getUnderlyingPersistentStorage<RxDocType>(
     instance: RxStorageInstance<RxDocType, any, any, any>
 ): RxStorageInstance<RxDocType, any, any, any> {