feat: chunk data store

.changeset/hip-rats-decide.md

@@ -0,0 +1,22 @@
+---
+'astro': minor
+---
+
+Adds a new experimental `collectionStorage` option for controlling how the content layer persists its data store
+
+By default, Astro serializes the entire content layer data store to a single file (`.astro/data-store.json`). For very large content collections, this file can grow large enough to hit platform file-size limits.
+
+Set `experimental.collectionStorage: 'chunked'` to instead split the data store across many smaller, content-addressed files inside a `.astro/data-store/` directory, described by a manifest:
+
+```js
+// astro.config.mjs
+import { defineConfig } from 'astro/config';
+
+export default defineConfig({
+  experimental: {
+    collectionStorage: 'chunked',
+  },
+});
+```
+
+Because each part file is named by a hash of its contents, unchanged parts keep the same name across builds and are not rewritten, and identical parts are deduplicated. The default value is `'single-file'`, which preserves the current behavior.

packages/astro/src/content/consts.ts

@@ -36,6 +36,8 @@ export const CONTENT_FLAGS = [
 
 export const CONTENT_TYPES_FILE = 'content.d.ts';
 export const DATA_STORE_FILE = 'data-store.json';
+export const DATA_STORE_DIR = 'data-store/';
+export const DATA_STORE_MANIFEST_FILE = 'manifest.json';
 export const ASSET_IMPORTS_FILE = 'content-assets.mjs';
 export const MODULES_IMPORTS_FILE = 'content-modules.mjs';
 export const COLLECTIONS_MANIFEST_FILE = 'collections/collections.json';

packages/astro/src/content/content-layer.ts

@@ -13,7 +13,6 @@ import {
 	ASSET_IMPORTS_FILE,
 	COLLECTIONS_MANIFEST_FILE,
 	CONTENT_LAYER_TYPE,
-	DATA_STORE_FILE,
 	MODULES_IMPORTS_FILE,
 } from './consts.js';
 import type { RenderedContent } from './data-store.js';
@@ -466,11 +465,3 @@ async function simpleLoader<TData extends { id: string }>(
 		),
 	});
 }
-/**
- * Get the path to the data store file.
- * During development, this is in the `.astro` directory so that the Vite watcher can see it.
- * In production, it's in the cache directory so that it's preserved between builds.
- */
-export function getDataStoreFile(settings: AstroSettings, isDev: boolean) {
-	return new URL(DATA_STORE_FILE, isDev ? settings.dotAstroDir : settings.config.cacheDir);
-}

packages/astro/src/content/data-store-source.ts

@@ -0,0 +1,64 @@
+import type { DataEntry, ImmutableDataStore } from './data-store.js';
+
+/**
+ * A read-only, async view over content collection data, used at runtime by
+ * `getCollection()` and `getEntry()`. It lets the runtime read from different
+ * data sources (an in-memory snapshot today, potentially a database or remote
+ * source in the future) through one interface, without depending on Node.js
+ * APIs.
+ *
+ * The query methods are async so a source can perform I/O when resolving data.
+ * The default {@link InMemorySource} resolves synchronously.
+ */
+export interface DataStoreSource {
+	hasCollection(collection: string): Promise<boolean> | boolean;
+	get<T = DataEntry>(collection: string, key: string): Promise<T | undefined> | T | undefined;
+	entries<T = DataEntry>(
+		collection: string,
+	): Promise<Array<[id: string, T]>> | Array<[id: string, T]>;
+	values<T = DataEntry>(collection: string): Promise<Array<T>> | Array<T>;
+	keys(collection: string): Promise<Array<string>> | Array<string>;
+	has(collection: string, key: string): Promise<boolean> | boolean;
+	collections(): Promise<Map<string, Map<string, any>>> | Map<string, Map<string, any>>;
+}
+
+/**
+ * A {@link DataStoreSource} backed by an in-memory {@link ImmutableDataStore}.
+ * All queries resolve synchronously; the async signatures exist to satisfy the
+ * {@link DataStoreSource} contract.
+ */
+export class InMemorySource implements DataStoreSource {
+	#store: ImmutableDataStore;
+
+	constructor(store: ImmutableDataStore) {
+		this.#store = store;
+	}
+
+	hasCollection(collection: string): boolean {
+		return this.#store.hasCollection(collection);
+	}
+
+	get<T = DataEntry>(collection: string, key: string): T | undefined {
+		return this.#store.get<T>(collection, key);
+	}
+
+	entries<T = DataEntry>(collection: string): Array<[id: string, T]> {
+		return this.#store.entries<T>(collection);
+	}
+
+	values<T = DataEntry>(collection: string): Array<T> {
+		return this.#store.values<T>(collection);
+	}
+
+	keys(collection: string): Array<string> {
+		return this.#store.keys(collection);
+	}
+
+	has(collection: string, key: string): boolean {
+		return this.#store.has(collection, key);
+	}
+
+	collections(): Map<string, Map<string, any>> {
+		return this.#store.collections();
+	}
+}

packages/astro/src/content/data-store-writer.ts

@@ -0,0 +1,181 @@
+import { promises as fs, type PathLike } from 'node:fs';
+import * as devalue from 'devalue';
+import xxhash, { type XXHashAPI } from 'xxhash-wasm';
+import { emptyDir } from '../core/fs/index.js';
+import { DATA_STORE_MANIFEST_FILE } from './consts.js';
+
+/** Maximum size, in UTF-8 bytes, of a single part file. */
+const CHUNK_SIZE_LIMIT = 20 * 1024 * 1024; // 20 MB
+
+/**
+ * A chunked store manifest: each collection maps to the list of part file names
+ * whose contents concatenate back into that collection's serialized string.
+ */
+export type DataStoreManifest = Record<string, string[]>;
+
+/**
+ * Persists the content collection data produced by the content layer. This is
+ * the write side that saves the data; {@link import('./data-store-source.js').DataStoreSource}
+ * is the read side that loads it back. Implementations run in Node.js
+ * (build/dev) and are never imported at runtime.
+ */
+export interface DataStoreWriter {
+	/** Serialize and persist the given collections. */
+	write(collections: Map<string, Map<string, any>>): Promise<void>;
+}
+
+/**
+ * Sort collections and their entries by key.
+ *
+ * Entry insertion order can vary between builds due to concurrent file
+ * processing (pLimit), so sorting here guarantees stable output regardless of
+ * processing order. Stable output keeps serialized strings (and the content
+ * hashes derived from them) deterministic across builds.
+ */
+function sortCollections(
+	collections: Map<string, Map<string, any>>,
+): Map<string, Map<string, any>> {
+	return new Map(
+		[...collections.entries()]
+			.sort(([a], [b]) => a.localeCompare(b))
+			.map(([key, collection]) => [
+				key,
+				new Map([...collection.entries()].sort(([a], [b]) => a.localeCompare(b))),
+			]),
+	);
+}
+
+/**
+ * Serialize collections to a deterministic devalue string.
+ */
+export function serializeDataStore(collections: Map<string, Map<string, any>>): string {
+	return devalue.stringify(sortCollections(collections));
+}
+
+const ENCODER = new TextEncoder();
+
+/**
+ * Split a string into parts each at most `maxBytes` UTF-8 bytes, never splitting
+ * a Unicode code point across parts.
+ *
+ * The store is serialized with devalue and written to disk as UTF-8. Splitting
+ * on UTF-16 code-unit boundaries (e.g. `String.prototype.slice`) can cut a
+ * surrogate pair in half; encoding a lone surrogate to UTF-8 substitutes U+FFFD,
+ * so concatenating the re-read parts would corrupt any astral-plane character
+ * (emoji, some CJK, etc.). Iterating with `for...of` yields whole code points,
+ * so `str.slice` is only ever called on code-point boundaries and the parts
+ * always rejoin to the exact original string.
+ */
+export function chunkString(str: string, maxBytes: number): string[] {
+	const chunks: string[] = [];
+	let startIndex = 0; // UTF-16 index where the current part starts
+	let index = 0; // current UTF-16 index (always on a code-point boundary)
+	let currentBytes = 0;
+	for (const char of str) {
+		const charBytes = ENCODER.encode(char).length;
+		// Close the current part before it would exceed the byte limit, but never
+		// emit an empty part (guards against a single code point over the limit).
+		if (currentBytes + charBytes > maxBytes && index > startIndex) {
+			chunks.push(str.slice(startIndex, index));
+			startIndex = index;
+			currentBytes = 0;
+		}
+		index += char.length; // 1 for BMP, 2 for a surrogate pair
+		currentBytes += charBytes;
+	}
+	if (startIndex < str.length) {
+		chunks.push(str.slice(startIndex));
+	}
+	return chunks;
+}
+
+/**
+ * Atomically write `data` to `file`.
+ *
+ * The data is written to a temporary file and then renamed into place to avoid
+ * partial reads. If the file already contains identical data, the write is
+ * skipped. Callers are responsible for serializing concurrent writes to the
+ * same file.
+ */
+export async function writeFileAtomic(file: PathLike, data: string): Promise<void> {
+	const tempFile = file instanceof URL ? new URL(`${file.href}.tmp`) : `${file}.tmp`;
+	const oldData = await fs.readFile(file, 'utf-8').catch(() => '');
+	if (oldData === data) {
+		// If the data hasn't changed, we can skip the write.
+		return;
+	}
+	// Write to a temporary file first and then move it to prevent partial reads.
+	await fs.writeFile(tempFile, data);
+	await fs.rename(tempFile, file);
+}
+
+/**
+ * A {@link DataStoreWriter} that serializes the whole store to a single file.
+ */
+export class FileWriter implements DataStoreWriter {
+	#file: PathLike;
+
+	constructor(file: PathLike) {
+		this.#file = file;
+	}
+
+	async write(collections: Map<string, Map<string, any>>): Promise<void> {
+		await writeFileAtomic(this.#file, serializeDataStore(collections));
+	}
+}
+
+/**
+ * A {@link DataStoreWriter} that splits the store across many content-addressed
+ * files inside a directory, described by a manifest.
+ *
+ * Each collection is serialized to a string and split into parts no larger than
+ * a fixed byte size, so no single file grows unbounded (platform file-size
+ * limits). Each part file is named by the xxhash of its contents, so unchanged
+ * parts keep the same name across builds and their writes are skipped, and two
+ * identical parts are naturally deduplicated. The manifest is written last as
+ * the commit point, and stale files are pruned afterwards. This is the inverse
+ * of {@link import('./data-store.js').ImmutableDataStore.manifestToMap}.
+ */
+export class ChunkedWriter implements DataStoreWriter {
+	#dir: URL;
+	#manifestFile: URL;
+	#hasher?: XXHashAPI;
+
+	constructor(dir: URL) {
+		this.#dir = dir;
+		this.#manifestFile = new URL(`./${DATA_STORE_MANIFEST_FILE}`, dir);
+	}
+
+	async write(collections: Map<string, Map<string, any>>): Promise<void> {
+		if (!this.#hasher) {
+			this.#hasher = await xxhash();
+		}
+		const { h64ToString } = this.#hasher;
+
+		// Track every file this snapshot references so the rest can be pruned.
+		const writtenFiles = new Set<string>();
+		const manifest: DataStoreManifest = {};
+
+		// Sorted iteration keeps file names deterministic across builds.
+		for (const [collectionName, entries] of sortCollections(collections)) {
+			const stringified = devalue.stringify(entries);
+			// Split the serialized collection so no single file grows unbounded.
+			const parts: string[] = [];
+			for (const part of chunkString(stringified, CHUNK_SIZE_LIMIT)) {
+				const fileName = `${h64ToString(part)}.txt`;
+				await writeFileAtomic(new URL(`./${fileName}`, this.#dir), part);
+				parts.push(fileName);
+				writtenFiles.add(fileName);
+			}
+			manifest[collectionName] = parts;
+		}
+
+		// The manifest is the commit point: every part it references already
+		// exists on disk, so a reader never sees a dangling reference.
+		await writeFileAtomic(this.#manifestFile, JSON.stringify(manifest));
+		writtenFiles.add(DATA_STORE_MANIFEST_FILE);
+
+		// Prune files left behind by previous snapshots.
+		emptyDir(this.#dir, writtenFiles);
+	}
+}

packages/astro/src/content/data-store.ts

@@ -1,5 +1,6 @@
 import type { MarkdownHeading } from '@astrojs/internal-helpers/markdown';
 import * as devalue from 'devalue';
+import { type DataStoreSource, InMemorySource } from './data-store-source.js';
 
 export interface RenderedContent {
 	/** Rendered HTML string. If present then `render(entry)` will return a component that renders this HTML. */
@@ -83,6 +84,31 @@ export class ImmutableDataStore {
 		return this._collections;
 	}
 
+	/**
+	 * Rebuilds a collections map from a chunked-store manifest whose part file
+	 * names have already been swapped for their contents.
+	 *
+	 * Each collection maps to a list of parts. A part is either a raw string
+	 * (when the store is loaded from disk) or an ESM namespace from a `?raw`
+	 * import (`{ default: string }`, when emitted into the virtual module at
+	 * runtime). A collection's parts are concatenated back into the exact
+	 * serialized string, then parsed with devalue. This is the inverse of
+	 * {@link import('./data-store-writer.js').ChunkedWriter} and stays free of
+	 * Node built-ins so it can run at runtime.
+	 */
+	static manifestToMap(manifest: Record<string, Array<string | { default: string }>>) {
+		const collections = new Map<string, Map<string, any>>();
+		for (const [collectionName, parts] of Object.entries(manifest)) {
+			let stringified = '';
+			for (const part of parts) {
+				stringified += typeof part === 'string' ? part : part.default;
+			}
+			const entries: Map<string, any> = devalue.parse(stringified);
+			collections.set(collectionName, entries);
+		}
+		return collections;
+	}
+
 	/**
 	 * Attempts to load a DataStore from the virtual module.
 	 * This only works in Vite.
@@ -94,7 +120,14 @@ export class ImmutableDataStore {
 			if (data.default instanceof Map) {
 				return ImmutableDataStore.fromMap(data.default);
 			}
-			const map = devalue.unflatten(data.default);
+			// A single-file store is emitted as a devalue-flattened array.
+			if (Array.isArray(data.default)) {
+				const map = devalue.unflatten(data.default);
+				return ImmutableDataStore.fromMap(map);
+			}
+			// A chunked store is emitted as a manifest object of collections to
+			// their (lazily imported) serialized parts.
+			const map = ImmutableDataStore.manifestToMap(data.default);
 			return ImmutableDataStore.fromMap(map);
 		} catch {}
 		return new ImmutableDataStore();
@@ -108,16 +141,17 @@ export class ImmutableDataStore {
 }
 
 function dataStoreSingleton() {
-	let instance: Promise<ImmutableDataStore> | ImmutableDataStore | undefined = undefined;
+	let instance: Promise<DataStoreSource> | DataStoreSource | undefined = undefined;
 	return {
-		get: async () => {
+		get: async (): Promise<DataStoreSource> => {
 			if (!instance) {
-				instance = ImmutableDataStore.fromModule();
+				instance = ImmutableDataStore.fromModule().then((store) => new InMemorySource(store));
 			}
 			return instance;
 		},
+		// Note: currently unused, but kept for API stability.
 		set: (store: ImmutableDataStore) => {
-			instance = store;
+			instance = new InMemorySource(store);
 		},
 	};
 }