feat: chunk data store

.changeset/hip-rats-decide.md

@@ -0,0 +1,22 @@
+---
+'astro': minor
+---
+
+Adds a new experimental `dataStore` option for controlling how the content layer persists its data store
+
+By default, Astro serializes the entire content layer data store to a single file (`.astro/data-store.json`). For very large content collections, this single serialized string and file can grow large enough to hit JavaScript string-length or file-size limits.
+
+Set `experimental.dataStore: 'chunked'` to instead split the data store across many smaller, content-addressed files inside a `.astro/data-store/` directory, described by a manifest:
+
+```js
+// astro.config.mjs
+import { defineConfig } from 'astro/config';
+
+export default defineConfig({
+  experimental: {
+    dataStore: 'chunked',
+  },
+});
+```
+
+Because each part file is named by a hash of its contents, unchanged parts keep the same name across builds and are not rewritten, and identical parts are deduplicated. The default value is `'file'`, which preserves the current behavior.

packages/astro/src/content/consts.ts

@@ -36,6 +36,8 @@ export const CONTENT_FLAGS = [
 
 export const CONTENT_TYPES_FILE = 'content.d.ts';
 export const DATA_STORE_FILE = 'data-store.json';
+export const DATA_STORE_DIR = 'data-store/';
+export const DATA_STORE_MANIFEST_FILE = '__manifest.json';
 export const ASSET_IMPORTS_FILE = 'content-assets.mjs';
 export const MODULES_IMPORTS_FILE = 'content-modules.mjs';
 export const COLLECTIONS_MANIFEST_FILE = 'collections/collections.json';

packages/astro/src/content/content-layer.ts

@@ -13,7 +13,6 @@ import {
 	ASSET_IMPORTS_FILE,
 	COLLECTIONS_MANIFEST_FILE,
 	CONTENT_LAYER_TYPE,
-	DATA_STORE_FILE,
 	MODULES_IMPORTS_FILE,
 } from './consts.js';
 import type { RenderedContent } from './data-store.js';
@@ -466,11 +465,3 @@ async function simpleLoader<TData extends { id: string }>(
 		),
 	});
 }
-/**
- * Get the path to the data store file.
- * During development, this is in the `.astro` directory so that the Vite watcher can see it.
- * In production, it's in the cache directory so that it's preserved between builds.
- */
-export function getDataStoreFile(settings: AstroSettings, isDev: boolean) {
-	return new URL(DATA_STORE_FILE, isDev ? settings.dotAstroDir : settings.config.cacheDir);
-}

packages/astro/src/content/data-store-source.ts

@@ -0,0 +1,64 @@
+import type { DataEntry, ImmutableDataStore } from './data-store.js';
+
+/**
+ * A read-only, async view over content collection data, used at runtime by
+ * `getCollection()` and `getEntry()`. It lets the runtime read from different
+ * data sources (an in-memory snapshot today, potentially a database or remote
+ * source in the future) through one interface, without depending on Node.js
+ * APIs.
+ *
+ * The query methods are async so a source can perform I/O when resolving data.
+ * The default {@link InMemorySource} resolves synchronously.
+ */
+export interface DataStoreSource {
+	hasCollection(collection: string): Promise<boolean> | boolean;
+	get<T = DataEntry>(collection: string, key: string): Promise<T | undefined> | T | undefined;
+	entries<T = DataEntry>(
+		collection: string,
+	): Promise<Array<[id: string, T]>> | Array<[id: string, T]>;
+	values<T = DataEntry>(collection: string): Promise<Array<T>> | Array<T>;
+	keys(collection: string): Promise<Array<string>> | Array<string>;
+	has(collection: string, key: string): Promise<boolean> | boolean;
+	collections(): Promise<Map<string, Map<string, any>>> | Map<string, Map<string, any>>;
+}
+
+/**
+ * A {@link DataStoreSource} backed by an in-memory {@link ImmutableDataStore}.
+ * All queries resolve synchronously; the async signatures exist to satisfy the
+ * {@link DataStoreSource} contract.
+ */
+export class InMemorySource implements DataStoreSource {
+	#store: ImmutableDataStore;
+
+	constructor(store: ImmutableDataStore) {
+		this.#store = store;
+	}
+
+	hasCollection(collection: string): boolean {
+		return this.#store.hasCollection(collection);
+	}
+
+	get<T = DataEntry>(collection: string, key: string): T | undefined {
+		return this.#store.get<T>(collection, key);
+	}
+
+	entries<T = DataEntry>(collection: string): Array<[id: string, T]> {
+		return this.#store.entries<T>(collection);
+	}
+
+	values<T = DataEntry>(collection: string): Array<T> {
+		return this.#store.values<T>(collection);
+	}
+
+	keys(collection: string): Array<string> {
+		return this.#store.keys(collection);
+	}
+
+	has(collection: string, key: string): boolean {
+		return this.#store.has(collection, key);
+	}
+
+	collections(): Map<string, Map<string, any>> {
+		return this.#store.collections();
+	}
+}

packages/astro/src/content/data-store-writer.ts

@@ -0,0 +1,209 @@
+import { promises as fs, type PathLike } from 'node:fs';
+import * as devalue from 'devalue';
+import xxhash, { type XXHashAPI } from 'xxhash-wasm';
+import { emptyDir } from '../core/fs/index.js';
+import { DATA_STORE_MANIFEST_FILE } from './consts.js';
+
+/** Maximum number of entries serialized into a single chunk. */
+const CHUNK_ENTRY_LIMIT = 1000;
+/** Maximum size, in UTF-8 bytes, of a single part file. */
+const CHUNK_SIZE_LIMIT = 20 * 1024 * 1024; // 20 MB
+
+/**
+ * A chunked store manifest: each collection maps to a list of chunks, and each
+ * chunk to the list of part file names whose contents concatenate back into the
+ * chunk's serialized string.
+ */
+export type DataStoreManifest = Record<string, string[][]>;
+
+/**
+ * Persists the content collection data produced by the content layer. This is
+ * the write side that saves the data; {@link import('./data-store-source.js').DataStoreSource}
+ * is the read side that loads it back. Implementations run in Node.js
+ * (build/dev) and are never imported at runtime.
+ */
+export interface DataStoreWriter {
+	/** Serialize and persist the given collections. */
+	write(collections: Map<string, Map<string, any>>): Promise<void>;
+}
+
+/**
+ * Sort collections and their entries by key.
+ *
+ * Entry insertion order can vary between builds due to concurrent file
+ * processing (pLimit), so sorting here guarantees stable output regardless of
+ * processing order. Stable output keeps serialized strings (and the content
+ * hashes derived from them) deterministic across builds.
+ */
+function sortCollections(
+	collections: Map<string, Map<string, any>>,
+): Map<string, Map<string, any>> {
+	return new Map(
+		[...collections.entries()]
+			.sort(([a], [b]) => a.localeCompare(b))
+			.map(([key, collection]) => [
+				key,
+				new Map([...collection.entries()].sort(([a], [b]) => a.localeCompare(b))),
+			]),
+	);
+}
+
+/**
+ * Serialize collections to a deterministic devalue string.
+ */
+export function serializeDataStore(collections: Map<string, Map<string, any>>): string {
+	return devalue.stringify(sortCollections(collections));
+}
+
+const ENCODER = new TextEncoder();
+
+/**
+ * Split a string into parts each at most `maxBytes` UTF-8 bytes, never splitting
+ * a Unicode code point across parts.
+ *
+ * The store is serialized with devalue and written to disk as UTF-8. Splitting
+ * on UTF-16 code-unit boundaries (e.g. `String.prototype.slice`) can cut a
+ * surrogate pair in half; encoding a lone surrogate to UTF-8 substitutes U+FFFD,
+ * so concatenating the re-read parts would corrupt any astral-plane character
+ * (emoji, some CJK, etc.). Iterating with `for...of` yields whole code points,
+ * so `str.slice` is only ever called on code-point boundaries and the parts
+ * always rejoin to the exact original string.
+ */
+export function chunkString(str: string, maxBytes: number): string[] {
+	const chunks: string[] = [];
+	let startIndex = 0; // UTF-16 index where the current part starts
+	let index = 0; // current UTF-16 index (always on a code-point boundary)
+	let currentBytes = 0;
+	for (const char of str) {
+		const charBytes = ENCODER.encode(char).length;
+		// Close the current part before it would exceed the byte limit, but never
+		// emit an empty part (guards against a single code point over the limit).
+		if (currentBytes + charBytes > maxBytes && index > startIndex) {
+			chunks.push(str.slice(startIndex, index));
+			startIndex = index;
+			currentBytes = 0;
+		}
+		index += char.length; // 1 for BMP, 2 for a surrogate pair
+		currentBytes += charBytes;
+	}
+	if (startIndex < str.length) {
+		chunks.push(str.slice(startIndex));
+	}
+	return chunks;
+}
+
+/**
+ * Split a Map into consecutive sub-maps of at most `chunkSize` entries each,
+ * preserving iteration order.
+ */
+export function chunkMap<T>(map: Map<string, T>, chunkSize: number): Array<Map<string, T>> {
+	const chunks: Array<Map<string, T>> = [];
+	let currentChunk = new Map<string, T>();
+	for (const [key, value] of map) {
+		currentChunk.set(key, value);
+		if (currentChunk.size >= chunkSize) {
+			chunks.push(currentChunk);
+			currentChunk = new Map<string, T>();
+		}
+	}
+	if (currentChunk.size > 0) {
+		chunks.push(currentChunk);
+	}
+	return chunks;
+}
+
+/**
+ * Atomically write `data` to `file`.
+ *
+ * The data is written to a temporary file and then renamed into place to avoid
+ * partial reads. If the file already contains identical data, the write is
+ * skipped. Callers are responsible for serializing concurrent writes to the
+ * same file.
+ */
+export async function writeFileAtomic(file: PathLike, data: string): Promise<void> {
+	const tempFile = file instanceof URL ? new URL(`${file.href}.tmp`) : `${file}.tmp`;
+	const oldData = await fs.readFile(file, 'utf-8').catch(() => '');
+	if (oldData === data) {
+		// If the data hasn't changed, we can skip the write.
+		return;
+	}
+	// Write to a temporary file first and then move it to prevent partial reads.
+	await fs.writeFile(tempFile, data);
+	await fs.rename(tempFile, file);
+}
+
+/**
+ * A {@link DataStoreWriter} that serializes the whole store to a single file.
+ */
+export class FileWriter implements DataStoreWriter {
+	#file: PathLike;
+
+	constructor(file: PathLike) {
+		this.#file = file;
+	}
+
+	async write(collections: Map<string, Map<string, any>>): Promise<void> {
+		await writeFileAtomic(this.#file, serializeDataStore(collections));
+	}
+}
+
+/**
+ * A {@link DataStoreWriter} that splits the store across many content-addressed
+ * files inside a directory, described by a manifest.
+ *
+ * Splitting avoids emitting one enormous serialized string (bounded by the JS
+ * string length limit) or one enormous file (bounded by platform file-size
+ * limits). Each part file is named by the xxhash of its contents, so unchanged
+ * parts keep the same name across builds and their writes are skipped, and two
+ * identical parts are naturally deduplicated. The manifest is written last as
+ * the commit point, and stale files are pruned afterwards. This is the inverse
+ * of {@link import('./data-store.js').ImmutableDataStore.manifestToMap}.
+ */
+export class ChunkedWriter implements DataStoreWriter {
+	#dir: URL;
+	#manifestFile: URL;
+	#hasher?: XXHashAPI;
+
+	constructor(dir: URL) {
+		this.#dir = dir;
+		this.#manifestFile = new URL(`./${DATA_STORE_MANIFEST_FILE}`, dir);
+	}
+
+	async write(collections: Map<string, Map<string, any>>): Promise<void> {
+		if (!this.#hasher) {
+			this.#hasher = await xxhash();
+		}
+		const { h64ToString } = this.#hasher;
+
+		// Track every file this snapshot references so the rest can be pruned.
+		const writtenFiles = new Set<string>();
+		const manifest: DataStoreManifest = {};
+
+		// Sorted iteration keeps file names deterministic across builds.
+		for (const [collectionName, entries] of sortCollections(collections)) {
+			const chunks: string[][] = [];
+			// Bound the size of any single serialized string.
+			for (const chunkedEntries of chunkMap(entries, CHUNK_ENTRY_LIMIT)) {
+				const stringified = devalue.stringify(chunkedEntries);
+				// Bound the size of any single file.
+				const parts: string[] = [];
+				for (const part of chunkString(stringified, CHUNK_SIZE_LIMIT)) {
+					const fileName = `${h64ToString(part)}.txt`;
+					await writeFileAtomic(new URL(`./${fileName}`, this.#dir), part);
+					parts.push(fileName);
+					writtenFiles.add(fileName);
+				}
+				chunks.push(parts);
+			}
+			manifest[collectionName] = chunks;
+		}
+
+		// The manifest is the commit point: every part it references already
+		// exists on disk, so a reader never sees a dangling reference.
+		await writeFileAtomic(this.#manifestFile, JSON.stringify(manifest));
+		writtenFiles.add(DATA_STORE_MANIFEST_FILE);
+
+		// Prune files left behind by previous snapshots.
+		emptyDir(this.#dir, writtenFiles);
+	}
+}