feat(docs): add docs sync system to keep docs/ in sync with database (#106)

Adds a CLI tool (pnpm sync:docs) that recursively syncs every .md file
under docs/ into content_items as a note with source='docs-sync', tagged
#project:shadowbrain, #docs, and a path-derived category tag.

- src/lib/docs-sync.ts: syncDocsDirectory engine (idempotent upsert,
  tag self-healing, prune of deleted files, --force/--dry-run)
- scripts/sync-docs.ts: CLI entry point mirroring import-markdown.ts
- 27 tests covering idempotency, tags, cleanup, dry-run, force, and the
  hidden/private re-sync regression
- version 0.8.0 -> 0.9.0 (new user-facing capability)

Author: mikkisguy

README.md

@@ -5,7 +5,7 @@
 
 # ShadowBrain
 
-[![Version](https://img.shields.io/badge/version-0.8.0-yellow)](CHANGELOG.md)
+[![Version](https://img.shields.io/badge/version-0.9.0-yellow)](CHANGELOG.md)
 
 </div>
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "shadowbrain",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "private": true,
   "scripts": {
     "dev": "next dev",
@@ -14,6 +14,7 @@
     "test:db:reset": "node scripts/test-db-reset.js",
     "test:db:cleanup": "node scripts/test-db-cleanup.js",
     "import:markdown": "node --env-file=.env --import tsx scripts/import-markdown.ts",
+    "sync:docs": "node --env-file=.env --import tsx scripts/sync-docs.ts",
     "migrate:journal-shadows": "node --env-file=.env --import tsx scripts/migrate-journal-shadows.ts",
     "hash:password": "tsx scripts/hash-password.ts",
     "test": "vitest",

scripts/sync-docs.ts

@@ -0,0 +1,126 @@
+/**
+ * CLI entry point for the docs sync system.
+ *
+ * Reads the `docs/` directory at the project root and upserts every
+ * `.md` file as a `content_item` with `type='note'` and
+ * `source='docs-sync'`, tagged `#project:shadowbrain`, `#docs`, and a
+ * path-derived category tag. Files removed from disk are pruned from the
+ * database. Re-running is safe and idempotent.
+ *
+ * Usage:
+ *   pnpm sync:docs
+ *   pnpm sync:docs --dir /path/to/docs
+ *   pnpm sync:docs --force        # re-write every file
+ *   pnpm sync:docs --dry-run      # preview without writing
+ *
+ * Exits non-zero when at least one file failed to sync, or when a
+ * transaction-level failure aborts the run. The summary is printed for
+ * per-file failures; a crash short-circuits to the error handler.
+ */
+import { resolve } from "path";
+import { getDb, closeDb } from "@/db/index";
+import { syncDocsDirectory, formatDocsSyncResult } from "@/lib/docs-sync";
+import { log } from "@/lib/logger";
+
+interface CliArgs {
+  dir: string;
+  force: boolean;
+  dryRun: boolean;
+}
+
+function parseArgs(argv: string[]): CliArgs {
+  let dir: string | null = null;
+  let force = false;
+  let dryRun = false;
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (arg === "--dir" || arg === "-d") {
+      const next = argv[i + 1];
+      if (!next) {
+        throw new Error("--dir requires a path argument");
+      }
+      dir = next;
+      i += 1;
+    } else if (arg.startsWith("--dir=")) {
+      dir = arg.slice("--dir=".length);
+    } else if (arg === "--force") {
+      force = true;
+    } else if (arg === "--dry-run") {
+      dryRun = true;
+    } else if (arg === "--help" || arg === "-h") {
+      printHelp();
+      process.exit(0);
+    } else {
+      throw new Error(`Unknown argument: ${arg}`);
+    }
+  }
+  return {
+    dir: dir ?? resolve(process.cwd(), "docs"),
+    force,
+    dryRun,
+  };
+}
+
+function printHelp(): void {
+  console.log(`Usage: pnpm sync:docs [--dir <path>] [--force] [--dry-run]
+
+Reads .md files from <path> (default: ./docs) and upserts each as a
+content_item with type='note' and source='docs-sync'. Each doc is tagged
+#project:shadowbrain, #docs, and a category tag derived from its path.
+Files removed from disk are pruned. Re-runs are idempotent.
+
+Options:
+  -d, --dir <path>   Directory to sync (default: ./docs)
+  --force            Re-write every file even if the stored content
+                     matches the on-disk version.
+  --dry-run          Preview changes without writing to the database.
+  -h, --help         Show this help
+`);
+}
+
+async function main(): Promise<void> {
+  let args: CliArgs;
+  try {
+    args = parseArgs(process.argv.slice(2));
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    console.error(`Argument error: ${message}`);
+    printHelp();
+    process.exit(2);
+  }
+
+  const db = getDb();
+  try {
+    const result = await syncDocsDirectory(db, args.dir, {
+      skipUnchanged: !args.force,
+      dryRun: args.dryRun,
+    });
+    console.log(formatDocsSyncResult(result));
+    log("info", "docs sync complete", {
+      event: "docs.sync.complete",
+      directory: result.directory,
+      created: result.created,
+      updated: result.updated,
+      skipped: result.skipped,
+      deleted: result.deleted,
+      failed: result.failed,
+      force: args.force,
+      dryRun: args.dryRun,
+    });
+    if (result.failed > 0) {
+      process.exitCode = 1;
+    }
+  } finally {
+    closeDb();
+  }
+}
+
+main().catch((err) => {
+  const message = err instanceof Error ? err.message : String(err);
+  console.error(`Docs sync failed: ${message}`);
+  log("error", "docs sync crashed", {
+    event: "docs.sync.crash",
+    error: err instanceof Error ? { message, stack: err.stack } : message,
+  });
+  process.exit(1);
+});