test(ci): gate cross-surface store/reference contract parity on the PR lane

.github/workflows/ci_test_lanes.yml

@@ -92,6 +92,45 @@ jobs:
       - name: Validate documentation dependencies
         run: npm run validate:doc-deps
 
+  contract_parity:
+    # Focused PR-time cross-surface contract-parity lane.
+    #
+    # Retrospective ent_68a9270e2e656da847c10ced: the `source_storage:'reference'`
+    # feature shipped incomplete to an evaluator across three releases because the
+    # covering integration tests ran only in the nightly remote_integration
+    # workflow — never on the PR baseline lane, which runs `test:unit` (no DB).
+    #
+    # This lane runs ONLY the parity-critical store/reference integration tests
+    # (MCP + REST surfaces driven through one shared scenario matrix) on every PR.
+    # It deliberately does NOT pull the whole nightly integration suite onto each
+    # PR — just the cross-surface parity gate — so it stays fast (~a few seconds
+    # of tests). The tests run against the same local SQLite backend the nightly
+    # job uses (default mode: RUN_REMOTE_TESTS unset), with file-parallelism
+    # disabled to avoid SQLite lock contention between the two HTTP-server files.
+    #
+    # Implements task_policy cross_surface_contract_parity_tested_all_surfaces
+    # (ent_2ad0677fe23c0c1878ae43e8) and
+    # fixed_means_behavior_verified_not_contract_accepted (ent_db0b7855d47012084477fb00).
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: "npm"
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Build server (integration tests boot the in-process Actions server)
+        run: npm run build:server
+
+      - name: Run cross-surface contract-parity integration tests
+        run: npm run test:contract-parity
+
   frontend:
     runs-on: ubuntu-latest
     steps:

docs/testing/automated_test_catalog.md

@@ -61,8 +61,8 @@ flowchart TD
 - Do not hand-edit suite inventory entries in this file. Update the generator or the repository tree, then regenerate.
 
 ## Repo-wide summary
-- Total automated test files: **517**
-- Backend and repo Vitest files: **483**
+- Total automated test files: **518**
+- Backend and repo Vitest files: **484**
 - Frontend Vitest files: **9**
 - Playwright spec files: **25**
 
@@ -72,7 +72,7 @@ flowchart TD
 | Vitest unit tests | 144 |
 | Vitest service tests | 35 |
 | Source-adjacent tests | 64 |
-| Vitest integration tests | 145 |
+| Vitest integration tests | 146 |
 | Vitest CLI tests | 65 |
 | Vitest contract tests | 14 |
 | Vitest security tests | 4 |
@@ -373,7 +373,7 @@ flowchart TD
 **Runner:** `vitest`
 **Command:** `npm run test:integration` or `npx vitest run tests/integration`
 **Requirements:** Database configured; remote-dependent subsets additionally need `RUN_REMOTE_TESTS=1`.
-**Files (145):**
+**Files (146):**
 - `tests/integration/aauth_attribution_stamping.test.ts`
 - `tests/integration/aauth_mcp_capability_parity.test.ts`
 - `tests/integration/aauth_mcp_initialize_admission.test.ts`
@@ -503,6 +503,7 @@ flowchart TD
 - `tests/integration/store_explicit_canonical_name.test.ts`
 - `tests/integration/store_external_link_schema.test.ts`
 - `tests/integration/store_prefix_duplicate_candidates.test.ts`
+- `tests/integration/store_reference_source_parity.test.ts`
 - `tests/integration/store_registered_schema_alias_precedence.test.ts`
 - `tests/integration/store_required_unknown_field_signals.test.ts`
 - `tests/integration/store_resolution_attributes_hint.test.ts`

package.json

@@ -94,6 +94,7 @@
     "test:bench": "cross-env RUN_BENCH=1 vitest run tests/performance",
     "test:cross-layer": "vitest run tests/integration/cli_to_mcp_store.test.ts tests/integration/cli_to_mcp_entities.test.ts tests/integration/cli_to_mcp_relationships.test.ts tests/integration/cli_to_mcp_schemas.test.ts",
     "test:parity": "vitest run tests/contract/contract_mcp_cli_parity.test.ts",
+    "test:contract-parity": "vitest run --no-file-parallelism tests/integration/store_reference_source_parity.test.ts tests/integration/http_store_reference_source.test.ts tests/integration/mcp_store_reference_source.test.ts",
     "test:e2e:coverage": "npm run test:contract && npm run test:cross-layer",
     "test:agent-mcp": "vitest run tests/agent",
     "test:coverage": "vitest run --coverage",

tests/helpers/store_reference_parity.ts

@@ -0,0 +1,165 @@
+/**
+ * Shared parity-matrix driver for `source_storage:'reference'` store scenarios.
+ *
+ * Retrospective ent_68a9270e2e656da847c10ced found that the by-reference
+ * source-storage feature shipped incomplete because contract parity across
+ * Neotoma's surfaces (MCP, HTTP/REST, CLI, SDK) was never tested on the same
+ * scenarios. This helper encodes a single scenario matrix and drives it across
+ * BOTH the MCP `store` tool dispatch and the REST `POST /store` route, so a
+ * regression on either surface (or a divergence between them) fails the same
+ * lane — implementing task_policy cross_surface_contract_parity_tested_all_surfaces
+ * (ent_2ad0677fe23c0c1878ae43e8).
+ *
+ * Each scenario asserts the EFFECT (storage_mode='reference' in the sources
+ * row), not merely that the input was accepted — per task_policy
+ * fixed_means_behavior_verified_not_contract_accepted (ent_db0b7855d47012084477fb00).
+ */
+
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import { db } from "../../src/db.js";
+import { NeotomaServer } from "../../src/server.js";
+
+export type StoreSurface = "mcp" | "rest";
+
+export interface ReferenceParityResult {
+  /** source_id of the file (reference) leg, used to inspect the sources row. */
+  sourceId: string;
+  /** storage_mode the surface reported in its response envelope. */
+  reportedStorageMode: unknown;
+}
+
+/**
+ * Drives the MCP `store` tool through the real CallToolRequestSchema dispatch
+ * surface (`executeTool`) — the same code path the `/mcp` JSON-RPC route
+ * routes a `tools/call` for `store` into. Returns the file-leg source_id and
+ * the storage_mode the MCP envelope reported.
+ */
+export async function storeReferenceViaMcp(
+  server: NeotomaServer,
+  args: {
+    userId: string;
+    filePath: string;
+    idempotencyKey: string;
+    withEntities: boolean;
+  }
+): Promise<ReferenceParityResult> {
+  const dispatch = server as unknown as {
+    executeTool: (
+      name: string,
+      args: unknown
+    ) => Promise<{ content: Array<{ type: string; text: string }> }>;
+  };
+
+  const toolArgs: Record<string, unknown> = {
+    user_id: args.userId,
+    file_path: args.filePath,
+    mime_type: "text/plain",
+    source_storage: "reference",
+    file_idempotency_key: `${args.idempotencyKey}-file`,
+  };
+  if (args.withEntities) {
+    toolArgs.idempotency_key = args.idempotencyKey;
+    toolArgs.entities = [{ entity_type: "note", title: "parity-mcp", content: "body" }];
+  } else {
+    toolArgs.idempotency_key = args.idempotencyKey;
+  }
+
+  const result = await dispatch.executeTool("store", toolArgs);
+  const payload = JSON.parse(result.content[0]!.text) as Record<string, unknown>;
+
+  if (args.withEntities) {
+    const unstructured = payload.unstructured as Record<string, unknown> | undefined;
+    return {
+      sourceId: unstructured?.source_id as string,
+      reportedStorageMode: unstructured?.storage_mode,
+    };
+  }
+  return {
+    sourceId: payload.source_id as string,
+    reportedStorageMode: payload.storage_mode,
+  };
+}
+
+/**
+ * Drives the REST `POST /store` route over real HTTP. Returns the file-leg
+ * source_id and the storage_mode the REST envelope reported.
+ */
+export async function storeReferenceViaRest(
+  apiBase: string,
+  args: {
+    userId: string;
+    filePath: string;
+    idempotencyKey: string;
+    withEntities: boolean;
+  }
+): Promise<ReferenceParityResult> {
+  const body: Record<string, unknown> = {
+    user_id: args.userId,
+    file_path: args.filePath,
+    mime_type: "text/plain",
+    source_storage: "reference",
+    file_idempotency_key: `${args.idempotencyKey}-file`,
+    idempotency_key: args.idempotencyKey,
+  };
+  if (args.withEntities) {
+    body.entities = [{ entity_type: "note", title: "parity-rest", content: "body" }];
+  }
+
+  const resp = await fetch(`${apiBase}/store`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify(body),
+  });
+  if (resp.status !== 200) {
+    throw new Error(`REST /store returned ${resp.status}: ${await resp.text()}`);
+  }
+  const payload = (await resp.json()) as Record<string, unknown>;
+
+  if (args.withEntities) {
+    const unstructured = payload.unstructured as Record<string, unknown> | undefined;
+    return {
+      sourceId: unstructured?.source_id as string,
+      reportedStorageMode: unstructured?.storage_mode,
+    };
+  }
+  return {
+    sourceId: payload.source_id as string,
+    reportedStorageMode: payload.storage_mode,
+  };
+}
+
+/** Reads back the storage_mode persisted on the sources row (the EFFECT). */
+export async function readSourceStorageMode(sourceId: string): Promise<string | undefined> {
+  const { data } = await db.from("sources").select("storage_mode").eq("id", sourceId).single();
+  return (data as { storage_mode?: string } | null)?.storage_mode;
+}
+
+/** Writes a temp file and tracks its directory for cleanup. */
+export function makeReferenceTempFile(
+  tempDirs: string[],
+  content: string,
+  filename = "parity-ref.txt"
+): string {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "neotoma-parity-ref-"));
+  tempDirs.push(dir);
+  const filePath = path.join(dir, filename);
+  fs.writeFileSync(filePath, content, "utf-8");
+  return filePath;
+}
+
+/** The shared scenario matrix: each (surface × shape) cell. */
+export interface ParityScenario {
+  surface: StoreSurface;
+  shape: "file-only" | "file+entities";
+  label: string;
+}
+
+export const REFERENCE_PARITY_MATRIX: ParityScenario[] = [
+  { surface: "mcp", shape: "file-only", label: "MCP store — file-only" },
+  { surface: "mcp", shape: "file+entities", label: "MCP store — file+entities" },
+  { surface: "rest", shape: "file-only", label: "REST /store — file-only" },
+  { surface: "rest", shape: "file+entities", label: "REST /store — file+entities" },
+];