fix(store,embed): honor source_storage:reference + disable graph polling (#1826, #1837)

docs/testing/automated_test_catalog.md

@@ -61,8 +61,8 @@ flowchart TD
 - Do not hand-edit suite inventory entries in this file. Update the generator or the repository tree, then regenerate.
 
 ## Repo-wide summary
-- Total automated test files: **516**
-- Backend and repo Vitest files: **482**
+- Total automated test files: **517**
+- Backend and repo Vitest files: **483**
 - Frontend Vitest files: **9**
 - Playwright spec files: **25**
 
@@ -72,7 +72,7 @@ flowchart TD
 | Vitest unit tests | 144 |
 | Vitest service tests | 35 |
 | Source-adjacent tests | 64 |
-| Vitest integration tests | 144 |
+| Vitest integration tests | 145 |
 | Vitest CLI tests | 65 |
 | Vitest contract tests | 14 |
 | Vitest security tests | 4 |
@@ -373,7 +373,7 @@ flowchart TD
 **Runner:** `vitest`
 **Command:** `npm run test:integration` or `npx vitest run tests/integration`
 **Requirements:** Database configured; remote-dependent subsets additionally need `RUN_REMOTE_TESTS=1`.
-**Files (144):**
+**Files (145):**
 - `tests/integration/aauth_attribution_stamping.test.ts`
 - `tests/integration/aauth_mcp_capability_parity.test.ts`
 - `tests/integration/aauth_mcp_initialize_admission.test.ts`
@@ -422,6 +422,7 @@ flowchart TD
 - `tests/integration/guest_write_rate_limit.test.ts`
 - `tests/integration/hook_failure_hint.test.ts`
 - `tests/integration/http_related_entities_multihop.test.ts`
+- `tests/integration/http_store_reference_source.test.ts`
 - `tests/integration/idempotency_collision.test.ts`
 - `tests/integration/idempotency_key_content_mismatch.test.ts`
 - `tests/integration/inspector_bundled_mount.test.ts`

inspector/src/hooks/use_graph.ts

@@ -9,6 +9,7 @@ export function useGraphNeighborhood(params: GraphNeighborhoodParams | null) {
     queryKey: ["graph-neighborhood", params],
     queryFn: ({ signal }) => retrieveGraphNeighborhood(params!, { signal }),
     enabled: isApiUrlConfigured() && !!params?.node_id,
+    refetchInterval: false,
   });
 }
 
@@ -28,6 +29,7 @@ export function useGraphNeighborhoodWithBase(
     queryFn: ({ signal }) =>
       retrieveGraphNeighborhoodWithBase(apiBase, params!, { signal }),
     enabled: !!apiBase.trim() && !!params?.node_id,
+    refetchInterval: false,
   });
 }
 
@@ -36,5 +38,6 @@ export function useRelatedEntities(params: RelatedEntitiesParams | null) {
     queryKey: ["related-entities", params],
     queryFn: ({ signal }) => retrieveRelatedEntities(params!, { signal }),
     enabled: isApiUrlConfigured() && !!params?.entity_id,
+    refetchInterval: false,
   });
 }

inspector/src/pages/embed_graph.tsx

@@ -140,7 +140,7 @@ function EmbedGraphView({ initialNodeId }: { initialNodeId: string }) {
 
   return (
     <div
-      className="flex flex-col h-full w-full"
+      className="flex flex-col min-h-screen w-full"
       data-testid="embed-graph-root"
       data-embed="graph"
     >
@@ -204,7 +204,7 @@ function EmbedGraphView({ initialNodeId }: { initialNodeId: string }) {
       </div>
 
       {/* Graph canvas — fills remaining height */}
-      <div className="flex-1 min-h-0 bg-background">
+      <div className="flex-1 bg-background min-h-[calc(100dvh-5rem)]">
         {showInitialQuerySkeleton(graph) ? (
           <GraphAreaSkeleton />
         ) : graph.error && activeNodeId ? (

src/actions.ts

@@ -1663,7 +1663,7 @@
       // Encryption off: local requests can use no-auth. HTTP (insecure) defaults to anonymous 000... user; HTTPS/localhost can use dev-local.
       if (!authHeader?.startsWith("Bearer ") && !connectionIdHeader) {
         if (isLocalRequest(req)) {
           const isInsecure = req.protocol === "http" || !(req as any).secure;
           if (isInsecure) {
             req.headers["x-connection-id"] = "dev-local-http";
             connectionIdHeader = "dev-local-http";
@@ -1953,7 +1953,7 @@
       },
       () => transport!.handleRequest(req, res, req.body)
     );
   } catch (error: any) {
     logger.error("[MCP HTTP] Request error:", error);
     if (!res.headersSent) {
       res.status(500).json({
@@ -2316,7 +2316,7 @@
     const result = await initiateOAuthFlow(connection_id, client_name, finalRedirectUri);
 
     return res.json(result);
   } catch (error: any) {
     logError("MCPOAuthInitiate", req, error);
 
     // Check if it's a structured OAuthError
@@ -2326,7 +2326,7 @@
         message: string;
         statusCode: number;
         retryable?: boolean;
         details?: Record<string, any>;
       };
 
       return res.status(oauthError.statusCode).json({
@@ -2393,7 +2393,7 @@
       process.env.NEOTOMA_FRONTEND_URL || process.env.FRONTEND_URL || "http://localhost:5195";
     const successUrl = `${frontendBase}/oauth?connection_id=${encodeURIComponent(connectionId)}&status=success`;
     return res.redirect(successUrl);
   } catch (error: any) {
     logError("MCPOAuthCallback", req, error);
 
     // Extract structured error information
@@ -2690,7 +2690,7 @@
     });
 
     return res.redirect(result.authUrl);
   } catch (error: any) {
     logError("MCPOAuthAuthorize", req, error);
     return res.status(500).send(error.message ?? "Authorization failed");
   }
@@ -2780,7 +2780,7 @@
     return res.redirect(
       `${frontendOauth}?connection_id=${encodeURIComponent(connectionId)}&status=success`
     );
   } catch (error: any) {
     logError("MCPLocalLoginDevStub", req, error);
     const status = error?.code === "OAUTH_STATE_INVALID" || error?.statusCode === 400 ? 400 : 401;
     return res.status(status).send(
@@ -2857,7 +2857,7 @@
 
       res.setHeader("Content-Type", "application/json");
       return res.json(token);
     } catch (error: any) {
       logError("MCPOAuthToken", req, error);
       return res.status(400).json({
         error: "invalid_grant",
@@ -2885,7 +2885,7 @@
     });
     res.setHeader("Content-Type", "application/json");
     return res.status(201).json(reg);
   } catch (error: any) {
     logError("MCPOAuthRegister", req, error);
     if (error instanceof OAuthError) {
       const oauth = error as { code?: string; message?: string; statusCode?: number };
@@ -2921,7 +2921,7 @@
     const status = await getConnectionStatus(connection_id);
 
     return res.json({ status, connection_id });
   } catch (error: any) {
     logError("MCPOAuthStatus", req, error);
     return sendError(res, 500, "DB_QUERY_FAILED", error.message);
   }
@@ -7613,32 +7613,65 @@
   userId: string;
   fileContent?: string;
   fileBuffer?: Buffer;
+  filePath?: string;
   mimeType: string;
   idempotencyKey?: string;
   originalFilename?: string;
   sourceType?: string;
+  storageMode?: "inline" | "reference";
 }) {
   const {
     fileContent,
     fileBuffer,
+    filePath,
     mimeType,
     idempotencyKey,
     originalFilename,
     sourceType,
     userId,
+    storageMode = "inline",
   } = params;
   const resolvedFileBuffer =
     fileBuffer ?? (fileContent !== undefined ? Buffer.from(fileContent, "base64") : undefined);
-  if (!resolvedFileBuffer) {
-    throw new Error("fileContent or fileBuffer is required for unstructured storage");
+  if (!resolvedFileBuffer && storageMode !== "reference") {
+    throw new Error("fileContent or fileBuffer is required for inline storage");
   }
+
+  if (storageMode === "reference") {
+    if (!filePath) {
+      throw new Error("filePath is required for reference storage mode");
+    }
+    const absolutePath = path.isAbsolute(filePath)
+      ? filePath
+      : path.resolve(process.cwd(), filePath);
+
+    const { storeRawReference } = await import("./services/raw_storage.js");
+    const referenceResult = await storeRawReference({
+      userId,
+      absolutePath,
+      mimeType,
+      originalFilename: originalFilename?.trim() || undefined,
+      idempotencyKey,
+      provenance: { upload_method: "api_store_reference", client: "api" },
+    });
+
+    return {
+      source_id: referenceResult.sourceId,
+      content_hash: referenceResult.contentHash,
+      storage_mode: "reference",
+      reference_path: referenceResult.path,
+      file_size: referenceResult.sizeBytes,
+      mime_type: referenceResult.mimeType,
+    };
+  }
+
   const resolvedIdempotencyKey =
     idempotencyKey ??
-    (await import("node:crypto")).createHash("sha256").update(resolvedFileBuffer).digest("hex");
+    (await import("node:crypto")).createHash("sha256").update(resolvedFileBuffer!).digest("hex");
 
   const storageResult = await storeRawContent({
     userId,
-    fileBuffer: resolvedFileBuffer,
+    fileBuffer: resolvedFileBuffer!,
     mimeType,
     originalFilename: originalFilename?.trim() || undefined,
     sourceType,
@@ -7694,15 +7727,29 @@
         const resolvedPath = path.isAbsolute(parsed.data.file_path as string)
           ? (parsed.data.file_path as string)
           : path.resolve(process.cwd(), parsed.data.file_path as string);
-        resolvedFileBuffer = fs.readFileSync(resolvedPath);
-        if (!mimeType) {
-          const ext = path.extname(resolvedPath).toLowerCase();
-          mimeType = getMimeTypeFromExtension(ext) || "application/octet-stream";
+
+        if (parsed.data.source_storage === "reference") {
+          // For reference mode, don't read the file buffer, just use the path
+          if (!mimeType) {
+            const ext = path.extname(resolvedPath).toLowerCase();
+            mimeType = getMimeTypeFromExtension(ext) || "application/octet-stream";
+          }
+          originalFilename = originalFilename || path.basename(resolvedPath);
+        } else {
+          // For inline mode, read the file buffer
+          resolvedFileBuffer = fs.readFileSync(resolvedPath);
+          if (!mimeType) {
+            const ext = path.extname(resolvedPath).toLowerCase();
+            mimeType = getMimeTypeFromExtension(ext) || "application/octet-stream";
+          }
+          originalFilename = originalFilename || path.basename(resolvedPath);
         }
-        originalFilename = originalFilename || path.basename(resolvedPath);
       }
 
-      if ((!fileContent && !resolvedFileBuffer) || !mimeType) {
+      if (
+        (!fileContent && !resolvedFileBuffer && parsed.data.source_storage !== "reference") ||
+        !mimeType
+      ) {
         sendError(
           res,
           400,
@@ -7716,12 +7763,14 @@
         userId,
         fileContent,
         fileBuffer: resolvedFileBuffer,
+        filePath: parsed.data.file_path,
         mimeType,
         idempotencyKey:
           parsed.data.file_idempotency_key ??
           (!hasEntities ? parsed.data.idempotency_key : undefined),
         originalFilename,
         sourceType: (parsed.data as Record<string, unknown>).source_type as string | undefined,
+        storageMode: parsed.data.source_storage,
       });
     };
 

src/services/observation_storage.ts

@@ -144,6 +144,28 @@ export async function createObservation(
     (row as Record<string, unknown>).provenance = attribution;
   }
 
+  // Content-addressed idempotency: the observation id is a deterministic hash
+  // of (source_id, interpretation_id, entity_id, fields, idempotency_key), so a
+  // re-store of identical content yields the same id. Return the existing row
+  // instead of letting the insert collide on the observations.id UNIQUE
+  // constraint (which surfaced as a 500 on the REST /store structured leg for
+  // repeated content — e.g. a "file + its entities" combined call). Scope the
+  // check to (id, user_id) so a same-content observation owned by another user
+  // never masks this user's write. Mirrors the MCP store path in server.ts.
+  const { data: existing, error: existingError } = await db
+    .from("observations")
+    .select("*")
+    .eq("id", observationId)
+    .eq("user_id", params.user_id)
+    .maybeSingle();
+
+  if (existingError) {
+    throw new Error(`Failed to check existing observation: ${existingError.message}`);
+  }
+  if (existing) {
+    return existing as ObservationRecord;
+  }
+
   const { data, error } = await db.from("observations").insert(row).select().single();
 
   if (error) {

src/shared/action_schemas.ts

@@ -543,6 +543,7 @@ export const StoreRequestSchema = z
     file_path: z.string().optional(),
     mime_type: z.string().min(1).optional(),
     original_filename: z.string().optional(),
+    source_storage: z.enum(["inline", "reference"]).optional().default("inline"),
     /** Plan/dry-run: resolve and report action per observation, skip inserts. */
     commit: z.boolean().optional().default(true),
     /** Refuse merges that resolve to an existing entity without a deterministic rule. */

tests/integration/http_store_reference_source.test.ts

@@ -0,0 +1,126 @@
+/**
+ * Integration regression: POST /store honors source_storage:'reference' (#1826).
+ *
+ * #1830 added source_storage to openapi.yaml but handleStorePost still routed
+ * every file leg through storeUnstructuredForApi (inline bytes). This test
+ * exercises the real Express handler over HTTP and asserts the sources row
+ * carries storage_mode='reference'.
+ */
+
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { createServer } from "node:http";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { app } from "../../src/actions.js";
+import { db } from "../../src/db.js";
+
+const TEST_USER_ID = "00000000-0000-0000-0000-000000000001";
+const API_PORT = 18122;
+const API_BASE = `http://127.0.0.1:${API_PORT}`;
+
+describe("POST /store — source_storage:'reference' (#1826 REST path)", () => {
+  let httpServer: ReturnType<typeof createServer>;
+  const createdSourceIds: string[] = [];
+  const tempDirs: string[] = [];
+
+  beforeAll(async () => {
+    httpServer = createServer(app);
+    await new Promise<void>((resolve, reject) => {
+      httpServer.listen(API_PORT, "127.0.0.1", () => resolve());
+      httpServer.once("error", reject);
+    });
+  });
+
+  afterAll(async () => {
+    if (createdSourceIds.length > 0) {
+      await db.from("observations").delete().in("source_id", createdSourceIds);
+      await db.from("raw_fragments").delete().in("source_id", createdSourceIds);
+      await db.from("sources").delete().in("id", createdSourceIds);
+    }
+    for (const dir of tempDirs) {
+      try {
+        if (fs.existsSync(dir)) fs.rmSync(dir, { recursive: true, force: true });
+      } catch {
+        // ignore cleanup errors
+      }
+    }
+    await new Promise<void>((resolve) => httpServer.close(() => resolve()));
+  });
+
+  function makeTempFile(content: string, filename = "http-ref-test.txt"): string {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), "neotoma-http-ref-"));
+    tempDirs.push(dir);
+    const filePath = path.join(dir, filename);
+    fs.writeFileSync(filePath, content, "utf-8");
+    return filePath;
+  }
+
+  it("file-only store writes storage_mode=reference on the sources row", async () => {
+    const filePath = makeTempFile(`http-ref-${Date.now()}`);
+    const idempotencyKey = `http-ref-only-${Date.now()}`;
+
+    const resp = await fetch(`${API_BASE}/store`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        user_id: TEST_USER_ID,
+        file_path: filePath,
+        mime_type: "text/plain",
+        source_storage: "reference",
+        file_idempotency_key: idempotencyKey,
+      }),
+    });
+
+    expect(resp.status).toBe(200);
+    const body = (await resp.json()) as Record<string, unknown>;
+    expect(body.storage_mode).toBe("reference");
+    expect(typeof body.source_id).toBe("string");
+    createdSourceIds.push(body.source_id as string);
+
+    const { data: sourceRow, error } = await db
+      .from("sources")
+      .select("storage_mode, reference_path, content_hash")
+      .eq("id", body.source_id as string)
+      .single();
+    expect(error).toBeNull();
+    expect(sourceRow?.storage_mode).toBe("reference");
+    expect(sourceRow?.reference_path).toBeTruthy();
+    expect(sourceRow?.content_hash).toBeTruthy();
+  });
+
+  it("combined entities+file store writes storage_mode=reference on the file leg", async () => {
+    const filePath = makeTempFile(`http-ref-combined-${Date.now()}`);
+    const idempotencyKey = `http-ref-combined-${Date.now()}`;
+
+    const resp = await fetch(`${API_BASE}/store`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        user_id: TEST_USER_ID,
+        idempotency_key: idempotencyKey,
+        file_path: filePath,
+        mime_type: "text/plain",
+        source_storage: "reference",
+        file_idempotency_key: `${idempotencyKey}-file`,
+        entities: [{ entity_type: "note", title: "HTTP ref combined", content: "body" }],
+      }),
+    });
+
+    expect(resp.status).toBe(200);
+    const body = (await resp.json()) as {
+      structured?: Record<string, unknown>;
+      unstructured?: Record<string, unknown>;
+    };
+    expect(body.unstructured?.storage_mode).toBe("reference");
+    expect(typeof body.unstructured?.source_id).toBe("string");
+    createdSourceIds.push(body.unstructured!.source_id as string);
+
+    const { data: sourceRow } = await db
+      .from("sources")
+      .select("storage_mode")
+      .eq("id", body.unstructured!.source_id as string)
+      .single();
+    expect(sourceRow?.storage_mode).toBe("reference");
+  });
+});