fix(docs): apply non-public folder exclusion on runtime /docs route (#1794)

docs/testing/automated_test_catalog.md

@@ -61,8 +61,8 @@ flowchart TD
 - Do not hand-edit suite inventory entries in this file. Update the generator or the repository tree, then regenerate.
 
 ## Repo-wide summary
-- Total automated test files: **501**
-- Backend and repo Vitest files: **467**
+- Total automated test files: **502**
+- Backend and repo Vitest files: **468**
 - Frontend Vitest files: **9**
 - Playwright spec files: **25**
 
@@ -72,7 +72,7 @@ flowchart TD
 | Vitest unit tests | 136 |
 | Vitest service tests | 35 |
 | Source-adjacent tests | 62 |
-| Vitest integration tests | 141 |
+| Vitest integration tests | 142 |
 | Vitest CLI tests | 65 |
 | Vitest contract tests | 14 |
 | Vitest security tests | 3 |
@@ -363,7 +363,7 @@ flowchart TD
 **Runner:** `vitest`
 **Command:** `npm run test:integration` or `npx vitest run tests/integration`
 **Requirements:** Database configured; remote-dependent subsets additionally need `RUN_REMOTE_TESTS=1`.
-**Files (141):**
+**Files (142):**
 - `tests/integration/aauth_attribution_stamping.test.ts`
 - `tests/integration/aauth_mcp_capability_parity.test.ts`
 - `tests/integration/aauth_mcp_initialize_admission.test.ts`
@@ -478,6 +478,7 @@ flowchart TD
 - `tests/integration/root_landing.test.ts`
 - `tests/integration/sandbox_mode.test.ts`
 - `tests/integration/sandbox_report.test.ts`
+- `tests/integration/sandbox_seed_token_bypass.test.ts`
 - `tests/integration/sandbox_stale_bearer_fallback.test.ts`
 - `tests/integration/schema_recommendation_integration.test.ts`
 - `tests/integration/session_introspection.test.ts`

scripts/build_bundled_docs.ts

@@ -24,29 +24,23 @@ import path from "node:path";
 import { fileURLToPath } from "node:url";
 import { resolveFrontmatter } from "../src/services/docs/doc_frontmatter.js";
 import { loadManifest } from "../src/services/docs/manifest_loader.js";
+import { NON_PUBLIC_TOP_FOLDERS } from "../src/services/docs/visibility.js";
 
 const here = path.dirname(fileURLToPath(import.meta.url));
 const repoRoot = path.resolve(here, "..");
 const docsRoot = path.join(repoRoot, "docs");
 const manifestPath = path.join(docsRoot, "site", "site_doc_manifest.yaml");
 const outRoot = path.join(repoRoot, "dist", "docs");
 
-// Top-level `docs/` folders excluded from the bundle. Visibility filtering
-// handles per-file internal docs inside kept folders.
-const EXCLUDED_TOP = new Set([
-  "site", // marketing-site MDX; the .md-only index does not serve it
-  "releases",
-  "feature_units",
-  "plans",
-  "proposals",
-  "prototypes",
-  "reports",
-  "implementation",
-  "private",
-  "assets",
-  "templates",
-  "research",
-]);
+// Top-level `docs/` folders excluded from the bundle. The non-public surface
+// set is shared with the runtime `/docs` route via `NON_PUBLIC_TOP_FOLDERS`.
+// The bundle additionally drops `site` (the marketing-site pages): the package
+// ships `.md` docs only, not the marketing tree, so npm installs never serve
+// `site/`. From-source hosts DO serve it (the root-landing footer links into
+// `site/pages/en/*`), which is why `site` is a bundler-only exclusion and not
+// in the shared set. Per-file `visibility` filtering (below) handles internal
+// docs inside kept folders.
+const BUNDLE_EXCLUDED_TOP = new Set([...NON_PUBLIC_TOP_FOLDERS, "site"]);
 const SKIP_DIRS = new Set(["node_modules", ".git", "archived", "private"]);
 
 function collectMarkdown(root: string): string[] {
@@ -62,7 +56,7 @@ function collectMarkdown(root: string): string[] {
       if (ent.isDirectory()) {
         if (SKIP_DIRS.has(ent.name)) continue;
         const childRel = rel ? `${rel}/${ent.name}` : ent.name;
-        if (EXCLUDED_TOP.has(childRel.split("/")[0])) continue;
+        if (BUNDLE_EXCLUDED_TOP.has(childRel.split("/")[0])) continue;
         walk(path.join(absDir, ent.name), childRel);
       } else if (ent.isFile() && ent.name.endsWith(".md")) {
         out.push(rel ? `${rel}/${ent.name}` : ent.name);

src/services/docs/index_builder.test.ts

@@ -38,10 +38,14 @@ beforeAll(() => {
   fs.mkdirSync(path.join(docsRoot, "architecture"), { recursive: true });
   fs.mkdirSync(path.join(docsRoot, "subsystems"), { recursive: true });
   fs.mkdirSync(path.join(docsRoot, "plans"), { recursive: true });
+  fs.mkdirSync(path.join(docsRoot, "releases"), { recursive: true });
   fs.writeFileSync(
     path.join(docsRoot, "foundation", "core_identity.md"),
     "# Core Identity\n\nState Layer.\n"
   );
+  // `releases` resolves to `visibility: public` via FOLDER_DEFAULTS, but it is a
+  // non-public top folder: excluded from the public surface unless show-internal.
+  fs.writeFileSync(path.join(docsRoot, "releases", "v1.md"), "# Release v1\n\nNotes.\n");
   fs.writeFileSync(
     path.join(docsRoot, "foundation", "philosophy.md"),
     "# Philosophy\n\nPrinciples.\n"
@@ -94,6 +98,27 @@ describe("buildDocsIndex", () => {
     ]);
   });
 
+  it("excludes non-public top folders by default", () => {
+    const idx = buildDocsIndex({ docsRoot, manifest: MANIFEST, env: { NODE_ENV: "production" } });
+    expect(idx.categories.find((c) => c.key === "releases")).toBeUndefined();
+    const slugs = idx.categories.flatMap((c) => [
+      ...c.uncategorized.map((d) => d.slug),
+      ...c.subcategories.flatMap((s) => s.docs.map((d) => d.slug)),
+    ]);
+    expect(slugs).not.toContain("releases/v1");
+  });
+
+  it("includes non-public top folders with the show-internal flag", () => {
+    const idx = buildDocsIndex({
+      docsRoot,
+      manifest: MANIFEST,
+      env: { NODE_ENV: "production", NEOTOMA_DOCS_SHOW_INTERNAL: "true" },
+    });
+    const releases = idx.categories.find((c) => c.key === "releases")!;
+    expect(releases).toBeDefined();
+    expect(releases.uncategorized.map((d) => d.slug)).toContain("releases/v1");
+  });
+
   it("populates the featured list in manifest order, filtered by visibility", () => {
     const idx = buildDocsIndex({ docsRoot, manifest: MANIFEST, env: { NODE_ENV: "production" } });
     expect(idx.featured.map((d) => d.repo_path)).toEqual(["docs/foundation/core_identity.md"]);

src/services/docs/index_builder.ts

@@ -14,7 +14,12 @@
 import fs from "node:fs";
 import path from "node:path";
 import { resolveFrontmatter, type DocFrontmatter } from "./doc_frontmatter.js";
-import { isVisible, type VisibilityEnv } from "./visibility.js";
+import {
+  isVisible,
+  isNonPublicTopFolder,
+  shouldShowInternal,
+  type VisibilityEnv,
+} from "./visibility.js";
 
 const SKIP_DIR_NAMES = new Set(["private", "node_modules", ".git", "archived"]);
 
@@ -124,10 +129,16 @@ function compareDocs(a: DocEntry, b: DocEntry): number {
 export function buildDocsIndex(opts: BuildDocsIndexOptions): DocsIndex {
   const { docsRoot, manifest, env, manifestEntries, includeDeprecated } = opts;
   const files = collectMarkdownFiles(docsRoot);
+  const showInternal = shouldShowInternal(env);
 
   // Resolve all entries up front (single pass).
   const all: DocEntry[] = [];
   for (const rel of files) {
+    // Non-public top-level folders (release history, feature units, the
+    // marketing site, internal-process trees) are not part of the public docs
+    // surface. Drop them unless show-internal is enabled, so a from-source host
+    // matches the curated npm bundle. See `NON_PUBLIC_TOP_FOLDERS`.
+    if (!showInternal && isNonPublicTopFolder(rel)) continue;
     const abs = path.join(docsRoot, rel);
     let source = "";
     try {

src/services/docs/render.test.ts

@@ -12,12 +12,19 @@ beforeAll(() => {
   docsRoot = path.join(tmpRoot, "docs");
   fs.mkdirSync(path.join(docsRoot, "foundation"), { recursive: true });
   fs.mkdirSync(path.join(docsRoot, "plans"), { recursive: true });
+  fs.mkdirSync(path.join(docsRoot, "releases"), { recursive: true });
   fs.mkdirSync(path.join(docsRoot, "private"), { recursive: true });
   fs.writeFileSync(
     path.join(docsRoot, "foundation", "core_identity.md"),
     "# Core Identity\n\nNeotoma is the State Layer.\n"
   );
   fs.writeFileSync(path.join(docsRoot, "plans", "next.md"), "# Next Plan\n\nDraft.\n");
+  // `releases` is `public` by FOLDER_DEFAULTS but is a non-public top folder:
+  // off the public docs surface unless show-internal is set.
+  fs.writeFileSync(
+    path.join(docsRoot, "releases", "changelog.md"),
+    "# Changelog\n\nRelease history.\n"
+  );
   fs.writeFileSync(
     path.join(docsRoot, "private", "secret.md"),
     "# Secret\n\nShould never render.\n"
@@ -89,4 +96,19 @@ describe("lookupDoc — visibility", () => {
     expect(r.ok).toBe(false);
     if (!r.ok) expect(r.error.kind).toBe("not_found");
   });
+
+  it("hides a public doc in a non-public top folder by default", () => {
+    const r = lookupDoc("releases/changelog", { docsRoot, env: { NODE_ENV: "production" } });
+    expect(r.ok).toBe(false);
+    if (!r.ok) expect(r.error.kind).toBe("hidden");
+  });
+
+  it("shows a non-public top folder doc with show-internal flag", () => {
+    const r = lookupDoc("releases/changelog", {
+      docsRoot,
+      env: { NODE_ENV: "production", NEOTOMA_DOCS_SHOW_INTERNAL: "true" },
+    });
+    expect(r.ok).toBe(true);
+    if (r.ok) expect(r.doc.frontmatter.title).toBe("Changelog");
+  });
 });

src/services/docs/render.ts

@@ -14,7 +14,12 @@ import fs from "node:fs";
 import path from "node:path";
 import { resolveFrontmatter, splitFrontmatter, type DocFrontmatter } from "./doc_frontmatter.js";
 import { renderMarkdown } from "./markdown_render.js";
-import { isVisible, type VisibilityEnv } from "./visibility.js";
+import {
+  isVisible,
+  isNonPublicTopFolder,
+  shouldShowInternal,
+  type VisibilityEnv,
+} from "./visibility.js";
 
 const SLUG_RE = /^[A-Za-z0-9_./-]+$/;
 
@@ -73,6 +78,12 @@ export function lookupDoc(
   if (relFromRoot.startsWith("private/")) {
     return { ok: false, error: { kind: "not_found" } };
   }
+  // Non-public top-level folders are off the public docs surface. Hide them
+  // unless show-internal is enabled, so direct slug lookup matches both the
+  // `/docs` index and the npm bundle (which never ships these folders).
+  if (!shouldShowInternal(env) && isNonPublicTopFolder(relFromRoot)) {
+    return { ok: false, error: { kind: "hidden" } };
+  }
   if (!fs.existsSync(abs) || !fs.statSync(abs).isFile()) {
     return { ok: false, error: { kind: "not_found" } };
   }

src/services/docs/visibility.test.ts

@@ -1,5 +1,10 @@
 import { describe, it, expect } from "vitest";
-import { isVisible, shouldShowInternal } from "./visibility.js";
+import {
+  isVisible,
+  shouldShowInternal,
+  isNonPublicTopFolder,
+  NON_PUBLIC_TOP_FOLDERS,
+} from "./visibility.js";
 
 describe("shouldShowInternal", () => {
   it("explicit true wins", () => {
@@ -39,3 +44,23 @@ describe("isVisible", () => {
     ).toBe(true);
   });
 });
+
+describe("isNonPublicTopFolder", () => {
+  it("matches paths under non-public top folders", () => {
+    expect(isNonPublicTopFolder("releases/in_progress/foo.md")).toBe(true);
+    expect(isNonPublicTopFolder("feature_units/bar.md")).toBe(true);
+    expect(isNonPublicTopFolder("plans/draft.md")).toBe(true);
+  });
+  it("does not match public-surface folders or top-level files", () => {
+    expect(isNonPublicTopFolder("foundation/core_identity.md")).toBe(false);
+    expect(isNonPublicTopFolder("getting_started/what_is_neotoma.md")).toBe(false);
+    expect(isNonPublicTopFolder("NEOTOMA_MANIFEST.md")).toBe(false);
+    // `site` is a bundler-only (packaging) exclusion, not a surface exclusion.
+    expect(isNonPublicTopFolder("site/pages/en/install.md")).toBe(false);
+  });
+  it("keeps the set and predicate in sync", () => {
+    for (const folder of NON_PUBLIC_TOP_FOLDERS) {
+      expect(isNonPublicTopFolder(`${folder}/x.md`)).toBe(true);
+    }
+  });
+});

src/services/docs/visibility.ts

@@ -30,3 +30,50 @@ export function isVisible(fm: Pick<DocFrontmatter, "visibility">, env: Visibilit
   if (fm.visibility === "public") return true;
   return shouldShowInternal(env);
 }
+
+/**
+ * Top-level `docs/` folders that are NOT part of the in-app public docs
+ * surface: high-volume non-in-app trees (release history, feature units) and
+ * the internal-process folders. These are dropped from the `/docs` index, from
+ * direct slug lookup, and from the npm bundle.
+ *
+ * Single source of truth shared by the runtime route (`index_builder`,
+ * `render`) and the build-time bundler (`scripts/build_bundled_docs.ts`), so a
+ * hosted-from-source instance and an npm install expose the same surface for
+ * these folders.
+ *
+ * On the runtime route the exclusion is gated by show-internal: the default
+ * (production) serves the curated public surface, and
+ * `NEOTOMA_DOCS_SHOW_INTERNAL=true` reveals the full checkout tree for local
+ * dev. The bundler applies the exclusion unconditionally (the package never
+ * ships internal content).
+ *
+ * NOTE: `site/` (the marketing-site pages) is intentionally NOT here. It is a
+ * *packaging* exclusion, not a surface exclusion: from-source hosts serve it
+ * (the root-landing footer links into `site/pages/en/*`), while the npm bundle
+ * drops it because the package ships `.md` docs only, not the marketing tree.
+ * The bundler adds `site` to its own exclusion set; see
+ * `scripts/build_bundled_docs.ts`.
+ */
+export const NON_PUBLIC_TOP_FOLDERS: ReadonlySet<string> = new Set([
+  "releases",
+  "feature_units",
+  "plans",
+  "proposals",
+  "prototypes",
+  "reports",
+  "implementation",
+  "private",
+  "assets",
+  "templates",
+  "research",
+]);
+
+/**
+ * True when a `docs/`-relative path lives under a non-public top-level folder.
+ * `relPath` is POSIX, `docs/`-relative (e.g. `releases/in_progress/foo.md`).
+ */
+export function isNonPublicTopFolder(relPath: string): boolean {
+  const top = relPath.split("/")[0];
+  return NON_PUBLIC_TOP_FOLDERS.has(top);
+}