feat: make the indexing file cap configurable

Add JCODEMUNCH_MAX_INDEX_FILES so large repos can be indexed with a higher file cap when needed.

Apply the limit to both local folder and GitHub repo indexing, and only show truncation notices when files were actually dropped.

Cover the env override and truncation behavior in tests.

Author: allenguarnes

README.md

@@ -374,6 +374,7 @@ For **LM Studio**, ensure the Local Server is running (usually on port 1234):
 | `OPENAI_MODEL`              | Model name for local LLMs (default: `qwen3-coder`) | No |
 | `OPENAI_TIMEOUT`            | Timeout in seconds for local requests (default: `60.0`) | No |
 | `CODE_INDEX_PATH`           | Custom cache path         | No       |
+| `JCODEMUNCH_MAX_INDEX_FILES`| Maximum files to index per repo/folder (default: `500`) | No |
 | `JCODEMUNCH_SHARE_SAVINGS`  | Set to `0` to disable anonymous community token savings reporting | No       |
 | `JCODEMUNCH_LOG_LEVEL`      | Log level: `DEBUG`, `INFO`, `WARNING`, `ERROR` (default: `WARNING`) | No       |
 | `JCODEMUNCH_LOG_FILE`       | Path to log file. If unset, logs go to stderr. Use a file to avoid polluting MCP stdio. | No       |

SECURITY.md

@@ -56,7 +56,7 @@ When a secret file is detected, a warning is included in the indexing response.
 
 * **Default maximum:** 500 KB per file (configurable via `max_file_size`).
 * Files exceeding the limit are skipped during discovery.
-* A configurable **file count limit** (default: 500 files) prevents runaway indexing of extremely large repositories.
+* A configurable **file count limit** (default: 500 files) prevents runaway indexing of extremely large repositories. Can be overridden using the `JCODEMUNCH_MAX_INDEX_FILES` environment variable.
 
 ---
 

src/jcodemunch_mcp/security.py

@@ -206,6 +206,38 @@ def safe_decode(data: bytes, encoding: str = "utf-8") -> str:
 # --- Composite Filters ---
 
 DEFAULT_MAX_FILE_SIZE = 500 * 1024  # 500KB
+DEFAULT_MAX_INDEX_FILES = 500
+MAX_INDEX_FILES_ENV_VAR = "JCODEMUNCH_MAX_INDEX_FILES"
+
+
+def get_max_index_files(max_files: Optional[int] = None) -> int:
+    """Resolve the maximum indexed file count from arg or environment.
+
+    Args:
+        max_files: Explicit override. Must be a positive integer when provided.
+
+    Returns:
+        Positive file-count limit. Falls back to the default if the environment
+        variable is unset or invalid.
+    """
+    if max_files is not None:
+        if max_files <= 0:
+            raise ValueError("max_files must be a positive integer")
+        return max_files
+
+    value = os.environ.get(MAX_INDEX_FILES_ENV_VAR)
+    if value is None:
+        return DEFAULT_MAX_INDEX_FILES
+
+    try:
+        parsed = int(value)
+    except ValueError:
+        return DEFAULT_MAX_INDEX_FILES
+
+    if parsed <= 0:
+        return DEFAULT_MAX_INDEX_FILES
+
+    return parsed
 
 
 def should_exclude_file(

src/jcodemunch_mcp/tools/index_folder.py

@@ -17,6 +17,7 @@
     is_binary_file,
     should_exclude_file,
     DEFAULT_MAX_FILE_SIZE,
+    get_max_index_files,
 )
 from ..storage import IndexStore
 from ..summarizer import summarize_symbols
@@ -60,7 +61,7 @@ def _load_gitignore(folder_path: Path) -> Optional[pathspec.PathSpec]:
 
 def discover_local_files(
     folder_path: Path,
-    max_files: int = 500,
+    max_files: Optional[int] = None,
     max_size: int = DEFAULT_MAX_FILE_SIZE,
     extra_ignore_patterns: Optional[list[str]] = None,
     follow_symlinks: bool = False,
@@ -77,6 +78,7 @@ def discover_local_files(
     Returns:
         Tuple of (list of Path objects for source files, list of warning strings).
     """
+    max_files = get_max_index_files(max_files)
     files = []
     warnings = []
     root = folder_path.resolve()
@@ -93,6 +95,7 @@ def discover_local_files(
         "too_large": 0,
         "unreadable": 0,
         "binary": 0,
+        "file_limit": 0,
     }
 
     # Load .gitignore
@@ -194,6 +197,7 @@ def discover_local_files(
 
     # File count limit with prioritization
     if len(files) > max_files:
+        skip_counts["file_limit"] = len(files) - max_files
         # Prioritize: src/, lib/, pkg/, cmd/, internal/ first
         priority_dirs = ["src/", "lib/", "pkg/", "cmd/", "internal/"]
 
@@ -247,11 +251,13 @@ def index_folder(
         return {"success": False, "error": f"Path is not a directory: {path}"}
 
     warnings = []
+    max_files = get_max_index_files()
 
     try:
         # Discover source files (with security filtering)
         source_files, discover_warnings, skip_counts = discover_local_files(
             folder_path,
+            max_files=max_files,
             extra_ignore_patterns=extra_ignore_patterns,
             follow_symlinks=follow_symlinks,
         )
@@ -430,8 +436,8 @@ def index_folder(
         if warnings:
             result["warnings"] = warnings
 
-        if len(source_files) >= 500:
-            result["note"] = "Folder has many files; indexed first 500"
+        if skip_counts.get("file_limit", 0) > 0:
+            result["note"] = f"Folder has many files; indexed first {max_files}"
 
         return result
 

src/jcodemunch_mcp/tools/index_repo.py

@@ -8,7 +8,7 @@
 import httpx
 
 from ..parser import parse_file, LANGUAGE_EXTENSIONS
-from ..security import is_secret_file, is_binary_extension
+from ..security import is_secret_file, is_binary_extension, get_max_index_files
 from ..storage import IndexStore
 from ..summarizer import summarize_symbols
 
@@ -86,9 +86,9 @@ def should_skip_file(path: str) -> bool:
 def discover_source_files(
     tree_entries: list[dict],
     gitignore_content: Optional[str] = None,
-    max_files: int = 500,
+    max_files: Optional[int] = None,
     max_size: int = 500 * 1024  # 500KB
-) -> list[str]:
+) -> tuple[list[str], bool]:
     """Discover source files from tree entries.
     
     Applies filtering pipeline:
@@ -100,6 +100,8 @@ def discover_source_files(
     6. File count limit
     """
     import pathspec
+
+    max_files = get_max_index_files(max_files)
 
     # Parse gitignore if provided
     gitignore_spec = None
@@ -149,8 +151,10 @@ def discover_source_files(
 
         files.append(path)
 
+    truncated = len(files) > max_files
+
     # File count limit with prioritization
-    if len(files) > max_files:
+    if truncated:
         # Prioritize: src/, lib/, pkg/, cmd/, internal/ first
         priority_dirs = ["src/", "lib/", "pkg/", "cmd/", "internal/"]
 
@@ -165,7 +169,7 @@ def priority_key(path):
         files.sort(key=priority_key)
         files = files[:max_files]
 
-    return files
+    return files, truncated
 
 
 async def fetch_file_content(
@@ -228,6 +232,7 @@ async def index_repo(
         github_token = os.environ.get("GITHUB_TOKEN")
 
     warnings = []
+    max_files = get_max_index_files()
 
     try:
         # Fetch tree
@@ -244,7 +249,11 @@ async def index_repo(
         gitignore_content = await fetch_gitignore(owner, repo, github_token)
 
         # Discover source files
-        source_files = discover_source_files(tree_entries, gitignore_content)
+        source_files, truncated = discover_source_files(
+            tree_entries,
+            gitignore_content,
+            max_files=max_files,
+        )
 
         if not source_files:
             return {"success": False, "error": "No source files found"}
@@ -381,8 +390,8 @@ async def fetch_with_limit(path: str) -> tuple[str, str]:
         if warnings:
             result["warnings"] = warnings
 
-        if len(source_files) >= 500:
-            result["warnings"] = warnings + ["Repository has many files; indexed first 500"]
+        if truncated:
+            result["warnings"] = warnings + [f"Repository has many files; indexed first {max_files}"]
 
         return result
 

tests/test_security.py

@@ -4,6 +4,7 @@
 import sys
 import pytest
 from pathlib import Path
+from unittest.mock import patch
 
 from jcodemunch_mcp.security import (
     validate_path,
@@ -16,6 +17,9 @@
     should_exclude_file,
     SECRET_PATTERNS,
     BINARY_EXTENSIONS,
+    DEFAULT_MAX_INDEX_FILES,
+    MAX_INDEX_FILES_ENV_VAR,
+    get_max_index_files,
 )
 
 
@@ -221,6 +225,24 @@ def test_checks_can_be_disabled(self, tmp_path):
         assert should_exclude_file(f, tmp_path, check_secrets=False) is None
 
 
+class TestMaxIndexFilesConfig:
+    def test_defaults_when_env_is_unset(self):
+        with patch.dict(os.environ, {}, clear=True):
+            assert get_max_index_files() == DEFAULT_MAX_INDEX_FILES
+
+    def test_reads_env_override(self):
+        with patch.dict(os.environ, {MAX_INDEX_FILES_ENV_VAR: "1234"}, clear=True):
+            assert get_max_index_files() == 1234
+
+    def test_invalid_env_falls_back_to_default(self):
+        with patch.dict(os.environ, {MAX_INDEX_FILES_ENV_VAR: "invalid"}, clear=True):
+            assert get_max_index_files() == DEFAULT_MAX_INDEX_FILES
+
+    def test_non_positive_explicit_value_is_rejected(self):
+        with pytest.raises(ValueError, match="positive integer"):
+            get_max_index_files(0)
+
+
 # --- Integration: discover_local_files with security ---
 
 class TestDiscoverLocalFilesSecure:
@@ -277,6 +299,31 @@ def test_extra_ignore_patterns(self, tmp_path):
         assert "main.py" in names
         assert "temp.py" not in names
 
+    def test_respects_env_file_limit(self, tmp_path):
+        """Environment override controls local folder file discovery limit."""
+        from jcodemunch_mcp.tools.index_folder import discover_local_files
+
+        for i in range(10):
+            (tmp_path / f"file{i}.py").write_text(f"x = {i}\n")
+
+        with patch.dict(os.environ, {MAX_INDEX_FILES_ENV_VAR: "3"}, clear=False):
+            files, *_ = discover_local_files(tmp_path)
+
+        assert len(files) == 3
+
+    def test_exact_env_file_limit_does_not_report_truncation(self, tmp_path):
+        """Exact file-count matches should not be treated as truncation."""
+        from jcodemunch_mcp.tools.index_folder import discover_local_files
+
+        for i in range(3):
+            (tmp_path / f"file{i}.py").write_text(f"x = {i}\n")
+
+        with patch.dict(os.environ, {MAX_INDEX_FILES_ENV_VAR: "3"}, clear=False):
+            files, _, skip_counts = discover_local_files(tmp_path)
+
+        assert len(files) == 3
+        assert skip_counts["file_limit"] == 0
+
     @pytest.mark.skipif(sys.platform == "win32", reason="Symlinks unreliable on Windows")
     def test_symlinks_skipped_by_default(self, tmp_path):
         """Symlinks are skipped when follow_symlinks=False."""
@@ -308,11 +355,12 @@ def test_secret_files_filtered_in_discovery(self):
             {"path": "src/utils.py", "type": "blob", "size": 500},
         ]
 
-        files = discover_source_files(tree_entries)
+        files, truncated = discover_source_files(tree_entries)
         assert "src/main.py" in files
         assert "src/utils.py" in files
         assert ".env" not in files
         assert "certs/server.pem" not in files
+        assert truncated is False
 
 
 # --- Encoding safety in index_store ---

tests/test_tools.py

@@ -1,11 +1,14 @@
 """Tests for tools module."""
 
 import pytest
+from unittest.mock import patch
+
 from jcodemunch_mcp.tools.index_repo import (
     parse_github_url,
     discover_source_files,
     should_skip_file,
 )
+from jcodemunch_mcp.security import MAX_INDEX_FILES_ENV_VAR
 
 
 def test_parse_github_url_full():
@@ -39,12 +42,13 @@ def test_discover_source_files():
         {"path": "src/utils.py", "type": "blob", "size": 500},
     ]
 
-    files = discover_source_files(tree_entries, gitignore_content=None)
+    files, truncated = discover_source_files(tree_entries, gitignore_content=None)
 
     assert "src/main.py" in files
     assert "src/utils.py" in files
     assert "node_modules/foo.js" not in files
     assert "README.md" not in files  # Not a source file
+    assert truncated is False
 
 
 def test_discover_source_files_respects_max():
@@ -54,8 +58,9 @@ def test_discover_source_files_respects_max():
         for i in range(1000)
     ]
 
-    files = discover_source_files(tree_entries, max_files=100)
+    files, truncated = discover_source_files(tree_entries, max_files=100)
     assert len(files) == 100
+    assert truncated is True
 
 
 def test_discover_source_files_prioritizes_src():
@@ -68,8 +73,49 @@ def test_discover_source_files_prioritizes_src():
         for i in range(300)
     ]
 
-    files = discover_source_files(tree_entries, max_files=100)
+    files, truncated = discover_source_files(tree_entries, max_files=100)
     # Most files should be from src/
     src_count = sum(1 for f in files if f.startswith("src/"))
     assert src_count > 50  # Majority should be src/
+    assert truncated is True
+
+
+def test_discover_source_files_uses_env_override():
+    """Test that environment override is used when max_files is omitted."""
+    tree_entries = [
+        {"path": f"file{i}.py", "type": "blob", "size": 100}
+        for i in range(20)
+    ]
+
+    with patch.dict("os.environ", {MAX_INDEX_FILES_ENV_VAR: "7"}, clear=False):
+        files, truncated = discover_source_files(tree_entries)
+
+    assert len(files) == 7
+    assert truncated is True
+
+
+def test_discover_source_files_explicit_max_overrides_env():
+    """Explicit max_files should win over environment configuration."""
+    tree_entries = [
+        {"path": f"file{i}.py", "type": "blob", "size": 100}
+        for i in range(20)
+    ]
+
+    with patch.dict("os.environ", {MAX_INDEX_FILES_ENV_VAR: "7"}, clear=False):
+        files, truncated = discover_source_files(tree_entries, max_files=5)
+
+    assert len(files) == 5
+    assert truncated is True
+
+
+def test_discover_source_files_exact_limit_is_not_truncated():
+    """An exact match to the limit should not be reported as truncation."""
+    tree_entries = [
+        {"path": f"file{i}.py", "type": "blob", "size": 100}
+        for i in range(5)
+    ]
+
+    files, truncated = discover_source_files(tree_entries, max_files=5)
 
+    assert len(files) == 5
+    assert truncated is False