Merge pull request #28 from allenguarnes/main

Make the indexing file cap configurable

Author: jgravelle

README.md

@@ -376,6 +376,7 @@ For **LM Studio**, ensure the Local Server is running (usually on port 1234):
 | `OPENAI_MODEL`              | Model name for local LLMs (default: `qwen3-coder`) | No |
 | `OPENAI_TIMEOUT`            | Timeout in seconds for local requests (default: `60.0`) | No |
 | `CODE_INDEX_PATH`           | Custom cache path         | No       |
+| `JCODEMUNCH_MAX_INDEX_FILES`| Maximum files to index per repo/folder (default: `500`) | No |
 | `JCODEMUNCH_SHARE_SAVINGS`  | Set to `0` to disable anonymous community token savings reporting | No       |
 | `JCODEMUNCH_LOG_LEVEL`      | Log level: `DEBUG`, `INFO`, `WARNING`, `ERROR` (default: `WARNING`) | No       |
 | `JCODEMUNCH_LOG_FILE`       | Path to log file. If unset, logs go to stderr. Use a file to avoid polluting MCP stdio. | No       |

SECURITY.md

@@ -56,7 +56,7 @@ When a secret file is detected, a warning is included in the indexing response.
 
 * **Default maximum:** 500 KB per file (configurable via `max_file_size`).
 * Files exceeding the limit are skipped during discovery.
-* A configurable **file count limit** (default: 500 files) prevents runaway indexing of extremely large repositories.
+* A configurable **file count limit** (default: 500 files) prevents runaway indexing of extremely large repositories. Can be overridden using the `JCODEMUNCH_MAX_INDEX_FILES` environment variable.
 
 ---
 

src/jcodemunch_mcp/security.py

@@ -206,6 +206,38 @@ def safe_decode(data: bytes, encoding: str = "utf-8") -> str:
 # --- Composite Filters ---
 
 DEFAULT_MAX_FILE_SIZE = 500 * 1024  # 500KB
+DEFAULT_MAX_INDEX_FILES = 500
+MAX_INDEX_FILES_ENV_VAR = "JCODEMUNCH_MAX_INDEX_FILES"
+
+
+def get_max_index_files(max_files: Optional[int] = None) -> int:
+    """Resolve the maximum indexed file count from arg or environment.
+
+    Args:
+        max_files: Explicit override. Must be a positive integer when provided.
+
+    Returns:
+        Positive file-count limit. Falls back to the default if the environment
+        variable is unset or invalid.
+    """
+    if max_files is not None:
+        if max_files <= 0:
+            raise ValueError("max_files must be a positive integer")
+        return max_files
+
+    value = os.environ.get(MAX_INDEX_FILES_ENV_VAR)
+    if value is None:
+        return DEFAULT_MAX_INDEX_FILES
+
+    try:
+        parsed = int(value)
+    except ValueError:
+        return DEFAULT_MAX_INDEX_FILES
+
+    if parsed <= 0:
+        return DEFAULT_MAX_INDEX_FILES
+
+    return parsed
 
 
 def should_exclude_file(

src/jcodemunch_mcp/tools/index_folder.py

@@ -18,6 +18,7 @@
     is_binary_file,
     should_exclude_file,
     DEFAULT_MAX_FILE_SIZE,
+    get_max_index_files,
 )
 from ..storage import IndexStore
 from ..summarizer import summarize_symbols
@@ -61,7 +62,7 @@ def _load_gitignore(folder_path: Path) -> Optional[pathspec.PathSpec]:
 
 def discover_local_files(
     folder_path: Path,
-    max_files: int = 500,
+    max_files: Optional[int] = None,
     max_size: int = DEFAULT_MAX_FILE_SIZE,
     extra_ignore_patterns: Optional[list[str]] = None,
     follow_symlinks: bool = False,
@@ -78,6 +79,7 @@ def discover_local_files(
     Returns:
         Tuple of (list of Path objects for source files, list of warning strings).
     """
+    max_files = get_max_index_files(max_files)
     files = []
     warnings = []
     root = folder_path.resolve()
@@ -94,6 +96,7 @@ def discover_local_files(
         "too_large": 0,
         "unreadable": 0,
         "binary": 0,
+        "file_limit": 0,
     }
 
     # Load .gitignore
@@ -195,6 +198,7 @@ def discover_local_files(
 
     # File count limit with prioritization
     if len(files) > max_files:
+        skip_counts["file_limit"] = len(files) - max_files
         # Prioritize: src/, lib/, pkg/, cmd/, internal/ first
         priority_dirs = ["src/", "lib/", "pkg/", "cmd/", "internal/"]
 
@@ -248,11 +252,13 @@ def index_folder(
         return {"success": False, "error": f"Path is not a directory: {path}"}
 
     warnings = []
+    max_files = get_max_index_files()
 
     try:
         # Discover source files (with security filtering)
         source_files, discover_warnings, skip_counts = discover_local_files(
             folder_path,
+            max_files=max_files,
             extra_ignore_patterns=extra_ignore_patterns,
             follow_symlinks=follow_symlinks,
         )
@@ -432,8 +438,8 @@ def index_folder(
         if warnings:
             result["warnings"] = warnings
 
-        if len(source_files) >= 500:
-            result["note"] = "Folder has many files; indexed first 500"
+        if skip_counts.get("file_limit", 0) > 0:
+            result["note"] = f"Folder has many files; indexed first {max_files}"
 
         return result
 

src/jcodemunch_mcp/tools/index_repo.py

@@ -9,7 +9,7 @@
 import httpx
 
 from ..parser import parse_file, LANGUAGE_EXTENSIONS
-from ..security import is_secret_file, is_binary_extension
+from ..security import is_secret_file, is_binary_extension, get_max_index_files
 from ..storage import IndexStore
 from ..summarizer import summarize_symbols
 
@@ -87,9 +87,9 @@ def should_skip_file(path: str) -> bool:
 def discover_source_files(
     tree_entries: list[dict],
     gitignore_content: Optional[str] = None,
-    max_files: int = 500,
+    max_files: Optional[int] = None,
     max_size: int = 500 * 1024  # 500KB
-) -> list[str]:
+) -> tuple[list[str], bool]:
     """Discover source files from tree entries.
     
     Applies filtering pipeline:
@@ -101,6 +101,8 @@ def discover_source_files(
     6. File count limit
     """
     import pathspec
+
+    max_files = get_max_index_files(max_files)
 
     # Parse gitignore if provided
     gitignore_spec = None
@@ -150,8 +152,10 @@ def discover_source_files(
 
         files.append(path)
 
+    truncated = len(files) > max_files
+
     # File count limit with prioritization
-    if len(files) > max_files:
+    if truncated:
         # Prioritize: src/, lib/, pkg/, cmd/, internal/ first
         priority_dirs = ["src/", "lib/", "pkg/", "cmd/", "internal/"]
 
@@ -166,7 +170,7 @@ def priority_key(path):
         files.sort(key=priority_key)
         files = files[:max_files]
 
-    return files
+    return files, truncated
 
 
 async def fetch_file_content(
@@ -229,6 +233,7 @@ async def index_repo(
         github_token = os.environ.get("GITHUB_TOKEN")
 
     warnings = []
+    max_files = get_max_index_files()
 
     try:
         # Fetch tree
@@ -245,7 +250,11 @@ async def index_repo(
         gitignore_content = await fetch_gitignore(owner, repo, github_token)
 
         # Discover source files
-        source_files = discover_source_files(tree_entries, gitignore_content)
+        source_files, truncated = discover_source_files(
+            tree_entries,
+            gitignore_content,
+            max_files=max_files,
+        )
 
         if not source_files:
             return {"success": False, "error": "No source files found"}
@@ -383,8 +392,8 @@ async def fetch_with_limit(path: str) -> tuple[str, str]:
         if warnings:
             result["warnings"] = warnings
 
-        if len(source_files) >= 500:
-            result["warnings"] = warnings + ["Repository has many files; indexed first 500"]
+        if truncated:
+            result["warnings"] = warnings + [f"Repository has many files; indexed first {max_files}"]
 
         return result
 

tests/test_security.py

@@ -4,6 +4,7 @@
 import sys
 import pytest
 from pathlib import Path
+from unittest.mock import patch
 
 from jcodemunch_mcp.security import (
     validate_path,
@@ -16,6 +17,9 @@
     should_exclude_file,
     SECRET_PATTERNS,
     BINARY_EXTENSIONS,
+    DEFAULT_MAX_INDEX_FILES,
+    MAX_INDEX_FILES_ENV_VAR,
+    get_max_index_files,
 )
 
 
@@ -221,6 +225,24 @@ def test_checks_can_be_disabled(self, tmp_path):
         assert should_exclude_file(f, tmp_path, check_secrets=False) is None
 
 
+class TestMaxIndexFilesConfig:
+    def test_defaults_when_env_is_unset(self):
+        with patch.dict(os.environ, {}, clear=True):
+            assert get_max_index_files() == DEFAULT_MAX_INDEX_FILES
+
+    def test_reads_env_override(self):
+        with patch.dict(os.environ, {MAX_INDEX_FILES_ENV_VAR: "1234"}, clear=True):
+            assert get_max_index_files() == 1234
+
+    def test_invalid_env_falls_back_to_default(self):
+        with patch.dict(os.environ, {MAX_INDEX_FILES_ENV_VAR: "invalid"}, clear=True):
+            assert get_max_index_files() == DEFAULT_MAX_INDEX_FILES
+
+    def test_non_positive_explicit_value_is_rejected(self):
+        with pytest.raises(ValueError, match="positive integer"):
+            get_max_index_files(0)
+
+
 # --- Integration: discover_local_files with security ---
 
 class TestDiscoverLocalFilesSecure:
@@ -277,6 +299,31 @@ def test_extra_ignore_patterns(self, tmp_path):
         assert "main.py" in names
         assert "temp.py" not in names
 
+    def test_respects_env_file_limit(self, tmp_path):
+        """Environment override controls local folder file discovery limit."""
+        from jcodemunch_mcp.tools.index_folder import discover_local_files
+
+        for i in range(10):
+            (tmp_path / f"file{i}.py").write_text(f"x = {i}\n")
+
+        with patch.dict(os.environ, {MAX_INDEX_FILES_ENV_VAR: "3"}, clear=False):
+            files, *_ = discover_local_files(tmp_path)
+
+        assert len(files) == 3
+
+    def test_exact_env_file_limit_does_not_report_truncation(self, tmp_path):
+        """Exact file-count matches should not be treated as truncation."""
+        from jcodemunch_mcp.tools.index_folder import discover_local_files
+
+        for i in range(3):
+            (tmp_path / f"file{i}.py").write_text(f"x = {i}\n")
+
+        with patch.dict(os.environ, {MAX_INDEX_FILES_ENV_VAR: "3"}, clear=False):
+            files, _, skip_counts = discover_local_files(tmp_path)
+
+        assert len(files) == 3
+        assert skip_counts["file_limit"] == 0
+
     @pytest.mark.skipif(sys.platform == "win32", reason="Symlinks unreliable on Windows")
     def test_symlinks_skipped_by_default(self, tmp_path):
         """Symlinks are skipped when follow_symlinks=False."""
@@ -308,11 +355,12 @@ def test_secret_files_filtered_in_discovery(self):
             {"path": "src/utils.py", "type": "blob", "size": 500},
         ]
 
-        files = discover_source_files(tree_entries)
+        files, truncated = discover_source_files(tree_entries)
         assert "src/main.py" in files
         assert "src/utils.py" in files
         assert ".env" not in files
         assert "certs/server.pem" not in files
+        assert truncated is False
 
 
 # --- Encoding safety in index_store ---

tests/test_tools.py

@@ -1,11 +1,14 @@
 """Tests for tools module."""
 
 import pytest
+from unittest.mock import patch
+
 from jcodemunch_mcp.tools.index_repo import (
     parse_github_url,
     discover_source_files,
     should_skip_file,
 )
+from jcodemunch_mcp.security import MAX_INDEX_FILES_ENV_VAR
 
 
 def test_parse_github_url_full():
@@ -41,14 +44,15 @@ def test_discover_source_files():
         {"path": "include/engine.hpp", "type": "blob", "size": 350},
     ]
 
-    files = discover_source_files(tree_entries, gitignore_content=None)
+    files, truncated = discover_source_files(tree_entries, gitignore_content=None)
 
     assert "src/main.py" in files
     assert "src/utils.py" in files
     assert "src/engine.cpp" in files
     assert "include/engine.hpp" in files
     assert "node_modules/foo.js" not in files
     assert "README.md" not in files  # Not a source file
+    assert truncated is False
 
 
 def test_discover_source_files_respects_max():
@@ -58,8 +62,9 @@ def test_discover_source_files_respects_max():
         for i in range(1000)
     ]
 
-    files = discover_source_files(tree_entries, max_files=100)
+    files, truncated = discover_source_files(tree_entries, max_files=100)
     assert len(files) == 100
+    assert truncated is True
 
 
 def test_discover_source_files_prioritizes_src():
@@ -72,8 +77,49 @@ def test_discover_source_files_prioritizes_src():
         for i in range(300)
     ]
 
-    files = discover_source_files(tree_entries, max_files=100)
+    files, truncated = discover_source_files(tree_entries, max_files=100)
     # Most files should be from src/
     src_count = sum(1 for f in files if f.startswith("src/"))
     assert src_count > 50  # Majority should be src/
+    assert truncated is True
+
+
+def test_discover_source_files_uses_env_override():
+    """Test that environment override is used when max_files is omitted."""
+    tree_entries = [
+        {"path": f"file{i}.py", "type": "blob", "size": 100}
+        for i in range(20)
+    ]
+
+    with patch.dict("os.environ", {MAX_INDEX_FILES_ENV_VAR: "7"}, clear=False):
+        files, truncated = discover_source_files(tree_entries)
+
+    assert len(files) == 7
+    assert truncated is True
+
+
+def test_discover_source_files_explicit_max_overrides_env():
+    """Explicit max_files should win over environment configuration."""
+    tree_entries = [
+        {"path": f"file{i}.py", "type": "blob", "size": 100}
+        for i in range(20)
+    ]
+
+    with patch.dict("os.environ", {MAX_INDEX_FILES_ENV_VAR: "7"}, clear=False):
+        files, truncated = discover_source_files(tree_entries, max_files=5)
+
+    assert len(files) == 5
+    assert truncated is True
+
+
+def test_discover_source_files_exact_limit_is_not_truncated():
+    """An exact match to the limit should not be reported as truncation."""
+    tree_entries = [
+        {"path": f"file{i}.py", "type": "blob", "size": 100}
+        for i in range(5)
+    ]
+
+    files, truncated = discover_source_files(tree_entries, max_files=5)
 
+    assert len(files) == 5
+    assert truncated is False