feat: diagnostic logging for index_folder — skip counts and no-symbol tracking

* feat: add per-reason discovery logging and diagnostic response fields to index_folder

Adds structured skip_counts tracking to discover_local_files() covering all
filter paths (symlink, path_traversal, skip_pattern, gitignore, extra_ignore,
secret, wrong_extension, too_large, unreadable, binary). Files that pass
discovery but yield no AST symbols are now tracked separately with DEBUG
logging. Both datasets are returned in the index_folder tool response as
discovery_skip_counts, no_symbols_count, and no_symbols_files, making it
straightforward to diagnose why files are absent from the index.

Adds --log-level / --log-file CLI args to server main() (also via
JCODEMUNCH_LOG_LEVEL / JCODEMUNCH_LOG_FILE env vars). Defaults to WARNING
so normal operation is silent. Updates README and MCP tool description
to document the new flags and response fields. Fixes test_security.py
call sites to unpack the new 3-tuple return value.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* refactor: trim index_folder tool description to reduce context bloat

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* fix: surface diagnostic fields in incremental index result

Add discovery_skip_counts, no_symbols_count, and no_symbols_files to
the incremental return path in index_folder, which previously omitted
them despite being computed. Also fix test unpacking __, __ -> *_.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: gebeer

README.md

@@ -173,6 +173,25 @@ Config file location:
 }
 ```
 
+**With debug logging (useful when diagnosing why files are not indexed):**
+
+```json
+{
+  "mcpServers": {
+    "jcodemunch": {
+      "command": "uvx",
+      "args": [
+        "jcodemunch-mcp",
+        "--log-level", "DEBUG",
+        "--log-file", "/tmp/jcodemunch.log"
+      ]
+    }
+  }
+}
+```
+
+> Logging flags can also be set via env vars `JCODEMUNCH_LOG_LEVEL` and `JCODEMUNCH_LOG_FILE`. Always use `--log-file` (or the env var) when debugging — writing logs to stderr can corrupt the MCP stdio stream in some clients.
+
 After saving the config, **restart Claude Desktop / Claude Code** for the server to appear.
 
 ### Google Antigravity
@@ -350,6 +369,8 @@ For **LM Studio**, ensure the Local Server is running (usually on port 1234):
 | `OPENAI_TIMEOUT`            | Timeout in seconds for local requests (default: `60.0`) | No |
 | `CODE_INDEX_PATH`           | Custom cache path         | No       |
 | `JCODEMUNCH_SHARE_SAVINGS`  | Set to `0` to disable anonymous community token savings reporting | No       |
+| `JCODEMUNCH_LOG_LEVEL`      | Log level: `DEBUG`, `INFO`, `WARNING`, `ERROR` (default: `WARNING`) | No       |
+| `JCODEMUNCH_LOG_FILE`       | Path to log file. If unset, logs go to stderr. Use a file to avoid polluting MCP stdio. | No       |
 
 ### Community Savings Meter
 

src/jcodemunch_mcp/server.py

@@ -3,7 +3,9 @@
 import argparse
 import asyncio
 import json
+import logging
 import os
+from pathlib import Path
 from typing import Any, Optional
 
 from mcp.server import Server
@@ -55,7 +57,7 @@ async def list_tools() -> list[Tool]:
         ),
         Tool(
             name="index_folder",
-            description="Index a local folder containing source code. Walks directory, parses ASTs, extracts symbols, and saves to local storage. Works with any folder containing supported language files.",
+            description="Index a local folder containing source code. Response includes `discovery_skip_counts` (files filtered per reason), `no_symbols_count`/`no_symbols_files` (files with no extractable symbols) for diagnosing missing files.",
             inputSchema={
                 "type": "object",
                 "properties": {
@@ -385,7 +387,34 @@ def main(argv: Optional[list[str]] = None):
         prog="jcodemunch-mcp",
         description="Run the jCodeMunch MCP stdio server.",
     )
-    parser.parse_args(argv)
+    parser.add_argument(
+        "--log-level",
+        default=os.environ.get("JCODEMUNCH_LOG_LEVEL", "WARNING"),
+        choices=["DEBUG", "INFO", "WARNING", "ERROR"],
+        help="Log level (also via JCODEMUNCH_LOG_LEVEL env var)",
+    )
+    parser.add_argument(
+        "--log-file",
+        default=os.environ.get("JCODEMUNCH_LOG_FILE"),
+        help="Log file path (also via JCODEMUNCH_LOG_FILE env var). Defaults to stderr.",
+    )
+    args = parser.parse_args(argv)
+
+    log_level = getattr(logging, args.log_level)
+    handlers: list[logging.Handler] = []
+    if args.log_file:
+        log_path = Path(args.log_file).expanduser()
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+        handlers.append(logging.FileHandler(log_path))
+    else:
+        handlers.append(logging.StreamHandler())
+
+    logging.basicConfig(
+        level=log_level,
+        format="%(asctime)s %(name)s %(levelname)s %(message)s",
+        handlers=handlers,
+    )
+
     asyncio.run(run_server())
 
 

src/jcodemunch_mcp/tools/index_folder.py

@@ -1,11 +1,14 @@
 """Index local folder tool - walk, parse, summarize, save."""
 
+import logging
 import os
 from pathlib import Path
 from typing import Optional
 
 import pathspec
 
+logger = logging.getLogger(__name__)
+
 from ..parser import parse_file, LANGUAGE_EXTENSIONS
 from ..security import (
     validate_path,
@@ -61,7 +64,7 @@ def discover_local_files(
     max_size: int = DEFAULT_MAX_FILE_SIZE,
     extra_ignore_patterns: Optional[list[str]] = None,
     follow_symlinks: bool = False,
-) -> tuple[list[Path], list[str]]:
+) -> tuple[list[Path], list[str], dict[str, int]]:
     """Discover source files in a local folder with security filtering.
 
     Args:
@@ -78,6 +81,20 @@ def discover_local_files(
     warnings = []
     root = folder_path.resolve()
 
+    skip_counts: dict[str, int] = {
+        "symlink": 0,
+        "symlink_escape": 0,
+        "path_traversal": 0,
+        "skip_pattern": 0,
+        "gitignore": 0,
+        "extra_ignore": 0,
+        "secret": 0,
+        "wrong_extension": 0,
+        "too_large": 0,
+        "unreadable": 0,
+        "binary": 0,
+    }
+
     # Load .gitignore
     gitignore_spec = _load_gitignore(root)
 
@@ -96,58 +113,85 @@ def discover_local_files(
 
         # Symlink protection
         if not follow_symlinks and file_path.is_symlink():
+            skip_counts["symlink"] += 1
+            logger.debug("SKIP symlink: %s", file_path)
             continue
         if file_path.is_symlink() and is_symlink_escape(root, file_path):
+            skip_counts["symlink_escape"] += 1
             warnings.append(f"Skipped symlink escape: {file_path}")
             continue
 
         # Path traversal check
         if not validate_path(root, file_path):
+            skip_counts["path_traversal"] += 1
             warnings.append(f"Skipped path traversal: {file_path}")
             continue
 
         # Get relative path for pattern matching
         try:
             rel_path = file_path.relative_to(root).as_posix()
         except ValueError:
+            skip_counts["path_traversal"] += 1
+            logger.debug("SKIP relative_to_failed: %s", file_path)
             continue
 
         # Skip patterns
         if should_skip_file(rel_path):
+            skip_counts["skip_pattern"] += 1
+            logger.debug("SKIP skip_pattern: %s", rel_path)
             continue
 
         # .gitignore matching
         if gitignore_spec and gitignore_spec.match_file(rel_path):
+            skip_counts["gitignore"] += 1
+            logger.debug("SKIP gitignore: %s", rel_path)
             continue
 
         # Extra ignore patterns
         if extra_spec and extra_spec.match_file(rel_path):
+            skip_counts["extra_ignore"] += 1
+            logger.debug("SKIP extra_ignore: %s", rel_path)
             continue
 
         # Secret detection
         if is_secret_file(rel_path):
+            skip_counts["secret"] += 1
             warnings.append(f"Skipped secret file: {rel_path}")
             continue
 
         # Extension filter
         ext = file_path.suffix
         if ext not in LANGUAGE_EXTENSIONS:
+            skip_counts["wrong_extension"] += 1
+            logger.debug("SKIP wrong_extension: %s", rel_path)
             continue
 
         # Size limit
         try:
             if file_path.stat().st_size > max_size:
+                skip_counts["too_large"] += 1
+                logger.debug("SKIP too_large: %s", rel_path)
                 continue
         except OSError:
+            skip_counts["unreadable"] += 1
+            logger.debug("SKIP unreadable (stat failed): %s", rel_path)
             continue
 
         # Binary detection (content sniff for files with source extensions)
         if is_binary_file(file_path):
+            skip_counts["binary"] += 1
             warnings.append(f"Skipped binary file: {rel_path}")
             continue
 
+        logger.debug("ACCEPT: %s", rel_path)
         files.append(file_path)
 
+    logger.info(
+        "Discovery complete — accepted: %d, skipped by reason: %s",
+        len(files),
+        skip_counts,
+    )
+
     # File count limit with prioritization
     if len(files) > max_files:
         # Prioritize: src/, lib/, pkg/, cmd/, internal/ first
@@ -169,7 +213,7 @@ def priority_key(file_path: Path) -> tuple:
         files.sort(key=priority_key)
         files = files[:max_files]
 
-    return files, warnings
+    return files, warnings, skip_counts
 
 
 def index_folder(
@@ -206,12 +250,13 @@ def index_folder(
 
     try:
         # Discover source files (with security filtering)
-        source_files, discover_warnings = discover_local_files(
+        source_files, discover_warnings, skip_counts = discover_local_files(
             folder_path,
             extra_ignore_patterns=extra_ignore_patterns,
             follow_symlinks=follow_symlinks,
         )
         warnings.extend(discover_warnings)
+        logger.info("Discovery skip counts: %s", skip_counts)
 
         if not source_files:
             return {"success": False, "error": "No source files found"}
@@ -259,6 +304,7 @@ def index_folder(
             languages: dict[str, int] = {}
             raw_files_subset: dict[str, str] = {}
 
+            incremental_no_symbols: list[str] = []
             for rel_path in files_to_parse:
                 content = current_files[rel_path]
                 ext = os.path.splitext(rel_path)[1]
@@ -270,8 +316,18 @@ def index_folder(
                     if symbols:
                         new_symbols.extend(symbols)
                         raw_files_subset[rel_path] = content
+                    else:
+                        incremental_no_symbols.append(rel_path)
+                        logger.debug("NO SYMBOLS (incremental): %s", rel_path)
                 except Exception as e:
                     warnings.append(f"Failed to parse {rel_path}: {e}")
+                    logger.debug("PARSE ERROR (incremental): %s — %s", rel_path, e)
+
+            logger.info(
+                "Incremental parsing — with symbols: %d, no symbols: %d",
+                len(new_symbols),
+                len(incremental_no_symbols),
+            )
 
             new_symbols = summarize_symbols(new_symbols, use_ai=use_ai_summaries)
 
@@ -300,6 +356,9 @@ def index_folder(
                 "changed": len(changed), "new": len(new), "deleted": len(deleted),
                 "symbol_count": len(updated.symbols) if updated else 0,
                 "indexed_at": updated.indexed_at if updated else "",
+                "discovery_skip_counts": skip_counts,
+                "no_symbols_count": len(incremental_no_symbols),
+                "no_symbols_files": incremental_no_symbols[:50],
             }
             if warnings:
                 result["warnings"] = warnings
@@ -311,6 +370,7 @@ def index_folder(
         raw_files = {}
         parsed_files = []
 
+        no_symbols_files: list[str] = []
         for rel_path, content in current_files.items():
             ext = os.path.splitext(rel_path)[1]
             language = LANGUAGE_EXTENSIONS.get(ext)
@@ -323,10 +383,20 @@ def index_folder(
                     languages[language] = languages.get(language, 0) + 1
                     raw_files[rel_path] = content
                     parsed_files.append(rel_path)
+                else:
+                    no_symbols_files.append(rel_path)
+                    logger.debug("NO SYMBOLS: %s", rel_path)
             except Exception as e:
                 warnings.append(f"Failed to parse {rel_path}: {e}")
+                logger.debug("PARSE ERROR: %s — %s", rel_path, e)
                 continue
 
+        logger.info(
+            "Parsing complete — with symbols: %d, no symbols: %d",
+            len(parsed_files),
+            len(no_symbols_files),
+        )
+
         if not all_symbols:
             return {"success": False, "error": "No symbols extracted from files"}
 
@@ -352,6 +422,9 @@ def index_folder(
             "symbol_count": len(all_symbols),
             "languages": languages,
             "files": parsed_files[:20],  # Limit files in response
+            "discovery_skip_counts": skip_counts,
+            "no_symbols_count": len(no_symbols_files),
+            "no_symbols_files": no_symbols_files[:50],  # Show up to 50 for inspection
         }
 
         if warnings:

tests/test_security.py

@@ -232,7 +232,7 @@ def test_excludes_secret_files(self, tmp_path):
         (tmp_path / ".env").write_text("SECRET=foo\n")
         (tmp_path / "config.pem").write_text("-----BEGIN CERTIFICATE-----\n")
 
-        files, warnings = discover_local_files(tmp_path)
+        files, warnings, _ = discover_local_files(tmp_path)
         rel_paths = [f.name for f in files]
         assert "main.py" in rel_paths
         assert ".env" not in rel_paths
@@ -247,7 +247,7 @@ def test_excludes_binary_files(self, tmp_path):
         binary = tmp_path / "bad.py"
         binary.write_bytes(b"import os\x00\nprint('hi')")
 
-        files, warnings = discover_local_files(tmp_path)
+        files, warnings, _ = discover_local_files(tmp_path)
         names = [f.name for f in files]
         assert "good.py" in names
         assert "bad.py" not in names
@@ -260,7 +260,7 @@ def test_respects_gitignore(self, tmp_path):
         (tmp_path / "kept.py").write_text("x = 1\n")
         (tmp_path / "ignored.py").write_text("y = 2\n")
 
-        files, _ = discover_local_files(tmp_path)
+        files, *_ = discover_local_files(tmp_path)
         names = [f.name for f in files]
         assert "kept.py" in names
         assert "ignored.py" not in names
@@ -272,7 +272,7 @@ def test_extra_ignore_patterns(self, tmp_path):
         (tmp_path / "main.py").write_text("x = 1\n")
         (tmp_path / "temp.py").write_text("y = 2\n")
 
-        files, _ = discover_local_files(tmp_path, extra_ignore_patterns=["temp.py"])
+        files, *_ = discover_local_files(tmp_path, extra_ignore_patterns=["temp.py"])
         names = [f.name for f in files]
         assert "main.py" in names
         assert "temp.py" not in names
@@ -287,7 +287,7 @@ def test_symlinks_skipped_by_default(self, tmp_path):
         link = tmp_path / "link.py"
         link.symlink_to(real)
 
-        files, _ = discover_local_files(tmp_path, follow_symlinks=False)
+        files, *_ = discover_local_files(tmp_path, follow_symlinks=False)
         names = [f.name for f in files]
         assert "real.py" in names
         assert "link.py" not in names