Merge pull request #1330 from mvalentsev/fix/convo-miner-skip-subagents

fix(convo-miner): skip Claude Code subagent transcripts by default (#1217)

Author: igorls

mempalace/cli.py

@@ -736,6 +736,7 @@ def cmd_mine(args):
                 limit=args.limit,
                 dry_run=args.dry_run,
                 extract_mode=args.extract,
+                include_subagents=getattr(args, "include_subagents", False),
             )
         elif args.mode == "extract":
             from .format_miner import mine_formats
@@ -2372,6 +2373,17 @@ def main():
             f"Windows if you hit ONNX bad_alloc (#1455)."
         ),
     )
+    p_mine.add_argument(
+        "--include-subagents",
+        action="store_true",
+        default=False,
+        help=(
+            "Also mine Claude Code subagent transcripts (subagents/ dirs). "
+            "Excluded by default: these are short ephemeral exchanges "
+            "(Explore/Plan/Grep agents) already summarized in the parent "
+            "session, and on typical workspaces they dominate file counts."
+        ),
+    )
 
     # sweep
     p_sweep = sub.add_parser(

mempalace/convo_miner.py

@@ -412,12 +412,21 @@ def detect_convo_room(content: str) -> str:
 # =============================================================================
 
 
-def scan_convos(convo_dir: str) -> list:
+def scan_convos(convo_dir: str, include_subagents: bool = False) -> list:
     """Find all potential conversation files.
 
     Skips symlinks and oversized files. Each skipped symlink is logged to
     ``sys.stderr`` with a ``  SKIP: <relative-path> (symlink)`` line so the
     caller can tell why an apparent conversation directory yielded no files.
+
+    By default, directories named ``subagents`` are skipped: Claude Code
+    records Explore/Plan/Grep subagent transcripts there, and on typical
+    workspaces they outnumber main session files by one to two orders of
+    magnitude. Pass ``include_subagents=True`` to mine them anyway.
+
+    The match is case-insensitive on the directory name only (``subagents``
+    or ``Subagents``), so directories like ``mysubagents`` or
+    ``subagentsbackup`` are not affected.
     """
     # A direct conversation file is a valid source. For a file, feed only
     # its basename through the existing directory validation loop.
@@ -429,7 +438,11 @@ def scan_convos(convo_dir: str) -> list:
     )
     files = []
     for root, dirs, filenames in scan_entries:
-        dirs[:] = [d for d in dirs if d not in CONVO_SKIP_DIRS]
+        dirs[:] = [
+            d
+            for d in dirs
+            if d not in CONVO_SKIP_DIRS and (include_subagents or d.lower() != "subagents")
+        ]
         for filename in filenames:
             if filename.endswith(".meta.json"):
                 continue
@@ -739,12 +752,16 @@ def mine_convos(
     limit: int = 0,
     dry_run: bool = False,
     extract_mode: str = "exchange",
+    include_subagents: bool = False,
 ):
     """Mine a directory of conversation files into the palace.
 
     extract_mode:
         "exchange" — default exchange-pair chunking (Q+A = one unit)
         "general"  — general extractor: decisions, preferences, milestones, problems, emotions
+    include_subagents:
+        False (default) — skip Claude Code ``subagents/`` directories
+        True            — also mine subagent transcripts
 
     The real work is in :func:`_mine_convos_impl`; this wrapper holds the
     per-palace flock around it so two concurrent ``mempalace mine --mode
@@ -771,6 +788,7 @@ def mine_convos(
             limit=limit,
             dry_run=dry_run,
             extract_mode=extract_mode,
+            include_subagents=include_subagents,
         )
 
     with mine_palace_lock(palace_path):
@@ -782,6 +800,7 @@ def mine_convos(
             limit=limit,
             dry_run=dry_run,
             extract_mode=extract_mode,
+            include_subagents=include_subagents,
         )
 
 
@@ -867,6 +886,7 @@ def _mine_convos_impl(
     limit: int = 0,
     dry_run: bool = False,
     extract_mode: str = "exchange",
+    include_subagents: bool = False,
 ):
     from .config import MempalaceConfig
 
@@ -886,7 +906,7 @@ def _mine_convos_impl(
     convo_path = Path(convo_dir).expanduser().resolve()
     wing = _resolve_wing(convo_path, wing)
 
-    files = scan_convos(convo_dir)
+    files = scan_convos(convo_dir, include_subagents=include_subagents)
 
     print(f"\n{'=' * 55}")
     print("  MemPalace Mine -- Conversations")

tests/test_cli.py

@@ -593,6 +593,7 @@ def test_cmd_mine_convos_mode(mock_config_cls):
         no_gitignore=False,
         include_ignored=[],
         extract="general",
+        include_subagents=False,
     )
     with patch("mempalace.convo_miner.mine_convos") as mock_mine:
         cmd_mine(args)
@@ -604,9 +605,32 @@ def test_cmd_mine_convos_mode(mock_config_cls):
             limit=10,
             dry_run=True,
             extract_mode="general",
+            include_subagents=False,
         )
 
 
+@patch("mempalace.cli.MempalaceConfig")
+def test_cmd_mine_convos_mode_threads_include_subagents_flag(mock_config_cls):
+    mock_config_cls.return_value.palace_path = "/fake/palace"
+    args = argparse.Namespace(
+        dir="/chats",
+        palace=None,
+        mode="convos",
+        wing="mywing",
+        agent="me",
+        limit=10,
+        dry_run=True,
+        no_gitignore=False,
+        include_ignored=[],
+        extract="exchange",
+        include_subagents=True,
+    )
+    with patch("mempalace.convo_miner.mine_convos") as mock_mine:
+        cmd_mine(args)
+        kwargs = mock_mine.call_args.kwargs
+        assert kwargs["include_subagents"] is True
+
+
 @patch("mempalace.cli.MempalaceConfig")
 def test_cmd_mine_include_ignored_comma_split(mock_config_cls):
     mock_config_cls.return_value.palace_path = "/fake/palace"

tests/test_convo_miner_unit.py

@@ -510,6 +510,101 @@ def selective_stat(self, *args, **kwargs):
         assert "SKIP: unreadable.txt" in err
         assert "stat error" in err
 
+    def test_scan_skips_subagent_dirs_by_default(self, tmp_path):
+        # Mimic Claude Code layout: ~/.claude/projects/<slug>/<session>/subagents/agent-*.jsonl
+        session_dir = tmp_path / "session-abc"
+        session_dir.mkdir()
+        (session_dir / "main.jsonl").write_text('{"type":"user"}\n', encoding="utf-8")
+        subagents_dir = session_dir / "subagents"
+        subagents_dir.mkdir()
+        (subagents_dir / "agent-abc.jsonl").write_text('{"type":"user"}\n', encoding="utf-8")
+        (subagents_dir / "agent-def.jsonl").write_text('{"type":"user"}\n', encoding="utf-8")
+
+        files = scan_convos(str(tmp_path))
+        names = [f.name for f in files]
+
+        assert "main.jsonl" in names
+        assert "agent-abc.jsonl" not in names
+        assert "agent-def.jsonl" not in names
+
+    def test_scan_includes_subagent_dirs_when_opted_in(self, tmp_path):
+        session_dir = tmp_path / "session-abc"
+        session_dir.mkdir()
+        (session_dir / "main.jsonl").write_text('{"type":"user"}\n', encoding="utf-8")
+        subagents_dir = session_dir / "subagents"
+        subagents_dir.mkdir()
+        (subagents_dir / "agent-abc.jsonl").write_text('{"type":"user"}\n', encoding="utf-8")
+
+        files = scan_convos(str(tmp_path), include_subagents=True)
+        names = [f.name for f in files]
+
+        assert "main.jsonl" in names
+        assert "agent-abc.jsonl" in names
+
+    def test_scan_skips_subagent_dirs_at_any_depth(self, tmp_path):
+        # The "subagents" name match is by directory name, not by depth: verify
+        # both shallow (top-level) and nested subagents/ get skipped.
+        (tmp_path / "subagents").mkdir()
+        (tmp_path / "subagents" / "agent-top.jsonl").write_text("{}", encoding="utf-8")
+        nested = tmp_path / "session" / "subagents"
+        nested.mkdir(parents=True)
+        (nested / "agent-deep.jsonl").write_text("{}", encoding="utf-8")
+        (tmp_path / "session" / "main.jsonl").write_text("{}", encoding="utf-8")
+
+        files = scan_convos(str(tmp_path))
+        names = [f.name for f in files]
+
+        assert "main.jsonl" in names
+        assert "agent-top.jsonl" not in names
+        assert "agent-deep.jsonl" not in names
+
+    def test_scan_does_not_skip_suffix_named_dirs(self, tmp_path):
+        # Exact name match only: 'mysubagents' or 'subagentsbackup' must still
+        # be mined. Guards against future regression to substring/regex match.
+        for dir_name in ("mysubagents", "subagentsbackup", "subagent"):
+            d = tmp_path / dir_name
+            d.mkdir()
+            (d / f"{dir_name}.jsonl").write_text("{}", encoding="utf-8")
+
+        files = scan_convos(str(tmp_path))
+        names = {f.name for f in files}
+
+        assert "mysubagents.jsonl" in names
+        assert "subagentsbackup.jsonl" in names
+        assert "subagent.jsonl" in names
+
+    def test_scan_skips_subagents_case_insensitive(self, tmp_path):
+        # On Windows + macOS APFS the filesystem is case-preserving; if Claude
+        # Code or a plugin ever emits 'Subagents' (capitalized), the filter
+        # must still match. Only one variant per tmp_path because case-
+        # insensitive filesystems collapse 'Subagents' and 'SUBAGENTS'.
+        d = tmp_path / "Subagents"
+        d.mkdir()
+        (d / "agent.jsonl").write_text("{}", encoding="utf-8")
+        (tmp_path / "main.jsonl").write_text("{}", encoding="utf-8")
+
+        files = scan_convos(str(tmp_path))
+        names = {f.name for f in files}
+
+        assert "main.jsonl" in names
+        assert "agent.jsonl" not in names
+
+    def test_scan_mines_an_explicitly_named_file_inside_subagents(self, tmp_path):
+        # The skip is directory pruning, so it cannot reach a caller who names
+        # one file: that path feeds a single synthetic entry with no directories
+        # to prune. The split is deliberate -- --include-subagents governs what a
+        # directory walk sweeps up, while naming a path is an explicit request
+        # and stays honored. Pinned because the two behaviours were written
+        # independently and nothing else exercises them together.
+        subagents_dir = tmp_path / "session-abc" / "subagents"
+        subagents_dir.mkdir(parents=True)
+        target = subagents_dir / "agent-abc.jsonl"
+        target.write_text('{"type":"user"}\n', encoding="utf-8")
+
+        files = scan_convos(str(target))
+
+        assert [f.name for f in files] == ["agent-abc.jsonl"]
+
 
 class TestFileChunksLocked:
     def test_uses_bounded_upsert_batches(self, monkeypatch):