fix(search): pass index dir as a parameter — delete AA_SEARCH_INDEX_DIR global (0.2.2)

Root-cause for the stage3d_hybrid_search flake. v0.2.1 serialized the tests
with a mutex (band-aid). The structural cause: do_search_codebase handed the
search-index dir to search::search_graph through the PROCESS-GLOBAL env var
AA_SEARCH_INDEX_DIR — a hidden channel between two functions that should
communicate by argument. Parallel callers/tests stomped it; build_search_index
wiped+rebuilt the dir mid-read → tantivy FileDoesNotExist.

search_graph now takes index_dir: Option<&Path>. find_search_index_dir and the
env var are deleted; main.rs passes graph_path.parent()/search_index; prd_input
+ the four integration tests pass None. The test mutex is removed — the four
hybrid-search tests run fully parallel, each passing its own index dir.

Verified: full suite green; hybrid+integration search tests 3× green under
parallelism with no mutex. source: dijkstra root-cause audit.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: cdeust

.claude-plugin/marketplace.json

@@ -6,14 +6,14 @@
   },
   "metadata": {
     "description": "automatised-pipeline — Rust MCP that indexes ANY codebase (every file type: source in 10 languages with full AST, plus docs/config/binaries as File nodes) into a LadybugDB property graph and exposes 24 tools (index_codebase, query_graph, get_impact, get_context, semantic-diff) to agents.",
-    "version": "0.2.1"
+    "version": "0.2.2"
   },
   "plugins": [
     {
       "name": "automatised-pipeline",
       "source": "./",
       "description": "24 MCP tools · all-file indexing (every file is a File node; .js→Imports, Markdown→References) · LadybugDB property graph · Leiden communities · BM25+TF-IDF+RRF search",
-      "version": "0.2.1",
+      "version": "0.2.2",
       "author": {
         "name": "Clement Deust",
         "email": "admin@ai-architect.tools"

.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "automatised-pipeline",
   "description": "Rust MCP server that indexes ANY codebase into a LadybugDB property graph — source in 10 languages with full AST, plus docs/config/binaries as File nodes (.js→Imports, Markdown→References). Resolves imports and call chains, detects communities via Leiden, traces execution flows from entry points, and exposes 24 MCP tools to agents — index_codebase, query_graph, get_symbol, get_impact, get_context, semantic_diff, and more.",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "author": {
     "name": "Clement Deust",
     "email": "admin@ai-architect.tools"

CHANGELOG.md

@@ -6,6 +6,21 @@ adheres to [Semantic Versioning](https://semver.org/).
 
 ## [Unreleased]
 
+## [0.2.2] — Remove the search-index env-var channel (flaky-test root cause)
+
+### Fixed
+
+- **Root-caused the `stage3d_hybrid_search` flake.** v0.2.1 serialized the
+  tests with a mutex — a band-aid. The structural cause was that
+  `do_search_codebase` passed the search-index directory to
+  `search::search_graph` through the PROCESS-GLOBAL env var
+  `AA_SEARCH_INDEX_DIR`, a hidden channel that races across any parallel
+  callers (and was wiped+rebuilt mid-read → tantivy `FileDoesNotExist`).
+  `search_graph` now takes `index_dir: Option<&Path>` as an explicit
+  parameter; the env var and `find_search_index_dir` are deleted. The test
+  mutex is removed — the four tests run fully parallel, each passing its own
+  index dir (verified 3× green). source: dijkstra root-cause audit.
+
 ## [0.2.1] — Release hygiene + flaky-test fix
 
 ### Fixed

Cargo.toml

@@ -9,7 +9,7 @@ members = [".", "benches/harness", "crates/zera"]
 
 [package]
 name = "ai-architect-mcp"
-version = "0.2.1"
+version = "0.2.2"
 edition = "2021"
 description = "Stage-by-stage rewrite of the ai-architect pipeline as an MCP server. One tool per stage. Grown by zetetic + genius agents."
 

src/main.rs

@@ -2443,14 +2443,13 @@ fn do_search_codebase(arguments: &Value) -> Result<Value, String> {
         return Err(format!("graph_path does not exist: {graph_str}"));
     }
 
-    // Set search index dir hint for hybrid search.
-    // Convention: search_index/ is a sibling of graph/ under the output dir.
-    if let Some(parent) = graph_path.parent() {
-        let search_index_dir = parent.join("search_index");
-        if search_index_dir.exists() {
-            std::env::set_var("AA_SEARCH_INDEX_DIR", search_index_dir.to_string_lossy().as_ref());
-        }
-    }
+    // The search index lives in a sibling ``search_index/`` of the graph dir.
+    // Pass it explicitly to search_graph — no process-global env hand-off
+    // (that channel raced across parallel callers; see search::search_graph).
+    let search_index_dir = graph_path
+        .parent()
+        .map(|p| p.join("search_index"))
+        .filter(|p| p.exists());
 
     let start = std::time::Instant::now();
     let store = graph_store::GraphStore::open_or_create(graph_path)?;
@@ -2459,7 +2458,8 @@ fn do_search_codebase(arguments: &Value) -> Result<Value, String> {
         label_filter,
         min_score: 0.01,
     };
-    let results = search::search_graph(&store, query, &options)?;
+    let results =
+        search::search_graph(&store, query, &options, search_index_dir.as_deref())?;
     let elapsed_ms = start.elapsed().as_millis() as u64;
 
     let items: Vec<Value> = results.iter().map(|r| json!({

src/prd_input.rs

@@ -242,7 +242,7 @@ fn search_and_enrich(store: &GraphStore, tokens: &[String]) -> Vec<MatchedSymbol
             label_filter: None,
             min_score: 0.0,
         };
-        let hits = match search::search_graph(store, token, &opts) {
+        let hits = match search::search_graph(store, token, &opts, None) {
             Ok(h) => h,
             Err(_) => continue,
         };

src/search/mod.rs

@@ -142,6 +142,7 @@ pub fn search_graph(
     store: &GraphStore,
     query: &str,
     options: &SearchOptions,
+    index_dir: Option<&std::path::Path>,
 ) -> Result<Vec<SearchResult>, String> {
     let _start = Instant::now();
     let query_lower = query.to_lowercase();
@@ -150,9 +151,13 @@ pub fn search_graph(
         return Ok(Vec::new());
     }
 
-    // Try to find search index directory relative to the graph path.
-    // Convention: search_index/ is a sibling of graph/ under the output_dir.
-    let index_dir = find_search_index_dir(store);
+    // The search-index directory is passed by the caller (sibling
+    // ``search_index/`` of the graph dir). It used to be smuggled through the
+    // process-global env var ``AA_SEARCH_INDEX_DIR``, which raced across
+    // parallel callers/tests (tantivy FileDoesNotExist). Passing it as an
+    // argument removes that hidden global channel entirely. source: dijkstra
+    // root-cause audit of the stage3d_hybrid_search flake.
+    let index_dir = index_dir.map(|p| p.to_path_buf());
 
     let has_bm25 = index_dir.as_ref()
         .map(|d| d.join("bm25").exists())
@@ -517,21 +522,6 @@ fn find_name_candidates(store: &GraphStore, name: &str, limit: usize) -> Vec<Str
 // Internal: find search index directory
 // ---------------------------------------------------------------------------
 
-fn find_search_index_dir(_store: &GraphStore) -> Option<std::path::PathBuf> {
-    // The GraphStore doesn't expose its path, but we can probe known locations.
-    // Convention: search_index/ is a sibling of graph/ under the output dir.
-    // The caller passes graph_path when opening the store. We use a probe:
-    // check if ../search_index/ exists relative to the DB path.
-    //
-    // Since GraphStore doesn't expose its path, we use an env-var hint
-    // set by the search tool handler, or probe common locations.
-    if let Ok(hint) = std::env::var("AA_SEARCH_INDEX_DIR") {
-        let p = std::path::PathBuf::from(hint);
-        if p.exists() { return Some(p); }
-    }
-    None
-}
-
 // ---------------------------------------------------------------------------
 // Internal: candidate fetching (for substring fallback)
 // ---------------------------------------------------------------------------

tests/stage3d_hybrid_search.rs

@@ -11,21 +11,13 @@ use ai_architect_mcp::search;
 use std::fs;
 use std::path::PathBuf;
 use std::sync::atomic::{AtomicU64, Ordering};
-use std::sync::{Mutex, MutexGuard, OnceLock};
-
-// search_graph reads the index location from the PROCESS-GLOBAL env var
-// `AA_SEARCH_INDEX_DIR`. cargo runs the tests in this binary on parallel
-// threads, so without serialization two tests stomp each other's env var
-// (and `build_search_index` wipes+rebuilds its dir), producing a tantivy
-// `FileDoesNotExist` on the BM25 store. Hold this lock for the whole of
-// each test so the env var + index belong to exactly one test at a time.
-// source: observed CI flake (stage3d_hybrid_search.rs, run 26824494088).
-fn search_test_guard() -> MutexGuard<'static, ()> {
-    static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
-    LOCK.get_or_init(|| Mutex::new(()))
-        .lock()
-        .unwrap_or_else(|poisoned| poisoned.into_inner())
-}
+
+// These tests run in parallel (cargo default). Each builds its own index in a
+// per-test temp dir and passes that dir EXPLICITLY to search_graph, so there
+// is no shared global state to race — the previous flake came from smuggling
+// the index dir through the process-global env var AA_SEARCH_INDEX_DIR, now
+// removed at the root (search_graph takes index_dir as a parameter).
+// source: dijkstra root-cause audit (run 26824494088).
 
 const FIXTURE_MAIN: &str = r#"
 fn main() {
@@ -58,12 +50,9 @@ pub fn route_incoming(path: &str) -> String {
 
 static COUNTER: AtomicU64 = AtomicU64::new(0);
 
-fn setup_with_search_index(
-    test_name: &str,
-) -> (MutexGuard<'static, ()>, PathBuf, GraphStore) {
-    // Acquire BEFORE touching the shared env var / index dir; the guard is
-    // returned so the caller holds it for the whole test.
-    let guard = search_test_guard();
+/// Returns (tmp_root, store, index_dir). The index_dir is passed explicitly to
+/// search_graph by each test — no shared env var, so the tests are parallel-safe.
+fn setup_with_search_index(test_name: &str) -> (PathBuf, GraphStore, PathBuf) {
     let n = COUNTER.fetch_add(1, Ordering::SeqCst);
     let tmp_root = std::env::temp_dir().join(format!(
         "stage3d_hybrid_{}_{n}_{}", test_name, std::process::id()
@@ -88,22 +77,19 @@ fn setup_with_search_index(
     assert!(si.bm25_doc_count > 0, "should index BM25 docs");
     assert!(si.vector_doc_count > 0, "should index vector docs");
 
-    // Set env hint for search_graph to find the index
     let idx_dir = tmp_root.join("search_index");
-    std::env::set_var("AA_SEARCH_INDEX_DIR", idx_dir.to_string_lossy().as_ref());
-
-    (guard, tmp_root, store)
+    (tmp_root, store, idx_dir)
 }
 
 #[test]
 fn test_hybrid_bm25_keyword_search() {
-    let (_guard, tmp_root, store) = setup_with_search_index("bm25");
+    let (tmp_root, store, idx_dir) = setup_with_search_index("bm25");
     let opts = search::SearchOptions {
         limit: 10,
         label_filter: None,
         min_score: 0.0,
     };
-    let results = search::search_graph(&store, "handle tool", &opts).unwrap();
+    let results = search::search_graph(&store, "handle tool", &opts, Some(&idx_dir)).unwrap();
     assert!(!results.is_empty(), "BM25 should find 'handle tool'");
     let found = results.iter().any(|r| r.name.contains("handle_tool"));
     assert!(found, "should find handle_tool_call via BM25: {:?}",
@@ -114,15 +100,15 @@ fn test_hybrid_bm25_keyword_search() {
 
 #[test]
 fn test_hybrid_semantic_search() {
-    let (_guard, tmp_root, store) = setup_with_search_index("semantic");
+    let (tmp_root, store, idx_dir) = setup_with_search_index("semantic");
     let opts = search::SearchOptions {
         limit: 10,
         label_filter: None,
         min_score: 0.0,
     };
     // "process incoming requests" should find process_request or route_incoming
     // via TF-IDF even though the exact phrase doesn't appear
-    let results = search::search_graph(&store, "process incoming requests", &opts).unwrap();
+    let results = search::search_graph(&store, "process incoming requests", &opts, Some(&idx_dir)).unwrap();
     assert!(!results.is_empty(),
         "semantic search should find results for 'process incoming requests'");
 
@@ -141,13 +127,13 @@ fn test_hybrid_semantic_search() {
 
 #[test]
 fn test_rrf_fusion_combines_rankings() {
-    let (_guard, tmp_root, store) = setup_with_search_index("rrf");
+    let (tmp_root, store, idx_dir) = setup_with_search_index("rrf");
     let opts = search::SearchOptions {
         limit: 10,
         label_filter: None,
         min_score: 0.0,
     };
-    let results = search::search_graph(&store, "handle", &opts).unwrap();
+    let results = search::search_graph(&store, "handle", &opts, Some(&idx_dir)).unwrap();
     assert!(!results.is_empty(), "should find results for 'handle'");
 
     // RRF scores are in the range ~0.01-0.03 (1/(60+rank))
@@ -163,7 +149,7 @@ fn test_rrf_fusion_combines_rankings() {
 
 #[test]
 fn test_build_search_index_creates_files() {
-    let (_guard, tmp_root, _store) = setup_with_search_index("files");
+    let (tmp_root, _store, _idx_dir) = setup_with_search_index("files");
 
     let bm25_dir = tmp_root.join("search_index/bm25");
     assert!(bm25_dir.exists(), "BM25 index directory should exist");

tests/stage3d_integration.rs

@@ -95,7 +95,7 @@ fn test_search_exact_name() {
         label_filter: None,
         min_score: 0.0,
     };
-    let results = search::search_graph(&store, "main", &opts).unwrap();
+    let results = search::search_graph(&store, "main", &opts, None).unwrap();
     assert!(!results.is_empty(), "search for 'main' should return results");
     assert_eq!(results[0].name, "main", "top result should be exact match");
     assert!(
@@ -114,7 +114,7 @@ fn test_search_partial_name() {
         label_filter: None,
         min_score: 0.0,
     };
-    let results = search::search_graph(&store, "handle", &opts).unwrap();
+    let results = search::search_graph(&store, "handle", &opts, None).unwrap();
     assert!(!results.is_empty(), "search for 'handle' should find handle_request");
     let found = results.iter().any(|r| r.name.contains("handle"));
     assert!(found, "should find a symbol containing 'handle'");
@@ -129,7 +129,7 @@ fn test_search_label_filter() {
         label_filter: Some("Function".to_string()),
         min_score: 0.0,
     };
-    let results = search::search_graph(&store, "process", &opts).unwrap();
+    let results = search::search_graph(&store, "process", &opts, None).unwrap();
     for r in &results {
         assert_eq!(r.label, "Function", "label filter should be respected");
     }
@@ -144,7 +144,7 @@ fn test_search_results_have_context() {
         label_filter: None,
         min_score: 0.0,
     };
-    let results = search::search_graph(&store, "main", &opts).unwrap();
+    let results = search::search_graph(&store, "main", &opts, None).unwrap();
     let main_result = results.iter().find(|r| r.name == "main").unwrap();
     assert!(!main_result.file_path.is_empty(), "should have file_path");
     assert!(main_result.community_id.is_some(), "main should have a community");