feat: silent-fix signal in the gate + bulk PR-ref warming for scale

- build_security_dataset.py: silent_fix_prob>=0.70 now also ADMITS a row (gate),
  not just tiers it — so classifying gate-dropped rows becomes a recall lever.
  Test updated.
- local_diffs.py: `warm-prs` bulk-fetches every PR head ref per repo (one fetch
  -> all PR diffs local & instant; geth = 14,456 refs / +120MB), and PR lookup
  now tolerates both ref layouts. Makes a full 17k-row classification LLM-bound
  instead of network-bound.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: grandchildrice

collection/local_diffs.py

@@ -76,13 +76,35 @@ def get_commit_diff(client: str, sha: str) -> str | None:
     return r.stdout if r.returncode == 0 and r.stdout.strip() else None
 
 
+def warm_prs(client: str) -> None:
+    """Bulk-fetch every PR head ref so all PR diffs are served locally/instantly.
+    One fetch per repo (~+120 MB blobless for geth) instead of a network call per
+    PR — turns the 17k-row classification into an LLM-bound-only job."""
+    p = ensure_clone(client)
+    r = _run(["git", "-C", str(p), "fetch", "--filter=blob:none", "--quiet",
+              "origin", "+refs/pull/*/head:refs/pull/*/head"], timeout=1200)
+    n = _run(["git", "-C", str(p), "for-each-ref", "refs/pull/", "--format=x"]).stdout.count("x")
+    print(f"[local_diffs] {client}: {n} PR refs local"
+          + ("" if r.returncode == 0 else " (some conflicts skipped)"), file=sys.stderr)
+
+
+def _resolve_pr_ref(p: Path, n: str) -> str | None:
+    """PR head ref, tolerating both layouts (refs/pull/N/head and refs/pull/N)."""
+    for ref in (f"refs/pull/{n}/head", f"refs/pull/{n}"):
+        if _run(["git", "-C", str(p), "rev-parse", "--verify", "--quiet", ref]).returncode == 0:
+            return ref
+    return None
+
+
 def get_pr_diff(client: str, n: str) -> str | None:
     p = ensure_clone(client)
-    ref = f"refs/pull/{n}/head"
-    if _run(["git", "-C", str(p), "rev-parse", "--verify", "--quiet", ref]).returncode != 0:
-        f = _run(["git", "-C", str(p), "fetch", "--quiet", "origin", f"{ref}:{ref}"], timeout=300)
-        if f.returncode != 0:
+    ref = _resolve_pr_ref(p, n)
+    if ref is None:
+        target = f"refs/pull/{n}/head"
+        if _run(["git", "-C", str(p), "fetch", "--quiet", "origin",
+                 f"{target}:{target}"], timeout=300).returncode != 0:
             return None
+        ref = target
     head = _run(["git", "-C", str(p), "rev-parse", ref]).stdout.strip()
     if not head:
         return None
@@ -153,6 +175,7 @@ def main() -> int:
     ap = argparse.ArgumentParser()
     sub = ap.add_subparsers(dest="cmd", required=True)
     w = sub.add_parser("warm"); w.add_argument("--client", required=True)
+    wp = sub.add_parser("warm-prs"); wp.add_argument("--client", required=True)
     rf = sub.add_parser("refresh"); rf.add_argument("--client", required=True)
     d = sub.add_parser("diff")
     d.add_argument("--client", required=True); d.add_argument("--url", required=True)
@@ -163,6 +186,10 @@ def main() -> int:
             ensure_clone(c)
             print(f"[local_diffs] {c} ready at {repo_path(c)}")
         return 0
+    if a.cmd == "warm-prs":
+        for c in clients:
+            warm_prs(c)
+        return 0
     if a.cmd == "refresh":
         for c in clients:
             refresh(c)

pipeline/build_security_dataset.py

@@ -344,7 +344,16 @@ def build(df: pd.DataFrame) -> tuple[pd.DataFrame, dict]:
     # match also fires on release notes that merely *list* a "fix crash". T2
     # already dropped dep-bump/CI titles upstream.
     has_fiximpact = df["title"].fillna("").str.contains(FIX_IMPACT_RE)
-    df["security_relevant"] = has_id | has_sev | has_kw | has_stride | has_cwe | has_fiximpact
+    # Recall expansion via the learned classifier: a row the LLM confidently
+    # calls a silent fix (silent_fix_prob >= 0.70) is admitted even if the
+    # deterministic keyword gate missed it. Only fires where a classification
+    # exists (column present); absent -> no effect.
+    if "silent_fix_prob" in df.columns:
+        has_silentfix = pd.to_numeric(df["silent_fix_prob"], errors="coerce").fillna(0) >= 0.70
+    else:
+        has_silentfix = pd.Series(False, index=df.index)
+    df["security_relevant"] = (has_id | has_sev | has_kw | has_stride | has_cwe
+                               | has_fiximpact | has_silentfix)
 
     sec = df[df["security_relevant"]].copy()
     # fix_commit (issue #89 field, method-2 backlink): the fixing commit SHA is

tests/test_security_dataset.py

@@ -58,7 +58,12 @@ def test_every_row_has_a_security_signal(df):
         r"|race condition)\b[^.\n]{0,25}"
         r"\b(?:fix|fixed|prevent|avoid|guard against|resolved|patch)\w*\b", re.I)
     has_fiximpact = df["title"].fillna("").str.contains(fiximpact)
-    assert bool((has_sev | has_kw | has_stride | has_cwe | has_id | has_fiximpact).all())
+    if "silent_fix_prob" in df.columns:
+        has_silentfix = pd.to_numeric(df["silent_fix_prob"], errors="coerce").fillna(0) >= 0.70
+    else:
+        has_silentfix = pd.Series(False, index=df.index)
+    assert bool((has_sev | has_kw | has_stride | has_cwe | has_id | has_fiximpact
+                 | has_silentfix).all())
 
 
 def test_confidence_values(df):