production: classify C_candidate with gemma4:31b, promote 166 silent fixes

Ran the training-free LLM classifier (gemma4:31b via Ollama Cloud) over 339
C_candidate PR/commit rows using local_diffs (rate-limit-free, cache-resumable).
Flagged 166 as real silent fixes (133 DoS / 16 consensus / 11 validation …),
promoting them C_corroborated -> B. Essential slice 1367 -> 1535.

- llm_classify_fixes.py: --apply mode (local_diffs + persistent diff/pred caches).
- data/silent_fix_llm.csv: classifier output, tracked so the build is reproducible.
- run_pipeline.sh: curate stage folds it in when present (--silent-fix-csv).
- Spot-check: flagged rows are genuine ("loop limit to prevent remote DoS",
  "OOB panic unmarshaling JSON", "nil deref at backend init"); non-fixes
  correctly rejected (CI/dep-bump/feature; even "panic added for diagnostics").

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: grandchildrice

collection/llm_classify_fixes.py

@@ -253,10 +253,81 @@ def evaluate(preds):
     return {"precision": prec, "recall": rec, "f1": f1, "tp": tp, "fp": fp, "tn": tn, "fn": fn}
 
 
+def apply_to_dataset(a) -> int:
+    """Classify real dataset rows and emit source_url -> silent_fix_prob.
+
+    Diffs come from local_diffs (bare clone + persistent cache, rate-limit-free);
+    LLM predictions are cached per URL so re-runs are resumable ("差分だけ").
+    """
+    import csv
+    sys.path.insert(0, str(Path(__file__).resolve().parent))
+    import local_diffs
+
+    df = pd.read_parquet(a.inp)
+    if a.tier != "all" and "authority_tier" in df.columns:
+        df = df[df["authority_tier"] == a.tier]
+    df = df[df["source_url"].str.contains(r"/pull/|/commit/", na=False)].copy()
+    if a.limit:
+        df = df.head(a.limit)
+    diff_cache = json.loads(a.cache.read_text()) if a.cache.exists() else {}
+    pred_cache = json.loads(a.pred_cache.read_text()) if a.pred_cache.exists() else {}
+    rows = df.to_dict("records")
+    print(f"[apply] {len(rows)} rows (tier={a.tier}); "
+          f"{sum(1 for r in rows if str(r['source_url']) in pred_cache)} already predicted",
+          file=sys.stderr)
+
+    def work(r):
+        url = str(r["source_url"])
+        if url in pred_cache:
+            return url, pred_cache[url]
+        diff = local_diffs.get_diff_cached(url, r["source_platform"], diff_cache)
+        if not diff:
+            return url, {"skip": "nodiff"}
+        it = {"title": str(r.get("title") or "")[:200],
+              "desc": str(r.get("description") or "")[:600], "diff": _diff_doc(diff)}
+        return url, classify(it)["pred"]
+
+    done = 0
+    with ThreadPoolExecutor(max_workers=a.workers) as ex:
+        for url, pred in ex.map(work, rows):
+            pred_cache[url] = pred
+            done += 1
+            if done % 40 == 0:
+                a.pred_cache.write_text(json.dumps(pred_cache))
+                a.cache.write_text(json.dumps(diff_cache))
+                print(f"  [apply] {done}/{len(rows)}", file=sys.stderr)
+    a.pred_cache.write_text(json.dumps(pred_cache))
+    a.cache.write_text(json.dumps(diff_cache))
+
+    a.apply_out.parent.mkdir(parents=True, exist_ok=True)
+    n_fix = 0
+    with a.apply_out.open("w", newline="", encoding="utf-8") as fh:
+        w = csv.writer(fh)
+        w.writerow(["source_url", "silent_fix_prob", "is_security_fix", "vuln_class", "reason"])
+        for r in rows:
+            url = str(r["source_url"]); pr = pred_cache.get(url, {})
+            if not isinstance(pr, dict) or "is_security_fix" not in pr:
+                continue
+            isfix = bool(pr.get("is_security_fix"))
+            conf = float(pr.get("confidence") or 0)
+            prob = conf if isfix else 1 - conf          # p(security fix)
+            if prob >= 0.70:
+                n_fix += 1
+            w.writerow([url, f"{prob:.3f}", int(isfix), pr.get("vuln_class", ""),
+                        str(pr.get("reason", ""))[:200]])
+    print(f"[apply] wrote {a.apply_out} — {n_fix} rows with silent_fix_prob>=0.70", file=sys.stderr)
+    return 0
+
+
 def main() -> int:
     ap = argparse.ArgumentParser()
     ap.add_argument("--build-eval", action="store_true")
     ap.add_argument("--run", action="store_true")
+    ap.add_argument("--apply", action="store_true", help="classify dataset rows -> silent_fix csv")
+    ap.add_argument("--tier", default="C_candidate", help="authority_tier to classify (or 'all')")
+    ap.add_argument("--apply-out", default=Path("scratchpad_crawl/supp/llm_silent_fix.csv"), type=Path)
+    ap.add_argument("--pred-cache", default=Path("scratchpad_crawl/llm_pred_cache.json"), type=Path)
+    ap.add_argument("--limit", type=int, default=0)
     ap.add_argument("--eval-set", default=Path("scratchpad_crawl/llm_eval_set.json"), type=Path)
     ap.add_argument("--out", default=Path("scratchpad_crawl/llm_preds.json"), type=Path)
     ap.add_argument("--in", dest="inp", default=Path("data/ethereum_vulns.parquet"), type=Path)
@@ -284,6 +355,9 @@ def main() -> int:
     if a.engine == "ollama" and a.workers > 2:
         a.workers = 2  # a single local model serializes; avoid thrashing
 
+    if a.apply:
+        return apply_to_dataset(a)
+
     if a.build_eval:
         cache = json.loads(a.cache.read_text())
         items = build_eval(pd.read_parquet(a.inp), pd.read_parquet(a.raw), cache, a.per_class, a.seed)

collection/run_pipeline.sh

@@ -145,9 +145,14 @@ mkdir -p data/raw
 stage publish_raw cp "$TRAIN" data/raw/train.classified.parquet
 
 # --- Stage 9: curate --------------------------------------------------------
+# Optionally fold in the learned silent-fix signal (gemma4:31b via
+# llm_classify_fixes.py --apply). Present -> promotes classified C_candidate
+# fixes to the corroborated tier; absent -> deterministic build only.
+SILENT_FIX_ARG=()
+[ -f data/silent_fix_llm.csv ] && SILENT_FIX_ARG=(--silent-fix-csv data/silent_fix_llm.csv)
 stage curate PY pipeline/build_security_dataset.py \
     --in data/raw/train.classified.parquet \
     --out data/ethereum_vulns.parquet \
-    --manifest data/manifest.json
+    --manifest data/manifest.json "${SILENT_FIX_ARG[@]}"
 
 echo "=== DONE. Curated -> data/ethereum_vulns.parquet ==="

data/BUILD_REPORT.md

@@ -41,11 +41,15 @@ Critical** (geth, besu, teku). Severities preserved through the canonical path.
 
 ## After (curated)
 
-- rows: **1,877**
+- rows: **1,880**
 - residual boilerplate FP: **0**  ✅
-- **essential slice (A+B): 1,367** (was 173 rated-only) — 7.9× larger clean high-precision core
-- by authority_tier: {'B_corroborated': 1134, 'C_candidate': 510, 'A_authoritative': 233}
-- by n_signals: {1: 555, 2: 907, 3: 345, 4: 65, 5: 5} — 60% of rows now carry ≥2 independent signals
+- **essential slice (A+B): 1,535** (was 173 rated-only) — clean high-precision core
+- by authority_tier: {'B_corroborated': 1296, 'C_candidate': 345, 'A_authoritative': 239}
+- **learned silent-fix signal (gemma4:31b):** classified 339 C_candidate diffs,
+  flagged 166 as real silent fixes (133 DoS / 16 consensus / 11 validation …),
+  promoting them C→B. Model chosen by an 80-item eval sweep (F1 0.872, precision
+  0.895); see `docs/model_evaluation.md`. Regenerate via
+  `collection/llm_classify_fixes.py --apply` → `data/silent_fix_llm.csv`.
 - by severity: {'Unrated': 963, 'Info': 773, 'High': 63, 'Medium': 54, 'Low': 21, 'Critical': 3}
   (High/Medium dropped vs iter-1 because T2b removed 49 unrelated CVEs' bogus CVSS severities)
 - by source:

data/ethereum_vulns.parquet

@@ -16,6 +16,7 @@
     "stride",
     "cwe_top25",
     "evidence",
+    "silent_fix_prob",
     "security_score",
     "fix_commit",
     "confidence",
@@ -40,13 +41,13 @@
       "low": 1
     },
     "by_authority_tier": {
-      "B_corroborated": 1130,
-      "C_candidate": 511,
+      "B_corroborated": 1296,
+      "C_candidate": 345,
       "A_authoritative": 239
     },
     "by_n_signals": {
-      "1": 556,
-      "2": 907,
+      "1": 390,
+      "2": 1073,
       "3": 343,
       "4": 69,
       "5": 5