|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""train_silent_fix_classifier.py — learned silent-fix detector (method 1). |
| 3 | +
|
| 4 | +Faithful, torch-free instantiation of the security-patch-classification line of |
| 5 | +silent-fix research (Sabetta & Bezzi, ESEM 2018: represent the patch as a |
| 6 | +*document* and train a supervised classifier; the deep-embedding successors — |
| 7 | +VulFixMiner ASE'21, GraphSPD S&P'23 — swap bag-of-words for CodeBERT/CPG but |
| 8 | +keep the same supervised-on-code-change setup). Regex proxies failed to |
| 9 | +discriminate (see detect_silent_fixes.py, validated negative); a *learned* |
| 10 | +weighting of diff tokens is the method the research actually endorses. |
| 11 | +
|
| 12 | +Pipeline: |
| 13 | + 1. Weak labels from the dataset itself: |
| 14 | + positive = confirmed security fix (advisory/CVE/GHSA id or rated severity) |
| 15 | + negative = dep-bump / docs / CI meta-work (never a client vuln) |
| 16 | + 2. Fetch each item's code diff (cached on disk; re-runs are free). |
| 17 | + 3. TF-IDF over the diff's changed lines (1–2 grams, code tokens). |
| 18 | + 4. Logistic regression, stratified 5-fold CV, report ROC-AUC + PR-AUC. |
| 19 | + 5. Persist model + metrics. Only worth wiring into curation if AUC beats the |
| 20 | + tier baseline (regex ≈ 0.5, i.e. no better than chance). |
| 21 | +
|
| 22 | +Usage: |
| 23 | + uv run python collection/train_silent_fix_classifier.py \ |
| 24 | + --in data/ethereum_vulns.parquet --raw data/raw/train.classified.parquet \ |
| 25 | + --cache scratchpad_crawl/diff_cache.json --per-class 150 |
| 26 | +""" |
| 27 | +from __future__ import annotations |
| 28 | + |
| 29 | +import argparse |
| 30 | +import json |
| 31 | +import re |
| 32 | +import subprocess |
| 33 | +import sys |
| 34 | +import time |
| 35 | +from pathlib import Path |
| 36 | + |
| 37 | +import numpy as np |
| 38 | +import pandas as pd |
| 39 | + |
| 40 | +CLIENT_REPOS: dict[str, str] = { |
| 41 | + "geth": "ethereum/go-ethereum", "nethermind": "NethermindEth/nethermind", |
| 42 | + "besu": "hyperledger/besu", "erigon": "erigontech/erigon", |
| 43 | + "reth": "paradigmxyz/reth", "lighthouse": "sigp/lighthouse", |
| 44 | + "lodestar": "ChainSafe/lodestar", "nimbus": "status-im/nimbus-eth2", |
| 45 | + "prysm": "prysmaticlabs/prysm", "teku": "Consensys/teku", |
| 46 | + "grandine": "grandinetech/grandine", |
| 47 | +} |
| 48 | +PR_RE = re.compile(r"/pull/(\d+)") |
| 49 | +SHA_RE = re.compile(r"/commit/([0-9a-f]{7,40})", re.IGNORECASE) |
| 50 | +NOISE_TITLE_RE = re.compile( |
| 51 | + r"\b(?:bump|chore\(deps|dependabot|renovate|docs?:|readme|changelog|typo" |
| 52 | + r"|lint|ci:|workflow|github actions|codeql|rename|comment|cleanup" |
| 53 | + r"|refactor|reword|polish|cosmetic|formatting|gofmt|clippy)\b", re.IGNORECASE) |
| 54 | + |
| 55 | + |
| 56 | +def _ident(row): |
| 57 | + u = str(row.get("source_url", "")) |
| 58 | + m = PR_RE.search(u) |
| 59 | + if m: |
| 60 | + return ("pr", m.group(1)) |
| 61 | + m = SHA_RE.search(u) |
| 62 | + if m: |
| 63 | + return ("sha", m.group(1)) |
| 64 | + return None |
| 65 | + |
| 66 | + |
| 67 | +def _fetch_diff(repo, kind, ident): |
| 68 | + try: |
| 69 | + if kind == "pr": |
| 70 | + cmd = ["gh", "pr", "diff", ident, "--repo", repo] |
| 71 | + else: |
| 72 | + cmd = ["gh", "api", f"/repos/{repo}/commits/{ident}", |
| 73 | + "--jq", ".files[] | .patch // empty"] |
| 74 | + r = subprocess.run(cmd, capture_output=True, text=True, timeout=45, |
| 75 | + encoding="utf-8", errors="replace") |
| 76 | + return r.stdout if r.returncode == 0 and r.stdout.strip() else None |
| 77 | + except (FileNotFoundError, subprocess.TimeoutExpired): |
| 78 | + return None |
| 79 | + |
| 80 | + |
| 81 | +def _diff_to_doc(diff: str) -> str: |
| 82 | + """Reduce a unified diff to its changed code lines (Sabetta&Bezzi doc).""" |
| 83 | + out = [] |
| 84 | + for ln in diff.splitlines(): |
| 85 | + if ln[:1] in "+-" and not ln.startswith(("+++", "---")): |
| 86 | + out.append(ln[1:]) |
| 87 | + elif ln.startswith("diff --git"): |
| 88 | + out.append(ln.split()[-1]) # keep the path token |
| 89 | + return "\n".join(out)[:20000] |
| 90 | + |
| 91 | + |
| 92 | +def build_labels(cur: pd.DataFrame, raw: pd.DataFrame, per_class: int): |
| 93 | + rated = {"critical", "high", "medium", "low"} |
| 94 | + idrx = re.compile(r"CVE-\d{4}-\d{4,7}|GHSA-", re.I) |
| 95 | + blob = cur["title"].fillna("") + " " + cur["description"].fillna("") |
| 96 | + pos = cur[(cur["severity"].str.lower().isin(rated) | blob.str.contains(idrx)) |
| 97 | + & cur["source_url"].str.contains(r"/pull/|/commit/", na=False)] |
| 98 | + rblob = raw["title"].fillna("") |
| 99 | + neg = raw[rblob.str.contains(NOISE_TITLE_RE) |
| 100 | + & raw["source_url"].str.contains(r"/pull/|/commit/", na=False) |
| 101 | + & ~(raw["title"].fillna("") + " " + raw["description"].fillna("")).str.contains(idrx)] |
| 102 | + pos = pos.head(per_class) |
| 103 | + neg = neg.head(per_class) |
| 104 | + items = [(r, 1) for _, r in pos.iterrows()] + [(r, 0) for _, r in neg.iterrows()] |
| 105 | + return items |
| 106 | + |
| 107 | + |
| 108 | +def main() -> int: |
| 109 | + ap = argparse.ArgumentParser(description=__doc__, |
| 110 | + formatter_class=argparse.RawDescriptionHelpFormatter) |
| 111 | + ap.add_argument("--in", dest="inp", required=True, type=Path) |
| 112 | + ap.add_argument("--raw", required=True, type=Path) |
| 113 | + ap.add_argument("--cache", default=Path("scratchpad_crawl/diff_cache.json"), type=Path) |
| 114 | + ap.add_argument("--per-class", type=int, default=150) |
| 115 | + ap.add_argument("--sleep", type=float, default=0.35) |
| 116 | + ap.add_argument("--model-out", type=Path) |
| 117 | + a = ap.parse_args() |
| 118 | + |
| 119 | + cur = pd.read_parquet(a.inp) |
| 120 | + raw = pd.read_parquet(a.raw) |
| 121 | + items = build_labels(cur, raw, a.per_class) |
| 122 | + print(f"[train] labeled items: {sum(y for _,y in items)} pos / " |
| 123 | + f"{sum(1 for _,y in items if not y)} neg", file=sys.stderr) |
| 124 | + |
| 125 | + cache = {} |
| 126 | + if a.cache.exists(): |
| 127 | + cache = json.loads(a.cache.read_text()) |
| 128 | + docs, ys = [], [] |
| 129 | + fetched = 0 |
| 130 | + for row, y in items: |
| 131 | + url = str(row["source_url"]) |
| 132 | + if url in cache: |
| 133 | + diff = cache[url] |
| 134 | + else: |
| 135 | + ident = _ident(row) |
| 136 | + repo = CLIENT_REPOS.get(row["source_platform"]) |
| 137 | + diff = _fetch_diff(repo, *ident) if (ident and repo) else None |
| 138 | + cache[url] = diff or "" |
| 139 | + fetched += 1 |
| 140 | + if fetched % 25 == 0: |
| 141 | + print(f" fetched {fetched} diffs…", file=sys.stderr) |
| 142 | + a.cache.parent.mkdir(parents=True, exist_ok=True) |
| 143 | + a.cache.write_text(json.dumps(cache)) |
| 144 | + time.sleep(a.sleep) |
| 145 | + if diff: |
| 146 | + docs.append(_diff_to_doc(diff)); ys.append(y) |
| 147 | + a.cache.parent.mkdir(parents=True, exist_ok=True) |
| 148 | + a.cache.write_text(json.dumps(cache)) |
| 149 | + print(f"[train] usable diffs: {len(docs)} " |
| 150 | + f"({sum(ys)} pos / {len(ys)-sum(ys)} neg)", file=sys.stderr) |
| 151 | + |
| 152 | + from sklearn.feature_extraction.text import TfidfVectorizer |
| 153 | + from sklearn.linear_model import LogisticRegression |
| 154 | + from sklearn.pipeline import Pipeline |
| 155 | + from sklearn.model_selection import cross_val_predict, StratifiedKFold |
| 156 | + from sklearn.metrics import roc_auc_score, average_precision_score, classification_report |
| 157 | + |
| 158 | + X, yv = docs, np.array(ys) |
| 159 | + pipe = Pipeline([ |
| 160 | + ("tfidf", TfidfVectorizer(token_pattern=r"[A-Za-z_][A-Za-z0-9_]{1,}", |
| 161 | + ngram_range=(1, 2), min_df=2, max_features=20000, |
| 162 | + sublinear_tf=True)), |
| 163 | + ("clf", LogisticRegression(max_iter=2000, class_weight="balanced", C=2.0)), |
| 164 | + ]) |
| 165 | + cv = StratifiedKFold(5, shuffle=True, random_state=0) |
| 166 | + proba = cross_val_predict(pipe, X, yv, cv=cv, method="predict_proba")[:, 1] |
| 167 | + auc = roc_auc_score(yv, proba) |
| 168 | + ap_ = average_precision_score(yv, proba) |
| 169 | + print("\n================ silent-fix classifier (5-fold CV) ================") |
| 170 | + print(f" ROC-AUC : {auc:.3f} (regex baseline ≈ 0.50 = chance)") |
| 171 | + print(f" PR-AUC : {ap_:.3f} (positive prevalence {yv.mean():.2f})") |
| 172 | + print(classification_report(yv, (proba >= 0.5).astype(int), |
| 173 | + target_names=["non-fix", "silent-fix"], digits=3)) |
| 174 | + verdict = ("BEATS baseline — worth wiring into curation" if auc >= 0.75 |
| 175 | + else "NOT better than baseline — do not ship" if auc < 0.65 |
| 176 | + else "marginal — needs more data / deep embeddings") |
| 177 | + print(f" VERDICT : {verdict}") |
| 178 | + |
| 179 | + if a.model_out and auc >= 0.75: |
| 180 | + import pickle |
| 181 | + pipe.fit(X, yv) |
| 182 | + a.model_out.write_bytes(pickle.dumps(pipe)) |
| 183 | + print(f" saved model -> {a.model_out}") |
| 184 | + return 0 |
| 185 | + |
| 186 | + |
| 187 | +if __name__ == "__main__": |
| 188 | + raise SystemExit(main()) |
0 commit comments