Corpus batch scan #638
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Corpus batch scan | |
| on: | |
| schedule: | |
| - cron: "17 */4 * * *" # every 4 hours, off-peak offset | |
| workflow_dispatch: | |
| inputs: | |
| batch: | |
| description: "Batch number to run (0-9, empty = read from corpus/batch_state.txt)" | |
| default: "" | |
| permissions: | |
| contents: write # needed to commit batch_state.txt + results | |
| jobs: | |
| scan: | |
| name: "Batch scan" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Install dependencies | |
| run: pip install z3-solver requests hypothesis pytest networkx | |
| - name: Resolve batch number and query | |
| id: batch | |
| run: | | |
| OVERRIDE="${{ github.event.inputs.batch }}" | |
| if [ -n "$OVERRIDE" ]; then | |
| NUM=$OVERRIDE | |
| else | |
| NUM=$(cat corpus/batch_state.txt 2>/dev/null || echo "0") | |
| fi | |
| NUM=$(( NUM % 10 )) | |
| echo "num=$NUM" >> "$GITHUB_OUTPUT" | |
| QUERIES=( | |
| "language:python topic:llm stars:>500" | |
| "language:python topic:langchain stars:>200" | |
| "language:python topic:openai stars:>300" | |
| "language:python topic:agent stars:>400" | |
| "language:python topic:rag stars:>200" | |
| "language:python topic:evaluation stars:>100" | |
| "language:python topic:fastapi stars:>1000" | |
| "language:python topic:celery stars:>200" | |
| "language:python topic:django stars:>500" | |
| "language:python topic:pydantic stars:>300" | |
| ) | |
| echo "query=${QUERIES[$NUM]}" >> "$GITHUB_OUTPUT" | |
| - name: Run scan (token from env — never in args) | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.CORPUS_GITHUB_TOKEN }} | |
| run: | | |
| mkdir -p corpus | |
| # Run from parent directory so `pact` is importable as a package | |
| cd .. | |
| python3 -m pact.scan_github \ | |
| --query "${{ steps.batch.outputs.query }}" \ | |
| --limit 20 \ | |
| --max-files 100 \ | |
| --out "pact/corpus/batch-${{ steps.batch.outputs.num }}.jsonl" | |
| cd pact | |
| - name: Append to rolling corpus | |
| run: | | |
| cat "corpus/batch-${{ steps.batch.outputs.num }}.jsonl" >> corpus/corpus.jsonl | |
| - name: Advance batch state | |
| run: | | |
| NEXT=$(( (${{ steps.batch.outputs.num }} + 1) % 10 )) | |
| echo "$NEXT" > corpus/batch_state.txt | |
| - name: Summarize and FP canary | |
| run: | | |
| BATCH_NUM="${{ steps.batch.outputs.num }}" | |
| python3 - <<EOF | |
| import json, collections | |
| batch = [json.loads(l) for l in open(f"corpus/batch-{$BATCH_NUM}.jsonl") if l.strip()] | |
| total_corpus = sum(1 for l in open("corpus/corpus.jsonl") if l.strip()) | |
| modes = collections.Counter(r["mode"] for r in batch) | |
| repos = len(set(r["repo"] for r in batch)) | |
| print(f"Batch {$BATCH_NUM}: {repos} repos, {len(batch)} violations — rolling corpus: {total_corpus}") | |
| print() | |
| print("| Mode | Count |") | |
| print("|------|-------|") | |
| for mode, count in modes.most_common(10): | |
| print(f"| {mode} | {count} |") | |
| # FP canary: any mode >70% of a single repo's hits | |
| by_repo = collections.defaultdict(list) | |
| for r in batch: | |
| by_repo[r["repo"]].append(r["mode"]) | |
| suspects = [] | |
| for repo, ms in by_repo.items(): | |
| if len(ms) < 5: | |
| continue | |
| top_mode, top_count = collections.Counter(ms).most_common(1)[0] | |
| if top_count / len(ms) > 0.70: | |
| suspects.append((top_count / len(ms), repo, top_mode, len(ms))) | |
| if suspects: | |
| print() | |
| print("### FP canary (>70% single mode per repo)") | |
| for ratio, repo, mode, total in sorted(suspects, reverse=True)[:5]: | |
| print(f" {ratio:.0%} {repo} {mode} ({total} violations)") | |
| EOF | |
| - name: Commit results | |
| run: | | |
| git config user.name "pact-corpus-bot" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git add corpus/ | |
| git diff --staged --quiet && echo "nothing to commit" && exit 0 | |
| git commit -m "corpus: batch ${{ steps.batch.outputs.num }} ($(date -u +%Y-%m-%d))" | |
| git push |