Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/architecture/analyzers/adjudicator.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ When enabled via the `--adjudicate` CLI flag (or `ScanPolicy.adjudicator.enabled
- **False positive demotion**: Uses an LLM to reason about whether a deterministic HIGH/CRITICAL match represents a real instance of the threat, or a coincidental regex hit on benign prose.
- **Cascade prevention**: Because it runs before the LLM analyzer, demoted findings never enter the LLM analyzer's static-finding enrichment context — so a wrong deterministic HIGH cannot be amplified into LLM findings citing the same pattern hit.
- **Audit trail**: Every finding it considers is recorded in `scan_metadata.adjudicator.audit` with the LLM's verdict, confidence, and reason.
- **Usage accounting**: Adjudication input, output, and total tokens are included in the scan result's aggregate `llm_usage` values.

The adjudicator only touches deterministic findings (static, pipeline, behavioral, bytecode, yara analyzers) at HIGH or CRITICAL severity. LLM and other advisory findings are outside its scope.

Expand Down
2 changes: 1 addition & 1 deletion docs/reference/output-formats.md
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ skill-scanner scan evals/skills/data-exfiltration/environment-secrets --format j
}
```

`llm_usage` is only present when at least one LLM call was made (`--use-llm` and/or `--enable-meta`), aggregated across every LLM analyzer and meta-analysis call for the scan. It's omitted entirely on static-only scans.
`llm_usage` is present when the configured provider reports non-zero token usage for an LLM call made by `--use-llm`, `--enable-meta`, and/or `--adjudicate`. It is aggregated across every LLM analyzer, meta-analysis, and adjudication call for the scan. The field is omitted when no usage is reported, including static-only scans and calls from providers that do not return usage metadata.

Use `--compact` to remove pretty-printing for machine pipelines.

Expand Down
19 changes: 19 additions & 0 deletions skill_scanner/core/analyzers/adjudicator.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,12 @@

from ..models import Finding, Severity, Skill
from ..rule_registry import PackLoader
from .llm_request_handler import (
LLMTokenUsage,
_add_token_usage,
_empty_token_usage,
_extract_token_usage,
)

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -277,6 +283,15 @@ def __init__(
# traceability.
self.audit: list[AdjudicationResult] = []

# Cumulative token usage across all LLM calls in the most recent
# adjudicate() run, including calls whose response cannot be parsed.
self._llm_usage: LLMTokenUsage = _empty_token_usage()

@property
def llm_usage(self) -> LLMTokenUsage:
"""Token usage from the most recent :meth:`adjudicate` run."""
return dict(self._llm_usage) # type: ignore[return-value]

def is_available(self) -> bool:
"""Whether the adjudicator has enough config to run.

Expand Down Expand Up @@ -346,6 +361,9 @@ def _call_llm(self, prompt: str) -> dict[str, Any] | None:
for attempt in range(self.max_retries + 1):
try:
response = litellm.completion(**request, drop_params=True)
# The provider charged for a successful completion even if
# its content is malformed and cannot yield a verdict.
_add_token_usage(self._llm_usage, _extract_token_usage(response))
content = response["choices"][0]["message"]["content"] or ""
content = content.strip()
break
Expand Down Expand Up @@ -498,6 +516,7 @@ def adjudicate(self, findings: list[Finding], skill: Skill) -> list[Finding]:

Returns the same list (mutated) so callers can chain.
"""
self._llm_usage = _empty_token_usage()
if not self.is_available():
logger.debug("adjudicator not configured (no model env var); skipping all findings")
return findings
Expand Down
47 changes: 27 additions & 20 deletions skill_scanner/core/scanner.py
Original file line number Diff line number Diff line change
Expand Up @@ -257,33 +257,39 @@ def _scan_single_skill(self, skill: Skill, skill_directory: Path) -> ScanResult:
# raised. LLM errors leave findings at their original severity,
# so enabling this pass cannot introduce false negatives.
adjudicator_audit: list[dict[str, Any]] = []
adjudicator_usage = _empty_token_usage()
if self.policy.adjudicator.enabled and all_findings:
try:
from .analyzers.adjudicator import Adjudicator

adj = Adjudicator(
min_fp_confidence=self.policy.adjudicator.min_fp_confidence,
)
if adj.is_available():
adj.adjudicate(all_findings, skill)
analyzer_names.append("adjudicator")
adjudicator_audit = [
{
"rule_id": r.rule_id,
"verdict": r.verdict,
"confidence": r.confidence,
"reason": r.reason,
"demoted_to": r.demoted_to,
"model_id": r.model_id,
}
for r in adj.audit
]
else:
logger.debug(
"adjudicator enabled but no LLM model configured; "
"set SKILL_SCANNER_LLM_MODEL or "
"SKILL_SCANNER_ADJUDICATOR_LLM_MODEL to activate"
)
try:
if adj.is_available():
adj.adjudicate(all_findings, skill)
analyzer_names.append("adjudicator")
adjudicator_audit = [
{
"rule_id": r.rule_id,
"verdict": r.verdict,
"confidence": r.confidence,
"reason": r.reason,
"demoted_to": r.demoted_to,
"model_id": r.model_id,
}
for r in adj.audit
]
else:
logger.debug(
"adjudicator enabled but no LLM model configured; "
"set SKILL_SCANNER_LLM_MODEL or "
"SKILL_SCANNER_ADJUDICATOR_LLM_MODEL to activate"
)
finally:
# Preserve billed usage even if a later adjudication
# step raises and findings remain fail-closed.
_add_token_usage(adjudicator_usage, adj.llm_usage)
except Exception as exc:
logger.warning("Adjudication failed: %s", exc)

Expand Down Expand Up @@ -328,6 +334,7 @@ def _scan_single_skill(self, skill: Skill, skill_directory: Path) -> ScanResult:

# Aggregate token usage across all LLM analyzers that ran.
aggregated_usage = _empty_token_usage()
_add_token_usage(aggregated_usage, adjudicator_usage)
for analyzer in llm_analyzers:
if hasattr(analyzer, "llm_usage"):
_add_token_usage(aggregated_usage, analyzer.llm_usage)
Expand Down
104 changes: 102 additions & 2 deletions tests/test_adjudicator.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@
Adjudicator,
)
from skill_scanner.core.models import Finding, Severity, Skill, SkillFile, SkillManifest
from skill_scanner.core.scan_policy import ScanPolicy
from skill_scanner.core.scanner import SkillScanner

# ----- Fixtures -------------------------------------------------------------

Expand Down Expand Up @@ -96,10 +98,23 @@ def _finding(
)


def _mock_litellm_response(verdict: str, confidence: int, reason: str = "test") -> Any:
def _mock_litellm_response(
verdict: str,
confidence: int,
reason: str = "test",
*,
prompt_tokens: int = 0,
completion_tokens: int = 0,
) -> Any:
"""Build a mock LiteLLM response whose choices[0].message.content is JSON."""
payload = json.dumps({"verdict": verdict, "confidence": confidence, "reason": reason})
resp: Any = {"choices": [{"message": {"content": payload}}]}
response_type = type("MockLiteLLMResponse", (dict,), {})
resp: Any = response_type({"choices": [{"message": {"content": payload}}]})
resp.usage = MagicMock(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
)
return resp


Expand Down Expand Up @@ -194,6 +209,91 @@ def test_llm_exception_keeps_original_severity(self, tmp_path: Path, with_model_
assert finding.severity == Severity.HIGH
assert "adjudication" not in (finding.metadata or {})


class TestAdjudicatorTokenUsage:
"""Adjudicator reports every billed LiteLLM completion."""

def test_accumulates_usage_across_findings(self, tmp_path: Path, with_model_env: None) -> None:
skill = _make_skill(tmp_path, "---\nname: test\n---\n\nSome content.\n")
findings = [
_finding("PROMPT_INJECTION_CONCEALMENT", Severity.HIGH, line_number=4),
_finding("PIPELINE_TAINT_FLOW", Severity.HIGH, analyzer="pipeline", line_number=4),
]

with patch(
"litellm.completion",
side_effect=[
_mock_litellm_response("real", 5, prompt_tokens=100, completion_tokens=20),
_mock_litellm_response("real", 5, prompt_tokens=40, completion_tokens=10),
],
):
adj = Adjudicator()
adj.adjudicate(findings, skill)

assert adj.llm_usage == {
"input_tokens": 140,
"output_tokens": 30,
"total_tokens": 170,
}

def test_counts_usage_when_response_content_is_malformed(self, tmp_path: Path, with_model_env: None) -> None:
skill = _make_skill(tmp_path, "---\nname: test\n---\n\nSome content.\n")
finding = _finding("PROMPT_INJECTION_CONCEALMENT", Severity.HIGH, line_number=4)
response = _mock_litellm_response("real", 5, prompt_tokens=75, completion_tokens=8)
response["choices"][0]["message"]["content"] = "not json"

with patch("litellm.completion", return_value=response):
adj = Adjudicator()
adj.adjudicate([finding], skill)

assert finding.severity == Severity.HIGH
assert adj.llm_usage == {
"input_tokens": 75,
"output_tokens": 8,
"total_tokens": 83,
}

def test_scanner_combines_adjudicator_and_analyzer_usage(self, tmp_path: Path, with_model_env: None) -> None:
skill = _make_skill(
tmp_path,
"---\nname: test-skill\ndescription: test skill.\n---\n\nSome content.\n",
)
finding = _finding("PROMPT_INJECTION_CONCEALMENT", Severity.HIGH, line_number=5)

class DeterministicAnalyzer:
def get_name(self) -> str:
return "static_analyzer"

def analyze(self, _skill: Skill) -> list[Finding]:
return [finding]

class UsageLLMAnalyzer:
llm_usage = {"input_tokens": 300, "output_tokens": 50, "total_tokens": 350}

def get_name(self) -> str:
return "llm_analyzer"

def analyze(self, _skill: Skill) -> list[Finding]:
return []

policy = ScanPolicy.default()
policy.adjudicator.enabled = True
scanner = SkillScanner(analyzers=[DeterministicAnalyzer(), UsageLLMAnalyzer()], policy=policy) # type: ignore[list-item]
response = _mock_litellm_response("real", 5, prompt_tokens=80, completion_tokens=20)

with patch("litellm.completion", return_value=response):
result = scanner._scan_single_skill(skill, Path(skill.directory))

assert result.llm_usage == {
"input_tokens": 380,
"output_tokens": 70,
"total_tokens": 450,
}


class TestAdjudicatorMalformedResponses:
"""Malformed adjudicator responses keep findings fail-closed."""

def test_malformed_json_keeps_original_severity(self, tmp_path: Path, with_model_env: None) -> None:
skill = _make_skill(tmp_path, "---\nname: test\n---\n\nSome content.\n")
finding = _finding("PROMPT_INJECTION_CONCEALMENT", Severity.HIGH, line_number=4)
Expand Down
Loading