Add sanitized evaluation measurement records

Signed-off-by: Aaron Gonzales <aagonzales@nvidia.com>

Author: binaryaaron

src/anonymizer/measurement/__init__.py

@@ -19,7 +19,7 @@
     record_stage,
     stage_timer,
 )
-from anonymizer.measurement.records.row import record_record_metrics
+from anonymizer.measurement.records.row import record_evaluation_metrics, record_record_metrics
 from anonymizer.measurement.session import configured_measurement_session, current_collector, measurement_session
 
 __all__ = [
@@ -35,6 +35,7 @@
     "measurement_session",
     "record_model_workflow",
     "record_ndd_workflow",
+    "record_evaluation_metrics",
     "record_record_metrics",
     "record_run_metadata",
     "record_stage",

src/anonymizer/measurement/records/row.py

@@ -4,10 +4,22 @@
 from __future__ import annotations
 
 from collections import Counter
-from typing import TYPE_CHECKING, Any
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, Literal, TypedDict
 
-from anonymizer.engine.constants import COL_FINAL_ENTITIES
+from anonymizer.engine.constants import (
+    COL_ATTRIBUTE_FIDELITY_INVALID_ENTITIES,
+    COL_ATTRIBUTE_FIDELITY_VALID,
+    COL_DETECTION_INVALID_ENTITIES,
+    COL_DETECTION_VALID,
+    COL_FINAL_ENTITIES,
+    COL_RELATIONAL_CONSISTENCY_INVALID_RELATIONS,
+    COL_RELATIONAL_CONSISTENCY_VALID,
+    COL_TYPE_FIDELITY_INVALID_REPLACEMENTS,
+    COL_TYPE_FIDELITY_VALID,
+)
 from anonymizer.measurement._coerce import (
+    _coerce_bool,
     _coerce_int,
     _count_items,
     _count_text_tokens,
@@ -34,6 +46,69 @@
     from anonymizer.measurement.collector import MeasurementCollector
 
 
+_EvaluationBoolField = Literal[
+    "detection_valid",
+    "type_fidelity_valid",
+    "relational_consistency_valid",
+    "attribute_fidelity_valid",
+]
+_EvaluationCountField = Literal[
+    "detection_invalid_entity_count",
+    "type_fidelity_invalid_replacement_count",
+    "relational_consistency_invalid_relation_count",
+    "attribute_fidelity_invalid_entity_count",
+]
+
+
+class _EvaluationRecordFields(TypedDict, total=False):
+    detection_valid: bool | None
+    type_fidelity_valid: bool | None
+    relational_consistency_valid: bool | None
+    attribute_fidelity_valid: bool | None
+    detection_invalid_entity_count: int
+    type_fidelity_invalid_replacement_count: int
+    relational_consistency_invalid_relation_count: int
+    attribute_fidelity_invalid_entity_count: int
+
+
+@dataclass(frozen=True)
+class _EvaluationBoolMetric:
+    source_column: str
+    output_field: _EvaluationBoolField
+
+
+@dataclass(frozen=True)
+class _EvaluationCountMetric:
+    source_column: str
+    output_field: _EvaluationCountField
+    primary_key: str
+
+
+_EVALUATION_BOOL_METRICS = (
+    _EvaluationBoolMetric(COL_DETECTION_VALID, "detection_valid"),
+    _EvaluationBoolMetric(COL_TYPE_FIDELITY_VALID, "type_fidelity_valid"),
+    _EvaluationBoolMetric(COL_RELATIONAL_CONSISTENCY_VALID, "relational_consistency_valid"),
+    _EvaluationBoolMetric(COL_ATTRIBUTE_FIDELITY_VALID, "attribute_fidelity_valid"),
+)
+
+_EVALUATION_COUNT_METRICS = (
+    _EvaluationCountMetric(COL_DETECTION_INVALID_ENTITIES, "detection_invalid_entity_count", "invalid_entities"),
+    _EvaluationCountMetric(
+        COL_TYPE_FIDELITY_INVALID_REPLACEMENTS,
+        "type_fidelity_invalid_replacement_count",
+        "invalid_replacements",
+    ),
+    _EvaluationCountMetric(
+        COL_RELATIONAL_CONSISTENCY_INVALID_RELATIONS,
+        "relational_consistency_invalid_relation_count",
+        "invalid_relations",
+    ),
+    _EvaluationCountMetric(
+        COL_ATTRIBUTE_FIDELITY_INVALID_ENTITIES, "attribute_fidelity_invalid_entity_count", "entities"
+    ),
+)
+
+
 def record_record_metrics(
     dataframe: pd.DataFrame,
     *,
@@ -76,6 +151,54 @@ def record_record_metrics(
         )
 
 
+def record_evaluation_metrics(
+    dataframe: pd.DataFrame,
+    *,
+    mode: str,
+    strategy: str,
+    text_column: str,
+) -> None:
+    """Record sanitized per-row LLM-as-judge verdict metrics from an evaluated trace dataframe."""
+    collector = current_collector()
+    if collector is None or not collector.record_level:
+        return
+
+    columns = set(dataframe.columns)
+    if not _has_evaluation_metrics(columns):
+        return
+
+    for row_index, row in dataframe.iterrows():
+        collector.record(
+            "evaluation_record",
+            **_base_record_fields(
+                collector=collector,
+                row_index=row_index,
+                row=row,
+                text_column=text_column,
+                mode=mode,
+                strategy=strategy,
+            ),
+            **_evaluation_record_fields(row, columns=columns),
+        )
+
+
+def _has_evaluation_metrics(columns: set[str]) -> bool:
+    return any(metric.source_column in columns for metric in _EVALUATION_BOOL_METRICS) or any(
+        metric.source_column in columns for metric in _EVALUATION_COUNT_METRICS
+    )
+
+
+def _evaluation_record_fields(row: pd.Series, *, columns: set[str]) -> _EvaluationRecordFields:
+    fields: _EvaluationRecordFields = {}
+    for metric in _EVALUATION_BOOL_METRICS:
+        if metric.source_column in columns:
+            fields[metric.output_field] = _coerce_bool(row.get(metric.source_column))
+    for metric in _EVALUATION_COUNT_METRICS:
+        if metric.source_column in columns:
+            fields[metric.output_field] = _count_items(row.get(metric.source_column), primary_key=metric.primary_key)
+    return fields
+
+
 def _base_record_fields(
     *,
     collector: MeasurementCollector,

tests/tools/test_measurement_tools.py

@@ -4,6 +4,7 @@
 from __future__ import annotations
 
 import importlib.util
+import json
 import sys
 from collections.abc import Iterator
 from contextlib import contextmanager
@@ -934,8 +935,16 @@ def test_benchmark_case_can_run_optional_evaluation(
     tmp_path: Path,
 ) -> None:
     tool = load_tool("measurement_benchmark_tool_evaluate", REPO_ROOT / "tools/measurement/run_benchmarks.py")
+    from anonymizer.interface.results import AnonymizerResult
+
     calls: list[Any] = []
-    run_result = object()
+    run_result = AnonymizerResult(
+        dataframe=pd.DataFrame({"text": ["Alice works at Acme"]}),
+        trace_dataframe=pd.DataFrame({"text": ["Alice works at Acme"]}),
+        resolved_text_column="text",
+        failed_records=[],
+        replace_method=None,
+    )
 
     @contextmanager
     def fake_measurement_session(_config: Any) -> Iterator[None]:
@@ -974,3 +983,143 @@ def evaluate(self, result: object) -> object:
     )
 
     assert calls == [("run", tool.Redact(), "text"), ("evaluate", run_result)]
+
+
+def test_benchmark_optional_evaluation_records_sanitized_judge_metrics(tmp_path: Path) -> None:
+    tool = load_tool("measurement_benchmark_tool_evaluate_metrics", REPO_ROOT / "tools/measurement/run_benchmarks.py")
+    from anonymizer.interface.results import AnonymizerResult
+
+    dangerous_values = [
+        "alice@example.com",
+        "bob@example.com",
+        "sk-secret-123",
+        "replacement-output-secret",
+        "nested-malformed-secret",
+        "raw judge prompt",
+        "raw judge response",
+    ]
+    run_result = AnonymizerResult(
+        dataframe=pd.DataFrame({"text": ["Alice has sk-secret-123"]}),
+        trace_dataframe=pd.DataFrame({"text": ["Alice has sk-secret-123"]}),
+        resolved_text_column="text",
+        failed_records=[],
+        replace_method=None,
+    )
+    evaluated_public_columns = {
+        "text": ["Alice has sk-secret-123"],
+        "text_replaced": ["Avery has replacement-output-secret"],
+        "final_entities": [[{"value": "alice@example.com", "label": "email"}]],
+        "detection_valid": [False],
+        "detection_invalid_entities": [{"invalid_entities": [{"value": "alice@example.com", "label": "email"}]}],
+        "type_fidelity_valid": [False],
+        "type_fidelity_invalid_replacements": [
+            {"invalid_replacements": [{"original": "alice@example.com", "synthetic": "bob@example.com"}]}
+        ],
+        "relational_consistency_valid": [False],
+        "relational_consistency_invalid_relations": [{"invalid_relations": [{"reasoning": "raw judge response"}]}],
+        "attribute_fidelity_valid": [False],
+        "attribute_fidelity_invalid_entities": ['[{"entity": "nested-malformed-secret"}'],
+    }
+    evaluated_result = AnonymizerResult(
+        dataframe=pd.DataFrame(evaluated_public_columns),
+        trace_dataframe=pd.DataFrame(
+            {
+                **evaluated_public_columns,
+                "_detection_judge": [
+                    {
+                        "prompt": "raw judge prompt",
+                        "response": "raw judge response",
+                        "invalid_entities": [{"value": "alice@example.com"}],
+                    }
+                ],
+                "_type_fidelity_judge": [
+                    {"invalid_replacements": [{"original": "alice@example.com", "synthetic": "bob@example.com"}]}
+                ],
+            }
+        ),
+        resolved_text_column="text",
+        failed_records=[],
+        replace_method=None,
+    )
+
+    class FakeAnonymizer:
+        def run(self, *, config: Any, data: Any) -> AnonymizerResult:
+            return run_result
+
+        def evaluate(self, result: AnonymizerResult) -> AnonymizerResult:
+            assert result is run_result
+            return evaluated_result
+
+    spec = _minimal_benchmark_spec(
+        tool,
+        suite_id="evaluate-suite",
+        configs=[
+            tool.ConfigSpec(
+                id="substitute",
+                replace=tool.ReplaceSpec(strategy=tool.ReplaceKind.substitute),
+                evaluate=True,
+            )
+        ],
+    )
+    _write_text_input(tmp_path, "Alice has sk-secret-123")
+    case = _minimal_benchmark_case(tool, suite_id="evaluate-suite", config_id="substitute")
+    measurement_path = tmp_path / "raw" / "input__substitute__r000.jsonl"
+
+    tool._execute_case(
+        FakeAnonymizer(),
+        spec.workloads[0],
+        spec.configs[0],
+        raw_path=measurement_path,
+        trace_path=None,
+        task_trace_path=None,
+        case=case,
+        spec=spec,
+        base_dir=tmp_path,
+        dd_trace=tool.DDTraceMode.none,
+    )
+
+    serialized = measurement_path.read_text(encoding="utf-8")
+    rows = [json.loads(line) for line in serialized.splitlines()]
+    evaluation_rows = [row for row in rows if row["record_type"] == "evaluation_record"]
+
+    assert len(evaluation_rows) == 1
+    assert {
+        "record_type": "evaluation_record",
+        "mode": "replace",
+        "strategy": "Substitute",
+        "row_index": 0,
+        "detection_valid": False,
+        "detection_invalid_entity_count": 1,
+        "type_fidelity_valid": False,
+        "type_fidelity_invalid_replacement_count": 1,
+        "relational_consistency_valid": False,
+        "relational_consistency_invalid_relation_count": 1,
+        "attribute_fidelity_valid": False,
+        "attribute_fidelity_invalid_entity_count": 0,
+    }.items() <= evaluation_rows[0].items()
+    forbidden_fields = {
+        "text",
+        "text_replaced",
+        "text_with_spans",
+        "final_entities",
+        "detection_invalid_entities",
+        "type_fidelity_invalid_replacements",
+        "relational_consistency_invalid_relations",
+        "attribute_fidelity_invalid_entities",
+        "_detection_judge",
+        "_type_fidelity_judge",
+        "_relational_consistency_judge",
+        "_attribute_fidelity_judge",
+    }
+    assert forbidden_fields.isdisjoint(evaluation_rows[0])
+    for raw_value in dangerous_values:
+        assert raw_value not in serialized
+
+    table_dir = tmp_path / "tables"
+    tool.export_measurement_tables(measurement_path, table_dir)
+    exported = pd.read_parquet(table_dir / "evaluation_record.parquet")
+    exported_text = str(exported.to_json(orient="records"))
+
+    assert forbidden_fields.isdisjoint(exported.columns)
+    for raw_value in dangerous_values:
+        assert raw_value not in exported_text

tools/measurement/README.md

@@ -28,6 +28,7 @@ plus `manifest.json`:
 - `run.parquet`
 - `stage.parquet`
 - `record.parquet`
+- `evaluation_record.parquet` when replace judge evaluation is enabled
 - `ndd_workflow.parquet` when DataDesigner adapter records are present
 - `model_workflow.parquet` when direct model workflow records are present
 
@@ -65,7 +66,8 @@ raw = pd.read_json("benchmark-runs/suite/measurements.jsonl", lines=True)
 The measurement system has three layers:
 
 - Instrumentation in Anonymizer emits JSONL records for runs, stages,
-  DataDesigner workflows, direct model workflows, and per-record safety metrics.
+  DataDesigner workflows, direct model workflows, per-record safety metrics,
+  and optional sanitized replace-judge evaluation metrics.
 - Benchmark runners create repeatable workloads and write those JSONL records
   plus optional sidecars such as detection artifacts and DataDesigner traces.
 - Analysis tools convert raw run artifacts into case, group, and model tables.
@@ -243,6 +245,12 @@ Set `evaluate: true` on a replace config when the benchmark should run
 same case. This is intentionally replace-only for now; rewrite runs already
 perform their internal evaluation/repair loop during `run()`.
 
+When evaluation is enabled, the safe measurement log includes
+`evaluation_record` rows with judge verdict booleans and invalid-item counts.
+It does not persist the evaluated result dataframe or trace dataframe. Those
+dataframes can contain original text, entity values, replacement values, raw
+judge outputs, prompts, and model responses.
+
 Before starting a real run, the benchmark runner performs cheap preflight
 checks: suite/config parsing, local dataset existence, CSV/Parquet text-column
 metadata, provider YAML shape, and active model-alias references. `--dry-run`
@@ -409,3 +417,15 @@ Safety and replacement:
 - `replacement_synthetic_original_collision_count`: final entity occurrences
   whose original value was reused as a synthetic replacement value elsewhere in
   the same record.
+
+Replace judge evaluation:
+
+- `detection_valid`, `type_fidelity_valid`,
+  `relational_consistency_valid`, and `attribute_fidelity_valid`: per-record
+  judge verdicts when `evaluate: true` is enabled.
+- `detection_invalid_entity_count`,
+  `type_fidelity_invalid_replacement_count`,
+  `relational_consistency_invalid_relation_count`, and
+  `attribute_fidelity_invalid_entity_count`: counts of invalid judge findings.
+  These fields count structures returned by the judges but do not include raw
+  values, replacement strings, or judge reasoning text.