chore: add json/ndjson/jsonl to regression test

.github/workflows/regression-tests.yml

@@ -4,6 +4,7 @@ on: [push, pull_request, workflow_dispatch]
 
 env:
   EVTX_BASELINE_VERSION: v0.8.4
+  JSON_MATCHER_VERSION: v0.0.2
 
 jobs:
   true-positive-tests:
@@ -19,13 +20,18 @@ jobs:
     - name: Install Python dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install pyyaml
+        pip install pyyaml sigma-cli pySigma-backend-golangexpr
 
     - name: Download evtx-sigma-checker
       run: |
         wget --no-verbose https://github.com/NextronSystems/evtx-baseline/releases/download/$EVTX_BASELINE_VERSION/evtx-sigma-checker
         chmod +x evtx-sigma-checker
+
+    - name: Download json-checker
+      run: |
+        wget --no-verbose https://github.com/SigmaHQ/json_matcher/releases/download/$JSON_MATCHER_VERSION/json_matcher_$JSON_MATCHER_VERSION -O json_checker
+        chmod +x json_checker
 
     - name: Run regression tests
       run: |
-        python tests/regression_tests_runner.py --rules-paths rules rules-emerging-threats rules-threat-hunting --evtx-checker ./evtx-sigma-checker --thor-config tests/thor.yml --ignore-validation
+        python tests/regression_tests_runner.py --rules-paths rules rules-emerging-threats rules-threat-hunting --evtx-checker ./evtx-sigma-checker --thor-config tests/thor.yml --ignore-validation --json-checker ./json_checker

regression_data/README.md

@@ -0,0 +1,112 @@
+# Sigma Regression Testing
+
+Regression tests verify that Sigma rules actually match the events they are
+supposed to detect. Each rule (rules with status `test` or `stable` must) points to an
+`info.yml` describing one or more test cases, each backed by a real event
+sample (EVTX or JSON). CI runs every sample against its rule and fails if the
+expected number of matches is not produced.
+
+The runner lives at [`tests/regression_tests_runner.py`](../tests/regression_tests_runner.py)
+and runs on every push/PR via [`.github/workflows/regression-tests.yml`](../.github/workflows/regression-tests.yml).
+
+## Layout
+
+```
+regression_data/
+├── pipelines/                 # Sigma conversion pipelines used by JSON tests
+└── rules/<product>/<category>/<rule_name>/
+    ├── info.yml               # test definitions for the rule
+    ├── <rule_id>.evtx         # EVTX sample (name must equal rule id)
+    └── <rule_id>.json         # optional JSON/NDJSON sample (name must equal rule id)
+```
+
+The rule file references its tests with:
+
+```yaml
+regression_tests_path: ../../regression_data/rules/windows/process_creation/<rule_name>/info.yml
+```
+
+## Supported Types
+
+### EVTX
+
+Runs [`evtx-sigma-checker`](https://github.com/NextronSystems/evtx-baseline)
+against the `.evtx` sample using the THOR log-source config and the rule
+directory.
+
+```yaml
+- name: Positive Detection Test
+  type: evtx
+  provider: Microsoft-Windows-Sysmon # Not used atm
+  match_count: 1
+  path: regression_data/rules/windows/process_creation/<rule_name>/<rule_id>.evtx
+```
+
+The `.evtx` file name must equal the rule `id`.
+
+### Json / NDJson / JsonL
+
+`json` means a single JSON object, while `ndjson`/`jsonl` mean newline-delimited
+JSON objects (one object per line).
+
+The rule is compiled to a `golang_expr` query with `sigma convert` (applying any
+listed `pipelines` and `filters`), then run against the event sample by
+`json_checker`.
+
+```yaml
+- name: Positive Detection Test
+  type: json          # or: ndjson, jsonl
+  match_count: 1
+  pipelines:
+      - regression_data/pipelines/process_creation_fieldmapping.yml
+  path: regression_data/rules/windows/process_creation/<rule_name>/<rule_id>.json
+```
+
+`pipelines` and `filters` are optional and with no pipeline, the rule is converted with `--without-pipeline`.
+
+## info.yml Format
+
+```yaml
+id: 242d26e0-1ce5-4a34-960d-144f34f60e37   # id of this test-info file
+description: N/A
+date: 2025-12-25
+author: Author Name
+rule_metadata:
+    - id: 7dbbcac2-57a0-45ac-b306-ff30a8bd2981   # must match the rule file id
+      title: Windows AMSI Related Registry Tampering Via CommandLine
+regression_tests_info:
+    - name: Positive Detection Test
+      type: evtx
+      provider: Microsoft-Windows-Sysmon # Not used atm
+      match_count: 1
+      path: regression_data/rules/.../<rule_id>.evtx
+    - name: Positive Detection Test
+      type: json
+      match_count: 1
+      pipelines:
+          - regression_data/pipelines/process_creation_fieldmapping.yml
+      path: regression_data/rules/.../<rule_id>.json
+```
+
+Fields per entry in `regression_tests_info`:
+
+| Field         | Required | Description                                                        |
+|---------------|----------|--------------------------------------------------------------------|
+| `name`        | no       | Human-readable test name.                                          |
+| `type`        | yes      | `evtx`, `json`, `ndjson`, or `jsonl`.                              |
+| `path`        | yes      | Path to the event sample.                                          |
+| `match_count` | no       | Expected number of matches. Fails if fewer; warns if more.        |
+| `provider`    | no       | Event provider (informational, used by EVTX tests).               |
+| `pipelines`   | no       | Sigma pipelines applied before conversion (JSON types).           |
+| `filters`     | no       | Sigma filters applied during conversion (JSON types).             |
+
+If `match_count` is omitted, the test passes when there is at least one match.
+
+## Validation rules
+
+- Rules with status `test` or `stable` must define `regression_tests_path`
+  (enforced unless `--ignore-validation`).
+- Referenced `info.yml` and sample files must exist.
+- Rule `id` == `info.yml`, `rule_metadata[0].id` == EVTX/JSON sample file name.
+
+The runner exits non-zero on any failed test, missing file, or inconsistency.

regression_data/pipelines/process_creation_fieldmapping.yml

@@ -0,0 +1,23 @@
+name: process_creation_fieldmapping
+priority: 100
+transformations:
+    - id: field_mapping_process_creation
+      type: field_name_mapping
+      mapping:
+          CommandLine: Event.EventData.CommandLine
+          Image: Event.EventData.Image
+          OriginalFileName: Event.EventData.OriginalFileName
+      rule_conditions:
+          - type: logsource
+            category: process_creation
+            product: windows
+    - id: field_mapping_registry_set
+      type: field_name_mapping
+      mapping:
+          TargetObject: Event.EventData.TargetObject
+          Details: Event.EventData.Details
+          Image: Event.EventData.Image
+      rule_conditions:
+          - type: logsource
+            category: registry_set
+            product: windows

regression_data/rules/windows/process_creation/proc_creation_win_amsi_registry_tampering/info.yml

@@ -11,3 +11,9 @@ regression_tests_info:
       provider: Microsoft-Windows-Sysmon
       match_count: 1
       path: regression_data/rules/windows/process_creation/proc_creation_win_amsi_registry_tampering/7dbbcac2-57a0-45ac-b306-ff30a8bd2981.evtx
+    - name: Positive Detection Test
+      type: json
+      match_count: 1
+      pipelines:
+          - regression_data/pipelines/process_creation_fieldmapping.yml
+      path: regression_data/rules/windows/process_creation/proc_creation_win_amsi_registry_tampering/7dbbcac2-57a0-45ac-b306-ff30a8bd2981.json

regression_data/rules/windows/process_creation/proc_creation_win_autologger_session_registry_modification/info.yml

@@ -11,3 +11,9 @@ regression_tests_info:
       provider: Microsoft-Windows-Sysmon
       match_count: 1
       path: regression_data/rules/windows/process_creation/proc_creation_win_autologger_session_registry_modification/d7b81144-b866-48a4-9bcc-275dc69d870e.evtx
+    - name: Positive Detection Test
+      type: json
+      match_count: 1
+      pipelines:
+          - regression_data/pipelines/process_creation_fieldmapping.yml
+      path: regression_data/rules/windows/process_creation/proc_creation_win_autologger_session_registry_modification/d7b81144-b866-48a4-9bcc-275dc69d870e.json

regression_data/rules/windows/registry/registry_set/registry_set_add_load_service_in_safe_mode/info.yml

@@ -11,3 +11,9 @@ regression_tests_info:
       provider: Microsoft-Windows-Sysmon
       match_count: 1
       path: regression_data/rules/windows/registry/registry_set/registry_set_add_load_service_in_safe_mode/1547e27c-3974-43e2-a7d7-7f484fb928ec.evtx
+    - name: Positive Detection Test
+      type: json
+      match_count: 1
+      pipelines:
+          - regression_data/pipelines/process_creation_fieldmapping.yml
+      path: regression_data/rules/windows/registry/registry_set/registry_set_add_load_service_in_safe_mode/1547e27c-3974-43e2-a7d7-7f484fb928ec.json

tests/regression_tests_runner.py

@@ -78,13 +78,23 @@ def load_info_yaml(
                     }
                 )
 
+            base_dir = os.path.dirname(regression_tests_path)
+            pipelines = [
+                get_absolute_path(base_dir, p) for p in test.get("pipelines", [])
+            ]
+            filters = [
+                get_absolute_path(base_dir, f) for f in test.get("filters", [])
+            ]
+
             test_data.append(
                 {
                     "type": test.get("type", "unknown"),
                     "path": test_path,
                     "name": test.get("name", "Unnamed Test"),
                     "provider": test.get("provider", ""),
                     "match_count": test.get("match_count"),
+                    "pipelines": pipelines,
+                    "filters": filters,
                 }
             )
         info_metadata_rule_id = None
@@ -293,12 +303,88 @@ def run_evtx_checker(
         return False, ""
 
 
+def compile_rule_to_expr(
+    rule_path: str, pipelines: List[str], filters: List[str]
+) -> str:
+    """Compile a Sigma rule to a golang_expr query via the sigma CLI."""
+    cmd = ["sigma", "convert", "-t", "golang_expr"]
+    for pipeline in pipelines:
+        cmd += ["-p", pipeline]
+    if len(pipelines) == 0:
+        cmd += ["--without-pipeline"]
+    for filter in filters:
+        cmd += ["--filter", filter]
+    cmd.append(rule_path)
+
+    result = subprocess.run(
+        cmd, capture_output=True, text=True, timeout=300, check=True
+    )
+
+    return result.stdout.strip()
+
+
+def run_json_checker(
+    test_type: str,
+    rule_path: str,
+    rule_id: str,
+    test_data: Dict,
+    json_checker_path: str,
+) -> tuple[bool, str]:
+    """Compile the rule to an expr query and run json_checker against the events."""
+    try:
+        expr_query = compile_rule_to_expr(
+            rule_path, test_data.get("pipelines", []), test_data.get("filters", [])
+        )
+    except subprocess.CalledProcessError as e:
+        print(f"  Error compiling rule {rule_id} with sigma: {e.stderr or e}")
+        return False, ""
+    except subprocess.TimeoutExpired:
+        print(f"  Timeout compiling rule {rule_id} with sigma")
+        return False, ""
+
+    if not expr_query:
+        print(f"  Error: sigma produced an empty expr query for {rule_id}")
+        return False, ""
+
+    cmd = [json_checker_path, "--event", test_data["path"], "--expr", expr_query, "--test-type", test_type]
+    try:
+        result = subprocess.run(
+            cmd, capture_output=True, text=True, timeout=300, check=True
+        )
+    except subprocess.TimeoutExpired:
+        print("  Timeout: json_checker timed out")
+        return False, ""
+    except subprocess.CalledProcessError as e:
+        print(f"  Error running json_checker: {e.stderr or e}")
+        return False, ""
+
+    match_lines = [ln for ln in result.stdout.splitlines() if ln.endswith("MATCH")]
+    match_count = len(match_lines)
+    all_output = "\n    ".join(match_lines)
+
+    expected_count = test_data.get("match_count")
+    if expected_count is not None:
+        if match_count < expected_count:
+            print(
+                f"  Error: {rule_id}: Match count too low: expected {expected_count}, got {match_count}"
+            )
+            return False, all_output
+        if match_count > expected_count:
+            print(
+                f"  Warning: {rule_id}: Got {match_count} matches but only {expected_count} expected - consider updating match_count in info.yml"
+            )
+        return True, all_output
+
+    return match_count > 0, all_output
+
+
 def run_test(
     rule_path: str,
     rule_id: str,
     test_data: Dict,
     evtx_checker_path: str,
     thor_config: str,
+    json_checker_path: str,
 ) -> tuple[bool, str]:
     """Run a test based on its type."""
     test_type = test_data.get("type", "unknown")
@@ -307,6 +393,12 @@ def run_test(
         return run_evtx_checker(
             rule_path, rule_id, test_data, evtx_checker_path, thor_config
         )
+
+    if test_type in {"json", "ndjson", "jsonl"}:
+        if not json_checker_path:
+            print("  Error: --json-checker is required for 'ndjson/json/jsonl' tests")
+            return False, ""
+        return run_json_checker(test_type, rule_path, rule_id, test_data, json_checker_path)
     print(f"  Warning: Unknown test type '{test_type}', skipping")
     return False, ""
 
@@ -335,6 +427,11 @@ def parse_arguments() -> argparse.Namespace:
         help="Path to thor.yml configuration file (required unless using --validate-only)",
     )
 
+    parser.add_argument(
+        "--json-checker",
+        help="Path to json_checker binary (required for 'ndjson/json/jsonl' tests)",
+    )
+
     parser.add_argument(
         "--validate-only",
         action="store_true",
@@ -379,6 +476,11 @@ def init_checks(args: argparse.Namespace) -> None:
         if not os.path.exists(args.thor_config):
             print(f"Error: Thor config not found at {args.thor_config}")
             sys.exit(1)
+
+        # json_checker is optional; only needed for 'ndjson/json/jsonl' tests
+        if args.json_checker and not os.path.exists(args.json_checker):
+            print(f"Error: json_checker not found at {args.json_checker}")
+            sys.exit(1)
         print(f"Rules paths: {args.rules_paths}")
 
     if not args.validate_only:
@@ -414,7 +516,12 @@ def run_tests(
             total_tests += 1
 
             success, output = run_test(
-                rule_path, rule_id, test_data, args.evtx_checker, args.thor_config
+                rule_path,
+                rule_id,
+                test_data,
+                args.evtx_checker,
+                args.thor_config,
+                args.json_checker,
             )
 
             if success: