chore: add json/ndjson/jsonl to regression test

Author: Paul Hager

.github/workflows/regression-tests.yml

@@ -4,6 +4,7 @@ on: [push, pull_request, workflow_dispatch]
 
 env:
   EVTX_BASELINE_VERSION: v0.8.4
+  JSON_MATCHER_VERSION: v0.0.2
 
 jobs:
   true-positive-tests:
@@ -19,13 +20,18 @@ jobs:
     - name: Install Python dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install pyyaml
+        pip install pyyaml sigma-cli pySigma-backend-golangexpr
 
     - name: Download evtx-sigma-checker
       run: |
         wget --no-verbose https://github.com/NextronSystems/evtx-baseline/releases/download/$EVTX_BASELINE_VERSION/evtx-sigma-checker
         chmod +x evtx-sigma-checker
+    
+    - name: Download json-checker
+      run: |
+        wget --no-verbose https://github.com/pH-T/json_matcher/releases/download/$JSON_MATCHER_VERSION/json_matcher_v0.0.1 -O json_checker
+        chmod +x json_checker
 
     - name: Run regression tests
       run: |
-        python tests/regression_tests_runner.py --rules-paths rules rules-emerging-threats rules-threat-hunting --evtx-checker ./evtx-sigma-checker --thor-config tests/thor.yml --ignore-validation
+        python tests/regression_tests_runner.py --rules-paths rules rules-emerging-threats rules-threat-hunting --evtx-checker ./evtx-sigma-checker --thor-config tests/thor.yml --ignore-validation --json-checker ./json_checker

regression_data/pipelines/process_creation_fieldmapping.yml

@@ -0,0 +1,23 @@
+name: process_creation_fieldmapping
+priority: 100
+transformations:
+    - id: field_mapping_process_creation
+      type: field_name_mapping
+      mapping:
+          CommandLine: Event.EventData.CommandLine
+          Image: Event.EventData.Image
+          OriginalFileName: Event.EventData.OriginalFileName
+          rule_conditions:
+              - type: logsource
+                category: process_creation
+                product: windows
+    - id: field_mapping_registry_set
+      type: field_name_mapping
+      mapping:
+          TargetObject: Event.EventData.TargetObject
+          Details: Event.EventData.Details
+          Image: Event.EventData.Image
+          rule_conditions:
+              - type: logsource
+                category: registry_set
+                product: windows

regression_data/rules/windows/process_creation/proc_creation_win_amsi_registry_tampering/info.yml

@@ -11,3 +11,9 @@ regression_tests_info:
       provider: Microsoft-Windows-Sysmon
       match_count: 1
       path: regression_data/rules/windows/process_creation/proc_creation_win_amsi_registry_tampering/7dbbcac2-57a0-45ac-b306-ff30a8bd2981.evtx
+    - name: Positive Detection Test
+      type: json
+      match_count: 1
+      pipelines:
+          - regression_data/pipelines/process_creation_fieldmapping.yml
+      path: regression_data/rules/windows/process_creation/proc_creation_win_amsi_registry_tampering/7dbbcac2-57a0-45ac-b306-ff30a8bd2981.json

regression_data/rules/windows/process_creation/proc_creation_win_autologger_session_registry_modification/info.yml

@@ -11,3 +11,9 @@ regression_tests_info:
       provider: Microsoft-Windows-Sysmon
       match_count: 1
       path: regression_data/rules/windows/process_creation/proc_creation_win_autologger_session_registry_modification/d7b81144-b866-48a4-9bcc-275dc69d870e.evtx
+    - name: Positive Detection Test
+      type: json
+      match_count: 1
+      pipelines:
+          - regression_data/pipelines/process_creation_fieldmapping.yml
+      path: regression_data/rules/windows/process_creation/proc_creation_win_autologger_session_registry_modification/d7b81144-b866-48a4-9bcc-275dc69d870e.json

regression_data/rules/windows/registry/registry_set/registry_set_add_load_service_in_safe_mode/info.yml

@@ -11,3 +11,9 @@ regression_tests_info:
       provider: Microsoft-Windows-Sysmon
       match_count: 1
       path: regression_data/rules/windows/registry/registry_set/registry_set_add_load_service_in_safe_mode/1547e27c-3974-43e2-a7d7-7f484fb928ec.evtx
+    - name: Positive Detection Test
+      type: json
+      match_count: 1
+      pipelines:
+          - regression_data/pipelines/process_creation_fieldmapping.yml
+      path: regression_data/rules/windows/registry/registry_set/registry_set_add_load_service_in_safe_mode/1547e27c-3974-43e2-a7d7-7f484fb928ec.json

tests/regression_tests_runner.py

@@ -78,13 +78,23 @@ def load_info_yaml(
                     }
                 )
 
+            base_dir = os.path.dirname(regression_tests_path)
+            pipelines = [
+                get_absolute_path(base_dir, p) for p in test.get("pipelines", [])
+            ]
+            filters = [
+                get_absolute_path(base_dir, f) for f in test.get("filters", [])
+            ]
+
             test_data.append(
                 {
                     "type": test.get("type", "unknown"),
                     "path": test_path,
                     "name": test.get("name", "Unnamed Test"),
                     "provider": test.get("provider", ""),
                     "match_count": test.get("match_count"),
+                    "pipelines": pipelines,
+                    "filters": filters,
                 }
             )
         info_metadata_rule_id = None
@@ -293,12 +303,88 @@ def run_evtx_checker(
         return False, ""
 
 
+def compile_rule_to_expr(
+    rule_path: str, pipelines: List[str], filters: List[str]
+) -> str:
+    """Compile a Sigma rule to a golang_expr query via the sigma CLI."""
+    cmd = ["sigma", "convert", "-t", "golang_expr"]
+    for pipeline in pipelines:
+        cmd += ["-p", pipeline]
+    if len(pipelines) == 0:
+        cmd += ["--without-pipeline"]
+    for filter in filters:
+        cmd += ["--filter", filter]
+    cmd.append(rule_path)
+
+    result = subprocess.run(
+        cmd, capture_output=True, text=True, timeout=300, check=True
+    )
+
+    return result.stdout.strip()
+
+
+def run_json_checker(
+    test_type: str,
+    rule_path: str,
+    rule_id: str,
+    test_data: Dict,
+    json_checker_path: str,
+) -> tuple[bool, str]:
+    """Compile the rule to an expr query and run json_checker against the events."""
+    try:
+        expr_query = compile_rule_to_expr(
+            rule_path, test_data.get("pipelines", []), test_data.get("filters", [])
+        )
+    except subprocess.CalledProcessError as e:
+        print(f"  Error compiling rule {rule_id} with sigma: {e.stderr or e}")
+        return False, ""
+    except subprocess.TimeoutExpired:
+        print(f"  Timeout compiling rule {rule_id} with sigma")
+        return False, ""
+
+    if not expr_query:
+        print(f"  Error: sigma produced an empty expr query for {rule_id}")
+        return False, ""
+
+    cmd = [json_checker_path, "--event", test_data["path"], "--expr", expr_query, "--test-type", test_type]
+    try:
+        result = subprocess.run(
+            cmd, capture_output=True, text=True, timeout=300, check=True
+        )
+    except subprocess.TimeoutExpired:
+        print("  Timeout: json_checker timed out")
+        return False, ""
+    except subprocess.CalledProcessError as e:
+        print(f"  Error running json_checker: {e.stderr or e}")
+        return False, ""
+
+    match_lines = [ln for ln in result.stdout.splitlines() if ln.endswith("MATCH")]
+    match_count = len(match_lines)
+    all_output = "\n    ".join(match_lines)
+
+    expected_count = test_data.get("match_count")
+    if expected_count is not None:
+        if match_count < expected_count:
+            print(
+                f"  Error: {rule_id}: Match count too low: expected {expected_count}, got {match_count}"
+            )
+            return False, all_output
+        if match_count > expected_count:
+            print(
+                f"  Warning: {rule_id}: Got {match_count} matches but only {expected_count} expected - consider updating match_count in info.yml"
+            )
+        return True, all_output
+
+    return match_count > 0, all_output
+
+
 def run_test(
     rule_path: str,
     rule_id: str,
     test_data: Dict,
     evtx_checker_path: str,
     thor_config: str,
+    json_checker_path: str,
 ) -> tuple[bool, str]:
     """Run a test based on its type."""
     test_type = test_data.get("type", "unknown")
@@ -307,6 +393,12 @@ def run_test(
         return run_evtx_checker(
             rule_path, rule_id, test_data, evtx_checker_path, thor_config
         )
+    
+    if test_type == "jsonl" or test_type == "ndjson" or test_type == "json" :
+        if not json_checker_path:
+            print("  Error: --json-checker is required for 'ndjson/json' tests")
+            return False, ""
+        return run_json_checker(test_type, rule_path, rule_id, test_data, json_checker_path)
     print(f"  Warning: Unknown test type '{test_type}', skipping")
     return False, ""
 
@@ -335,6 +427,11 @@ def parse_arguments() -> argparse.Namespace:
         help="Path to thor.yml configuration file (required unless using --validate-only)",
     )
 
+    parser.add_argument(
+        "--json-checker",
+        help="Path to json_checker binary (required for 'ndjsonjson' tests)",
+    )
+
     parser.add_argument(
         "--validate-only",
         action="store_true",
@@ -379,6 +476,11 @@ def init_checks(args: argparse.Namespace) -> None:
         if not os.path.exists(args.thor_config):
             print(f"Error: Thor config not found at {args.thor_config}")
             sys.exit(1)
+
+        # json_checker is optional; only needed for 'ndjson/json' tests
+        if args.json_checker and not os.path.exists(args.json_checker):
+            print(f"Error: json_checker not found at {args.json_checker}")
+            sys.exit(1)
         print(f"Rules paths: {args.rules_paths}")
 
     if not args.validate_only:
@@ -414,7 +516,12 @@ def run_tests(
             total_tests += 1
 
             success, output = run_test(
-                rule_path, rule_id, test_data, args.evtx_checker, args.thor_config
+                rule_path,
+                rule_id,
+                test_data,
+                args.evtx_checker,
+                args.thor_config,
+                args.json_checker,
             )
 
             if success: