Remove config.py in favor of diskcache-based JSON loading

.github/workflows/schema.yml

@@ -33,4 +33,5 @@ jobs:
             run: |
                 poetry run check-jsonschema --schemafile ./json-schema/schema_sigmahq_taxonomy.json ./tools/sigmahq_taxonomy.json
                 poetry run check-jsonschema --schemafile ./json-schema/schema_sigmahq_filename.json ./tools/sigmahq_filename.json
-                poetry run check-jsonschema --schemafile ./json-schema/schema_sigmahq_windows_validator.json ./tools/sigmahq_windows_validator.json
+                poetry run check-jsonschema --schemafile ./json-schema/schema_sigmahq_windows_eventid.json ./tools/sigmahq_windows_eventid.json
+                poetry run check-jsonschema --schemafile ./json-schema/schema_sigmahq_windows_provider.json ./tools/sigmahq_windows_provider.json

CHANGELOG.md

@@ -0,0 +1,9 @@
+# CHANGELOG
+
+## 2024-08-09
+
+- Update taxonomy
+- Use json for data instead of config.py
+- Use diskcache for caching remote data
+- Fix invalid sigma syntax
+- Move field validators to validator files

CONTRIBUTING.md

@@ -0,0 +1,19 @@
+# Contributing
+
+## How to update the validator data
+
+The validators rely on the JSON files located in the `tools/` directory.
+
+- `sigmahq_taxonomy.json` - Sigma taxonomy data
+- `sigmahq_filename.json` - Sigma rule filename prefix patterns
+- `sigmahq_windows_eventid.json` - Windows event ID categories
+- `sigmahq_windows_provider.json` - Windows provider names
+
+These files are validated against JSON schemas in `json-schema/`. After updating any JSON file, run the schema validation:
+
+```bash
+poetry run check-jsonschema --schemafile ./json-schema/schema_sigmahq_taxonomy.json ./tools/sigmahq_taxonomy.json
+poetry run check-jsonschema --schemafile ./json-schema/schema_sigmahq_filename.json ./tools/sigmahq_filename.json
+poetry run check-jsonschema --schemafile ./json-schema/schema_sigmahq_windows_eventid.json ./tools/sigmahq_windows_eventid.json
+poetry run check-jsonschema --schemafile ./json-schema/schema_sigmahq_windows_provider.json ./tools/sigmahq_windows_provider.json
+```

README.md

@@ -14,9 +14,7 @@ TBD
 
 ## 🧬 Data
 
-All the data value are in the config.py
-
-To use a local json version, you need to put them in a `validator_json` folder visible from the launch directory.
+All the data value are in the json files in the tools directory
 
 ## 📜 Maintainer
 

json-schema/schema_sigmahq_windows_eventid.json

@@ -0,0 +1,13 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "type": "object",
+    "properties": {
+        "title": { "type": "string" },
+        "version": { "type": "string" },
+        "category_no_eventid": {
+            "type":"array",
+            "items": { "type": "string" }
+        }
+    },
+    "required": ["title","version","category_no_eventid"]
+}

json-schema/schema_sigmahq_windows_validator.json → json-schema/schema_sigmahq_windows_provider.json

@@ -4,10 +4,6 @@
     "properties": {
         "title": { "type": "string" },
         "version": { "type": "string" },
-        "category_no_eventid": {
-            "type":"array",
-            "items": { "type": "string" }
-        },
         "category_provider_name": {
             "type": "object",
             "additionalProperties": {
@@ -16,6 +12,5 @@
             }
         }
     },
-    "required": ["title","version","category_no_eventid","category_provider_name"]
+    "required": ["title","version","category_provider_name"]
 }
-

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pySigma-validators-sigmahq"
-version = "0.20.3"
+version = "0.21.0"
 description = "pySigma SigmaHQ validators"
 license = "LGPL-2.1-only"
 readme = "README.md"
@@ -19,6 +19,7 @@ keywords = [ "pySigma", "SigmaHQ", "validators"]
 
 dependencies = [
     "pysigma >=1.3",
+    "diskcache >=5.6",
     "requests >=2.34"
 ]
 
@@ -36,7 +37,7 @@ defusedxml = "^0.7"
 pytest = "^9.1"
 pytest-cov = "^7.1"
 mypy = "^2.0"
-check-jsonschema = "^0.29"
+check-jsonschema = "^0.30"
 
 [tool.ruff]
 line-length = 100
@@ -58,7 +59,7 @@ select = ["E", "F", "W", "I001", "S", "PLR"]
 "tools/**" = ["S101"]
 
 # PLR0912: Too many branches - complex validation logic in field validators
-"sigma/validators/sigmahq/field.py" = ["PLR0912"]
+"sigma/validators/sigmahq/fieldname.py" = ["PLR0912"]
 
 [tool.ruff.format]
 quote-style = "double"