👷 ci: gate the schema on SchemaStore's own checks

The schema reached SchemaStore only on a release tag, and nothing on the
way there compiled it the way SchemaStore does: the freshness test
compares generator output against the committed file, and the tombi
tests validate TOML documents rather than the schema itself. A schema
that ajv strict mode rejects therefore surfaced as a failed sync pull
request after the release had shipped.

The sync workflow now clones SchemaStore, stages tox's schema and runs
their checker for it, on every pull request touching the schema and
again before the release sync opens a pull request.

Author: gaborbernat

.github/workflows/update-schemastore.yaml

@@ -2,10 +2,42 @@ name: Update SchemaStore
 on:
   push:
     tags: ["*"]
+  pull_request:
+    paths:
+      - "src/tox/tox.schema.json"
+      - ".github/workflows/update-schemastore.yaml"
 permissions:
   contents: read
 jobs:
+  validate:
+    name: schema passes SchemaStore's checks
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Clone SchemaStore
+        run: git clone --depth 1 https://github.com/SchemaStore/schemastore /tmp/schemastore
+      - name: Install SchemaStore's dependencies
+        run: npm ci
+        working-directory: /tmp/schemastore
+      - name: Stage tox's schema
+        run: |
+          python3 -c "
+          import json
+          with open('${{ github.workspace }}/src/tox/tox.schema.json') as f:
+              schema = json.load(f)
+          schema['\$id'] = 'https://json.schemastore.org/tox.json'
+          with open('/tmp/schemastore/src/schemas/json/tox.json', 'w') as f:
+              json.dump(schema, f, indent=2)
+              f.write('\n')
+          "
+      - name: Run SchemaStore's checks against it
+        run: node ./cli.js check --schema-name=tox.json
+        working-directory: /tmp/schemastore
   update-schemastore:
+    needs: validate
+    if: github.event_name == 'push'
     runs-on: ubuntu-24.04
     environment: schemastore
     env:

docs/changelog/4051.contrib.rst

@@ -0,0 +1,2 @@
+Check tox's JSON Schema against SchemaStore's own validator on every pull request that touches it, and before the
+release sync opens a pull request there - by :user:`gaborbernat`.

tests/session/cmd/test_schema.py

@@ -4,6 +4,7 @@
 import re
 import shutil
 import subprocess
+from collections.abc import Mapping, Sequence
 from pathlib import Path
 from textwrap import dedent
 from typing import TYPE_CHECKING, Any, Protocol
@@ -14,6 +15,7 @@
     from collections.abc import Iterator
 
     from tox.pytest import MonkeyPatch, ToxProjectCreator
+    from tox.util.json_types import JsonValue
 
 
 class LintWorkspace(Protocol):
@@ -145,26 +147,27 @@ def test_schema_freshness(
     )
 
 
-def _subschemas(node: Any, pointer: str = "#") -> Iterator[tuple[str, dict[str, Any]]]:
-    if isinstance(node, dict):
+def _subschemas(node: JsonValue, pointer: str = "#") -> Iterator[tuple[str, Mapping[str, JsonValue]]]:
+    if isinstance(node, Mapping):
         yield pointer, node
         for key, value in node.items():
             yield from _subschemas(value, f"{pointer}/{key}")
-    elif isinstance(node, list):
+    elif isinstance(node, Sequence) and not isinstance(node, str):
         for index, value in enumerate(node):
             yield from _subschemas(value, f"{pointer}/{index}")
 
 
-def test_schema_required_properties_are_declared(committed_schema: dict[str, Any]) -> None:
+def test_schema_required_properties_are_declared(committed_schema: Mapping[str, JsonValue]) -> None:
     # SchemaStore compiles the published schema with ajv strict mode, which rejects a required property that the
     # schema never declares, so catch it here rather than in a release-time sync PR (see tox-dev/tox#4051)
-    undeclared = [
-        f"{pointer} requires {name!r}"
-        for pointer, schema in _subschemas(committed_schema)
-        if isinstance(schema.get("required"), list)
-        for name in schema["required"]
-        if name not in schema.get("properties", {})
-    ]
+    undeclared: list[str] = []
+    for pointer, schema in _subschemas(committed_schema):
+        required = schema.get("required")
+        if not isinstance(required, Sequence) or isinstance(required, str):
+            continue
+        properties = schema.get("properties")
+        declared = set(properties) if isinstance(properties, Mapping) else set()
+        undeclared += [f"{pointer} requires {name!r}" for name in required if name not in declared]
     assert undeclared == []