Add 60 public-registry verification tasks #5
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: validate-task | |
| on: | |
| pull_request: | |
| branches: [main] | |
| paths: | |
| - ".github/workflows/validate-task.yml" | |
| - "test-cases/task.schema.json" | |
| - "test-cases/**/task.json" | |
| - "test-cases/**/extra_info/**" | |
| push: | |
| branches: [main] | |
| paths: | |
| - ".github/workflows/validate-task.yml" | |
| - "test-cases/task.schema.json" | |
| - "test-cases/**/task.json" | |
| - "test-cases/**/extra_info/**" | |
| workflow_dispatch: | |
| jobs: | |
| validate-task: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.11" | |
| - name: Install validator | |
| run: python -m pip install "jsonschema==4.26.0" | |
| - name: Collect changed task files | |
| id: changed | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then | |
| find test-cases -path "*/task.json" -o -path "*/extra_info/*" > changed-files.txt | |
| elif [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| base="${{ github.event.pull_request.base.sha }}" | |
| git diff --name-only "$base"...HEAD > changed-files.txt | |
| elif [[ "${{ github.event_name }}" == "push" && "${{ github.event.before }}" != "0000000000000000000000000000000000000000" ]]; then | |
| git diff --name-only "${{ github.event.before }}" "${{ github.sha }}" > changed-files.txt | |
| else | |
| find test-cases -path "*/task.json" -o -path "*/extra_info/*" > changed-files.txt | |
| fi | |
| echo "Changed files:" | |
| cat changed-files.txt | |
| - name: Validate changed tasks | |
| run: | | |
| python - <<'PY' | |
| import json | |
| import sys | |
| from pathlib import Path | |
| from jsonschema import Draft202012Validator | |
| repo = Path(".") | |
| schema_path = repo / "test-cases" / "task.schema.json" | |
| changed_paths = [ | |
| Path(line.strip()) | |
| for line in Path("changed-files.txt").read_text().splitlines() | |
| if line.strip() | |
| ] | |
| schema = json.loads(schema_path.read_text()) | |
| validator = Draft202012Validator(schema) | |
| validate_all = schema_path in changed_paths | |
| task_files: set[Path] = set() | |
| changed_json_files: set[Path] = set() | |
| if validate_all: | |
| task_files.update(repo.glob("test-cases/**/task.json")) | |
| for path in changed_paths: | |
| if not str(path).startswith("test-cases/"): | |
| continue | |
| if path.name == "task.json" and path.exists(): | |
| task_files.add(path) | |
| changed_json_files.add(path) | |
| continue | |
| if "extra_info" in path.parts: | |
| extra_index = path.parts.index("extra_info") | |
| task_dir = Path(*path.parts[:extra_index]) | |
| task_file = task_dir / "task.json" | |
| if task_file.exists(): | |
| task_files.add(task_file) | |
| if path.suffix == ".json" and path.exists(): | |
| changed_json_files.add(path) | |
| errors: list[str] = [] | |
| for json_file in sorted(changed_json_files): | |
| try: | |
| json.loads(json_file.read_text()) | |
| except Exception as exc: | |
| errors.append(f"{json_file}: invalid JSON: {exc}") | |
| for task_file in sorted(task_files): | |
| try: | |
| task = json.loads(task_file.read_text()) | |
| except Exception as exc: | |
| errors.append(f"{task_file}: invalid JSON: {exc}") | |
| continue | |
| for error in sorted(validator.iter_errors(task), key=lambda item: list(item.path)): | |
| location = "/" + "/".join(str(part) for part in error.path) | |
| errors.append(f"{task_file}{location}: {error.message}") | |
| extra_info = task.get("extra_info") or [] | |
| if not isinstance(extra_info, list): | |
| continue | |
| for index, item in enumerate(extra_info): | |
| if not isinstance(item, dict) or not item.get("path"): | |
| continue | |
| extra_path = task_file.parent / item["path"] | |
| if not extra_path.exists(): | |
| errors.append( | |
| f"{task_file}: extra_info[{index}].path does not exist: {item['path']}" | |
| ) | |
| if errors: | |
| print("Task validation failed:") | |
| for error in errors: | |
| print(f"- {error}") | |
| sys.exit(1) | |
| print(f"Validated {len(task_files)} task file(s) and {len(changed_json_files)} changed JSON file(s).") | |
| PY |