Sync component catalog #48
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sync component catalog | |
| # Re-runs ``script/sync_components.py`` against the schema version | |
| # matching the dashboard's installed ``esphome`` (or a specific | |
| # version on manual dispatch) and opens a pull request when the | |
| # generated ``definitions/components.index.json`` or per-id body | |
| # files under ``definitions/components/`` change. | |
| # | |
| # Triggers | |
| # -------- | |
| # - schedule : nightly at 03:00 UTC. The script is fully cached when | |
| # nothing has changed upstream, so this is cheap and a | |
| # no-op on most days. | |
| # - manual : ``workflow_dispatch`` with an optional ``version`` input | |
| # (e.g. ``2026.4.3``). When empty, the workflow defaults | |
| # to the version of ``esphome`` installed in CI to avoid | |
| # schema/runtime drift (the live introspection step needs | |
| # the matching esphome to load new components). | |
| # | |
| # Output | |
| # ------ | |
| # Always pushes to a stable branch named ``catalog/sync`` so the | |
| # scheduled run keeps updating the same in-flight PR rather than | |
| # spawning a new one every night. ``peter-evans/create-pull-request`` | |
| # closes the PR (and deletes the branch) automatically when the | |
| # rebuild produces no diff. | |
| on: | |
| schedule: | |
| - cron: "0 3 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| version: | |
| description: "ESPHome schema version (e.g. 2026.4.3). Leave empty to match the dashboard's installed esphome." | |
| required: false | |
| type: string | |
| include_prereleases: | |
| description: "Include pre-releases when auto-selecting the latest schema." | |
| required: false | |
| type: boolean | |
| default: false | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| concurrency: | |
| group: sync-component-catalog | |
| cancel-in-progress: false | |
| jobs: | |
| sync: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version: "3.13" | |
| cache: pip | |
| - name: Install package (with esphome extra) | |
| run: | | |
| python -m pip install --upgrade pip | |
| # ``[esphome]`` pulls in the esphome package so the narrow | |
| # introspection in sync_components (multi_conf, | |
| # platform_defaults, supported_platforms, type refinement) | |
| # can run. | |
| pip install -e '.[esphome]' | |
| - name: Resolve schema version | |
| id: version | |
| # Prefer an explicit dispatch input. Otherwise pin to the | |
| # installed esphome version so live introspection lines up | |
| # exactly — every component the schema lists is importable | |
| # for ``multi_conf`` / ``platform_defaults`` extraction. | |
| run: | | |
| set -euo pipefail | |
| if [ -n "${{ inputs.version }}" ]; then | |
| echo "version=${{ inputs.version }}" >> "$GITHUB_OUTPUT" | |
| echo "source=manual dispatch" >> "$GITHUB_OUTPUT" | |
| else | |
| INSTALLED=$(python -c "from esphome.const import __version__; print(__version__)") | |
| echo "version=$INSTALLED" >> "$GITHUB_OUTPUT" | |
| echo "source=installed esphome" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Run sync_components | |
| run: | | |
| set -euo pipefail | |
| ARGS=(--version "${{ steps.version.outputs.version }}") | |
| if [ "${{ inputs.include_prereleases }}" = "true" ]; then | |
| ARGS+=(--include-prereleases) | |
| fi | |
| python script/sync_components.py "${ARGS[@]}" | |
| - name: Smoke-test catalog | |
| # Catches regressions in popular components (missing fields, | |
| # type flips, id-vs-reference confusion). Runs BEFORE the | |
| # diff check so a broken catalog never gets proposed for | |
| # merge. | |
| run: python script/check_catalog.py | |
| - name: Detect catalog changes + summarise diff | |
| id: diff | |
| run: | | |
| set -euo pipefail | |
| if git diff --quiet -- \ | |
| esphome_device_builder/definitions/components.index.json \ | |
| esphome_device_builder/definitions/components/ \ | |
| && [ -z "$(git ls-files --others --exclude-standard esphome_device_builder/definitions/components/)" ]; then | |
| echo "changed=false" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| echo "changed=true" >> "$GITHUB_OUTPUT" | |
| # Build a human-friendly delta summary the PR body embeds. | |
| # We compare the freshly-generated catalog against the one | |
| # currently on main so the reviewer sees component-count | |
| # drift, per-type entry drift, and the populated-field | |
| # totals at a glance. | |
| python <<'PY' > /tmp/catalog-diff.md | |
| import json | |
| import subprocess | |
| from collections import Counter | |
| from pathlib import Path | |
| NEW_INDEX = Path("esphome_device_builder/definitions/components.index.json") | |
| BODIES_DIR = Path("esphome_device_builder/definitions/components") | |
| def load_bodies(index_blob: str, head_ref: str | None) -> tuple[dict, list[dict]]: | |
| meta = json.loads(index_blob) if index_blob else {"components": []} | |
| bodies: list[dict] = [] | |
| for entry in meta.get("components", []): | |
| cid = entry.get("id") | |
| if not cid: | |
| continue | |
| rel = BODIES_DIR / f"{cid}.json" | |
| if head_ref is None: | |
| if rel.is_file(): | |
| bodies.append({**entry, **json.loads(rel.read_text())}) | |
| else: | |
| bodies.append(entry) | |
| else: | |
| try: | |
| body_blob = subprocess.check_output( | |
| ["git", "show", f"{head_ref}:{rel}"], | |
| text=True, | |
| ) | |
| bodies.append({**entry, **json.loads(body_blob)}) | |
| except subprocess.CalledProcessError: | |
| bodies.append(entry) | |
| return meta, bodies | |
| new_meta, new_components = load_bodies(NEW_INDEX.read_text(), None) | |
| try: | |
| old_index_blob = subprocess.check_output( | |
| ["git", "show", f"HEAD:{NEW_INDEX}"], | |
| text=True, | |
| ) | |
| _, old_components = load_bodies(old_index_blob, "HEAD") | |
| except subprocess.CalledProcessError: | |
| old_components = [] | |
| new_data = new_meta | |
| def count_types(components: list[dict]) -> Counter: | |
| counts: Counter[str] = Counter() | |
| def walk(entries: list[dict]) -> None: | |
| for entry in entries: | |
| counts[entry.get("type") or "unknown"] += 1 | |
| walk(entry.get("config_entries") or []) | |
| for component in components: | |
| walk(component.get("config_entries") or []) | |
| return counts | |
| old_ids = {c["id"] for c in old_components} | |
| new_ids = {c["id"] for c in new_components} | |
| added = sorted(new_ids - old_ids) | |
| removed = sorted(old_ids - new_ids) | |
| old_types = count_types(old_components) | |
| new_types = count_types(new_components) | |
| all_types = sorted(set(old_types) | set(new_types)) | |
| old_total = sum(old_types.values()) | |
| new_total = sum(new_types.values()) | |
| # Headline includes the config-entry total so a refresh that | |
| # leaves the component count stable but adds thousands of | |
| # nested fields (e.g. a more complete MQTT_COMPONENT_SCHEMA | |
| # bundle landing upstream) doesn't read as "no change". | |
| lines = [ | |
| f"**Schema version**: `{new_data.get('esphome_schema_version', '?')}` ", | |
| f"**Components**: {len(old_components)} → {len(new_components)} " | |
| f"({len(new_components) - len(old_components):+d}) ", | |
| f"**Config entries**: {old_total} → {new_total} " | |
| f"({new_total - old_total:+d}) ", | |
| f"**Added**: {len(added)} · **Removed**: {len(removed)}", | |
| "", | |
| ] | |
| if added or removed: | |
| lines.append("<details><summary>Component churn</summary>") | |
| lines.append("") | |
| if added: | |
| lines.append(f"**Added ({len(added)}):** " + ", ".join(f"`{i}`" for i in added[:30])) | |
| if len(added) > 30: | |
| lines.append(f" _…and {len(added) - 30} more_") | |
| if removed: | |
| lines.append(f"**Removed ({len(removed)}):** " + ", ".join(f"`{i}`" for i in removed[:30])) | |
| if len(removed) > 30: | |
| lines.append(f" _…and {len(removed) - 30} more_") | |
| lines.append("") | |
| lines.append("</details>") | |
| lines.append("") | |
| lines.append("<details><summary>Config-entry type distribution</summary>") | |
| lines.append("") | |
| lines.append("| Type | Old | New | Δ |") | |
| lines.append("|------|----:|----:|---:|") | |
| for t in all_types: | |
| o = old_types.get(t, 0) | |
| n = new_types.get(t, 0) | |
| if o == n: | |
| continue | |
| lines.append(f"| `{t}` | {o} | {n} | {n - o:+d} |") | |
| lines.append("") | |
| lines.append("</details>") | |
| print("\n".join(lines)) | |
| PY | |
| { | |
| echo "summary<<DIFF_EOF" | |
| cat /tmp/catalog-diff.md | |
| echo "DIFF_EOF" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Open / update pull request | |
| if: steps.diff.outputs.changed == 'true' | |
| uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 | |
| with: | |
| branch: catalog/sync | |
| base: main | |
| commit-message: | | |
| Sync component catalog from schema ${{ steps.version.outputs.version }} | |
| Auto-generated by .github/workflows/sync-component-catalog.yml. | |
| title: "Sync component catalog from schema ${{ steps.version.outputs.version }}" | |
| body: | | |
| Automated catalog refresh. | |
| Schema source: **${{ steps.version.outputs.source }}** (version `${{ steps.version.outputs.version }}`). | |
| Triggered by: **${{ github.event_name == 'schedule' && 'nightly schedule' || format('manual dispatch by @{0}', github.actor) }}**. | |
| ${{ steps.diff.outputs.summary }} | |
| **Smoke test:** ✅ catalog passes [`script/check_catalog.py`](../blob/main/script/check_catalog.py) — every well-known component has the expected shape. | |
| --- | |
| Review checklist: | |
| - Skim the **Added** / **Removed** lists above for anything unexpected. | |
| - Check the type-distribution table for outsized drift in any single bucket (a sudden drop in `boolean` or `pin` likely means a sync regression rather than an upstream change). | |
| - If the diff looks weird, run `script/sync_components.py --version ${{ steps.version.outputs.version }}` locally and compare. The script is deterministic given a schema version + installed esphome. | |
| - Merge to ship the new catalog. | |
| labels: | | |
| catalog | |
| automated | |
| delete-branch: true | |
| - name: No-op summary | |
| if: steps.diff.outputs.changed == 'false' | |
| run: | | |
| echo "::notice::Component catalog is already up to date for schema ${{ steps.version.outputs.version }} - no PR opened." |