Init schedule docs update job

.github/actions/webchanges/action.yml

@@ -0,0 +1,72 @@
+name: "Web changes detector"
+author: "Dmitry Vasiliev"
+description: "Detect changes in web pages and notify via GitHub Actions"
+branding:
+  icon: "check-circle"
+  color: "blue"
+
+inputs:
+  config:
+    description: "Configuration for the web changes detector"
+    required: true
+  jobs:
+    description: "List of jobs to run for web changes detection"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: "Set up environment"
+      shell: bash
+      run: |
+        echo "CONFIG_PATH=${{ github.workspace }}/config.yaml" >> $GITHUB_ENV
+        echo "JOBS_PATH=${{ github.workspace }}/jobs.yaml" >> $GITHUB_ENV
+        echo "SNAPSHOTS_DB=${{ github.workspace }}/snapshots.db" >> $GITHUB_ENV
+        echo "HOOKS_PATH=${{ github.action_path }}/hooks.py" >> $GITHUB_ENV
+        echo "WEBCHANGES_VERSION=3.30.0" >> $GITHUB_ENV
+        echo "PYTHON_VERSION=3.10" >> $GITHUB_ENV
+    - name: "Write configs to files"
+      shell: bash
+      run: |
+        echo '${{ inputs.config }}' > $CONFIG_PATH
+        echo '${{ inputs.jobs }}' > $JOBS_PATH
+    - name: "Setup Python"
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
+        cache: "pip"
+    - name: "Install webchanges package"
+      shell: bash
+      run: |
+        pip install webchanges==${{ env.WEBCHANGES_VERSION }}
+    - name: "Compute SHA-256 of configs input"
+      shell: bash
+      run: |
+        # Compute the hash of the configs input and store it in an environment variable
+        JOB_HASH=$(sha256sum ${{ env.JOBS_PATH }} | awk '{ print $1 }')
+
+        # Compute cache key based on the hashes
+        echo "CACHE_KEY=${{ runner.os }}-snapshots-${JOB_HASH}" >> $GITHUB_ENV
+    - name: "Restore cache snapshots database"
+      id: cache-db
+      uses: actions/cache/restore@v4
+      with:
+        path: "${{ env.SNAPSHOTS_DB }}"
+        key: "${{ env.CACHE_KEY }}"
+    - name: "Check if snapshots database exists"
+      shell: bash
+      run: |
+        if [ ! -f $SNAPSHOTS_DB ]; then
+          echo "No snapshots database found, creating a new one."
+        else
+          echo "Snapshots database found, using existing one."
+        fi
+    - name: "Run web changes detection"
+      shell: bash
+      run: |
+        webchanges --config ${{ env.CONFIG_PATH }} --jobs ${{ env.JOBS_PATH }} --hooks ${{ env.HOOKS_PATH }} --database ${{ env.SNAPSHOTS_DB }}
+    - name: "Cache snapshots database"
+      uses: actions/cache/save@v4
+      with:
+        path: "${{ env.SNAPSHOTS_DB }}"
+        key: "${{ env.CACHE_KEY }}"

.github/actions/webchanges/hooks.py

@@ -0,0 +1,163 @@
+"""Custom reporters and filters for webchanges utility."""
+
+import json
+import logging
+import re
+import typing
+from datetime import datetime
+from datetime import timezone
+from http import HTTPStatus
+from typing import TypedDict
+
+from webchanges.filters import FilterBase
+from webchanges.reporters import MarkdownReporter
+
+logger = logging.getLogger(__name__)
+
+
+class GitHubIssueReporter(MarkdownReporter):
+    """Reporter that submits reports as issues to a GitHub repository."""
+
+    __kind__ = "github_issue"
+
+    config: TypedDict(
+        "_ConfigReportGithubIssue",
+        {
+            "enabled": bool,
+            "token": str,
+            "owner": str,
+            "repo": str,
+            "title": typing.Optional[str],
+            "labels": list[str],
+            "format_dt": typing.Optional[str],
+            "format_content": typing.Optional[str],
+            "assignees": typing.Optional[typing.Sequence[str]],
+            "type": typing.Optional[str],
+            "milestone": typing.Optional[str],
+        },
+    )
+
+    _API_URL = "https://api.github.com/repos/{owner}/{repo}/issues"
+    _CONTENT_LIMIT = 65536  # GitHub issue body limit
+
+    def _format_title(self) -> str:
+        """Format the title of the issue."""
+        now = datetime.now(tz=timezone.utc)
+        format_dt = self.config.get("format_dt", "%Y-%m-%d %H:%M:%S")
+
+        title = self.config.get("title")
+        if not title:
+            title = "WebChanges report"
+        else:
+            dt = now.strftime(format_dt)
+            title = title.format(dt=dt)
+
+        return title
+
+    def _format_text(self, content: str) -> str:
+        """Format the content of the issue."""
+        format_content = self.config.get("format_content")
+        content = content[: self._CONTENT_LIMIT]
+        if format_content:
+            placeholder = "{content}"
+            max_content_length = self._CONTENT_LIMIT - (len(format_content) - len(placeholder))
+            content = content[:max_content_length]
+            content = format_content.format(content=content)
+
+        return content
+
+    def _create_issue(self, content: str):
+        """Create an issue on GitHub."""
+        url = self._API_URL.format(owner=self.config["owner"], repo=self.config["repo"])
+        headers = {"Authorization": f"Bearer {self.config['token']}", "Accept": "application/json"}
+
+        title = self._format_title()
+        content = self._format_text(content)
+        issue_data = {"title": title, "body": content, "labels": self.config.get("labels", [])}
+
+        assignees = self.config.get("assignees", [])
+        if assignees:
+            issue_data["assignees"] = assignees
+
+        type_ = self.config.get("type")
+        if type_:
+            issue_data["type"] = type_
+
+        milestone = self.config.get("milestone")
+        if milestone:
+            issue_data["milestone"] = milestone
+
+        response = self.post_client(url, headers=headers, json=issue_data)
+
+        if response.status_code == HTTPStatus.CREATED:
+            logger.info("Issue created successfully.")
+        else:
+            json_object = json.loads(response.text)
+            json_formatted_str = json.dumps(json_object, indent=2)
+            raise RuntimeError(f"Failed to create issue: {json_formatted_str}")
+
+    def submit(
+        self,
+        max_length: int | None = None,
+        **kwargs: typing.Any,
+    ) -> typing.Iterator[str]:
+        """Submit the report to GitHub as an issue."""
+        lines = super().submit(max_length, **kwargs)
+        content = "\n".join(lines)
+        if not content:
+            logger.info("No content to submit.")
+            return
+
+        logger.info("Submitting issue to GitHub...")
+        self._create_issue(content)
+
+        return lines
+
+
+def get_lines_between(
+    lines: typing.Sequence[str],
+    start_pattern: str | None = None,
+    end_pattern: str | None = None,
+) -> typing.Generator[str, None, None]:
+    """Yield lines between start and end patterns."""
+    started = False
+    for line in lines:
+        if not started:
+            if start_pattern and re.search(start_pattern, line):
+                started = True
+            continue
+        if end_pattern and re.search(end_pattern, line):
+            break
+
+        yield line
+
+
+class BetweenLinesFilter(FilterBase):
+    """Filter to extract lines between two patterns."""
+
+    __kind__ = "between"
+
+    __supported_subfilters__ = {
+        "start": "Pattern to match the start line.",
+        "end": "Pattern to match the end line.",
+    }
+
+    __default_subfilter__ = "indent"
+
+    @staticmethod
+    def filter(
+        data: str | bytes,
+        mime_type: str,
+        subfilter: dict[str, typing.Any],
+    ) -> tuple[str | bytes, str]:
+        """Filter lines between start and end patterns."""
+        start_pattern = subfilter.get("start")
+        end_pattern = subfilter.get("end")
+
+        lines = data.splitlines(keepends=True)
+        filtered_lines = get_lines_between(lines, start_pattern, end_pattern)
+
+        return (
+            "\n".join(filtered_lines),
+            mime_type,
+        )

.github/workflows/check-update.yml

@@ -0,0 +1,42 @@
+name: "Check Telegram Mini App documentation updates"
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 * * * *'
+
+permissions:
+  issues: write
+
+jobs:
+  check-telegram-docs-updates:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check Telegram Mini App documentation updates
+        uses: ./.github/actions/webchanges
+        with:
+          jobs: |
+            name: Telegram Mini App documentation
+            url: https://core.telegram.org/bots/webapps
+            use_browser: false
+            filters:
+              - html2text
+              - between:
+                  start: "#### __WebAppInitData"
+                  end: "#### __Events Available for Mini Apps"
+          config: |
+            report:
+              tz: UTC
+              github_issue:
+                enabled: true
+                owner: "${{ github.repository_owner }}"
+                repo: "${{ github.event.repository.name }}"
+                token: "${{ secrets.GITHUB_TOKEN }}"
+                format_dt: "%Y-%m-%d"
+                title: "Telegram Mini App documentation update: {dt}"
+                assignees: 
+                  - "swimmwatch"
+                format_content: |
+                  ```diff
+                  {content}
+                  ```