zeeschuimer_map_item_sync.yml

# Auto-translate Zeeschuimer datasource `map_item` functions from Python to JS
# and open a draft PR per module against digitalmethodsinitiative/zeeschuimer.
#
# Triggers on pushes to master that touch any Zeeschuimer datasource (or the
# helper script itself). Also exposes a `workflow_dispatch` trigger with a
# `bootstrap` input for the initial run that translates every Zeeschuimer
# datasource at once (single PR). Datasources without a matching Zeeschuimer
# module — e.g. facebook — are skipped automatically.
#
# Architecture: a `detect` job groups changed files by module and emits a
# matrix; a `sync` job fans out one parallel run per module, each opening
# (or updating) its own PR on a stable per-module branch. The planning
# and PR-body logic are in `helper-scripts/map_item_ci.py` (# unit-tested 
# in tests/test_map_item_sync.py); the LLM translation and JS
# splicing live in `helper-scripts/map_item_converter.py`.
#
# Required secrets (configured in repo Settings -> Secrets and variables -> Actions):
#   DMI_OLLAMA_KEY               - API key for https://ollama.digitalmethods.net
#   ZEESCHUIMER_APP_ID           - numeric App ID of the GitHub App installed on
#                                  digitalmethodsinitiative/zeeschuimer with permissions
#                                  contents:write + pull-requests:write (and nothing else)
#   ZEESCHUIMER_APP_PRIVATE_KEY  - full PEM private key for that App (including BEGIN/END lines)

name: Sync Zeeschuimer map_item from 4CAT

on:
  push:
    branches: [master]
    paths:
      # Only datasource changes drive a push-triggered translation: the detect
      # job's plan-matrix diffs `datasources/**` and nothing else
      - 'datasources/**/search_*.py'
      - '.github/workflows/zeeschuimer_map_item_sync.yml'
  workflow_dispatch:
    inputs:
      bootstrap:
        description: 'Translate every Zeeschuimer datasource (initial sync, single PR). Ignored if "files" is set.'
        type: boolean
        default: false
      files:
        description: 'Space-separated list of datasource files to translate (e.g. "datasources/tiktok/search_tiktok.py"). Overrides bootstrap. One PR per module.'
        type: string
        default: ''
      llm_provider:
        description: 'LLM provider type used by LLMAdapter (default: ollama)'
        type: string
        default: 'ollama'
      llm_base_url:
        description: 'LLM provider URL (default: https://ollama.digitalmethods.net)'
        type: string
        default: 'https://ollama.digitalmethods.net'
      llm_api_key:
        description: 'LLM API key (default uses GitHub secret DMI_OLLAMA_KEY)'
        type: string
        default: ''
      model:
        description: 'Override LLM model on LLM provider (default: qwen2.5-coder:14b). Examples: qwen2.5-coder:7b, deepseek-coder-v2:16b, gemma3:27b'
        type: string
        default: ''

# Least privilege: this workflow's own GITHUB_TOKEN only needs to read the 4CAT
permissions:
  contents: read

jobs:
  detect:
    name: Detect modules to translate
    runs-on: ubuntu-latest
    outputs:
      mode: ${{ steps.plan.outputs.mode }}
      matrix: ${{ steps.plan.outputs.matrix }}
    steps:
      - name: Checkout 4CAT
        uses: actions/checkout@v4
        with:
          # Full history: the push-event plan diffs `github.event.before`
          # against `github.sha`. A shallow clone may not contain `before` for
          # a multi-commit push, in which case the diff resolves to nothing and
          # the change is silently skipped.
          fetch-depth: 0

      - name: Set up Python
        # `detect` runs map_item_ci.py (stdlib only — no LLM deps installed
        # here), but still needs a `python` on PATH; don't rely on the runner
        # image happening to provide one.
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Plan translation matrix
        id: plan
        env:
          EVENT_NAME: ${{ github.event_name }}
          INPUTS_FILES: ${{ inputs.files }}
          INPUTS_BOOTSTRAP: ${{ inputs.bootstrap }}
          BEFORE_SHA: ${{ github.event.before }}
          AFTER_SHA: ${{ github.sha }}
        # Validates paths against a strict datasource shape (dropping anything
        # else) and writes `mode` + `matrix` to $GITHUB_OUTPUT. See
        # helper-scripts/map_item_ci.py.
        run: python helper-scripts/map_item_ci.py plan-matrix

  sync:
    name: Sync ${{ matrix.target.module }}
    needs: detect
    if: needs.detect.outputs.mode != 'none'
    runs-on: ubuntu-latest
    # Per-module concurrency: a newer push to master supersedes any in-flight
    # sync for the same module (LLM run gets cancelled, latest run wins).
    # Each matrix instance gets its own group, so different modules don't block.
    concurrency:
      group: zeeschuimer-sync-${{ matrix.target.module }}
      cancel-in-progress: true
    strategy:
      fail-fast: false
      matrix:
        target: ${{ fromJson(needs.detect.outputs.matrix) }}
    steps:
      - name: Checkout 4CAT
        uses: actions/checkout@v4
        with:
          # Full history so the PR-body builder can `git diff before..after`
          # for the changed Python file (see map_item_ci.py build-pr-body).
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install LLM dependencies
        run: |
          # LLMAdapter (common/lib/llm.py) imports every provider's langchain
          # package at module load, so all of them are required even though we
          # only use the Ollama provider at runtime. Derive the exact list from
          # setup.py (single source of truth) so it can't drift from what 4CAT
          # declares; we install only this LLM subset, not all of 4CAT, to keep
          # the job light. Write the specs to a requirements file (one per line)
          # and install with `-r`, rather than an unquoted `pip install $VAR`:
          # that way a version specifier that contains a shell metacharacter
          # (e.g. a future `langchain_core>=0.3` pin — `>` is redirection) can't
          # be misparsed by the shell.
          python helper-scripts/map_item_ci.py llm-requirements > llm-requirements.txt
          echo "Installing from setup.py:"
          cat llm-requirements.txt
          pip install -r llm-requirements.txt

      - name: Mint Zeeschuimer App token
        id: app_token
        uses: actions/create-github-app-token@v1
        with:
          app-id: ${{ secrets.ZEESCHUIMER_APP_ID }}
          private-key: ${{ secrets.ZEESCHUIMER_APP_PRIVATE_KEY }}
          owner: digitalmethodsinitiative
          repositories: zeeschuimer

      - name: Checkout Zeeschuimer
        uses: actions/checkout@v4
        with:
          repository: digitalmethodsinitiative/zeeschuimer
          path: zeeschuimer-checkout
          token: ${{ steps.app_token.outputs.token }}

      - name: Run translation
        env:
          PROVIDER_API_KEY: ${{ inputs.llm_api_key || secrets.DMI_OLLAMA_KEY }}
          LLM_PROVIDER: ${{ inputs.llm_provider || 'ollama' }}
          LLM_BASE_URL: ${{ inputs.llm_base_url || 'https://ollama.digitalmethods.net' }}
          LLM_MODEL: ${{ inputs.model || 'qwen2.5-coder:14b' }}
          # Pass matrix values through env rather than interpolating. 
          # IS_BOOTSTRAP is always the literal true/false the detect job emitted
          # MODULE_FILES is a list of paths the detect job already validated against 
          # `datasources/<module>/search_<name>.py` shape (no shell
          # metacharacters), so the unquoted `$MODULE_FILES` expansion is safe
          # and still word-splits into multiple --files arguments.
          IS_BOOTSTRAP: ${{ matrix.target.bootstrap }}
          MODULE_FILES: ${{ matrix.target.files }}
        run: |
          if [ "$IS_BOOTSTRAP" = "true" ]; then
            # Bootstrap translates every datasource in one run; --no-fail-fast
            # so one datasource failing doesn't abort the whole initial sync.
            python helper-scripts/map_item_converter.py \
              --bootstrap \
              --no-fail-fast \
              --zeeschuimer-checkout ./zeeschuimer-checkout \
              --output-manifest ./manifest.json
          else
            python helper-scripts/map_item_converter.py \
              --files $MODULE_FILES \
              --zeeschuimer-checkout ./zeeschuimer-checkout \
              --output-manifest ./manifest.json
          fi

      - name: Build PR body
        id: pr_body
        env:
          MODULE: ${{ matrix.target.module }}
          BOOTSTRAP: ${{ matrix.target.bootstrap }}
          BEFORE_SHA: ${{ github.event.before }}
          AFTER_SHA: ${{ github.sha }}
          RUN_ID: ${{ github.run_id }}
          EVENT_NAME: ${{ github.event_name }}
          REPO: ${{ github.repository }}
        # Reads manifest.json, writes pr_body.md, and writes `title` to
        # $GITHUB_OUTPUT (delimiter form, injection-safe). See map_item_ci.py.
        run: python helper-scripts/map_item_ci.py build-pr-body --manifest manifest.json --out pr_body.md

      - name: Check there are JS changes to PR
        id: have_changes
        working-directory: zeeschuimer-checkout
        run: |
          if [ -z "$(git status --porcelain)" ]; then
            echo "has_changes=false" >> "$GITHUB_OUTPUT"
            echo "No JS changes produced by translation; not opening a PR."
          else
            echo "has_changes=true" >> "$GITHUB_OUTPUT"
          fi

      - name: Open or update Zeeschuimer PR
        if: steps.have_changes.outputs.has_changes == 'true'
        # Third-party action that operates with a write token to the Zeeschuimer
        # repo — pinned to a full commit SHA (the v6 release) rather than the
        # mutable `@v6` tag, so a tag move can't silently change what runs here.
        uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6
        with:
          path: zeeschuimer-checkout
          token: ${{ steps.app_token.outputs.token }}
          # Stable per-module branch: a fresh push that retranslates the same
          # module updates the same PR. Different modules never share a branch.
          branch: auto/4cat-map-item-sync-${{ matrix.target.module }}
          title: ${{ steps.pr_body.outputs.title }}
          commit-message: "chore: sync map_item for ${{ matrix.target.module }} from 4CAT ${{ github.sha }}"
          body-path: pr_body.md
          draft: true