gendoc-runtime.yaml

# Copyright 2026 FlagOS Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Generate docs for runtime images

# Post-build: verify each runtime image best-effort, compose a per-image
# description markdown (single source for the docs site, the in-repo
# runtime/<name>.md file, and the Harbor repository description), then
# commit + upload.
#
# Aligned with gendoc-base.yaml: the runtime description refresh uses the same
# upload-artifact transport, per-backend extract jobs on hardware runners, and
# self-healing retry loop. Verify counts in the PR body come from the extract
# jobs' real e2e verification (scripts/verify_runtime.py), not hand-entered
# workflow inputs.
#
# Manual-only (no workflow_run trigger): runtime images are rebuilt repeatedly
# during the FlagGems testing phase, so auto-triggering would produce noisy
# empty PRs.
#
# Transport: each extract job writes an empty version TSV and annotates it with
# metadata headers (run id, verify outcome) via scripts/annotate_version_tsv.py,
# then uploads it as a per-backend artifact (versions-<backend>) with
# upload-artifact. The TSV is a metadata carrier only — the description CONTENT
# is configs-driven: docs/gen_descriptions.py's render_runtime() does not read
# TSV data, so finalize regenerates it fresh from configs.yaml via gen_data.py.
# Accumulate downloads all matching artifacts with download-artifact, merges
# them, and saves the cumulative set to a git state branch
# (auto/versions-<label>-state) so retries don't lose already-collected data.
#
# Self-healing: self-hosted runner connectivity to GitHub is unreliable across
# vendors. The accumulate job collects whatever extractors succeed, saves partial
# results to the state branch, then self-triggers with only the missing backends.
# This loops until all are collected or the retry cap (50) is hit.

on:
  workflow_dispatch:
    inputs:
      backend:
        description: 'Backends to process — "all" or space-separated names'
        type: string
        default: 'all'
      retry_count:
        description: 'Number of retries so far (set by the workflow itself)'
        type: number
        default: 0

permissions:
  contents: write
  pull-requests: write
  actions: write

defaults:
  run:
    shell: bash

jobs:
  authorize:
    # Only accounts listed in .github/builders.txt may trigger this workflow
    # manually; see .github/actions/check-trigger-author for the check.
    # actions/checkout is required: local composite actions are resolved from
    # the workspace, so the repo must exist before the action step runs.
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
      - uses: ./.github/actions/check-trigger-author

  set-matrix:
    needs: authorize
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.m.outputs.matrix }}
    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: 0
      - run: python3 -m pip install --user pyyaml

      - id: m
        env:
          BACKEND: ${{ github.event.inputs.backend || 'all' }}
        run: |
          if [[ "${BACKEND}" == "all" ]]; then
            python3 scripts/generate_matrix.py --runtime > matrix.json
          else
            python3 scripts/generate_matrix.py --runtime ${BACKEND} > matrix.json
          fi
          echo "matrix=$(jq -c . matrix.json)" >> "$GITHUB_OUTPUT"

  extract:
    needs: set-matrix
    # Self-hosted runners are best-effort. When a runner is unreachable
    # (checkout / Harbor login / docker permissions), the job fails here.
    # The accumulate job detects the gap and re-dispatches only that backend.
    continue-on-error: true
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.set-matrix.outputs.matrix) }}
    runs-on: ${{ fromJSON(matrix.runson) }}
    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: 0

      - name: Install PyYAML
        run: |
          python3 -c "import yaml" 2>/dev/null \
            || python3 -m pip install --user --break-system-packages pyyaml

      - name: Harbor login (for image pull)
        env:
          HARBOR_USER: ${{ secrets.HARBOR_USER }}
          HARBOR_PW: ${{ secrets.HARBOR_PASSWORD }}
          # Harbor is internal — bypass any HTTP_PROXY/HTTPS_PROXY the runner
          # may have set for GitHub access, otherwise docker login routes
          # Harbor traffic through the proxy and fails.
          # Only uppercase HTTP_PROXY/HTTPS_PROXY (lowercase clash with runner
          # env vars set by the GitHub Actions runner config).
          HTTP_PROXY: ""
          HTTPS_PROXY: ""
          no_proxy: "harbor.baai.ac.cn,.baai.ac.cn"
        run: |
          host=$(python3 -c "import yaml;print(yaml.safe_load(open('.github/build-config.yml'))['registry']['host'])")
          for i in 1 2 3; do
            if printf '%s' "$HARBOR_PW" | docker login "$host" -u "$HARBOR_USER" --password-stdin; then
              exit 0
            fi
            echo "Harbor login attempt $i failed, retrying in 30s..."
            sleep 30
          done
          echo "::warning::Harbor login failed after 3 attempts — skipping ${{ matrix.name }}"
          exit 1

      - name: Verify runtime image (best-effort)
        continue-on-error: true
        id: verify
        run: |
          python3 docs/gen_data.py
          python3 scripts/verify_runtime.py "${{ matrix.name }}"

      # The TSV is a metadata transport only — description content is
      # configs-driven and regenerated at finalize. An empty TSV still carries
      # the # verify: outcome (real e2e result → PR body counts) and # run: id,
      # and marks the backend as collected (annotate_version_tsv.py requires
      # the file to exist; collect_version_tsvs.py counts .tsv files).
      - name: Create version TSV (transport)
        run: |
          mkdir -p versions
          : > "versions/${{ matrix.name }}.tsv"

      - name: Record verify status
        if: always()
        run: |
          mkdir -p versions
          echo "${{ steps.verify.outcome }}" > "versions/${{ matrix.name }}.verify_outcome"

      - name: Annotate version TSV
        run: python3 scripts/annotate_version_tsv.py "${{ matrix.name }}" versions

      - name: Upload version artifact
        uses: actions/upload-artifact@v7
        with:
          name: versions-${{ matrix.name }}
          path: versions/${{ matrix.name }}.tsv
          if-no-files-found: warn

  accumulate:
    needs: [set-matrix, extract]
    if: always() && needs.set-matrix.result == 'success'
    runs-on: ubuntu-latest
    env:
      GH_TOKEN: ${{ github.token }}
      RETRY_COUNT: ${{ github.event.inputs.retry_count || 0 }}
      MAX_RETRIES: 50
    outputs:
      done: ${{ steps.collect.outputs.done }}
      missing: ${{ steps.collect.outputs.missing }}
      count: ${{ steps.collect.outputs.count }}
      label: ${{ steps.collect.outputs.label }}
      verify_ok: ${{ steps.collect.outputs.verify_ok }}
      verify_fail: ${{ steps.collect.outputs.verify_fail }}
      verify_skip: ${{ steps.collect.outputs.verify_skip }}
    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: 0
      - run: python3 -m pip install --user pyyaml

      # merge-multiple flattens the per-backend artifact contents into
      # versions-dl/<backend>.tsv, which collect_version_tsvs.py reads.
      - name: Download version artifacts
        continue-on-error: true
        uses: actions/download-artifact@v8
        with:
          path: versions-dl
          pattern: versions-*
          merge-multiple: true

      # Restore previously-collected TSVs from the state branch so that
      # backends not refreshed this retry retain their data across runs.
      # On retry=0 the state branch doesn't exist yet — that's fine, we
      # start fresh.
      # For scoped runs, only restore TSVs for the backends being collected.
      - name: Restore state from previous retries
        env:
          GIT_AUTHOR_NAME: flagos-ci
          GIT_AUTHOR_EMAIL: noreply@flagos.net
          GIT_COMMITTER_NAME: flagos-ci
          GIT_COMMITTER_EMAIL: noreply@flagos.net
          SCOPE: ${{ github.event.inputs.backend || 'all' }}
        run: |
          label="$(python3 -c 'import yaml;print(yaml.safe_load(open("configs.yaml"))["version"])' 2>/dev/null || echo 'unknown')"
          state_branch="auto/versions-${label}-state"
          if git ls-remote --heads origin "refs/heads/${state_branch}" | grep -q refs; then
            echo "Restoring state from ${state_branch}"
            git fetch origin "${state_branch}"
            mkdir -p versions
            if [[ "${SCOPE}" == "all" ]]; then
              git checkout FETCH_HEAD -- versions/
            else
              for bk in ${SCOPE}; do
                f="versions/${bk}.tsv"
                if git show "FETCH_HEAD:${f}" > "${f}" 2>/dev/null; then
                  echo "  Restored ${bk}.tsv"
                fi
              done
            fi
          else
            echo "No state branch (${state_branch}) — starting fresh"
            mkdir -p versions
          fi

      # collect: merge TSVs → check completeness.
      # State was restored above; this step overwrites any stale entries
      # with freshly-downloaded data. Always exits 0 — the next step
      # (Finalize or Retry) acts on the done/missing outputs.
      - id: collect
        env:
          GIT_AUTHOR_NAME: flagos-ci
          GIT_AUTHOR_EMAIL: noreply@flagos.net
          GIT_COMMITTER_NAME: flagos-ci
          GIT_COMMITTER_EMAIL: noreply@flagos.net
        run: |
          # Collect: merge TSVs → check completeness.
          # Determine the expected backend set — the same scope set-matrix
          # used, so scoped runs don't auto-retry for backends that were
          # never requested.
          SCOPE="${{ github.event.inputs.backend || 'all' }}"
          if [[ "${SCOPE}" == "all" ]]; then
            EXPECTED=""
          else
            EXPECTED="${SCOPE}"
          fi
          result=$(python3 scripts/collect_version_tsvs.py \
            --versions versions --remote versions-dl \
            --retry "${RETRY_COUNT:-0}" \
            --expected "${EXPECTED}")
          echo "$result"

          # Parse JSON once, write each field to GITHUB_OUTPUT.
          echo "$result" | python3 -c "
          import sys, json
          d = json.load(sys.stdin)
          for k in ('done', 'count', 'label', 'missing',
                    'verify_ok', 'verify_fail', 'verify_skip'):
              print(f'{k}={d[k]}')
          " | tee -a "$GITHUB_OUTPUT"

      # Persist collected TSVs to the state branch so they survive
      # across retries (each accumulate runs on a fresh ubuntu-latest VM).
      # Each save is a full snapshot of versions/ — no incremental merge.
      - name: Save state for next retry
        if: always() && steps.collect.outcome == 'success'
        env:
          GIT_AUTHOR_NAME: flagos-ci
          GIT_AUTHOR_EMAIL: noreply@flagos.net
          GIT_COMMITTER_NAME: flagos-ci
          GIT_COMMITTER_EMAIL: noreply@flagos.net
        run: |
          label="$(python3 -c 'import yaml;print(yaml.safe_load(open("configs.yaml"))["version"])' 2>/dev/null || echo 'unknown')"
          state_branch="auto/versions-${label}-state"

          if [ ! -d versions ] || [ -z "$(ls -A versions 2>/dev/null)" ]; then
            echo "No version data to save"
            exit 0
          fi

          # Stage all TSV files so write-tree captures them (collect step
          # copies them from versions-dl/ — they are untracked).
          git add -f versions/*.tsv 2>/dev/null || true

          # Build a tree of the versions/ directory contents, then wrap it
          # in an outer tree so restore via checkout -- versions/ works.
          inner_tree=$(git write-tree --prefix=versions/)
          if [ -z "$inner_tree" ]; then
            echo "Failed to create inner tree"
            git restore --staged versions/ 2>/dev/null || true
            exit 0
          fi
          outer_tree=$(printf "040000 tree %s\tversions" "$inner_tree" | git mktree)

          # Use previous state commit as parent if it exists.
          parent=""
          if git ls-remote --heads origin "refs/heads/${state_branch}" | grep -q refs; then
            parent=$(git rev-parse "origin/${state_branch}" 2>/dev/null || echo "")
          fi
          parent_opt=""
          if [ -n "$parent" ]; then
            parent_opt="-p $parent"
          fi
          commit=$(echo "state:${label}" | git commit-tree "$outer_tree" $parent_opt)
          git push -f origin "${commit}:refs/heads/${state_branch}"
          echo "Saved state to ${state_branch} (commit ${commit:0:12})"

          # Unstage to keep working tree clean for the finalize step.
          git restore --staged versions/ 2>/dev/null || true

      - name: Finalize (descriptions + PR) or retry
        if: always() && steps.collect.outcome == 'success'
        env:
          GH_TOKEN: ${{ github.token }}
          VERSIONS_DIR: ${{ github.workspace }}/versions
          GITHUB_REF_NAME: ${{ github.ref_name }}
        run: |
          # collect poisons the index (git add -f versions/); unstage it.
          git restore --staged .
          git clean -fd -- versions-dl 2>/dev/null || true

          python3 scripts/finalize_descriptions.py \
            --mode runtime \
            --done "${{ steps.collect.outputs.done }}" \
            --count "${{ steps.collect.outputs.count || 0 }}" \
            --label "${{ steps.collect.outputs.label || 'unknown' }}" \
            --missing "${{ steps.collect.outputs.missing || '' }}" \
            --verify-ok "${{ steps.collect.outputs.verify_ok || 0 }}" \
            --verify-fail "${{ steps.collect.outputs.verify_fail || 0 }}" \
            --verify-skip "${{ steps.collect.outputs.verify_skip || 0 }}" \
            --retry "${{ env.RETRY_COUNT || 0 }}" \
            --max-retries "${MAX_RETRIES:-50}"