Generate docs for runtime images #7
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2026 FlagOS Contributors | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| name: Generate docs for runtime images | |
| # Post-build: verify each runtime image best-effort, compose a per-image | |
| # description markdown (single source for the docs site, the in-repo | |
| # runtime/<name>.md file, and the Harbor repository description), then | |
| # commit + upload. | |
| # | |
| # Aligned with gendoc-base.yaml: the runtime description refresh uses the same | |
| # upload-artifact transport, per-backend extract jobs on hardware runners, and | |
| # self-healing retry loop. Verify counts in the PR body come from the extract | |
| # jobs' real e2e verification (scripts/verify_runtime.py), not hand-entered | |
| # workflow inputs. | |
| # | |
| # Manual-only (no workflow_run trigger): runtime images are rebuilt repeatedly | |
| # during the FlagGems testing phase, so auto-triggering would produce noisy | |
| # empty PRs. | |
| # | |
| # Transport: each extract job writes an empty version TSV and annotates it with | |
| # metadata headers (run id, verify outcome) via scripts/annotate_version_tsv.py, | |
| # then uploads it as a per-backend artifact (versions-<backend>) with | |
| # upload-artifact. The TSV is a metadata carrier only — the description CONTENT | |
| # is configs-driven: docs/gen_descriptions.py's render_runtime() does not read | |
| # TSV data, so finalize regenerates it fresh from configs.yaml via gen_data.py. | |
| # Accumulate downloads all matching artifacts with download-artifact, merges | |
| # them, and saves the cumulative set to a git state branch | |
| # (auto/versions-<label>-state) so retries don't lose already-collected data. | |
| # | |
| # Self-healing: self-hosted runner connectivity to GitHub is unreliable across | |
| # vendors. The accumulate job collects whatever extractors succeed, saves partial | |
| # results to the state branch, then self-triggers with only the missing backends. | |
| # This loops until all are collected or the retry cap (50) is hit. | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| backend: | |
| description: 'Backends to process — "all" or space-separated names' | |
| type: string | |
| default: 'all' | |
| retry_count: | |
| description: 'Number of retries so far (set by the workflow itself)' | |
| type: number | |
| default: 0 | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| actions: write | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| authorize: | |
| # Only accounts listed in .github/builders.txt may trigger this workflow | |
| # manually; see .github/actions/check-trigger-author for the check. | |
| # actions/checkout is required: local composite actions are resolved from | |
| # the workspace, so the repo must exist before the action step runs. | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - uses: ./.github/actions/check-trigger-author | |
| set-matrix: | |
| needs: authorize | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.m.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: 0 | |
| - run: python3 -m pip install --user pyyaml | |
| - id: m | |
| env: | |
| BACKEND: ${{ github.event.inputs.backend || 'all' }} | |
| run: | | |
| if [[ "${BACKEND}" == "all" ]]; then | |
| python3 scripts/generate_matrix.py --runtime > matrix.json | |
| else | |
| python3 scripts/generate_matrix.py --runtime ${BACKEND} > matrix.json | |
| fi | |
| echo "matrix=$(jq -c . matrix.json)" >> "$GITHUB_OUTPUT" | |
| extract: | |
| needs: set-matrix | |
| # Self-hosted runners are best-effort. When a runner is unreachable | |
| # (checkout / Harbor login / docker permissions), the job fails here. | |
| # The accumulate job detects the gap and re-dispatches only that backend. | |
| continue-on-error: true | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.set-matrix.outputs.matrix) }} | |
| runs-on: ${{ fromJSON(matrix.runson) }} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install PyYAML | |
| run: | | |
| python3 -c "import yaml" 2>/dev/null \ | |
| || python3 -m pip install --user --break-system-packages pyyaml | |
| - name: Harbor login (for image pull) | |
| env: | |
| HARBOR_USER: ${{ secrets.HARBOR_USER }} | |
| HARBOR_PW: ${{ secrets.HARBOR_PASSWORD }} | |
| # Harbor is internal — bypass any HTTP_PROXY/HTTPS_PROXY the runner | |
| # may have set for GitHub access, otherwise docker login routes | |
| # Harbor traffic through the proxy and fails. | |
| # Only uppercase HTTP_PROXY/HTTPS_PROXY (lowercase clash with runner | |
| # env vars set by the GitHub Actions runner config). | |
| HTTP_PROXY: "" | |
| HTTPS_PROXY: "" | |
| no_proxy: "harbor.baai.ac.cn,.baai.ac.cn" | |
| run: | | |
| host=$(python3 -c "import yaml;print(yaml.safe_load(open('.github/build-config.yml'))['registry']['host'])") | |
| for i in 1 2 3; do | |
| if printf '%s' "$HARBOR_PW" | docker login "$host" -u "$HARBOR_USER" --password-stdin; then | |
| exit 0 | |
| fi | |
| echo "Harbor login attempt $i failed, retrying in 30s..." | |
| sleep 30 | |
| done | |
| echo "::warning::Harbor login failed after 3 attempts — skipping ${{ matrix.name }}" | |
| exit 1 | |
| - name: Verify runtime image (best-effort) | |
| continue-on-error: true | |
| id: verify | |
| run: | | |
| python3 docs/gen_data.py | |
| python3 scripts/verify_runtime.py "${{ matrix.name }}" | |
| # The TSV is a metadata transport only — description content is | |
| # configs-driven and regenerated at finalize. An empty TSV still carries | |
| # the # verify: outcome (real e2e result → PR body counts) and # run: id, | |
| # and marks the backend as collected (annotate_version_tsv.py requires | |
| # the file to exist; collect_version_tsvs.py counts .tsv files). | |
| - name: Create version TSV (transport) | |
| run: | | |
| mkdir -p versions | |
| : > "versions/${{ matrix.name }}.tsv" | |
| - name: Record verify status | |
| if: always() | |
| run: | | |
| mkdir -p versions | |
| echo "${{ steps.verify.outcome }}" > "versions/${{ matrix.name }}.verify_outcome" | |
| - name: Annotate version TSV | |
| run: python3 scripts/annotate_version_tsv.py "${{ matrix.name }}" versions | |
| - name: Upload version artifact | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: versions-${{ matrix.name }} | |
| path: versions/${{ matrix.name }}.tsv | |
| if-no-files-found: warn | |
| accumulate: | |
| needs: [set-matrix, extract] | |
| if: always() && needs.set-matrix.result == 'success' | |
| runs-on: ubuntu-latest | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| RETRY_COUNT: ${{ github.event.inputs.retry_count || 0 }} | |
| MAX_RETRIES: 50 | |
| outputs: | |
| done: ${{ steps.collect.outputs.done }} | |
| missing: ${{ steps.collect.outputs.missing }} | |
| count: ${{ steps.collect.outputs.count }} | |
| label: ${{ steps.collect.outputs.label }} | |
| verify_ok: ${{ steps.collect.outputs.verify_ok }} | |
| verify_fail: ${{ steps.collect.outputs.verify_fail }} | |
| verify_skip: ${{ steps.collect.outputs.verify_skip }} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: 0 | |
| - run: python3 -m pip install --user pyyaml | |
| # merge-multiple flattens the per-backend artifact contents into | |
| # versions-dl/<backend>.tsv, which collect_version_tsvs.py reads. | |
| - name: Download version artifacts | |
| continue-on-error: true | |
| uses: actions/download-artifact@v8 | |
| with: | |
| path: versions-dl | |
| pattern: versions-* | |
| merge-multiple: true | |
| # Restore previously-collected TSVs from the state branch so that | |
| # backends not refreshed this retry retain their data across runs. | |
| # On retry=0 the state branch doesn't exist yet — that's fine, we | |
| # start fresh. | |
| # For scoped runs, only restore TSVs for the backends being collected. | |
| - name: Restore state from previous retries | |
| env: | |
| GIT_AUTHOR_NAME: flagos-ci | |
| GIT_AUTHOR_EMAIL: noreply@flagos.net | |
| GIT_COMMITTER_NAME: flagos-ci | |
| GIT_COMMITTER_EMAIL: noreply@flagos.net | |
| SCOPE: ${{ github.event.inputs.backend || 'all' }} | |
| run: | | |
| label="$(python3 -c 'import yaml;print(yaml.safe_load(open("configs.yaml"))["version"])' 2>/dev/null || echo 'unknown')" | |
| state_branch="auto/versions-${label}-state" | |
| if git ls-remote --heads origin "refs/heads/${state_branch}" | grep -q refs; then | |
| echo "Restoring state from ${state_branch}" | |
| git fetch origin "${state_branch}" | |
| mkdir -p versions | |
| if [[ "${SCOPE}" == "all" ]]; then | |
| git checkout FETCH_HEAD -- versions/ | |
| else | |
| for bk in ${SCOPE}; do | |
| f="versions/${bk}.tsv" | |
| if git show "FETCH_HEAD:${f}" > "${f}" 2>/dev/null; then | |
| echo " Restored ${bk}.tsv" | |
| fi | |
| done | |
| fi | |
| else | |
| echo "No state branch (${state_branch}) — starting fresh" | |
| mkdir -p versions | |
| fi | |
| # collect: merge TSVs → check completeness. | |
| # State was restored above; this step overwrites any stale entries | |
| # with freshly-downloaded data. Always exits 0 — the next step | |
| # (Finalize or Retry) acts on the done/missing outputs. | |
| - id: collect | |
| env: | |
| GIT_AUTHOR_NAME: flagos-ci | |
| GIT_AUTHOR_EMAIL: noreply@flagos.net | |
| GIT_COMMITTER_NAME: flagos-ci | |
| GIT_COMMITTER_EMAIL: noreply@flagos.net | |
| run: | | |
| # Collect: merge TSVs → check completeness. | |
| # Determine the expected backend set — the same scope set-matrix | |
| # used, so scoped runs don't auto-retry for backends that were | |
| # never requested. | |
| SCOPE="${{ github.event.inputs.backend || 'all' }}" | |
| if [[ "${SCOPE}" == "all" ]]; then | |
| EXPECTED="" | |
| else | |
| EXPECTED="${SCOPE}" | |
| fi | |
| result=$(python3 scripts/collect_version_tsvs.py \ | |
| --versions versions --remote versions-dl \ | |
| --retry "${RETRY_COUNT:-0}" \ | |
| --expected "${EXPECTED}") | |
| echo "$result" | |
| # Parse JSON once, write each field to GITHUB_OUTPUT. | |
| echo "$result" | python3 -c " | |
| import sys, json | |
| d = json.load(sys.stdin) | |
| for k in ('done', 'count', 'label', 'missing', | |
| 'verify_ok', 'verify_fail', 'verify_skip'): | |
| print(f'{k}={d[k]}') | |
| " | tee -a "$GITHUB_OUTPUT" | |
| # Persist collected TSVs to the state branch so they survive | |
| # across retries (each accumulate runs on a fresh ubuntu-latest VM). | |
| # Each save is a full snapshot of versions/ — no incremental merge. | |
| - name: Save state for next retry | |
| if: always() && steps.collect.outcome == 'success' | |
| env: | |
| GIT_AUTHOR_NAME: flagos-ci | |
| GIT_AUTHOR_EMAIL: noreply@flagos.net | |
| GIT_COMMITTER_NAME: flagos-ci | |
| GIT_COMMITTER_EMAIL: noreply@flagos.net | |
| run: | | |
| label="$(python3 -c 'import yaml;print(yaml.safe_load(open("configs.yaml"))["version"])' 2>/dev/null || echo 'unknown')" | |
| state_branch="auto/versions-${label}-state" | |
| if [ ! -d versions ] || [ -z "$(ls -A versions 2>/dev/null)" ]; then | |
| echo "No version data to save" | |
| exit 0 | |
| fi | |
| # Stage all TSV files so write-tree captures them (collect step | |
| # copies them from versions-dl/ — they are untracked). | |
| git add -f versions/*.tsv 2>/dev/null || true | |
| # Build a tree of the versions/ directory contents, then wrap it | |
| # in an outer tree so restore via checkout -- versions/ works. | |
| inner_tree=$(git write-tree --prefix=versions/) | |
| if [ -z "$inner_tree" ]; then | |
| echo "Failed to create inner tree" | |
| git restore --staged versions/ 2>/dev/null || true | |
| exit 0 | |
| fi | |
| outer_tree=$(printf "040000 tree %s\tversions" "$inner_tree" | git mktree) | |
| # Use previous state commit as parent if it exists. | |
| parent="" | |
| if git ls-remote --heads origin "refs/heads/${state_branch}" | grep -q refs; then | |
| parent=$(git rev-parse "origin/${state_branch}" 2>/dev/null || echo "") | |
| fi | |
| parent_opt="" | |
| if [ -n "$parent" ]; then | |
| parent_opt="-p $parent" | |
| fi | |
| commit=$(echo "state:${label}" | git commit-tree "$outer_tree" $parent_opt) | |
| git push -f origin "${commit}:refs/heads/${state_branch}" | |
| echo "Saved state to ${state_branch} (commit ${commit:0:12})" | |
| # Unstage to keep working tree clean for the finalize step. | |
| git restore --staged versions/ 2>/dev/null || true | |
| - name: Finalize (descriptions + PR) or retry | |
| if: always() && steps.collect.outcome == 'success' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| VERSIONS_DIR: ${{ github.workspace }}/versions | |
| GITHUB_REF_NAME: ${{ github.ref_name }} | |
| run: | | |
| # collect poisons the index (git add -f versions/); unstage it. | |
| git restore --staged . | |
| git clean -fd -- versions-dl 2>/dev/null || true | |
| python3 scripts/finalize_descriptions.py \ | |
| --mode runtime \ | |
| --done "${{ steps.collect.outputs.done }}" \ | |
| --count "${{ steps.collect.outputs.count || 0 }}" \ | |
| --label "${{ steps.collect.outputs.label || 'unknown' }}" \ | |
| --missing "${{ steps.collect.outputs.missing || '' }}" \ | |
| --verify-ok "${{ steps.collect.outputs.verify_ok || 0 }}" \ | |
| --verify-fail "${{ steps.collect.outputs.verify_fail || 0 }}" \ | |
| --verify-skip "${{ steps.collect.outputs.verify_skip || 0 }}" \ | |
| --retry "${{ env.RETRY_COUNT || 0 }}" \ | |
| --max-retries "${MAX_RETRIES:-50}" |