heavy-build #32
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: heavy-build | |
| # One job per package, for the packages that cannot share a runner. | |
| # | |
| # build-repo puts ~70 members in a single container. allow-failure contains a | |
| # package that fails, but not one that is OOM-killed or never finishes -- both | |
| # take the container down and stop every other package from updating. A matrix | |
| # gives each of these its own runner, so a death is contained to the package | |
| # that caused it. | |
| # | |
| # They publish into the same custom-repo release as everything else, so this is | |
| # an implementation detail rather than another repo for anyone to add. | |
| # | |
| # Rebuilds are avoided three ways, cheapest first: | |
| # 1. the release itself -- if that exact pkgver is published, skip entirely | |
| # 2. cached sources, so a rebuild does not re-download gigabytes | |
| # 3. ccache, for the ones that genuinely recompile | |
| on: | |
| schedule: | |
| - cron: "20 4 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| packages: | |
| description: "Only these (space separated). Empty means all in the manifest." | |
| required: false | |
| default: "" | |
| force: | |
| description: "Rebuild even if the current version is already published" | |
| type: boolean | |
| default: false | |
| permissions: | |
| contents: write | |
| concurrency: | |
| group: "${{ github.workflow }}" | |
| cancel-in-progress: false | |
| env: | |
| REPO_NAME: custom | |
| RELEASE_TAG: custom-repo | |
| GH_REPO: ${{ github.repository }} | |
| jobs: | |
| plan: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.plan.outputs.matrix }} | |
| count: ${{ steps.plan.outputs.count }} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Ensure PyYAML | |
| # Present on the current runner images, but this job is the gate for | |
| # every build below it and is not worth leaving to chance. | |
| run: python3 -c 'import yaml' 2>/dev/null || pip install --quiet pyyaml | |
| - id: plan | |
| env: | |
| ONLY: ${{ inputs.packages }} | |
| run: | | |
| set -euo pipefail | |
| python3 - <<'PYEOF' >> "$GITHUB_OUTPUT" | |
| import json, os, yaml | |
| pkgs = yaml.safe_load(open(".github/heavy-packages.yaml"))["packages"] | |
| only = (os.environ.get("ONLY") or "").split() | |
| if only: | |
| pkgs = [p for p in pkgs if p["name"] in only] | |
| missing = set(only) - {p["name"] for p in pkgs} | |
| if missing: | |
| raise SystemExit(f"::error::not in the manifest: {' '.join(sorted(missing))}") | |
| out = [{ | |
| "name": p["name"], | |
| "jobs": p.get("jobs", 2), | |
| "timeout": p.get("timeout", 180), | |
| "ccache": bool(p.get("ccache", False)), | |
| "srccache": bool(p.get("srccache", False)), | |
| # Strings, not booleans: a matrix value reaches the shell as text | |
| # either way, and "false" reads better in the job name than False. | |
| "lto": "false" if p.get("lto", True) is False else "true", | |
| "preinstall": " ".join(p.get("preinstall", []) or []), | |
| } for p in pkgs] | |
| print("matrix=" + json.dumps({"include": out})) | |
| print(f"count={len(out)}") | |
| PYEOF | |
| build: | |
| needs: plan | |
| if: needs.plan.outputs.count != '0' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: ${{ matrix.timeout }} | |
| strategy: | |
| # One package's failure must never cancel the others -- that is the whole | |
| # reason these are not in build-repo. | |
| fail-fast: false | |
| # Two at a time. These are the memory-hungry ones and the runner pool is | |
| # shared with build-repo; saturating it just makes everything slower. | |
| max-parallel: 2 | |
| matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Free Disk Space (Ubuntu) | |
| uses: BRAINSia/free-disk-space@v2 | |
| with: | |
| tool-cache: false | |
| mandb: true | |
| android: true | |
| dotnet: true | |
| haskell: true | |
| large-packages: true | |
| docker-images: true | |
| # These are the builds that were being OOM-killed. Keep the swapfile. | |
| swap-storage: false | |
| - name: What is already published? | |
| id: pub | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| set -euo pipefail | |
| assets="$(gh release view "$RELEASE_TAG" --json assets --jq '.assets[].name' 2>/dev/null || true)" | |
| { | |
| echo 'list<<ASSETS' | |
| echo "$assets" | |
| echo 'ASSETS' | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Restore the source cache | |
| if: matrix.srccache | |
| uses: actions/cache@v6 | |
| with: | |
| path: cache/src | |
| # Sources change with the package, not with the run, so a single | |
| # restore-key per package keeps picking up the last one. | |
| key: src-${{ matrix.name }}-${{ github.run_id }} | |
| restore-keys: src-${{ matrix.name }}- | |
| - name: Restore the compiler cache | |
| if: matrix.ccache | |
| uses: actions/cache@v6 | |
| with: | |
| path: cache/ccache | |
| key: ccache-${{ matrix.name }}-${{ github.run_id }} | |
| restore-keys: ccache-${{ matrix.name }}- | |
| - name: Build ${{ matrix.name }} | |
| id: build | |
| env: | |
| SKIP_IF: ${{ inputs.force && '' || steps.pub.outputs.list }} | |
| run: | | |
| set -euo pipefail | |
| mkdir -p out cache/src cache/ccache | |
| docker run --rm \ | |
| -v "${{ github.workspace }}:/work" \ | |
| -e MAKE_JOBS="${{ matrix.jobs }}" \ | |
| -e USE_CCACHE="${{ matrix.ccache }}" \ | |
| -e USE_LTO="${{ matrix.lto }}" \ | |
| -e PREINSTALL="${{ matrix.preinstall }}" \ | |
| -e SKIP_IF \ | |
| -e GH_REPO \ | |
| -e RELEASE_TAG \ | |
| -e GITHUB_OUTPUT=/work/.gh-output \ | |
| -w /work \ | |
| archlinux:base-devel \ | |
| bash .github/scripts/build-one.sh "${{ matrix.name }}" | |
| [ -f .gh-output ] && cat .gh-output >> "$GITHUB_OUTPUT" || true | |
| - name: Sign and publish | |
| if: steps.build.outputs.skipped != 'true' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} | |
| GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} | |
| run: | | |
| set -euo pipefail | |
| shopt -s nullglob | |
| built=(out/*.pkg.tar.zst) | |
| if [ ${#built[@]} -eq 0 ]; then | |
| echo "Nothing produced." | |
| exit 0 | |
| fi | |
| # Sign only what this job built. The directory holds no database, so | |
| # sign-pacman-repo.sh has nothing else to touch. | |
| bash .github/scripts/sign-pacman-repo.sh out "$REPO_NAME" | |
| uploads=(out/*.pkg.tar.zst out/*.sig) | |
| printf 'uploading: %s\n' "${uploads[@]}" | |
| gh release upload "$RELEASE_TAG" "${uploads[@]}" --clobber | |
| - name: Record what was published | |
| if: steps.build.outputs.skipped != 'true' | |
| run: | | |
| shopt -s nullglob | |
| mkdir -p manifest | |
| for f in out/*.pkg.tar.zst; do basename "$f"; done > "manifest/${{ matrix.name }}.txt" | |
| cat "manifest/${{ matrix.name }}.txt" || true | |
| - name: Upload the manifest | |
| if: steps.build.outputs.skipped != 'true' | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: built-${{ matrix.name }} | |
| path: manifest/ | |
| if-no-files-found: ignore | |
| # Separate, and single. Every build job could repo-add its own result, but | |
| # they would race on one database and the last writer would drop the others. | |
| index: | |
| needs: build | |
| if: always() && needs.build.result != 'skipped' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Collect what the matrix published | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: built-* | |
| path: manifests | |
| merge-multiple: true | |
| - name: Add the new packages to the database | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} | |
| GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} | |
| run: | | |
| set -euo pipefail | |
| shopt -s nullglob | |
| mapfile -t NEW < <(cat manifests/*.txt 2>/dev/null | sort -u) | |
| echo "Built this run: ${#NEW[@]} package(s)" | |
| mkdir -p work && cd work | |
| # The database, plus only the packages being added. repo-add keeps the | |
| # entries of packages that are not present locally, so there is no | |
| # reason to pull down the whole repo. | |
| for f in "${REPO_NAME}.db.tar.gz" "${REPO_NAME}.files.tar.gz"; do | |
| gh release download "$RELEASE_TAG" --pattern "$f" --clobber || true | |
| done | |
| # Tolerant on purpose. This job is the only thing that puts any of | |
| # the matrix's work into the database, and a single missing asset | |
| # used to abort it under set -e -- losing every other package built | |
| # this run, not just the missing one. Add what is there and say what | |
| # is not. | |
| have=() | |
| for f in "${NEW[@]}"; do | |
| if gh release download "$RELEASE_TAG" --pattern "$f" --clobber; then | |
| have+=( "$f" ) | |
| else | |
| echo "::warning::$f was uploaded by its build job but is not on the release now; skipping it" | |
| fi | |
| done | |
| if [ ${#have[@]} -eq 0 ] && [ ${#NEW[@]} -gt 0 ]; then | |
| echo "::error::none of the ${#NEW[@]} newly built package(s) could be downloaded" | |
| exit 1 | |
| fi | |
| # Re-add any published package the database has lost track of. | |
| # | |
| # This job used to stop early when nothing was built, on the reasoning | |
| # that the database must then already be correct. It is not always: a | |
| # build-repo run that reads the database before this workflow | |
| # publishes, and writes it back afterwards, silently drops whatever | |
| # landed in between. Nothing else repairs that -- on the next run | |
| # every package is already published, so nothing is built, and the | |
| # loss becomes permanent rather than self-healing. | |
| # | |
| # The release is the ground truth for what exists; the database is | |
| # only an index of it. Anything published that the database does not | |
| # name gets added back. | |
| mapfile -t INDEXED < <(python3 ../.github/scripts/db-entries.py "$REPO_NAME") | |
| indexed=$'\n' | |
| for f in "${INDEXED[@]}"; do indexed+="${f}"$'\n'; done | |
| seen=$'\n' | |
| for f in "${NEW[@]}"; do seen+="${f}"$'\n'; done | |
| orphans=() | |
| while read -r asset; do | |
| case "$asset" in *.pkg.tar.zst|*.pkg.tar.xz) ;; *) continue ;; esac | |
| grep -qxF "$asset" <<<"$indexed" && continue | |
| grep -qxF "$asset" <<<"$seen" && continue | |
| orphans+=( "$asset" ) | |
| done < <(gh release view "$RELEASE_TAG" --json assets --jq '.assets[].name') | |
| for f in "${orphans[@]}"; do | |
| if gh release download "$RELEASE_TAG" --pattern "$f" --clobber; then | |
| echo " recovering $f, published but absent from the database" | |
| have+=( "$f" ) | |
| else | |
| echo "::warning::could not download $f to re-index it" | |
| fi | |
| done | |
| if [ ${#have[@]} -eq 0 ]; then | |
| echo "Nothing built and nothing missing; the database is already correct." | |
| exit 0 | |
| fi | |
| echo "Adding ${#have[@]} package(s) to the database." | |
| docker run --rm -v "$PWD:/w" -w /w archlinux:base-devel bash -c ' | |
| set -e | |
| # Without this the unmatched *.pkg.tar.xz reaches repo-add as a | |
| # literal path and set -e ends the run. | |
| shopt -s nullglob | |
| pacman -Sy --noconfirm --needed pacman-contrib >/dev/null 2>&1 || true | |
| repo-add --quiet --nocolor "'"${REPO_NAME}"'.db.tar.gz" *.pkg.tar.zst *.pkg.tar.xz | |
| for ext in db files; do | |
| rm -f "'"${REPO_NAME}"'.$ext" | |
| cp "'"${REPO_NAME}"'.$ext.tar.gz" "'"${REPO_NAME}"'.$ext" | |
| done | |
| chmod -R a+rwX . | |
| ' | |
| # The packages this job downloaded were already renamed by build-one.sh, | |
| # so repo-add recorded them correctly. The seeded database can still | |
| # carry a colon from before that fix, though, so run the sanitizer over | |
| # the result -- and run it before signing, since it rewrites the | |
| # database. | |
| python3 ../.github/scripts/sanitize-epoch-filenames.py . "$REPO_NAME" | |
| # Re-sign: the database changed, so its old signature no longer matches | |
| # and pacman rejects a bad signature harder than a missing one. | |
| rm -f ./*.db.sig ./*.files.sig ./*.db.tar.gz.sig ./*.files.tar.gz.sig | |
| bash ../.github/scripts/sign-pacman-repo.sh . "$REPO_NAME" | |
| # Build the list from what exists. nullglob covers the globs but not | |
| # the four literal names, and `gh release upload` fails the whole | |
| # command on one missing path -- that is how a finished build has | |
| # been thrown away before. | |
| uploads=() | |
| for f in "${REPO_NAME}.db" "${REPO_NAME}.files" \ | |
| "${REPO_NAME}.db.tar.gz" "${REPO_NAME}.files.tar.gz"; do | |
| [ -f "$f" ] && uploads+=( "$f" ) | |
| [ -f "$f.sig" ] && uploads+=( "$f.sig" ) | |
| done | |
| if [ ${#uploads[@]} -eq 0 ]; then | |
| echo "::error::repo-add produced no database to upload" | |
| exit 1 | |
| fi | |
| printf 'uploading: %s\n' "${uploads[@]}" | |
| gh release upload "$RELEASE_TAG" "${uploads[@]}" --clobber | |
| { | |
| echo "### heavy-build" | |
| echo | |
| printf -- '- %s\n' "${NEW[@]}" | |
| } >> "$GITHUB_STEP_SUMMARY" |