nightly-build #352
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: nightly-build | ||
| on: | ||
| schedule: | ||
| - cron: '35 8 * * *' # 08:35 UTC (01:35 PST / 04:35 EST) — original nightly | ||
| - cron: '0 0 * * *' # 00:00 UTC (~5:00 PM PT during DST) — tests-only preflight | ||
| workflow_dispatch: | ||
| inputs: | ||
| skip_buildkite: | ||
| description: 'Skip Buildkite tests (smoke, backward-compat, etc.)' | ||
| required: false | ||
| default: false | ||
| type: boolean | ||
| skip_test_pypi: | ||
| description: 'Skip publishing to Test PyPI' | ||
| required: false | ||
| default: false | ||
| type: boolean | ||
| jobs: | ||
| check-date: | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| should_run: ${{ steps.compute_should_run.outputs.should_run }} | ||
| is_preflight: ${{ steps.compute_should_run.outputs.is_preflight }} | ||
| steps: | ||
| - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 | ||
| - name: print latest_commit | ||
| run: echo ${{ github.sha }} | ||
| - id: compute_should_run | ||
| name: determine whether nightly should run | ||
| shell: bash | ||
| run: | | ||
| if [[ "${{ github.event_name }}" == "schedule" ]]; then | ||
| if git rev-list --after="24 hours" ${{ github.sha }} | grep -q .; then | ||
| echo "should_run=true" >> "$GITHUB_OUTPUT" | ||
| else | ||
| echo "should_run=false" >> "$GITHUB_OUTPUT" | ||
| fi | ||
| # The 0 0 * * * schedule is the tests-only preflight; it should not publish or release | ||
| if [[ "${{ github.event.schedule }}" == "0 0 * * *" ]]; then | ||
| echo "is_preflight=true" >> "$GITHUB_OUTPUT" | ||
| else | ||
| echo "is_preflight=false" >> "$GITHUB_OUTPUT" | ||
| fi | ||
| else | ||
| # For manual dispatches, always run and never treat as preflight | ||
| echo "should_run=true" >> "$GITHUB_OUTPUT" | ||
| echo "is_preflight=false" >> "$GITHUB_OUTPUT" | ||
| fi | ||
| gate-tests: | ||
| runs-on: ubuntu-latest | ||
| needs: [check-date] | ||
|
Check failure on line 53 in .github/workflows/nightly-build.yml
|
||
| outputs: | ||
| run_tests: ${{ steps.compute_gate.outputs.run_tests }} | ||
| publish_without_tests: ${{ steps.compute_gate.outputs.publish_without_tests }} | ||
| steps: | ||
| - id: compute_gate | ||
| name: compute run_tests gate for smoke tests | ||
| shell: bash | ||
| env: | ||
| EVENT: ${{ github.event_name }} | ||
| SKIP_BUILDKITE: ${{ inputs.skip_buildkite }} | ||
| SHOULD_RUN: ${{ needs.check-date.outputs.should_run }} | ||
| run: | | ||
| if [[ "$EVENT" == "workflow_dispatch" ]]; then | ||
| if [[ "$SKIP_BUILDKITE" == "true" ]]; then | ||
| echo "run_tests=false" >> "$GITHUB_OUTPUT" | ||
| echo "publish_without_tests=true" >> "$GITHUB_OUTPUT" | ||
| else | ||
| echo "run_tests=true" >> "$GITHUB_OUTPUT" | ||
| echo "publish_without_tests=false" >> "$GITHUB_OUTPUT" | ||
| fi | ||
| else | ||
| # For scheduled runs, rely on check-date | ||
| if [[ "$SHOULD_RUN" == "true" ]]; then | ||
| echo "run_tests=true" >> "$GITHUB_OUTPUT" | ||
| echo "publish_without_tests=false" >> "$GITHUB_OUTPUT" | ||
| else | ||
| echo "run_tests=false" >> "$GITHUB_OUTPUT" | ||
| echo "publish_without_tests=false" >> "$GITHUB_OUTPUT" | ||
| fi | ||
| fi | ||
| nightly-build-pypi: | ||
| runs-on: ubuntu-latest | ||
| needs: check-date | ||
| if: ${{ needs.check-date.outputs.should_run != 'false' }} | ||
| outputs: | ||
| version: ${{ steps.set-release-version.outputs.version }} | ||
| steps: | ||
| - name: Clone repository | ||
| uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 | ||
| - name: Set up Node.js | ||
| uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 | ||
| with: | ||
| node-version: '23' | ||
| cache: 'npm' | ||
| cache-dependency-path: 'sky/dashboard/package-lock.json' | ||
| - name: Install dashboard dependencies | ||
| run: | | ||
| cd sky/dashboard | ||
| npm ci | ||
| - name: Build dashboard | ||
| run: | | ||
| cd sky/dashboard | ||
| npm run build | ||
| - name: Set up Python 3.10 | ||
| uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4.9.1 | ||
| with: | ||
| python-version: '3.10' | ||
| cache: 'pip' # caching pip dependencies | ||
| - name: Install pypa/build | ||
| run: >- | ||
| python -m | ||
| pip install | ||
| build | ||
| --user | ||
| - name: Set release version | ||
| id: set-release-version | ||
| run: | | ||
| RELEASE_VERSION=$(date +%Y%m%d) | ||
| sed -i "s/{{SKYPILOT_COMMIT_SHA}}/${{ github.sha }}/g" sky/__init__.py | ||
| sed -i "s/__version__ = '.*'/__version__ = '1.0.0.dev${RELEASE_VERSION}'/g" sky/__init__.py | ||
| sed -i "s/name='skypilot',/name='skypilot-nightly',/g" sky/setup_files/setup.py | ||
| echo "version=1.0.0.dev${RELEASE_VERSION}" >> $GITHUB_OUTPUT | ||
| echo "Version: 1.0.0.dev${RELEASE_VERSION}" | ||
| - name: Build a binary wheel and a source tarball | ||
| run: >- | ||
| python -m | ||
| build | ||
| --sdist | ||
| --wheel | ||
| --outdir dist/ | ||
| . | ||
| - name: Upload distribution artifacts | ||
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | ||
| with: | ||
| name: skypilot-artifacts-skypilot-nightly | ||
| path: dist/ | ||
| smoke-tests-aws: | ||
| needs: [gate-tests, nightly-build-pypi] | ||
| if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | ||
| uses: ./.github/workflows/buildkite-trigger-wait.yml | ||
| with: | ||
| commit: ${{ github.sha }} | ||
| branch: ${{ github.ref_name }} | ||
| message: "nightly-build-pypi --aws" | ||
| pipeline: "smoke-tests" | ||
| build_env_vars: '{"ARGS": "--aws"}' | ||
| timeout_minutes: 90 # Increased from 60 (actual: ~70min) | ||
| secrets: | ||
| BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | ||
| backward-compat-test-nightly: | ||
| needs: [gate-tests, nightly-build-pypi] | ||
| if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | ||
| uses: ./.github/workflows/buildkite-trigger-wait.yml | ||
| with: | ||
| commit: ${{ github.sha }} | ||
| branch: ${{ github.ref_name }} | ||
| message: "nightly-build-pypi backward compatibility test --base-branch pypi/skypilot-nightly" | ||
| pipeline: "quicktest-core" | ||
| build_env_vars: '{"ARGS": "--kubernetes --base-branch pypi/skypilot-nightly"}' | ||
| timeout_minutes: 60 | ||
| secrets: | ||
| BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | ||
| backward-compat-test-stable: | ||
| needs: [gate-tests, nightly-build-pypi] | ||
| if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | ||
| uses: ./.github/workflows/buildkite-trigger-wait.yml | ||
| with: | ||
| commit: ${{ github.sha }} | ||
| branch: ${{ github.ref_name }} | ||
| message: "nightly-build-pypi backward compatibility test --base-branch pypi/skypilot" | ||
| pipeline: "quicktest-core" | ||
| build_env_vars: '{"ARGS": "--base-branch pypi/skypilot"}' | ||
| timeout_minutes: 60 | ||
| secrets: | ||
| BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | ||
| smoke-tests-kubernetes-resource-heavy: | ||
| needs: [gate-tests, nightly-build-pypi] | ||
| if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | ||
| uses: ./.github/workflows/buildkite-trigger-wait.yml | ||
| with: | ||
| commit: ${{ github.sha }} | ||
| branch: ${{ github.ref_name }} | ||
| message: "nightly-build-pypi --kubernetes --resource-heavy" | ||
| pipeline: "smoke-tests" | ||
| build_env_vars: '{"ARGS": "--kubernetes --resource-heavy"}' | ||
| timeout_minutes: 100 | ||
| secrets: | ||
| BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | ||
| smoke-tests-kubernetes-no-resource-heavy: | ||
| needs: [gate-tests, nightly-build-pypi] | ||
| if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | ||
| uses: ./.github/workflows/buildkite-trigger-wait.yml | ||
| with: | ||
| commit: ${{ github.sha }} | ||
| branch: ${{ github.ref_name }} | ||
| message: "nightly-build-pypi --kubernetes --no-resource-heavy" | ||
| pipeline: "smoke-tests" | ||
| build_env_vars: '{"ARGS": "--kubernetes --no-resource-heavy"}' | ||
| timeout_minutes: 180 # Increased from 80 (actual: ~163min with queue delays) | ||
| secrets: | ||
| BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | ||
| smoke-tests-kubernetes-no-resource-heavy-limit-deps: | ||
| needs: [gate-tests, nightly-build-pypi] | ||
| if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | ||
| uses: ./.github/workflows/buildkite-trigger-wait.yml | ||
| with: | ||
| commit: ${{ github.sha }} | ||
| branch: ${{ github.ref_name }} | ||
| message: "nightly-build-pypi --kubernetes --no-resource-heavy --dependency" | ||
| pipeline: "smoke-tests" | ||
| build_env_vars: '{"ARGS": "--kubernetes --no-resource-heavy --dependency"}' | ||
| timeout_minutes: 120 # Increased from 80 (actual: ~99min) | ||
| secrets: | ||
| BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | ||
| smoke-tests-remote-server-kubernetes: | ||
| needs: [gate-tests, nightly-build-pypi] | ||
| if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | ||
| uses: ./.github/workflows/buildkite-trigger-wait.yml | ||
| with: | ||
| commit: ${{ github.sha }} | ||
| branch: ${{ github.ref_name }} | ||
| message: "nightly-build-pypi --remote-server --kubernetes" | ||
| pipeline: "smoke-tests" | ||
| build_env_vars: '{"ARGS": "--remote-server --kubernetes"}' | ||
| timeout_minutes: 120 # Increased from 80 (actual: ~97min) | ||
| sleep_seconds: 600 # 10 minute delay to reduce buildkite resource usage | ||
| secrets: | ||
| BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | ||
| smoke-tests-kubernetes-jobs-consolidation: | ||
| needs: [gate-tests, nightly-build-pypi] | ||
| if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | ||
| uses: ./.github/workflows/buildkite-trigger-wait.yml | ||
| with: | ||
| commit: ${{ github.sha }} | ||
| branch: ${{ github.ref_name }} | ||
| message: "nightly-build-pypi --kubernetes --jobs-consolidation --no-resource-heavy" | ||
| pipeline: "smoke-tests" | ||
| build_env_vars: '{"ARGS": "--kubernetes --jobs-consolidation --no-resource-heavy"}' | ||
| timeout_minutes: 90 # Increased from 60 (actual: ~79min) | ||
| secrets: | ||
| BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | ||
| smoke-tests-shared-gke-api-server: | ||
| needs: [gate-tests, nightly-build-pypi] | ||
| if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | ||
| uses: ./.github/workflows/buildkite-trigger-wait.yml | ||
| with: | ||
| commit: ${{ github.sha }} | ||
| branch: ${{ github.ref_name }} | ||
| message: "nightly-build-pypi shared-gke-api-server with postgres tests" | ||
| pipeline: "nightly-build-shared-gke-api-server" | ||
| timeout_minutes: 180 # Increased from 120 to account for queue delays | ||
| secrets: | ||
| BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | ||
| smoke-tests-slurm-minimal: | ||
| needs: [gate-tests, nightly-build-pypi] | ||
| if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | ||
| uses: ./.github/workflows/buildkite-trigger-wait.yml | ||
| with: | ||
| commit: ${{ github.sha }} | ||
| branch: ${{ github.ref_name }} | ||
| message: "nightly-build-pypi test_minimal --slurm" | ||
| pipeline: "smoke-tests" | ||
| build_env_vars: '{"ARGS": "-k test_minimal --slurm", "FILE_PATTERN": "test_basic.py"}' | ||
| timeout_minutes: 60 | ||
| secrets: | ||
| BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | ||
| # Disable runpod until #8293 is fixed. | ||
| # smoke-tests-runpod-minimal: | ||
| # needs: [gate-tests, nightly-build-pypi] | ||
| # if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | ||
| # uses: ./.github/workflows/buildkite-trigger-wait.yml | ||
| # with: | ||
| # commit: ${{ github.sha }} | ||
| # branch: ${{ github.ref_name }} | ||
| # message: "nightly-build-pypi test_minimal --runpod" | ||
| # pipeline: "smoke-tests" | ||
| # build_env_vars: '{"ARGS": "-k test_minimal --runpod", "FILE_PATTERN": "test_basic.py"}' | ||
| # timeout_minutes: 60 | ||
| # secrets: | ||
| # BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | ||
| publish-and-validate-both: | ||
| needs: [check-date, gate-tests, nightly-build-pypi, smoke-tests-aws, smoke-tests-kubernetes-resource-heavy, smoke-tests-kubernetes-no-resource-heavy, smoke-tests-kubernetes-no-resource-heavy-limit-deps, smoke-tests-remote-server-kubernetes, smoke-tests-kubernetes-jobs-consolidation, smoke-tests-shared-gke-api-server, smoke-tests-slurm-minimal, backward-compat-test-nightly, backward-compat-test-stable] | ||
| # Allow publish/validate for manual dispatch or the original nightly cron; skip for the 5PM PT preflight | ||
| # Use always() so this job evaluates even if some test jobs were skipped when skip_buildkite is selected | ||
| if: ${{ always() && needs.check-date.outputs.is_preflight != 'true' && needs.nightly-build-pypi.result == 'success' && (needs.gate-tests.outputs.publish_without_tests == 'true' || (needs.gate-tests.outputs.run_tests == 'true' && needs.smoke-tests-aws.result == 'success' && needs.smoke-tests-kubernetes-resource-heavy.result == 'success' && needs.smoke-tests-kubernetes-no-resource-heavy.result == 'success' && needs.smoke-tests-kubernetes-no-resource-heavy-limit-deps.result == 'success' && needs.smoke-tests-remote-server-kubernetes.result == 'success' && needs.smoke-tests-kubernetes-jobs-consolidation.result == 'success' && needs.smoke-tests-shared-gke-api-server.result == 'success' && needs.smoke-tests-slurm-minimal.result == 'success' && needs.backward-compat-test-nightly.result == 'success' && needs.backward-compat-test-stable.result == 'success')) }} | ||
| uses: ./.github/workflows/publish-and-validate-both.yml | ||
| with: | ||
| package_name: skypilot-nightly | ||
| expected_version: ${{ needs.nightly-build-pypi.outputs.version }} | ||
| skip_test_pypi: ${{ github.event_name == 'workflow_dispatch' && inputs.skip_test_pypi || false }} | ||
| secrets: inherit | ||
| trigger-docker-and-helm-release: | ||
| needs: [check-date, publish-and-validate-both] | ||
| # Allow helm release for manual dispatch or the original nightly cron; skip for the 5PM PT preflight | ||
| if: ${{ always() && needs.check-date.outputs.is_preflight != 'true' && needs.publish-and-validate-both.result == 'success' }} | ||
| uses: ./.github/workflows/helm-docker-release.yaml | ||
| with: | ||
| package_name: skypilot-nightly | ||
| secrets: inherit | ||
| summary: | ||
| runs-on: ubuntu-latest | ||
| needs: [check-date, nightly-build-pypi, smoke-tests-aws, smoke-tests-kubernetes-resource-heavy, smoke-tests-kubernetes-no-resource-heavy, smoke-tests-kubernetes-no-resource-heavy-limit-deps, smoke-tests-remote-server-kubernetes, smoke-tests-kubernetes-jobs-consolidation, smoke-tests-shared-gke-api-server, smoke-tests-slurm-minimal, backward-compat-test-nightly, backward-compat-test-stable] | ||
| if: always() | ||
| steps: | ||
| - name: Summary | ||
| run: | | ||
| cat <<EOF >> "$GITHUB_STEP_SUMMARY" | ||
| # Nightly Build Summary | ||
| ## Buildkite Test Links | ||
| EOF | ||
| if [ "${{ needs.smoke-tests-aws.result }}" != "skipped" ] && [ -n "${{ needs.smoke-tests-aws.outputs.build_number }}" ]; then | ||
| cat <<EOF >> "$GITHUB_STEP_SUMMARY" | ||
| - [Smoke Tests AWS](https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-aws.outputs.build_number }}) - $([ "${{ needs.smoke-tests-aws.outputs.build_status }}" == "success" ] && echo "✅ Success" || echo "❌ Failed") | ||
| EOF | ||
| fi | ||
| if [ "${{ needs.smoke-tests-kubernetes-resource-heavy.result }}" != "skipped" ] && [ -n "${{ needs.smoke-tests-kubernetes-resource-heavy.outputs.build_number }}" ]; then | ||
| cat <<EOF >> "$GITHUB_STEP_SUMMARY" | ||
| - [Smoke Tests Kubernetes (Resource Heavy)](https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-kubernetes-resource-heavy.outputs.build_number }}) - $([ "${{ needs.smoke-tests-kubernetes-resource-heavy.outputs.build_status }}" == "success" ] && echo "✅ Success" || echo "❌ Failed") | ||
| EOF | ||
| fi | ||
| if [ "${{ needs.smoke-tests-kubernetes-no-resource-heavy.result }}" != "skipped" ] && [ -n "${{ needs.smoke-tests-kubernetes-no-resource-heavy.outputs.build_number }}" ]; then | ||
| cat <<EOF >> "$GITHUB_STEP_SUMMARY" | ||
| - [Smoke Tests Kubernetes (No Resource Heavy)](https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-kubernetes-no-resource-heavy.outputs.build_number }}) - $([ "${{ needs.smoke-tests-kubernetes-no-resource-heavy.outputs.build_status }}" == "success" ] && echo "✅ Success" || echo "❌ Failed") | ||
| EOF | ||
| fi | ||
| if [ "${{ needs.smoke-tests-kubernetes-no-resource-heavy-limit-deps.result }}" != "skipped" ] && [ -n "${{ needs.smoke-tests-kubernetes-no-resource-heavy-limit-deps.outputs.build_number }}" ]; then | ||
| cat <<EOF >> "$GITHUB_STEP_SUMMARY" | ||
| - [Smoke Tests Kubernetes (No Resource Heavy, Limit Deps)](https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-kubernetes-no-resource-heavy-limit-deps.outputs.build_number }}) - $([ "${{ needs.smoke-tests-kubernetes-no-resource-heavy-limit-deps.outputs.build_status }}" == "success" ] && echo "✅ Success" || echo "❌ Failed") | ||
| EOF | ||
| fi | ||
| if [ "${{ needs.smoke-tests-remote-server-kubernetes.result }}" != "skipped" ] && [ -n "${{ needs.smoke-tests-remote-server-kubernetes.outputs.build_number }}" ]; then | ||
| cat <<EOF >> "$GITHUB_STEP_SUMMARY" | ||
| - [Smoke Tests Remote Server Kubernetes](https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-remote-server-kubernetes.outputs.build_number }}) - $([ "${{ needs.smoke-tests-remote-server-kubernetes.outputs.build_status }}" == "success" ] && echo "✅ Success" || echo "❌ Failed") | ||
| EOF | ||
| fi | ||
| if [ "${{ needs.smoke-tests-kubernetes-jobs-consolidation.result }}" != "skipped" ] && [ -n "${{ needs.smoke-tests-kubernetes-jobs-consolidation.outputs.build_number }}" ]; then | ||
| cat <<EOF >> "$GITHUB_STEP_SUMMARY" | ||
| - [Smoke Tests Kubernetes (Jobs Consolidation)](https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-kubernetes-jobs-consolidation.outputs.build_number }}) - $([ "${{ needs.smoke-tests-kubernetes-jobs-consolidation.outputs.build_status }}" == "success" ] && echo "✅ Success" || echo "❌ Failed") | ||
| EOF | ||
| fi | ||
| if [ "${{ needs.smoke-tests-shared-gke-api-server.result }}" != "skipped" ] && [ -n "${{ needs.smoke-tests-shared-gke-api-server.outputs.build_number }}" ]; then | ||
| cat <<EOF >> "$GITHUB_STEP_SUMMARY" | ||
| - [Smoke Tests Shared GKE API Server](https://buildkite.com/skypilot-1/nightly-build-shared-gke-api-server/builds/${{ needs.smoke-tests-shared-gke-api-server.outputs.build_number }}) - $([ "${{ needs.smoke-tests-shared-gke-api-server.outputs.build_status }}" == "success" ] && echo "✅ Success" || echo "❌ Failed") | ||
| EOF | ||
| fi | ||
| if [ "${{ needs.smoke-tests-slurm-minimal.result }}" != "skipped" ] && [ -n "${{ needs.smoke-tests-slurm-minimal.outputs.build_number }}" ]; then | ||
| cat <<EOF >> "$GITHUB_STEP_SUMMARY" | ||
| - [Smoke Tests Slurm Minimal](https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-slurm-minimal.outputs.build_number }}) - $([ "${{ needs.smoke-tests-slurm-minimal.outputs.build_status }}" == "success" ] && echo "✅ Success" || echo "❌ Failed") | ||
| EOF | ||
| fi | ||
| # if [ "${{ needs.smoke-tests-runpod-minimal.result }}" != "skipped" ] && [ -n "${{ needs.smoke-tests-runpod-minimal.outputs.build_number }}" ]; then | ||
| # cat <<EOF >> "$GITHUB_STEP_SUMMARY" | ||
| # - [Smoke Tests RunPod Minimal](https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-runpod-minimal.outputs.build_number }}) - $([ "${{ needs.smoke-tests-runpod-minimal.outputs.build_status }}" == "success" ] && echo "✅ Success" || echo "❌ Failed") | ||
| # EOF | ||
| # fi | ||
| if [ "${{ needs.backward-compat-test-nightly.result }}" != "skipped" ] && [ -n "${{ needs.backward-compat-test-nightly.outputs.build_number }}" ]; then | ||
| cat <<EOF >> "$GITHUB_STEP_SUMMARY" | ||
| - [Backward Compat Test (Nightly)](https://buildkite.com/skypilot-1/quicktest-core/builds/${{ needs.backward-compat-test-nightly.outputs.build_number }}) - $([ "${{ needs.backward-compat-test-nightly.outputs.build_status }}" == "success" ] && echo "✅ Success" || echo "❌ Failed") | ||
| EOF | ||
| fi | ||
| if [ "${{ needs.backward-compat-test-stable.result }}" != "skipped" ] && [ -n "${{ needs.backward-compat-test-stable.outputs.build_number }}" ]; then | ||
| cat <<EOF >> "$GITHUB_STEP_SUMMARY" | ||
| - [Backward Compat Test (Stable)](https://buildkite.com/skypilot-1/quicktest-core/builds/${{ needs.backward-compat-test-stable.outputs.build_number }}) - $([ "${{ needs.backward-compat-test-stable.outputs.build_status }}" == "success" ] && echo "✅ Success" || echo "❌ Failed") | ||
| EOF | ||
| fi | ||
| notify-slack-failure: | ||
| runs-on: ubuntu-latest | ||
| needs: [check-date, nightly-build-pypi, smoke-tests-aws, smoke-tests-kubernetes-resource-heavy, smoke-tests-kubernetes-no-resource-heavy, smoke-tests-kubernetes-no-resource-heavy-limit-deps, smoke-tests-remote-server-kubernetes, smoke-tests-kubernetes-jobs-consolidation, smoke-tests-shared-gke-api-server, smoke-tests-slurm-minimal, backward-compat-test-nightly, backward-compat-test-stable, publish-and-validate-both, trigger-docker-and-helm-release] | ||
| # Only run this job if any of the previous jobs failed | ||
| if: failure() | ||
| steps: | ||
| - name: Prepare failure message | ||
| id: message_content | ||
| run: | | ||
| TITLE_TEXT="Workflow ${{ github.workflow }} failed." | ||
| COMMIT_SHA="${{ github.sha }}" | ||
| COMMIT_URL="${{ github.server_url }}/${{ github.repository }}/commit/${COMMIT_SHA}" | ||
| SHORT_SHA=$(echo "$COMMIT_SHA" | cut -c1-7) | ||
| EMOJI="🚨" | ||
| PRECHECK_TAG="" | ||
| if [[ "${{ needs.check-date.outputs.is_preflight }}" == "true" ]]; then | ||
| PRECHECK_TAG="[Preflight check, won't release] " | ||
| EMOJI="🔔" | ||
| fi | ||
| if [[ -n "$PRECHECK_TAG" ]]; then | ||
| TITLE_TEXT="${PRECHECK_TAG}${TITLE_TEXT}" | ||
| fi | ||
| BUILDKITE_MSG="" | ||
| if [[ "${{ needs.smoke-tests-aws.result }}" == "failure" || "${{ needs.smoke-tests-kubernetes-resource-heavy.result }}" == "failure" || "${{ needs.smoke-tests-kubernetes-no-resource-heavy.result }}" == "failure" || "${{ needs.smoke-tests-kubernetes-no-resource-heavy-limit-deps.result }}" == "failure" || "${{ needs.smoke-tests-remote-server-kubernetes.result }}" == "failure" || "${{ needs.smoke-tests-kubernetes-jobs-consolidation.result }}" == "failure" || "${{ needs.smoke-tests-shared-gke-api-server.result }}" == "failure" || "${{ needs.smoke-tests-slurm-minimal.result }}" == "failure" || "${{ needs.backward-compat-test-nightly.result }}" == "failure" || "${{ needs.backward-compat-test-stable.result }}" == "failure" ]]; then | ||
| if [[ "${{ needs.smoke-tests-aws.result }}" == "failure" ]]; then | ||
| BUILDKITE_MSG="<https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-aws.outputs.build_number }}|Buildkite Log(--aws)>" | ||
| fi | ||
| if [[ "${{ needs.smoke-tests-kubernetes-resource-heavy.result }}" == "failure" ]]; then | ||
| if [[ ! -z "$BUILDKITE_MSG" ]]; then | ||
| BUILDKITE_MSG="${BUILDKITE_MSG} and" | ||
| fi | ||
| BUILDKITE_MSG="${BUILDKITE_MSG} <https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-kubernetes-resource-heavy.outputs.build_number }}|Buildkite Log(--kubernetes --resource-heavy)>" | ||
| fi | ||
| if [[ "${{ needs.smoke-tests-kubernetes-no-resource-heavy.result }}" == "failure" ]]; then | ||
| if [[ ! -z "$BUILDKITE_MSG" ]]; then | ||
| BUILDKITE_MSG="${BUILDKITE_MSG} and" | ||
| fi | ||
| BUILDKITE_MSG="${BUILDKITE_MSG} <https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-kubernetes-no-resource-heavy.outputs.build_number }}|Buildkite Log(--kubernetes --no-resource-heavy)>" | ||
| fi | ||
| if [[ "${{ needs.smoke-tests-kubernetes-no-resource-heavy-limit-deps.result }}" == "failure" ]]; then | ||
| if [[ ! -z "$BUILDKITE_MSG" ]]; then | ||
| BUILDKITE_MSG="${BUILDKITE_MSG} and" | ||
| fi | ||
| BUILDKITE_MSG="${BUILDKITE_MSG} <https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-kubernetes-no-resource-heavy-limit-deps.outputs.build_number }}|Buildkite Log(--kubernetes --no-resource-heavy --dependency kubernetes)>" | ||
| fi | ||
| if [[ "${{ needs.smoke-tests-remote-server-kubernetes.result }}" == "failure" ]]; then | ||
| if [[ ! -z "$BUILDKITE_MSG" ]]; then | ||
| BUILDKITE_MSG="${BUILDKITE_MSG} and" | ||
| fi | ||
| BUILDKITE_MSG="${BUILDKITE_MSG} <https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-remote-server-kubernetes.outputs.build_number }}|Buildkite Log(--remote-server --kubernetes)>" | ||
| fi | ||
| if [[ "${{ needs.smoke-tests-kubernetes-jobs-consolidation.result }}" == "failure" ]]; then | ||
| if [[ ! -z "$BUILDKITE_MSG" ]]; then | ||
| BUILDKITE_MSG="${BUILDKITE_MSG} and" | ||
| fi | ||
| BUILDKITE_MSG="${BUILDKITE_MSG} <https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-kubernetes-jobs-consolidation.outputs.build_number }}|Buildkite Log(--kubernetes --jobs-consolidation --no-resource-heavy)>" | ||
| fi | ||
| if [[ "${{ needs.smoke-tests-shared-gke-api-server.result }}" == "failure" ]]; then | ||
| if [[ ! -z "$BUILDKITE_MSG" ]]; then | ||
| BUILDKITE_MSG="${BUILDKITE_MSG} and" | ||
| fi | ||
| BUILDKITE_MSG="${BUILDKITE_MSG} <https://buildkite.com/skypilot-1/nightly-build-shared-gke-api-server/builds/${{ needs.smoke-tests-shared-gke-api-server.outputs.build_number }}|Buildkite Log(shared-gke-api-server)>" | ||
| fi | ||
| if [[ "${{ needs.smoke-tests-slurm-minimal.result }}" == "failure" ]]; then | ||
| if [[ ! -z "$BUILDKITE_MSG" ]]; then | ||
| BUILDKITE_MSG="${BUILDKITE_MSG} and" | ||
| fi | ||
| BUILDKITE_MSG="${BUILDKITE_MSG} <https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-slurm-minimal.outputs.build_number }}|Buildkite Log(test_minimal --slurm)>" | ||
| fi | ||
| # if [[ "${{ needs.smoke-tests-runpod-minimal.result }}" == "failure" ]]; then | ||
| # if [[ ! -z "$BUILDKITE_MSG" ]]; then | ||
| # BUILDKITE_MSG="${BUILDKITE_MSG} and" | ||
| # fi | ||
| # BUILDKITE_MSG="${BUILDKITE_MSG} <https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-runpod-minimal.outputs.build_number }}|Buildkite Log(test_minimal --runpod)>" | ||
| # fi | ||
| if [[ "${{ needs.backward-compat-test-nightly.result }}" == "failure" ]]; then | ||
| if [[ ! -z "$BUILDKITE_MSG" ]]; then | ||
| BUILDKITE_MSG="${BUILDKITE_MSG} and" | ||
| fi | ||
| BUILDKITE_MSG="${BUILDKITE_MSG} <https://buildkite.com/skypilot-1/quicktest-core/builds/${{ needs.backward-compat-test-nightly.outputs.build_number }}|Buildkite Log(backward-compat-nightly)>" | ||
| fi | ||
| if [[ "${{ needs.backward-compat-test-stable.result }}" == "failure" ]]; then | ||
| if [[ ! -z "$BUILDKITE_MSG" ]]; then | ||
| BUILDKITE_MSG="${BUILDKITE_MSG} and" | ||
| fi | ||
| BUILDKITE_MSG="${BUILDKITE_MSG} <https://buildkite.com/skypilot-1/quicktest-core/builds/${{ needs.backward-compat-test-stable.outputs.build_number }}|Buildkite Log(backward-compat-stable)>" | ||
| fi | ||
| MARKDOWN_TEXT="${EMOJI} ${PRECHECK_TAG}Workflow <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|*${{ github.workflow }}*> failed at Buildkite step for commit <${COMMIT_URL}|${SHORT_SHA}>. ${BUILDKITE_MSG}" | ||
| else | ||
| MARKDOWN_TEXT="${EMOJI} ${PRECHECK_TAG}Workflow <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|*${{ github.workflow }}*> failed for commit <${COMMIT_URL}|${SHORT_SHA}>." | ||
| fi | ||
| echo "message_text=${TITLE_TEXT}" >> $GITHUB_OUTPUT | ||
| echo "message_block=${MARKDOWN_TEXT}" >> $GITHUB_OUTPUT | ||
| - name: Notify Slack channel on failure | ||
| uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0 | ||
| with: | ||
| method: chat.postMessage | ||
| token: ${{ secrets.SLACK_BOT_TOKEN }} | ||
| payload: | | ||
| { | ||
| "channel": "${{ secrets.SLACK_CHANNEL_ID }}", | ||
| "text": "${{ steps.message_content.outputs.message_text }}", | ||
| "blocks": [ | ||
| { | ||
| "type": "section", | ||
| "text": { | ||
| "type": "mrkdwn", | ||
| "text": "${{ steps.message_content.outputs.message_block }}" | ||
| } | ||
| } | ||
| ] | ||
| } | ||
