Notebook Execution and Test — Germline, Pangenome, VEP #25
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Notebook Execution and Test — Germline, Pangenome, VEP | |
| on: | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| concurrency: | |
| group: genomics-notebook-gpu-${{ github.ref }} | |
| cancel-in-progress: false | |
| env: | |
| PYTHON_VERSION: "3.12" | |
| PARABRICKS_VERSION: "4.6.0-1" | |
| jobs: | |
| germline-wes: | |
| runs-on: arc-runners-org-nvidia-ai-bp-2-gpu | |
| steps: | |
| - name: Checkout BP repository | |
| uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Get System Info | |
| run: | | |
| echo "===================== System Info (germline) =====================" | |
| more /etc/os-release | |
| nvidia-smi || echo "nvidia-smi not available" | |
| docker version | |
| - name: Run germline notebook in Parabricks container | |
| env: | |
| NGC_API_KEY: ${{ secrets.NGC_API_KEY }} | |
| run: | | |
| NOTEBOOK_DIR="/workspace/notebooks" | |
| NOTEBOOK_PATH="germline_wes.ipynb" | |
| OUTPUT_NOTEBOOK="/workspace/germline_wes_result.ipynb" | |
| echo "${NGC_API_KEY}" | docker login nvcr.io -u '$oauthtoken' --password-stdin | |
| mkdir -p "$(pwd)/notebook_data/germline" | |
| docker run --rm --gpus all \ | |
| -e NGC_API_KEY \ | |
| -e CI_PATCH_NOTEBOOKS=1 \ | |
| -e HOST_DATA_DIR="$(pwd)/notebook_data/germline" \ | |
| -e NOTEBOOK_DATA_DIR="/workspace/notebook_data/germline" \ | |
| -v $(pwd):/workspace \ | |
| -v /var/run/docker.sock:/var/run/docker.sock \ | |
| -w /workspace \ | |
| nvcr.io/nvidia/clara/clara-parabricks:${{ env.PARABRICKS_VERSION }} \ | |
| bash -c " | |
| apt-get update && apt-get install -y wget pigz docker.io | |
| if ! command -v sudo >/dev/null 2>&1; then | |
| printf '%s\n' '#!/bin/sh' 'exec "$@"' >/usr/local/bin/sudo | |
| chmod +x /usr/local/bin/sudo | |
| fi | |
| export USER="${USER:-root}" | |
| pip install papermill jupyter ipykernel | |
| ipython kernel install --name 'python3' --user | |
| echo \"\$NGC_API_KEY\" | docker login nvcr.io -u '\$oauthtoken' --password-stdin | |
| python3 /workspace/scripts/ci_patch_notebooks.py ${NOTEBOOK_DIR}/${NOTEBOOK_PATH} | |
| cd '$NOTEBOOK_DIR' | |
| papermill '$NOTEBOOK_PATH' '$OUTPUT_NOTEBOOK' --log-output --log-level DEBUG | |
| if [ \$? -ne 0 ]; then | |
| echo 'Notebook execution failed with exit code '\$? | |
| exit 1 | |
| fi | |
| if [ ! -f '$OUTPUT_NOTEBOOK' ]; then | |
| echo 'Error: Output notebook was not created' | |
| exit 1 | |
| fi | |
| " | |
| - name: Convert result to html format | |
| if: always() | |
| run: | | |
| if [ -f "germline_wes_result.ipynb" ]; then | |
| jupyter nbconvert --to html "germline_wes_result.ipynb" | |
| else | |
| echo "Notebook output file not found, skipping conversion" | |
| fi | |
| - name: Run Test Code | |
| env: | |
| NGC_API_KEY: ${{ secrets.NGC_API_KEY }} | |
| run: | | |
| echo "$NGC_API_KEY" | docker login nvcr.io --username '$oauthtoken' --password-stdin | |
| if [ ! -f "./germline_wes_result.html" ]; then | |
| echo "Error: germline_wes_result.html not found" | |
| exit 1 | |
| fi | |
| docker run \ | |
| -v ./germline_wes_result.html:/app/input/germline_wes.html \ | |
| -v "$(pwd):/workspace" \ | |
| nvcr.io/rw983xdqtcdp/auto_test_team/blueprint-github-test-image:latest \ | |
| pytest testcases/test_genomics_analysis_notebook.py -m germline --disable-warnings --html=/workspace/genomics.html | |
| - name: Upload artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: germline notebook and pytest | |
| path: | | |
| germline_wes_result.html | |
| genomics.html | |
| retention-days: 14 | |
| - name: Cleanup Docker resources | |
| if: always() | |
| run: | | |
| docker ps --filter "ancestor=nvcr.io/nvidia/clara/clara-parabricks:${{ env.PARABRICKS_VERSION }}" -q | xargs -r docker stop 2>/dev/null || true | |
| docker ps --filter "ancestor=nvcr.io/rw983xdqtcdp/auto_test_team/blueprint-github-test-image:latest" -q | xargs -r docker stop 2>/dev/null || true | |
| docker ps -a --filter "ancestor=nvcr.io/nvidia/clara/clara-parabricks:${{ env.PARABRICKS_VERSION }}" -q | xargs -r docker rm 2>/dev/null || true | |
| docker ps -a --filter "ancestor=nvcr.io/rw983xdqtcdp/auto_test_team/blueprint-github-test-image:latest" -q | xargs -r docker rm 2>/dev/null || true | |
| docker rmi nvcr.io/nvidia/clara/clara-parabricks:${{ env.PARABRICKS_VERSION }} 2>/dev/null || true | |
| docker rmi nvcr.io/rw983xdqtcdp/auto_test_team/blueprint-github-test-image:latest 2>/dev/null || true | |
| docker image prune -f | |
| - name: Set result output | |
| id: set_result | |
| if: always() | |
| run: | | |
| echo "RESULT=$(if [ ${{ job.status }} == 'success' ]; then echo 'PASS'; else echo 'FAIL'; fi)" >> $GITHUB_OUTPUT | |
| - name: Send mail | |
| uses: dawidd6/action-send-mail@6e71c855c9a091d80a519621b9fd3e8d252ca40c | |
| if: always() | |
| with: | |
| server_address: smtp.gmail.com | |
| server_port: 587 | |
| username: ${{ secrets.SMTP_USERNAME }} | |
| password: ${{ secrets.SMTP_PASSWORD }} | |
| subject: "QA Test (Germline) for ${{ github.repository }}" | |
| to: Github-Action-Blueprint-QA@nvidia.com | |
| from: github-workflow-notification@gmail.com | |
| html_body: | | |
| <p>Hello,</p> | |
| <p>Germline job for <strong>${{ github.repository }}</strong> completed.<br> | |
| <strong>Result:</strong> ${{ steps.set_result.outputs.RESULT }}</p> | |
| <p><a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}">View run</a></p> | |
| <p>Thanks!</p> | |
| pangenome: | |
| needs: germline-wes | |
| runs-on: arc-runners-org-nvidia-ai-bp-2-gpu | |
| steps: | |
| - name: Checkout BP repository | |
| uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Get System Info | |
| run: | | |
| echo "===================== System Info (pangenome) =====================" | |
| more /etc/os-release | |
| nvidia-smi || echo "nvidia-smi not available" | |
| docker version | |
| - name: Run pangenome notebook in Parabricks container | |
| env: | |
| NGC_API_KEY: ${{ secrets.NGC_API_KEY }} | |
| run: | | |
| NOTEBOOK_DIR="/workspace/notebooks" | |
| NOTEBOOK_PATH="pangenome.ipynb" | |
| OUTPUT_NOTEBOOK="/workspace/pangenome_result.ipynb" | |
| echo "${NGC_API_KEY}" | docker login nvcr.io -u '$oauthtoken' --password-stdin | |
| mkdir -p "$(pwd)/notebook_data/pangenome" | |
| docker run --rm --gpus all \ | |
| -e NGC_API_KEY \ | |
| -e CI_PATCH_NOTEBOOKS=1 \ | |
| -e HOST_DATA_DIR="$(pwd)/notebook_data/pangenome" \ | |
| -e NOTEBOOK_DATA_DIR="/workspace/notebook_data/pangenome" \ | |
| -v $(pwd):/workspace \ | |
| -v /var/run/docker.sock:/var/run/docker.sock \ | |
| -w /workspace \ | |
| nvcr.io/nvidia/clara/clara-parabricks:${{ env.PARABRICKS_VERSION }} \ | |
| bash -c " | |
| apt-get update && apt-get install -y wget pigz docker.io | |
| if ! command -v sudo >/dev/null 2>&1; then | |
| printf '%s\n' '#!/bin/sh' 'exec "$@"' >/usr/local/bin/sudo | |
| chmod +x /usr/local/bin/sudo | |
| fi | |
| export USER="${USER:-root}" | |
| pip install papermill jupyter ipykernel | |
| ipython kernel install --name 'python3' --user | |
| echo \"\$NGC_API_KEY\" | docker login nvcr.io -u '\$oauthtoken' --password-stdin | |
| python3 /workspace/scripts/ci_patch_notebooks.py ${NOTEBOOK_DIR}/${NOTEBOOK_PATH} | |
| cd '$NOTEBOOK_DIR' | |
| papermill '$NOTEBOOK_PATH' '$OUTPUT_NOTEBOOK' --log-output --log-level DEBUG | |
| if [ \$? -ne 0 ]; then | |
| echo 'Notebook execution failed with exit code '\$? | |
| exit 1 | |
| fi | |
| if [ ! -f '$OUTPUT_NOTEBOOK' ]; then | |
| echo 'Error: Output notebook was not created' | |
| exit 1 | |
| fi | |
| " | |
| - name: Convert result to HTML | |
| if: always() | |
| run: | | |
| if [ -f "pangenome_result.ipynb" ]; then | |
| jupyter nbconvert --to html "pangenome_result.ipynb" | |
| else | |
| echo "Output notebook not found, skipping conversion" | |
| fi | |
| - name: Run test code | |
| env: | |
| NGC_API_KEY: ${{ secrets.NGC_API_KEY }} | |
| run: | | |
| echo "$NGC_API_KEY" | docker login nvcr.io --username '$oauthtoken' --password-stdin | |
| if [ ! -f "./pangenome_result.html" ]; then | |
| echo "Error: pangenome_result.html not found" | |
| exit 1 | |
| fi | |
| docker run \ | |
| -v ./pangenome_result.html:/app/input/pangenome.html \ | |
| -v "$(pwd):/workspace" \ | |
| nvcr.io/rw983xdqtcdp/auto_test_team/blueprint-github-test-image:latest \ | |
| pytest testcases/test_genomics_analysis_notebook.py -m pangenome --disable-warnings --html=/workspace/genomics.html | |
| - name: Upload artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pangenome notebook and pytest | |
| path: | | |
| pangenome_result.html | |
| genomics.html | |
| retention-days: 14 | |
| - name: Cleanup Docker resources | |
| if: always() | |
| run: | | |
| docker ps --filter "ancestor=nvcr.io/nvidia/clara/clara-parabricks:${{ env.PARABRICKS_VERSION }}" -q | xargs -r docker stop 2>/dev/null || true | |
| docker ps --filter "ancestor=nvcr.io/rw983xdqtcdp/auto_test_team/blueprint-github-test-image:latest" -q | xargs -r docker stop 2>/dev/null || true | |
| docker ps -a --filter "ancestor=nvcr.io/nvidia/clara/clara-parabricks:${{ env.PARABRICKS_VERSION }}" -q | xargs -r docker rm 2>/dev/null || true | |
| docker ps -a --filter "ancestor=nvcr.io/rw983xdqtcdp/auto_test_team/blueprint-github-test-image:latest" -q | xargs -r docker rm 2>/dev/null || true | |
| docker rmi nvcr.io/nvidia/clara/clara-parabricks:${{ env.PARABRICKS_VERSION }} 2>/dev/null || true | |
| docker rmi nvcr.io/rw983xdqtcdp/auto_test_team/blueprint-github-test-image:latest 2>/dev/null || true | |
| docker image prune -f | |
| - name: Set result output | |
| id: set_result | |
| if: always() | |
| run: | | |
| echo "RESULT=$(if [ ${{ job.status }} == 'success' ]; then echo 'PASS'; else echo 'FAIL'; fi)" >> $GITHUB_OUTPUT | |
| - name: Send mail | |
| uses: dawidd6/action-send-mail@6e71c855c9a091d80a519621b9fd3e8d252ca40c | |
| if: always() | |
| with: | |
| server_address: smtp.gmail.com | |
| server_port: 587 | |
| username: ${{ secrets.SMTP_USERNAME }} | |
| password: ${{ secrets.SMTP_PASSWORD }} | |
| subject: "QA Test (Pangenome) for ${{ github.repository }}" | |
| to: Github-Action-Blueprint-QA@nvidia.com | |
| from: github-workflow-notification@gmail.com | |
| html_body: | | |
| <p>Hello,</p> | |
| <p>Pangenome job for <strong>${{ github.repository }}</strong> completed.<br> | |
| <strong>Result:</strong> ${{ steps.set_result.outputs.RESULT }}</p> | |
| <p><a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}">View run</a></p> | |
| <p>Thanks!</p> | |
| variant-effect-prediction: | |
| needs: pangenome | |
| runs-on: arc-runners-org-nvidia-ai-bp-2-gpu | |
| steps: | |
| - name: Checkout BP repository (with CodonFM submodule) | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Get System Info | |
| run: | | |
| echo "===================== System Info (VEP) =====================" | |
| more /etc/os-release | |
| nvidia-smi || echo "nvidia-smi not available" | |
| docker version | |
| - name: Run VEP notebook in Parabricks container | |
| env: | |
| NGC_API_KEY: ${{ secrets.NGC_API_KEY }} | |
| run: | | |
| NOTEBOOK_DIR="/workspace/notebooks" | |
| NOTEBOOK_PATH="variant_effect_prediction.ipynb" | |
| OUTPUT_NOTEBOOK="/workspace/variant_effect_prediction_result.ipynb" | |
| echo "${NGC_API_KEY}" | docker login nvcr.io -u '$oauthtoken' --password-stdin | |
| mkdir -p "$(pwd)/notebook_data/vep" | |
| docker run --rm --gpus all \ | |
| -e NGC_API_KEY \ | |
| -e CI_PATCH_NOTEBOOKS=1 \ | |
| -e HOST_DATA_DIR="$(pwd)/notebook_data/vep" \ | |
| -e NOTEBOOK_DATA_DIR="/workspace/notebook_data/vep" \ | |
| -v $(pwd):/workspace \ | |
| -v /var/run/docker.sock:/var/run/docker.sock \ | |
| -w /workspace \ | |
| nvcr.io/nvidia/clara/clara-parabricks:${{ env.PARABRICKS_VERSION }} \ | |
| bash -c " | |
| apt-get update && apt-get install -y wget pigz git docker.io tabix | |
| if ! command -v sudo >/dev/null 2>&1; then | |
| printf '%s\n' '#!/bin/sh' 'exec "$@"' >/usr/local/bin/sudo | |
| chmod +x /usr/local/bin/sudo | |
| fi | |
| export USER="${USER:-root}" | |
| pip install papermill jupyter ipykernel | |
| ipython kernel install --name 'python3' --user | |
| echo \"\$NGC_API_KEY\" | docker login nvcr.io -u '\$oauthtoken' --password-stdin | |
| if [ -f /workspace/CodonFM/requirements.txt ]; then | |
| pip install -r /workspace/CodonFM/requirements.txt | |
| fi | |
| python3 /workspace/scripts/ci_patch_notebooks.py ${NOTEBOOK_DIR}/${NOTEBOOK_PATH} | |
| cd '$NOTEBOOK_DIR' | |
| papermill '$NOTEBOOK_PATH' '$OUTPUT_NOTEBOOK' --log-output --log-level DEBUG | |
| if [ \$? -ne 0 ]; then | |
| echo 'Notebook execution failed with exit code '\$? | |
| exit 1 | |
| fi | |
| if [ ! -f '$OUTPUT_NOTEBOOK' ]; then | |
| echo 'Error: Output notebook was not created' | |
| exit 1 | |
| fi | |
| " | |
| - name: Convert result to HTML | |
| if: always() | |
| run: | | |
| if [ -f "variant_effect_prediction_result.ipynb" ]; then | |
| jupyter nbconvert --to html "variant_effect_prediction_result.ipynb" | |
| else | |
| echo "Output notebook not found, skipping conversion" | |
| fi | |
| - name: Run test code | |
| env: | |
| NGC_API_KEY: ${{ secrets.NGC_API_KEY }} | |
| run: | | |
| echo "$NGC_API_KEY" | docker login nvcr.io --username '$oauthtoken' --password-stdin | |
| if [ ! -f "./variant_effect_prediction_result.html" ]; then | |
| echo "Error: variant_effect_prediction_result.html not found" | |
| exit 1 | |
| fi | |
| docker run \ | |
| -v ./variant_effect_prediction_result.html:/app/input/variant_effect_prediction.html \ | |
| -v "$(pwd):/workspace" \ | |
| nvcr.io/rw983xdqtcdp/auto_test_team/blueprint-github-test-image:latest \ | |
| pytest testcases/test_genomics_analysis_notebook.py -m vep --disable-warnings --html=/workspace/genomics.html | |
| - name: Upload artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: variant effect prediction notebook and pytest | |
| path: | | |
| variant_effect_prediction_result.html | |
| genomics.html | |
| retention-days: 14 | |
| - name: Cleanup Docker resources | |
| if: always() | |
| run: | | |
| docker ps --filter "ancestor=nvcr.io/nvidia/clara/clara-parabricks:${{ env.PARABRICKS_VERSION }}" -q | xargs -r docker stop 2>/dev/null || true | |
| docker ps --filter "ancestor=nvcr.io/rw983xdqtcdp/auto_test_team/blueprint-github-test-image:latest" -q | xargs -r docker stop 2>/dev/null || true | |
| docker ps -a --filter "ancestor=nvcr.io/nvidia/clara/clara-parabricks:${{ env.PARABRICKS_VERSION }}" -q | xargs -r docker rm 2>/dev/null || true | |
| docker ps -a --filter "ancestor=nvcr.io/rw983xdqtcdp/auto_test_team/blueprint-github-test-image:latest" -q | xargs -r docker rm 2>/dev/null || true | |
| docker rmi nvcr.io/nvidia/clara/clara-parabricks:${{ env.PARABRICKS_VERSION }} 2>/dev/null || true | |
| docker rmi nvcr.io/rw983xdqtcdp/auto_test_team/blueprint-github-test-image:latest 2>/dev/null || true | |
| docker image prune -f | |
| - name: Set result output | |
| id: set_result | |
| if: always() | |
| run: | | |
| echo "RESULT=$(if [ ${{ job.status }} == 'success' ]; then echo 'PASS'; else echo 'FAIL'; fi)" >> $GITHUB_OUTPUT | |
| - name: Send mail | |
| uses: dawidd6/action-send-mail@6e71c855c9a091d80a519621b9fd3e8d252ca40c | |
| if: always() | |
| with: | |
| server_address: smtp.gmail.com | |
| server_port: 587 | |
| username: ${{ secrets.SMTP_USERNAME }} | |
| password: ${{ secrets.SMTP_PASSWORD }} | |
| subject: "QA Test (Variant Effect) for ${{ github.repository }}" | |
| to: Github-Action-Blueprint-QA@nvidia.com | |
| from: github-workflow-notification@gmail.com | |
| html_body: | | |
| <p>Hello,</p> | |
| <p>Variant effect prediction job for <strong>${{ github.repository }}</strong> completed.<br> | |
| <strong>Result:</strong> ${{ steps.set_result.outputs.RESULT }}</p> | |
| <p><a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}">View run</a></p> | |
| <p>Thanks!</p> |