Update notebooks for cloud environments #320
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Update notebooks for cloud environments" | |
| on: | |
| push: | |
| branches: | |
| - "**" | |
| - "!gh-pages" | |
| - "!open-in-colab" | |
| - "!open-in-kaggle" | |
| pull_request: | |
| branches: | |
| - main | |
| schedule: | |
| - cron: "0 4 * * MON" | |
| workflow_dispatch: | |
| inputs: | |
| branch: | |
| description: "Branch on viskex repository" | |
| type: string | |
| jobs: | |
| open_in_colab: | |
| strategy: | |
| matrix: | |
| include: | |
| - backend: dolfinx | |
| fem_on_colab_packages: | | |
| gmsh@current%dolfinx.io | |
| fenicsx==development-real@current$dolfinx | |
| notebook_pattern: | | |
| "**/tutorial_*_dolfinx*.ipynb" | |
| - backend: firedrake | |
| fem_on_colab_packages: | | |
| firedrake==development-real@current | |
| notebook_pattern: | | |
| "**/tutorial_*_firedrake*.ipynb" | |
| fail-fast: false | |
| uses: fem-on-colab/open-in-colab-workflow/.github/workflows/workflow_call.yml@main | |
| with: | |
| source_directory: website_source | |
| work_directory: open_in_colab | |
| notebook_pattern: ${{ matrix.notebook_pattern }} | |
| notebook_preparation: | | |
| BRANCH="${{ (inputs || github.event.inputs).branch }}" | |
| if [ -z "${BRANCH}" ]; then | |
| BRANCH=main | |
| fi | |
| NOTEBOOK_PATTERN="${{ matrix.notebook_pattern }}" | |
| python3 -m pip install --no-dependencies nbvalx simpleeval | |
| git clone https://github.com/viskex/viskex.git | |
| cd viskex | |
| git checkout ${BRANCH} | |
| NO_TESTS_COLLECTED=5 | |
| python3 -m pytest --ipynb-action=create-notebooks --collapse --work-dir=.ipynb_colab tutorials || (($?==$NO_TESTS_COLLECTED)) | |
| find tutorials -type d -name .ipynb_colab -exec rsync -avz --remove-source-files --include="*.ipynb" --exclude="*" {}/ {}/.. \; | |
| find tutorials -type f -name '*\[*].ipynb' -exec bash -c 'for f; do dir=$(dirname "$f"); base=$(basename "$f"); stem="${base%%[*}"; plain="$dir/$stem.ipynb"; if [ -f "$plain" ]; then rm "$plain"; fi; done' _ {} + | |
| NOTEBOOKS_TO_RUN=$(while read -r PATTERN; do | |
| for FILE in $(find tutorials -wholename $(echo ${PATTERN} | sed 's|\"||g')); do | |
| if ! grep -q '%%px' "${FILE}"; then | |
| echo "${FILE}" | |
| fi | |
| done | |
| done <<< $(printf "%s" "${NOTEBOOK_PATTERN}")) | |
| python3 ../website_source/share/open_in_cloud/replace_common.py ${BRANCH} ${NOTEBOOKS_TO_RUN} | |
| NOTEBOOKS_TO_INCLUDE=$(for FILE in ${NOTEBOOKS_TO_RUN}; do | |
| echo -n "--include=${FILE} " | |
| done) | |
| rsync -avz --delete --include "*/" ${NOTEBOOKS_TO_INCLUDE} --exclude="*" tutorials ../open_in_colab/ | |
| find ../open_in_colab/ -type f -name "*.ipynb" -exec bash -c 'for f; do new=${f//[\[\]=]/_}; if [[ "$f" != "$new" && ! -e "$new" ]]; then mv "$f" "$new" && ln -s "$(basename "$new")" "$f"; fi; done' _ {} + | |
| fem_on_colab_packages: ${{ matrix.fem_on_colab_packages }} | |
| pip_packages: | | |
| viskex@https://github.com/viskex/viskex.git@current | |
| test_script: | | |
| NOTEBOOKS_TO_TEST="" | |
| while read -r PATTERN; do | |
| NOTEBOOKS_TO_TEST="${NOTEBOOKS_TO_TEST} $(find open_in_colab -type f -wholename $(echo ${PATTERN} | sed 's|\"||g'))" | |
| done <<< $(printf "%s" "${{ matrix.notebook_pattern }}") | |
| python3 -m pytest --nbval ${NOTEBOOKS_TO_TEST} | |
| find open_in_colab -type l -name "*.ipynb" -exec bash -c 'for link; do target=$(readlink -f "$link"); [[ -f "$target" ]] && rm "$link" && mv "$target" "$link"; done' _ {} + | |
| publish_on: github@viskex/viskex.github.io@open-in-colab | |
| publish_if_repository: viskex/viskex.github.io | |
| open_in_kaggle: | |
| strategy: | |
| matrix: | |
| include: | |
| - backend: dolfinx | |
| fem_on_kaggle_packages: | | |
| gmsh@current%dolfinx.io | |
| fenicsx==development-real@current$dolfinx | |
| notebook_pattern: | | |
| "**/tutorial_*_dolfinx*.ipynb" | |
| - backend: firedrake | |
| fem_on_kaggle_packages: | | |
| firedrake==development-real@current | |
| notebook_pattern: | | |
| "**/tutorial_*_firedrake*.ipynb" | |
| fail-fast: false | |
| uses: fem-on-kaggle/open-in-kaggle-workflow/.github/workflows/workflow_call.yml@main | |
| with: | |
| source_directory: website_source | |
| work_directory: open_in_kaggle | |
| notebook_pattern: ${{ matrix.notebook_pattern }} | |
| notebook_preparation: | | |
| BRANCH="${{ (inputs || github.event.inputs).branch }}" | |
| if [ -z "${BRANCH}" ]; then | |
| BRANCH=main | |
| fi | |
| NOTEBOOK_PATTERN="${{ matrix.notebook_pattern }}" | |
| python3 -m pip install --no-dependencies nbvalx simpleeval | |
| git clone https://github.com/viskex/viskex.git | |
| cd viskex | |
| git checkout ${BRANCH} | |
| NO_TESTS_COLLECTED=5 | |
| python3 -m pytest --ipynb-action=create-notebooks --collapse --work-dir=.ipynb_kaggle tutorials || (($?==$NO_TESTS_COLLECTED)) | |
| find tutorials -type d -name .ipynb_kaggle -exec rsync -avz --remove-source-files --include="*.ipynb" --exclude="*" {}/ {}/.. \; | |
| find tutorials -type f -name '*\[*].ipynb' -exec bash -c 'for f; do dir=$(dirname "$f"); base=$(basename "$f"); stem="${base%%[*}"; plain="$dir/$stem.ipynb"; if [ -f "$plain" ]; then rm "$plain"; fi; done' _ {} + | |
| NOTEBOOKS_TO_RUN=$(while read -r PATTERN; do | |
| for FILE in $(find tutorials -wholename $(echo ${PATTERN} | sed 's|\"||g')); do | |
| if ! grep -q '%%px' "${FILE}"; then | |
| echo "${FILE}" | |
| fi | |
| done | |
| done <<< $(printf "%s" "${NOTEBOOK_PATTERN}")) | |
| python3 ../website_source/share/open_in_cloud/replace_common.py ${BRANCH} ${NOTEBOOKS_TO_RUN} | |
| NOTEBOOKS_TO_INCLUDE=$(for FILE in ${NOTEBOOKS_TO_RUN}; do | |
| echo -n "--include=${FILE} " | |
| done) | |
| rsync -avz --delete --include "*/" ${NOTEBOOKS_TO_INCLUDE} --exclude="*" tutorials ../open_in_kaggle/ | |
| find ../open_in_kaggle/ -type f -name "*.ipynb" -exec bash -c 'for f; do new=${f//[\[\]=]/_}; if [[ "$f" != "$new" && ! -e "$new" ]]; then mv "$f" "$new" && ln -s "$(basename "$new")" "$f"; fi; done' _ {} + | |
| fem_on_kaggle_packages: ${{ matrix.fem_on_kaggle_packages }} | |
| pip_packages: | | |
| viskex@https://github.com/viskex/viskex.git@current | |
| test_script: | | |
| NOTEBOOKS_TO_TEST="" | |
| while read -r PATTERN; do | |
| NOTEBOOKS_TO_TEST="${NOTEBOOKS_TO_TEST} $(find open_in_kaggle -type f -wholename $(echo ${PATTERN} | sed 's|\"||g'))" | |
| done <<< $(printf "%s" "${{ matrix.notebook_pattern }}") | |
| python3 -m pytest --nbval ${NOTEBOOKS_TO_TEST} | |
| find open_in_kaggle -type l -name "*.ipynb" -exec bash -c 'for link; do target=$(readlink -f "$link"); [[ -f "$target" ]] && rm "$link" && mv "$target" "$link"; done' _ {} + | |
| publish_on: github@viskex/viskex.github.io@open-in-kaggle | |
| publish_if_repository: viskex/viskex.github.io | |
| warn: | |
| runs-on: ubuntu-latest | |
| if: github.repository == 'viskex/viskex.github.io' && github.ref == 'refs/heads/main' && github.event_name == 'schedule' | |
| steps: | |
| - name: Warn if scheduled workflow is about to be disabled | |
| uses: fem-on-colab/warn-workflow-about-to-be-disabled-action@58c6022d9aeb22accfa8df9bf5b58c3d8867208c # v1.0.0 | |
| with: | |
| workflow-filename: open_in_cloud.yml | |
| days-elapsed: 50 |