#530: Migrated notebook deployment to Notebook Connector 3.0.0

.github/workflows/ci.yaml

@@ -113,26 +113,14 @@ jobs:
     needs: approval-for-notebook-tests
 
   run-stable-notebook-tests:
-    uses: ./.github/workflows/notebook_tests.yaml
-    strategy:
-      fail-fast: false
-      matrix:
-        class: [ normal, large ]
-    secrets: inherit
-    with:
-      status-filter: stable
-      test-classification: ${{ matrix.class }}
-    needs: run-notebook-docker-image-builder
-
-  run-unstable-notebook-tests:
     uses: ./.github/workflows/notebook_tests.yaml
     strategy:
       fail-fast: false
       matrix:
         class: [ normal ]
     secrets: inherit
     with:
-      status-filter: unstable
+      status-filter: stable
       test-classification: ${{ matrix.class }}
     needs: run-notebook-docker-image-builder
 

.github/workflows/notebook_docker_image_builder.yaml

@@ -5,27 +5,18 @@ on:
 jobs:
   notebook-tests-docker-image-builder:
     runs-on: ubuntu-24.04
-    strategy:
-      fail-fast: true
-      matrix:
-        wip: [no-wip, wip]
-    name: Running Build Docker image - ${{ matrix.wip }}
+    name: Running Build Docker image
     env:
-      DOCKER_TAG: "${{ matrix.wip }}-${{ github.sha }}"
-      DOCKER_IMG_ARTIFACT: "ai-lab-${{ matrix.wip }}-${{ github.sha }}"
-      DOCKER_IMG_FILENAME: "ai-lab-${{ matrix.wip }}-${{ github.sha }}.tar"
+      DOCKER_TAG: "${{ github.sha }}"
+      DOCKER_IMG_ARTIFACT: "ai-lab-${{ github.sha }}"
+      DOCKER_IMG_FILENAME: "ai-lab-${{ github.sha }}.tar"
     steps:
       - uses: actions/checkout@v5
         with:
           fetch-depth: 0
       - uses: ./.github/actions/prepare_test_env/
       - name: Create Docker Image
-        run: |
-            additional_options=""
-            if [[ "${{ matrix.wip }}" == "wip" ]]; then
-              additional_options="--work-in-progress-notebooks"
-            fi
-            poetry run ai-lab create-docker-image --version "$DOCKER_TAG" --log-level info $additional_options
+        run: poetry run ai-lab create-docker-image --version "$DOCKER_TAG" --log-level info
       - name: Export docker image
         run: docker save -o "./$DOCKER_IMG_FILENAME" "exasol/ai-lab:$DOCKER_TAG"
       - uses: actions/upload-artifact@v4

.github/workflows/notebook_tests.yaml

@@ -50,9 +50,9 @@ jobs:
     needs: [fetch-notebook-tests]
     runs-on: ${{ needs.fetch-notebook-tests.outputs.runner }}
     env:
-      DOCKER_TAG: "${{ matrix.nb_test.wip }}-${{ github.sha }}"
-      DOCKER_IMG_ARTIFACT: "ai-lab-${{ matrix.nb_test.wip }}-${{ github.sha }}"
-      DOCKER_IMG_FILENAME: "ai-lab-${{ matrix.nb_test.wip }}-${{ github.sha }}.tar"
+      DOCKER_TAG: "${{ github.sha }}"
+      DOCKER_IMG_ARTIFACT: "ai-lab-${{ github.sha }}"
+      DOCKER_IMG_FILENAME: "ai-lab-${{ github.sha }}.tar"
     strategy:
       fail-fast: false
       matrix:
@@ -65,14 +65,14 @@ jobs:
 
       - uses: ./.github/actions/prepare_test_env/
 
-      - name: Download AI Lab image ${{matrix.nb_test.wip}}
+      - name: Download AI Lab image
         id: download-image
         uses: actions/download-artifact@v5
         with:
           name: "${{ env.DOCKER_IMG_ARTIFACT }}"
           path: "."
 
-      - name: Load AI Lab image ${{matrix.nb_test.wip}}
+      - name: Load AI Lab image
         run: docker load -i "./$DOCKER_IMG_FILENAME"
 
       - name: Run notebook tests

doc/changes/changes_5.1.0.md

@@ -21,6 +21,7 @@ This release removes the SageMaker Notebooks. Furthermore, it replaces the forme
 * #521: Removed some Ansible unit tests
 * #525: Moved the release workflow into the package CLI layer and removed the legacy release script tree
 * #526: Migrated the AWS-backed CI from CodeBuild to GitHub Actions
+* #530: Migrated notebook deployment to Notebook Connector 3.0.0 from PyPI and removed the repo-local notebook source tree
 
 ## Bug Fixes
 

doc/developer_guide/hyperlink-checking.md

@@ -1,10 +1,10 @@
 # Check for Broken Hyperlinks
 
-AI Lab's GitHub workflow [hyperlinks.yaml](https://github.com/exasol/ai-lab/blob/main/.github/workflows/hyperlinks.yaml) checks the validity of hyperlinks using different tools.
+AI Lab checks the validity of hyperlinks locally using different tools.
 
 If a specific link temporarily is unreachable, you can ignore it as specified by the following table.
 
 | Files | Tool | How to ignore broken links? |
 |-------|------|-----------------------------|
 | Markdown files, e.g. README or Changelog | GitHub action [lycheeverse/lychee-action](https://github.com/lycheeverse/lychee-action) | Add regexp to file `.lycheeignore` |
-| Jupyter notebooks | `pytest --check-links` | Add CLI option `--check-links-ignore <REGEXP>` to the GitHub workflow file |
+| Jupyter notebooks | `pytest --check-links` | Add CLI option `--check-links-ignore <REGEXP>` when running the test locally |

doc/developer_guide/notebooks.md

@@ -1,47 +1,37 @@
-# Notebook Files
+# Notebook Deployment
 
-AI Lab repository includes some Jupyter notebooks and scripts to add these notebooks to AI Lab images (the old name - DSS images), e.g. AMI or Docker Images.
-
-Please add or update the notebook files in folder [exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook](../../exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook).
+AI Lab notebooks are deployed from the Notebook Connector package during image builds.
+The Docker image build runs the Notebook Connector CLI command `ai-lab deploy-notebooks`
+to populate the initial notebook directory used by Jupyter.
 
 ## Notebook Connector
 
-Most notebooks use the `exasol-notebook-connector` - the component providing various helper functions for notebooks.
-We typically put there the code we don't want to show in a notebook. Logically, this component is a part of the AI Lab.
-It is listed in the [notebook_requirements.txt](../../exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook_requirements.txt) and gets installed in the container with pip.
-In order to simplify making simultaneous changes to both notebooks and the notebook-connector we allow referencing the latter temporarily through a git dependency,
-e.g. `exasol-notebook-connector @ git+https://github.com/exasol/notebook-connector@e3a8525`.
-It is important to reference a commit in the `main` branch, not the `main` branch itself. The git dependency should be
-replaced with the proper pypi dependency before releasing of the AI Lab.
-
-## Work-in-progress Notebooks
-
-Sometimes, we might have notebooks which we don't want to release yet, either because they are incomplete or 
-only a pre-release. However, we already want to test these notebooks using the notebook tests or build 
-Docker images that contain them.
-These work-in-progress notebooks should be located in the following directory:
+The `exasol-notebook-connector` package provides the notebook helper functions and the
+packaged notebooks that AI Lab ships in its images.
+Its dependencies are still listed in
+[notebook_requirements.txt](../../exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook_requirements.txt)
+and are installed into the Jupyter virtual environment during the build.
 
-```exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/work_in_progress```
-
-The directory then only gets copied into the Docker image when either the test parameter `work_in_progress_notebooks`
-is requesting it or if you use the commandline option `--work-in-progress-notebooks` is added to the 
-`create-docker-image` command in the CLI.
+When you need to update notebooks, change them in the Notebook Connector project and
+rebuild the AI Lab image. This repository no longer carries a separate notebook source tree.
 
 ## Notebook Testing
 
-We are running tests for the notebooks in the Docker Edition of the AI Lab. For this we are creating a Docker test setup in 
-[test_notebooks_in_dss_docker_image.py](../../test/notebook_test_runner/test_notebooks_in_dss_docker_image.py) which installs test libraries into the AI Lab Docker Image.
-It further creates a new test and Docker Container for each notebook test in [test/notebooks](../../test/notebooks). 
-Notebook test names need to fit the pattern `nbtest_*.py`, to prevent pytest running them outside of Docker setup.
+We run notebook tests in the Docker edition of AI Lab by building a test image on top of
+the AI Lab Docker image and installing the test dependencies from
+[test/notebooks](../../test/notebooks).
+Notebook test names need to fit the pattern `nbtest_*.py`, so pytest does not run them
+outside the notebook test runner.
 
-Environment variables with the prefix `NBTEST_` with which you call
-[test_notebooks_in_dss_docker_image.py](../../test/notebook_test_runner/test_notebooks_in_dss_docker_image.py) are forwarded 
-into the Docker container and to the notebook test. You can use this to forward secrets to the notebook tests.
+Environment variables with the prefix `NBTEST_` are forwarded into the Docker container and
+to the notebook test. You can use this to forward secrets to the notebook tests.
 
-By default all created containers and images are removed after running the tests regardless of success or failure.
-However, with the following pytest commandline parameters you can keep them or reuse them to speed up local testing:
+By default all created containers and images are removed after running the tests regardless
+of success or failure.
+However, with the following pytest command line parameters you can keep them or reuse them
+to speed up local testing:
 
-```
+```text
   --dss-docker-image=DSS_DOCKER_IMAGE
                         Name and version of existing Docker image to use for tests
   --keep-dss-docker-image

doc/user_guide/docker/docker-usage.md

@@ -31,7 +31,7 @@ The AI Lab can connect to an _External Exasol database_, as well as using an _In
 
 You can use the command shown below with some limitations:
 * Does not support Integrated Exasol Docker-DB.
-* Does not allow creating Script Language Containers (SLCs), as is done in the examples in the notebooks [Exporting a flavor](https://github.com/exasol/ai-lab/blob/main/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/script_languages_container/export_as_is.ipynb) and [Customizing a flavor](https://github.com/exasol/ai-lab/blob/main/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/script_languages_container/customize.ipynb).
+* Does not allow creating Script Language Containers (SLCs), as shown in the notebooks "Exporting a flavor" and "Customizing a flavor".
 
 The command will
 * Run a Docker container using the specified version of the AI Lab.

exasol/ds/sandbox/cli/commands/create_docker_image.py

@@ -45,9 +45,6 @@
         "--keep-container", type=bool, is_flag=True,
         help="""Keep the Docker Container running after creating the image.
         Otherwise stop and remove the container."""),
-    click.option(
-        "--work-in-progress-notebooks", type=bool, is_flag=True, default=False,
-        help="""Adds work in progress notebooks to the container."""),
 ])
 @add_options(logging_options)
 def create_docker_image(
@@ -56,7 +53,6 @@ def create_docker_image(
         publish: bool,
         registry_user: str,
         keep_container: bool,
-        work_in_progress_notebooks: bool,
         log_level: str,
 ):
     """
@@ -73,7 +69,7 @@ def registry_password():
         return os.environ.get(PASSWORD_ENV, None)
 
     set_log_level(log_level)
-    creator = DssDockerImage(repository, version, keep_container, work_in_progress_notebooks)
+    creator = DssDockerImage(repository, version, keep_container)
     if publish:
         creator.registry = DockerRegistry(registry_user, registry_password())
     creator.create()

exasol/ds/sandbox/lib/dss_docker/create_image.py

@@ -92,9 +92,7 @@ def __init__(
             repository: str,
             version: str = None,
             keep_container: bool = False,
-            work_in_progress_notebooks: bool = False
     ):
-        self.work_in_progress_notebooks = work_in_progress_notebooks
         version = version if version else AI_LAB_VERSION
         self.container_name = f"ds-sandbox-{DssDockerImage.timestamp()}"
         self.repository = repository
@@ -108,10 +106,7 @@ def image_name(self):
         return f"{self.repository}:{self.version}"
 
     def _ansible_playbook(self) -> ansible.Playbook:
-        extra_vars = {
-            "docker_container": self.container_name,
-            "work_in_progress_notebooks": self.work_in_progress_notebooks
-        }
+        extra_vars = {"docker_container": self.container_name}
         return ansible.Playbook("ai_lab_docker_playbook.yml", extra_vars)
 
     def _ansible_config(self) -> ConfigObject:

exasol/ds/sandbox/lib/setup_ec2/run_install_dependencies.py

@@ -25,7 +25,6 @@ def run_install_dependencies(
     """
     extra_vars = {
         "ai_lab_version": configuration.ai_lab_version,
-        "work_in_progress_notebooks": False
     }
     extra_vars.update(playbook.vars)
     enhanced_playbook = ansible.Playbook(playbook.file, extra_vars)

exasol/ds/sandbox/runtime/ansible/ai_lab_docker_playbook.yml

@@ -36,7 +36,6 @@
     docker_group: docker
     user_home: /home/jupyter
     initial_notebook_folder: "{{ user_home }}/notebook-defaults"
-    work_in_progress_notebooks: "{{ work_in_progress_notebooks }}"
     need_sudo: yes
     docker_integration_test: true
   tasks:

exasol/ds/sandbox/runtime/ansible/general_setup_tasks.yml

@@ -39,7 +39,6 @@
     jupyterlab_password: "{{ dss_facts.jupyter.password }}"
     jupyterlab_notebook_folder_initial: "{{ dss_facts.notebook_folder.initial }}"
     jupyterlab_notebook_folder: "{{ dss_facts.notebook_folder.final }}"
-    work_in_progress_notebooks: "{{ work_in_progress_notebooks }}"
 - name: Change Owner of All Files and Dirs in Home Directory
   ansible.builtin.file:
     path: "{{ user_home }}"

exasol/ds/sandbox/runtime/ansible/roles/docker/defaults/main.yml

@@ -3,5 +3,5 @@
 apt_dependencies:
   - apt-transport-https=2.4.14
   - gpg-agent=2.2.27-3ubuntu2.5
-  - ca-certificates=20240203~22.04.1
+  - ca-certificates=20260601~22.04.1
   - software-properties-common=0.99.22.9

exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/jupyter_requirements.txt

@@ -1,2 +1,2 @@
-jupyterlab==4.4.8
-pexpect==4.8.0
+jupyterlab >=4.4.8,<5
+pexpect >=4.8.0,<5