Pixi dependency management

.github/actions/create_python_environment/action.yml

@@ -10,16 +10,18 @@ runs:
   using: composite
   steps:
     - uses: conda-incubator/setup-miniconda@v3
-    - name: Create environment and install QUEENS
+      with:
+        miniforge-version: "latest"
+    - name: Create conda-lock environment and install QUEENS
       id: environment
       shell: bash -l {0}
       env:
         PYTHON_PACKAGE_MANAGER: conda
       run: |
-        $PYTHON_PACKAGE_MANAGER env create -f environment.yml
+        $PYTHON_PACKAGE_MANAGER install -c conda-forge conda-lock -y
+        conda-lock install --conda mamba -n queens composed.conda-lock.yml
         $PYTHON_PACKAGE_MANAGER activate queens
-        pip install torch==2.5.1 --index-url https://download.pytorch.org/whl/cpu
-        pip install -e .[safe_develop,fourc]
+        pip install --no-deps -e .
         $PYTHON_PACKAGE_MANAGER env export > pipeline_conda_environment.yml
         $PYTHON_PACKAGE_MANAGER list
         echo "ppm=$PYTHON_PACKAGE_MANAGER" >> $GITHUB_OUTPUT

.github/workflows/build_documentation.yml

@@ -23,7 +23,15 @@ jobs:
       run:
         shell: bash -l {0}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
+      - uses: prefix-dev/setup-pixi@v0.9.4
+        with:
+          pixi-version: v0.67.0
+          cache: true
+          cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
+          environments: queens-dev
+          frozen: true
+          activate-environment: queens-dev
       - name: Mark repo as safe for git
         run: |
           git config --global --add safe.directory "$GITHUB_WORKSPACE"
@@ -34,27 +42,18 @@ jobs:
       - name: Create links to 4C
         run: |
           ln -s /home/user/4C/bin/ config/4C_build
-      - name: Create Python environment
-        id: environment
-        uses: ./.github/actions/create_python_environment
-      - name: Add tutorial dependencies
-        env:
-          PYTHON_PACKAGE_MANAGER: ${{steps.environment.outputs.ppm}}
+      - name: Register Jupyter kernel
         run: |
-          $PYTHON_PACKAGE_MANAGER activate queens
-          pip install -e .[tutorial]
           python -m ipykernel install --user --name queens --display-name "Python (queens)"
       - name: Install xvfb
         run: |
           apt-get update
           apt-get install -y xvfb
       - name: Sphinx build
         env:
-          PYTHON_PACKAGE_MANAGER: ${{steps.environment.outputs.ppm}}
           PYTHONPATH: ${{ github.workspace }}
         run: |
           set -euxo pipefail
-          $PYTHON_PACKAGE_MANAGER activate queens
           sphinx-apidoc -o doc/source src/ -fMT
           cd doc
           xvfb-run -a sphinx-build -b html -d build/doctrees source build/html -W

.github/workflows/tests_local.yml

@@ -10,6 +10,10 @@ on:
       - main
   workflow_dispatch:
 
+permissions:
+  contents: read
+  pull-requests: write
+
 env:
   # Set TEST_TIMING_OPTION if local test should be timed. Default is off.
   TEST_TIMING_OPTION: ""
@@ -24,23 +28,152 @@ jobs:
       run:
         shell: bash -l {0}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
+      - uses: prefix-dev/setup-pixi@v0.9.4
+        with:
+          pixi-version: v0.67.0
+          cache: true
+          cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
+          environments: >-
+            queens-base
+            queens-all
+            queens-dev
+          frozen: true
+          activate-environment: queens-dev
+      - name: Mark repo as safe for git
+        run: |
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
       - name: Install rsync
         run: |
           sudo apt-get update
           sudo apt-get install -y rsync
       - name: Create links to 4C
         run: |
           ln -s /home/user/4C/bin/ config/4C_build
-      - name: Create Python environment
-        id: environment
-        uses: ./.github/actions/create_python_environment
-      - name: Get Python package manager
-        run: echo "PYTHON_PACKAGE_MANAGER=${{steps.environment.outputs.ppm}}" >> $GITHUB_ENV
+      - name: Check dependency declaration integrity
+        run: |
+          set -euo pipefail
+          # Validate that the PEP-style dependency declarations in pyproject.toml
+          # stay aligned with the pixi dependency sections.
+          python3 tools/dependencies/check_pyproject_dependency_integrity.py \
+            --path pyproject.toml \
+            --allow-version-mismatch gpflow
+      - name: Check pixi environment integrity
+        id: environment_integrity
+        run: |
+          set -euo pipefail
+
+          base_ref="main"
+          pyproject_diff_file="$(mktemp)"
+          git fetch --no-tags --depth=1 origin "${base_ref}"
+
+          echo "::group::Check if dependency declarations changed..."
+          pyproject_changed=false
+          # The diff helper returns:
+          #   0 if dependency-related declarations are unchanged,
+          #   1 if there is a relevant dependency diff,
+          #   >1 if the diff command itself failed.
+          if python3 tools/dependencies/diff_pyproject_dependency_declarations.py \
+            --base-ref "origin/${base_ref}" \
+            --head-ref HEAD \
+            > "${pyproject_diff_file}"; then
+            pyproject_changed=false
+          else
+            # $? is the exit status of the last command.
+            # when arriving here, the diff command found a diff (exit status 1) or
+            # the diff command itself failed (exit status >1).
+            pyproject_check_status=$?
+            if [ "${pyproject_check_status}" -eq 1 ]; then
+              pyproject_changed=true
+            else
+              echo "Failed to compare dependency-related pyproject.toml sections."
+              exit "${pyproject_check_status}"
+            fi
+          fi
+          echo "Pyproject.toml dependency sections changed compared to ${base_ref}: ${pyproject_changed}"
+          echo "::endgroup::"
+
+          echo "::group::Check if pixi.lock would be updated by pixi lock..."
+          # Detect any change (modified, deleted, untracked, etc.)
+          if [ -n "$(git status --porcelain -- pixi.lock)" ]; then
+            echo "Restoring pixi.lock from HEAD..."
+            git restore --source=HEAD --staged --worktree pixi.lock 2>/dev/null
+          fi
+          lock_update_possible=false
+          # deactivate exit on error to allow `pixi lock --check --dry-run` to fail
+          set +e
+          # `pixi lock --check --dry-run` exits non-zero when the current pyproject.toml
+          # would produce a different pixi.lock without writing the lockfile to disk.
+          pixi lock --check --dry-run
+          exit_code_pixi_lock_check=$?
+          # reactivate exit on error
+          set -e
+          echo "exit code: $exit_code_pixi_lock_check"
+          if [ $exit_code_pixi_lock_check -ne 0 ]; then
+            lock_update_possible=true
+          fi
+          echo "pixi.lock would be updated by pixi lock: ${lock_update_possible}"
+          echo "::endgroup::"
+
+          {
+            echo "pyproject_changed=${pyproject_changed}"
+            echo "lock_update_possible=${lock_update_possible}"
+            echo "compare_ref=${base_ref}"
+          } >> "$GITHUB_OUTPUT"
+
+
+          # ---- Write rich Markdown to Step Summary ----
+          {
+            echo "### Pixi Environment Integrity"
+            echo ""
+            echo "- dependency-relevant \`pyproject.toml\` changes vs \`${base_ref}\`: ${pyproject_changed}"
+            echo "- \`pixi.lock\` would change if refreshed: ${lock_update_possible}"
+
+            if [ "${pyproject_changed}" = "true" ]; then
+              echo ""
+              echo "<details>"
+              echo "<summary>Dependency diff</summary>"
+              echo ""
+              echo '```diff'
+              cat "${pyproject_diff_file}"
+              echo '```'
+              echo "</details>"
+            fi
+          } >> "$GITHUB_STEP_SUMMARY"
+
+
+          # ---- Write clean, readable logs ----
+          echo "::group::Summary Pixi Environment Integrity"
+          echo "----------------------------------------"
+          echo "pyproject changes vs ${base_ref}: ${pyproject_changed}"
+          echo "pixi.lock update possible: ${lock_update_possible}"
+
+          if [ "${pyproject_changed}" = "true" ]; then
+            echo "::group::Dependency diff"
+            cat "${pyproject_diff_file}"
+            echo "::endgroup::"
+          fi
+          echo "::endgroup::"
+
+          echo "::group::Do integrity check..."
+          # Only fail if the dependency declarations changed and the lockfile is
+          # stale. A stale lockfile without dependency declaration changes is
+          # reported in the summary but does not block the workflow here.
+          if [ "${pyproject_changed}" = "true" ] && [ "${lock_update_possible}" = "true" ]; then
+            echo "pyproject.toml dependency sections changed compared to main"
+            echo "AND pixi.lock would be updated by pixi lock."
+            echo "Please regenerate and commit pixi.lock for this change."
+            exit 1
+          else
+            echo "Pixi environment integrity is OK."
+          fi
+          echo "::endgroup::"
       - name: Code checks
         run: |
-          $PYTHON_PACKAGE_MANAGER activate queens
+          set -euo pipefail
 
+          # Run all checks in one fail-fast block so a tool failure is not
+          # hidden by later bookkeeping commands such as echo/endgroup lines.
           echo "::group::Run isort..."
           isort --check-only src/* tests
           echo "::endgroup::"
@@ -62,7 +195,8 @@ jobs:
           echo "::endgroup::"
 
           echo "::group::Run nbstripout..."
-          find . -name '*.ipynb' -exec nbstripout --verify {} +
+          find . -path './.pixi' -prune -o -name '*.ipynb' -exec nbstripout --verify {} +
+
           echo "::endgroup::"
 
           # echo "::group::Create code quality report..."
@@ -71,7 +205,7 @@ jobs:
           # echo "::endgroup::"
 
           echo "::group::Check compatibility with licenses of dependencies..."
-          liccheck -r requirements.txt
+          pip-licenses --python=.pixi/envs/queens-all/bin/python
           echo "::endgroup::"
 
           echo "::group::Create an rc file for the license header check..."
@@ -82,23 +216,19 @@ jobs:
           mypy .
           echo "::endgroup::"
       - name: Check license headers
-        uses: apache/skywalking-eyes/header@v0.4.0
+        uses: apache/skywalking-eyes/header@v0.8.0
       - name: Check conventional commit messages
         uses: webiny/action-conventional-commits@v1.3.0
       - name: Run pytest
         run: |
-          $PYTHON_PACKAGE_MANAGER activate queens
+          set -euo pipefail
+
           pytest -v -m "unit_tests or integration_tests or integration_tests_fourc" --cov --cov-report=term --cov-report=html:html_coverage_report --cov-report=xml:xml_coverage_report.xml $TEST_TIMING_OPTION --color=yes -o junit_logging=all --junitxml=test_junit.xml
           python .github/xml_summaries_to_md.py test_junit.xml xml_coverage_report.xml >> $GITHUB_STEP_SUMMARY
-      - name: Add tutorial dependencies
-        env:
-          PYTHON_PACKAGE_MANAGER: ${{steps.environment.outputs.ppm}}
-        run: |
-            $PYTHON_PACKAGE_MANAGER activate queens
-            pip install -e .[tutorial]
       - name: Run pytest for tutorials
         run: |
-          $PYTHON_PACKAGE_MANAGER activate queens
+          set -euo pipefail
+
           pytest -v -m "tutorial_tests"
       - name: Upload coverage report
         uses: actions/upload-pages-artifact@v3

.gitignore

@@ -82,3 +82,6 @@ venv.bak/
 
 # best editor ever files
 *.swp
+# pixi environments
+.pixi/*
+!.pixi/config.toml