code_test_and_deploy.yml

name: tests

on:
  push:
    branches:
      - main
    tags:
      - 'v*'
  pull_request:

  schedule:
    # Cron runs on the 1st and 15th of every month.
    # This will only run on main by default.
    - cron: "0 0 1,15 * *"

# Cancel in-progress runs if a new run is triggered, except for main branch.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

jobs:
  linting:
    runs-on: ubuntu-latest
    steps:
      - uses: neuroinformatics-unit/actions/lint@v2

  test:
    if: github.event_name != 'pull_request' || github.event.pull_request.draft == false
    needs: [linting]
    name: ${{ matrix.os }} py${{ matrix.python-version }}
    runs-on: ${{ matrix.os }}

    defaults:
      run:
        shell: bash -l {0}

    strategy:
      fail-fast: true
      matrix:
        # macos-14 is M1, macos-13 is intel. Run on earliest and
        # latest python versions. All python versions are tested in
        # the weekly cron job.
         # Test all Python versions for cron job, and only first/last for other triggers
        os: ${{ fromJson(github.event_name == 'schedule'
              && '["ubuntu-latest","windows-latest","macos-14","macos-15-intel"]'
              || '["ubuntu-latest","windows-latest","macos-latest"]') }}
        python-version: ${{ fromJson(github.event_name == 'schedule'
              && '["3.9","3.10","3.11","3.12","3.13"]'
              || '["3.9","3.13"]') }}

    steps:
      - uses: actions/checkout@v6
      - name: Set up Conda
        uses: conda-incubator/setup-miniconda@v4
        with:
          python-version: ${{ matrix.python-version }}
          auto-update-conda: true
          channels: conda-forge
          activate-environment: "datashuttle-test"

      - name: Debug environment
        run: |
          echo "=== Active conda env ==="
          echo $CONDA_DEFAULT_ENV
          conda info
          
          echo "=== Which python/pip ==="
          which python || echo "python not found"
          which pip || echo "pip not found"
          
          echo "=== Conda envs ==="
          conda env list
          
          echo "=== Python in datashuttle-test env ==="
          conda run -n datashuttle-test which python
          conda run -n datashuttle-test which pip || echo "pip not in datashuttle-test"
          
          echo "=== pip via python -m ==="
          python -m pip --version || echo "python -m pip also broken"
          
      # The recommended installation is via conda, but we need to test
      # against dependencies from the pyproject.toml on the branch
      # to ensure dependency changes in a PR are reflected.
      - name: Install rclone
        run: |
          conda activate datashuttle-test
          conda install -c conda-forge rclone

      - name: Install datashuttle from repo
        run: |
          python -m pip install --upgrade pip
          pip install .[dev]

      - name: Install pass on Linux
        # this is required for Rclone config encryption
        if: runner.os == 'Linux'
        run: |
          set -euo pipefail
          sudo apt-get update
          sudo apt-get install -y pass gnupg git

          # Create a dedicated GPG home for this job
          export GNUPGHOME="$(mktemp -d)"
          echo "GNUPGHOME=$GNUPGHOME" >> "$GITHUB_ENV"   # <-- make it available to later steps

          # Generate a non-interactive key (no passphrase), no expiry
          gpg --batch --yes --pinentry-mode loopback --passphrase '' \
              --quick-gen-key "CI Key <ci@example.invalid>" default default 0

          # Initialize pass with the key fingerprint (more robust than UID)
          FPR="$(gpg --list-secret-keys --with-colons | awk -F: '/^fpr:/ {print $10; exit}')"
          pass init "$FPR"

        # run SSH tests only on Linux because Windows and macOS
        # are already run within a virtual container and so cannot
        # run Linux containers because nested containerisation is disabled.
      - name: Test SSH (Linux only)
        if: runner.os == 'Linux'
        run: |
          sudo service mysql stop  # free up port 3306 for ssh tests
          pytest tests/tests_transfers/ssh

      - name: Test all except central connections
        run: |
          pytest --ignore=tests/tests_transfers/ssh --ignore=tests/tests_transfers/gdrive --ignore=tests/tests_transfers/aws

      - name: Test Google Drive
        env:
          GDRIVE_CLIENT_ID: ${{ secrets.GDRIVE_CLIENT_ID }}
          GDRIVE_CLIENT_SECRET: ${{ secrets.GDRIVE_CLIENT_SECRET }}
          GDRIVE_ROOT_FOLDER_ID: ${{ secrets.GDRIVE_ROOT_FOLDER_ID }}
          GDRIVE_CONFIG_TOKEN: ${{ secrets.GDRIVE_CONFIG_TOKEN }}
        run: |
          pytest tests/tests_transfers/gdrive

      - name: Test AWS
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_REGION: ${{ secrets.AWS_REGION }}
          AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }}
        run: |
          pytest tests/tests_transfers/aws

      - name: Upload coverage reports to Codecov
        uses: codecov/codecov-action@v5
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          slug: neuroinformatics-unit/datashuttle

  build_sdist_wheels:
    name: Build source distribution
    needs: [test]
    if: github.event_name == 'push' && github.ref_type == 'tag'
    runs-on: ubuntu-latest
    steps:
    - uses: neuroinformatics-unit/actions/build_sdist_wheels@v2


  upload_all:
    name: Publish build distributions
    needs: [build_sdist_wheels]
    if: github.event_name == 'push' && github.ref_type == 'tag'
    runs-on: ubuntu-latest
    steps:
    - uses: actions/download-artifact@v8
      with:
        name: artifact
        path: dist
    - uses: pypa/gh-action-pypi-publish@release/v1
      with:
        user: __token__
        password: ${{ secrets.TWINE_API_KEY }}