testing-pipeline.yml

name: OpenCue Testing Pipeline

on:
  push:
    branches: [master]
  pull_request:
    branches: [master]

jobs:
  check_opencue_licenses:
    name: Check Licenses of OpenCue Dependencies
    runs-on: ubuntu-22.04
    container: python:3.9
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true

      - name: Run license check
        shell: bash
        run: ci/check_licenses.sh

  build_opencue_packages:
    name: Build Python Packages
    runs-on: ubuntu-22.04
    container: python:3.7
    outputs:
      opencue_proto_path: ${{ steps.package_outputs.outputs.opencue_proto_path }}
      opencue_pycue_path: ${{ steps.package_outputs.outputs.opencue_pycue_path }}
      opencue_pyoutline_path: ${{ steps.package_outputs.outputs.opencue_pyoutline_path }}
      opencue_cueadmin_path: ${{ steps.package_outputs.outputs.opencue_cueadmin_path }}
      opencue_cueman_path: ${{ steps.package_outputs.outputs.opencue_cueman_path }}
      opencue_cuecmd_path: ${{ steps.package_outputs.outputs.opencue_cuecmd_path }}
      opencue_cuesubmit_path: ${{ steps.package_outputs.outputs.opencue_cuesubmit_path }}
      opencue_rqd_path: ${{ steps.package_outputs.outputs.opencue_rqd_path }}
      opencue_cuenimby_path: ${{ steps.package_outputs.outputs.opencue_cuenimby_path }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-tags: true
          fetch-depth: 0

      - name: Mark repository as safe (Fix for https://github.com/actions/checkout/issues/1048)
        run: git config --global --add safe.directory $GITHUB_WORKSPACE

      - name: Set build ID
        run: |
          set -e
          echo "Build ID: $(ci/generate_version_number.py)"
          echo "BUILD_ID=$(ci/generate_version_number.py)" >> ${GITHUB_ENV}

      - uses: ./.github/actions/build-python-packages

      - name: Gather package paths
        id: package_outputs
        run: |
          find . -name *.whl
          echo "opencue_proto_path=$(find ./packages -name 'opencue_proto-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_rqd_path=$(find ./packages -name 'opencue_rqd-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_pycue_path=$(find ./packages -name 'opencue_pycue-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_pyoutline_path=$(find ./packages -name 'opencue_pyoutline-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_cuesubmit_path=$(find ./packages -name 'opencue_cuesubmit-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_cueadmin_path=$(find ./packages -name 'opencue_cueadmin-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_cueman_path=$(find ./packages -name 'opencue_cueman-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_cuecmd_path=$(find ./packages -name 'opencue_cuecmd-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_cuegui_path=$(find ./packages -name 'opencue_cuegui-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_cuenimby_path=$(find ./packages -name 'opencue_cuenimby-*.whl' -print -quit)" >> $GITHUB_OUTPUT

  install_opencue_packages:
    needs: build_opencue_packages
    name: Test installing packages with python ${{ matrix.python-version }}
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.7", "3.9", "3.10", "3.11"]
    container: python:${{ matrix.python-version }}
    steps:
      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          name: opencue_packages
          path: packages

      - name: Install package
        run: |
          set -e
          pip install \
              ${{ needs.build_opencue_packages.outputs.opencue_proto_path }} \
              ${{ needs.build_opencue_packages.outputs.opencue_pycue_path }} \
              ${{ needs.build_opencue_packages.outputs.opencue_pyoutline_path }} \
              ${{ needs.build_opencue_packages.outputs.opencue_cueadmin_path }} \
              ${{ needs.build_opencue_packages.outputs.opencue_cueman_path }} \
              ${{ needs.build_opencue_packages.outputs.opencue_cuecmd_path }} \
              ${{ needs.build_opencue_packages.outputs.opencue_cuesubmit_path }} \
              ${{ needs.build_opencue_packages.outputs.opencue_cuegui_path }} \
              ${{ needs.build_opencue_packages.outputs.opencue_rqd_path }} \
              ${{ needs.build_opencue_packages.outputs.opencue_cuenimby_path }}
          python -c "import opencue_proto"
          python -c "import opencue_proto.report_pb2"
          python -c "import opencue_proto.report_pb2_grpc"

  test_python_2023:
    needs: build_opencue_packages
    name: Run Python Unit Tests (CY2023)
    runs-on: ubuntu-22.04
    container: aswf/ci-opencue:2023
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true

      - name: Mark repository as safe (Fix for https://github.com/actions/checkout/issues/1048)
        run: git config --global --add safe.directory $GITHUB_WORKSPACE

      - name: Download a single artifact
        uses: actions/download-artifact@v4
        with:
          name: opencue_packages
          path: packages

      - name: Run Python Tests
        run: |
          export OPENCUE_PROTO_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_proto_path }}"
          export OPENCUE_PYCUE_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_pycue_path }}"
          export OPENCUE_PYOUTLINE_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_pyoutline_path }}"
          export OPENCUE_CUEADMIN_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cueadmin_path }}"
          export OPENCUE_CUEMAN_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cueman_path }}"
          export OPENCUE_CUESUBMIT_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuesubmit_path }}"
          export OPENCUE_CUEGUI_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuegui_path }}"
          export OPENCUE_RQD_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_rqd_path }}"
          ci/run_python_tests.sh

  test_cuebot_2023:
    needs: build_opencue_packages
    name: Build Cuebot and Run Unit Tests (CY2023)
    runs-on: ubuntu-22.04
    container:
      image: aswf/ci-opencue:2023
    steps:
      - uses: actions/checkout@v4
      - name: Build with Gradle
        run: |
          chown -R aswfuser:aswfgroup .
          su -c "cd cuebot && ./gradlew build --stacktrace" aswfuser

  test_python_2024:
    needs: build_opencue_packages
    name: Run Python Unit Tests (CY2024)
    runs-on: ubuntu-22.04
    container: aswf/ci-opencue:2024
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true

      - name: Mark repository as safe (Fix for https://github.com/actions/checkout/issues/1048)
        run: git config --global --add safe.directory $GITHUB_WORKSPACE

      - name: Download a single artifact
        uses: actions/download-artifact@v4
        with:
          name: opencue_packages
          path: packages

      - name: Run Python Tests
        run: |
          export OPENCUE_PROTO_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_proto_path }}"
          export OPENCUE_PYCUE_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_pycue_path }}"
          export OPENCUE_PYOUTLINE_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_pyoutline_path }}"
          export OPENCUE_CUEADMIN_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cueadmin_path }}"
          export OPENCUE_CUEMAN_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cueman_path }}"
          export OPENCUE_CUESUBMIT_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuesubmit_path }}"
          export OPENCUE_CUEGUI_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuegui_path }}"
          export OPENCUE_RQD_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_rqd_path }}"
          export OPENCUE_CUENIMBY_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuenimby_path }}"
          ci/run_python_tests.sh

  test_cuebot_2024:
    needs: build_opencue_packages
    name: Build Cuebot and Run Unit Tests (CY2024)
    runs-on: ubuntu-22.04
    container:
      image: aswf/ci-opencue:2024
    steps:
      - uses: actions/checkout@v4
      - name: Build with Gradle
        run: |
          chown -R aswfuser:aswfgroup .
          su -c "cd cuebot && ./gradlew build --stacktrace" aswfuser

  integration_test:
    needs: build_opencue_packages
    name: Run Integration Test
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true

      - name: Mark repository as safe (Fix for https://github.com/actions/checkout/issues/1048)
        run: git config --global --add safe.directory $GITHUB_WORKSPACE

      - name: Download a single artifact
        uses: actions/download-artifact@v4
        with:
          name: opencue_packages
          path: packages

      - name: Run test
        run: |
          export OPENCUE_PROTO_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_proto_path }}"
          export OPENCUE_PYCUE_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_pycue_path }}"
          export OPENCUE_PYOUTLINE_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_pyoutline_path }}"
          export OPENCUE_CUEADMIN_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cueadmin_path }}"
          export OPENCUE_CUEMAN_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cueman_path }}"
          export OPENCUE_CUESUBMIT_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuesubmit_path }}"
          export OPENCUE_CUEGUI_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuegui_path }}"
          export OPENCUE_RQD_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_rqd_path }}"
          export OPENCUE_CUENIMBY_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuenimby_path }}"
          ci/run_integration_test.sh

      - name: Archive log files
        uses: actions/upload-artifact@v4
        if: ${{ always() }}
        with:
          name: test-logs
          path: /tmp/opencue-test/*.log

  lint_python:
    needs: build_opencue_packages
    name: Lint Python Code
    runs-on: ubuntu-22.04
    container: aswf/ci-opencue:2024.1
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true

      - name: Mark repository as safe (Fix for https://github.com/actions/checkout/issues/1048)
        run: git config --global --add safe.directory $GITHUB_WORKSPACE

      - name: Download a single artifact
        uses: actions/download-artifact@v4
        with:
          name: opencue_packages
          path: packages

      - name: Set package vars from parent action
        run: |
          export OPENCUE_PROTO_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_proto_path }}"
          export OPENCUE_PYCUE_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_pycue_path }}"
          export OPENCUE_PYOUTLINE_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_pyoutline_path }}"
          export OPENCUE_CUEADMIN_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cueadmin_path }}"
          export OPENCUE_CUEMAN_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cueman_path }}"
          export OPENCUE_CUESUBMIT_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuesubmit_path }}"
          export OPENCUE_CUEGUI_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuegui_path }}"
          export OPENCUE_RQD_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_rqd_path }}"
          export OPENCUE_CUENIMBY_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuenimby_path }}"

      - name: Lint Python Code
        run: ci/run_python_lint.sh

  test_sphinx:
    needs: build_opencue_packages
    name: Test Documentation Build
    runs-on: ubuntu-22.04
    container:
      image: aswf/ci-opencue:2023
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true

      - name: Mark repository as safe (Fix for https://github.com/actions/checkout/issues/1048)
        run: git config --global --add safe.directory $GITHUB_WORKSPACE

      - name: Run Sphinx build
        run: ci/build_sphinx_docs.sh

  check_changed_files:
    name: Check Changed Files
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2
          set-safe-directory: true
          fetch-tags: true

      - name: Check for Version Change
        run: |
          git diff --name-status HEAD^1 HEAD| grep '^M' | awk '{print $2}'
          readarray -t changed_files < <(git diff --name-status HEAD^1 HEAD| grep '^M' | awk '{print $2}')
          ci/check_changed_files.py "${changed_files[@]}"

  check_migration_files:
    name: Check Database Migration Files
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4
      - name: Check Migration Files
        run: ci/check_database_migrations.py

  check_for_version_bump:
    name: Check for Version Bump
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2
          set-safe-directory: true
          fetch-tags: true

      - name: Check for Version Change
        run: |
          git diff --name-status HEAD^1 HEAD| awk '{print $2}'
          readarray -t changed_files < <(git diff --name-status HEAD^1 HEAD| awk '{print $2}')
          ci/check_version_bump.py "${changed_files[@]}"

  test_rest_gateway:
    name: Test REST Gateway
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true

      - name: Mark repository as safe
        run: git config --global --add safe.directory $GITHUB_WORKSPACE

      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version: '1.21'

      - name: Install Protocol Buffers Compiler
        run: |
          sudo apt-get update
          sudo apt-get install -y protobuf-compiler

      - name: Install Go tools
        run: |
          go install github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway@latest
          go install github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2@latest
          go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest

      - name: Generate protobuf code
        working-directory: rest_gateway/opencue_gateway
        run: |
          mkdir -p gen/go
          protoc -I ../../proto/src/ \
            --go_out ./gen/go/ \
            --go_opt paths=source_relative \
            --go-grpc_out ./gen/go/ \
            --go-grpc_opt paths=source_relative \
            ../../proto/src/*.proto
          protoc -I ../../proto/src/ \
            --grpc-gateway_out ./gen/go \
            --grpc-gateway_opt paths=source_relative \
            --grpc-gateway_opt generate_unbound_methods=true \
            ../../proto/src/*.proto

      - name: Install Go dependencies
        working-directory: rest_gateway/opencue_gateway
        run: |
          if [ ! -f go.mod ]; then go mod init opencue_gateway; fi
          go mod tidy

      - name: Run unit tests with coverage
        working-directory: rest_gateway/opencue_gateway
        run: |
          go test -v -coverprofile=coverage.out -covermode=atomic .
          go tool cover -func=coverage.out

      - name: Upload coverage report
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: rest-gateway-coverage
          path: rest_gateway/opencue_gateway/coverage.out

  build_cueweb:
    name: Build CueWeb Docker Image
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true

      - name: Mark repository as safe
        run: git config --global --add safe.directory $GITHUB_WORKSPACE

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Build CueWeb Docker image
        working-directory: cueweb
        run: |
          docker buildx build . -f Dockerfile -t opencue/cueweb:test --load
          echo "CueWeb Docker image built successfully"

  build_rest_gateway:
    name: Build REST Gateway Docker Image
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true

      - name: Mark repository as safe
        run: git config --global --add safe.directory $GITHUB_WORKSPACE

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Build REST Gateway Docker image
        run: |
          docker buildx build . -f rest_gateway/Dockerfile -t opencue/rest-gateway:test --load
          echo "REST Gateway Docker image built successfully"