packaging-pipeline.yml

name: OpenCue Packaging Pipeline

# Trigger this pipeline on new commits to master.
on:
  push:
    branches: [master]

jobs:
  build_opencue_packages:
    name: Build Python Packages
    runs-on: ubuntu-22.04
    container: python:3.7
    outputs:
      opencue_proto_path: ${{ steps.package_outputs.outputs.opencue_proto_path }}
      opencue_rqd_path: ${{ steps.package_outputs.outputs.opencue_rqd_path }}
      opencue_pycue_path: ${{ steps.package_outputs.outputs.opencue_pycue_path }}
      opencue_pyoutline_path: ${{ steps.package_outputs.outputs.opencue_pyoutline_path }}
      opencue_cueadmin_path: ${{ steps.package_outputs.outputs.opencue_cueadmin_path }}
      opencue_cueman_path: ${{ steps.package_outputs.outputs.opencue_cueman_path }}
      opencue_cuecmd_path: ${{ steps.package_outputs.outputs.opencue_cuecmd_path }}
      opencue_cuesubmit_path: ${{ steps.package_outputs.outputs.opencue_cuesubmit_path }}
      opencue_cuegui_path: ${{ steps.package_outputs.outputs.opencue_cuegui_path }}
      opencue_cuenimby_path: ${{ steps.package_outputs.outputs.opencue_cuenimby_path }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-tags: true
          fetch-depth: 0

      - name: Mark repository as safe (Fix for https://github.com/actions/checkout/issues/1048)
        run: git config --global --add safe.directory $GITHUB_WORKSPACE

      - uses: ./.github/actions/build-python-packages

      - name: Gather package paths
        id: package_outputs
        run: |
          echo "opencue_proto_path=$(find ./packages -name 'opencue_proto-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_rqd_path=$(find ./packages -name 'opencue_rqd-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_pycue_path=$(find ./packages -name 'opencue_pycue-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_pyoutline_path=$(find ./packages -name 'opencue_pyoutline-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_cuesubmit_path=$(find ./packages -name 'opencue_cuesubmit-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_cueadmin_path=$(find ./packages -name 'opencue_cueadmin-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_cueman_path=$(find ./packages -name 'opencue_cueman-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_cuecmd_path=$(find ./packages -name 'opencue_cuecmd-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_cuegui_path=$(find ./packages -name 'opencue_cuegui-*.whl' -print -quit)" >> $GITHUB_OUTPUT
          echo "opencue_cuenimby_path=$(find ./packages -name 'opencue_cuenimby-*.whl' -print -quit)" >> $GITHUB_OUTPUT

  build_rust_binaries:
    name: Build Rust Binaries
    strategy:
      matrix:
        include:
          - target: x86_64-unknown-linux-gnu
            os: ubuntu-22.04
          - target: x86_64-unknown-linux-musl
            os: ubuntu-22.04
          - target: x86_64-apple-darwin
            os: macos-13
          - target: aarch64-apple-darwin
            os: macos-13
    runs-on: ${{ matrix.os }}
    outputs:
      rust_binaries_artifact: rust-binaries-${{ github.sha }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-tags: true
          fetch-depth: 0

      - name: Mark repository as safe
        run: git config --global --add safe.directory $GITHUB_WORKSPACE

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable
        with:
          targets: ${{ matrix.target }}

      - name: Install dependencies (Ubuntu)
        if: matrix.os == 'ubuntu-22.04'
        run: |
          sudo apt-get update
          sudo apt-get install -y libx11-dev protobuf-compiler
          if [ "${{ matrix.target }}" = "x86_64-unknown-linux-musl" ]; then
            sudo apt-get install -y musl-tools
          fi

      - name: Install dependencies (macOS)
        if: matrix.os == 'macos-13'
        run: |
          brew install protobuf

      - name: Set build ID
        run: |
          echo "BUILD_ID=$(ci/generate_version_number.py)" >> $GITHUB_ENV

      - name: Build Rust binaries
        run: |
          cd rust
          cargo build --release --target ${{ matrix.target }} --no-default-features

      - name: Create release directory
        run: mkdir -p release

      - name: Copy binaries (Linux)
        if: matrix.os == 'ubuntu-22.04'
        run: |
          cp rust/target/${{ matrix.target }}/release/openrqd release/openrqd-${{ env.BUILD_ID }}-${{ matrix.target }}

      - name: Copy binaries (macOS)
        if: matrix.os == 'macos-13'
        run: |
          cp rust/target/${{ matrix.target }}/release/openrqd release/openrqd-${{ env.BUILD_ID }}-${{ matrix.target }}

      - name: Upload binaries
        uses: actions/upload-artifact@v4
        with:
          name: rust-binaries-${{ matrix.target }}
          path: release/

  integration_test:
    needs: build_opencue_packages
    name: Run Integration Test
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true

      - name: Mark repository as safe (Fix for https://github.com/actions/checkout/issues/1048)
        run: git config --global --add safe.directory $GITHUB_WORKSPACE

      - name: Download Python packages artifact
        uses: actions/download-artifact@v4
        with:
          name: opencue_packages
          path: packages

      - name: Download Rust binaries artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: rust-binaries-*
          path: rust-binaries
          merge-multiple: true

      - name: Run test
        run: |
          export OPENCUE_PROTO_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_proto_path }}"
          export OPENCUE_PYCUE_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_pycue_path }}"
          export OPENCUE_PYOUTLINE_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_pyoutline_path }}"
          export OPENCUE_CUEADMIN_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cueadmin_path }}"
          export OPENCUE_CUEMAN_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cueman_path }}"
          export OPENCUE_CUESUBMIT_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuesubmit_path }}"
          export OPENCUE_CUEGUI_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuegui_path }}"
          export OPENCUE_RQD_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_rqd_path }}"
          ci/run_integration_test.sh

      - name: Archive log files
        uses: actions/upload-artifact@v4
        if: ${{ always() }}
        with:
          name: test-logs
          path: /tmp/opencue-test/*.log

  build_components:
    needs:
      - integration_test
      - build_opencue_packages
    strategy:
      matrix:
        include:
          - component: cuebot
            NAME: Cuebot
            ARTIFACTS: cuebot-${BUILD_ID}-all.jar opencue-cuebot-${BUILD_ID}-1.noarch.rpm

          - component: rqd
            NAME: RQD

    name: Build ${{ matrix.NAME }}
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          # Fetch all Git history, otherwise the current version number will
          # not be correctly calculated.
          fetch-depth: 0

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.S3_REGION }}
          role-to-assume: ${{ secrets.AWS_S3_ROLE }}
          role-duration-seconds: 1800
        continue-on-error: true

      - name: Login to Docker Hub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKER_USER }}
          password: ${{ secrets.DOCKER_PASS }}
        continue-on-error: true

      - name: Download Python packages artifact
        uses: actions/download-artifact@v4
        with:
          name: opencue_packages
          path: packages

      - name: Download Rust binaries artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: rust-binaries-*
          path: rust-binaries
          merge-multiple: true

      - name: Set build ID
        run: |
          set -e
          echo "Build ID: $(ci/generate_version_number.py)"
          echo "BUILD_ID=$(ci/generate_version_number.py)" >> ${GITHUB_ENV}

      - name: Build Docker image
        run: |
          image_name="opencuebuild/${{ matrix.component }}:${BUILD_ID}"
          docker buildx build . -f ${{ matrix.component }}/Dockerfile -t ${image_name} \
            --build-arg OPENCUE_PROTO_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_proto_path }}" \
            --build-arg OPENCUE_RQD_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_rqd_path }}" \
            --build-arg OPENCUE_PYCUE_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_pycue_path }}" \
            --build-arg OPENCUE_PYOUTLINE_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_pyoutline_path }}" \
            --build-arg OPENCUE_CUEADMIN_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cueadmin_path }}" \
            --build-arg OPENCUE_CUEMAN_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cueman_path }}" \
            --build-arg OPENCUE_CUESUBMIT_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuesubmit_path }}" \
            --build-arg BUILD_ID="${BUILD_ID}"
          docker push ${image_name}

      - name: Extract Artifacts
        if: ${{ matrix.ARTIFACTS }}
        run: |
          set -e
          image_name="opencuebuild/${{ matrix.component }}:${BUILD_ID}"
          container_id=$(docker create ${image_name})
          artifacts="${{ matrix.ARTIFACTS }}"
          mkdir -p "${GITHUB_WORKSPACE}/artifacts/"
          for artifact in $artifacts; do
            docker cp ${container_id}:/opt/opencue/${artifact} "${GITHUB_WORKSPACE}/artifacts/"
          done
          docker rm $container_id

      - name: Upload Artifacts
        continue-on-error: true
        if: ${{ matrix.ARTIFACTS }}
        env:
          S3_BUCKET: ${{ secrets.S3_BUCKET }}
        run: |
          artifacts="${{ matrix.ARTIFACTS }}"
          for artifact in $artifacts; do
            aws s3 cp ${GITHUB_WORKSPACE}/artifacts/${artifact} s3://${S3_BUCKET}/opencue/${BUILD_ID}/
          done

  upload_python_packages_test:
    needs:
      - integration_test
      - build_opencue_packages
    runs-on: ubuntu-22.04
    container: python:3.11
    name: "Upload python packages to testpypi repo"
    environment: testpypi
    continue-on-error: true
    env:
      TWINE_USERNAME: "__token__"
      TWINE_PASSWORD: "${{ secrets.TEST_PYPI_API_TOKEN }}"
      TWINE_REPOSITORY: "testpypi"

    steps:
      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          name: opencue_packages
          path: packages

      - name: Upload packages
        run: |
          python3 -m pip install --upgrade twine
          python3 -m twine upload --verbose --skip-existing packages/opencue_*

  create_other_artifacts:
    name: Create Other Build Artifacts
    needs:
      - build_components
      - build_opencue_packages
      - build_rust_binaries
    runs-on: ubuntu-22.04
    continue-on-error: true
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          # Fetch all Git history, otherwise the current version number will
          # not be correctly calculated.
          fetch-depth: 0

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.S3_REGION }}
          role-to-assume: ${{ secrets.AWS_S3_ROLE }}
          role-duration-seconds: 1800
        continue-on-error: true

      - name: Set build ID
        run: |
          set -e
          echo "Build ID: $(ci/generate_version_number.py)"
          echo "BUILD_ID=$(ci/generate_version_number.py)" >> ${GITHUB_ENV}
          echo "BUILD_ID=$(ci/generate_version_number.py)" >VERSION

      - name: Extract database schema
        run: |
          mkdir -p "${GITHUB_WORKSPACE}/artifacts/"
          ci/extract_schema.sh ${BUILD_ID} "${GITHUB_WORKSPACE}/artifacts/"

      - name: Create build metadata
        run: |
          mkdir -p "${GITHUB_WORKSPACE}/artifacts/"
          echo "{\"git_commit\": \"$(BUILD_SOURCEVERSION)\"}" | tee "${GITHUB_WORKSPACE}/artifacts/build_metadata.json"

      - name: Download Python packages artifact
        uses: actions/download-artifact@v4
        with:
          name: opencue_packages
          path: packages

      - name: Download Rust binaries artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: rust-binaries-*
          path: rust-binaries
          merge-multiple: true

      - name: Upload artifacts
        env:
          S3_BUCKET: ${{ secrets.S3_BUCKET }}
        run: |
          aws s3 cp ${{ needs.build_opencue_packages.outputs.opencue_proto_path }} s3://${S3_BUCKET}/opencue/${BUILD_ID}/
          aws s3 cp ${{ needs.build_opencue_packages.outputs.opencue_rqd_path }} s3://${S3_BUCKET}/opencue/${BUILD_ID}/
          aws s3 cp ${{ needs.build_opencue_packages.outputs.opencue_pycue_path }} s3://${S3_BUCKET}/opencue/${BUILD_ID}/
          aws s3 cp ${{ needs.build_opencue_packages.outputs.opencue_pyoutline_path }} s3://${S3_BUCKET}/opencue/${BUILD_ID}/
          aws s3 cp ${{ needs.build_opencue_packages.outputs.opencue_cueadmin_path }} s3://${S3_BUCKET}/opencue/${BUILD_ID}/
          aws s3 cp ${{ needs.build_opencue_packages.outputs.opencue_cueman_path }} s3://${S3_BUCKET}/opencue/${BUILD_ID}/
          aws s3 cp ${{ needs.build_opencue_packages.outputs.opencue_cuesubmit_path }} s3://${S3_BUCKET}/opencue/${BUILD_ID}/
          aws s3 cp ${{ needs.build_opencue_packages.outputs.opencue_cuegui_path }} s3://${S3_BUCKET}/opencue/${BUILD_ID}/
          aws s3 cp LICENSE s3://${S3_BUCKET}/opencue/${BUILD_ID}/
          aws s3 cp VERSION s3://${S3_BUCKET}/opencue/${BUILD_ID}/
          aws s3 cp "${GITHUB_WORKSPACE}/artifacts/schema-${BUILD_ID}.sql" s3://${S3_BUCKET}/opencue/${BUILD_ID}/
          aws s3 cp "${GITHUB_WORKSPACE}/artifacts/seed_data-${BUILD_ID}.sql" s3://${S3_BUCKET}/opencue/${BUILD_ID}/
          aws s3 cp "${GITHUB_WORKSPACE}/artifacts/build_metadata.json" s3://${S3_BUCKET}/opencue/${BUILD_ID}/
          # Upload Rust binaries
          for binary in rust-binaries/*; do
            if [ -f "$binary" ]; then
              aws s3 cp "$binary" s3://${S3_BUCKET}/opencue/${BUILD_ID}/
            fi
          done

      - name: Display artifacts
        env:
          S3_BUCKET: ${{ secrets.S3_BUCKET }}
        run: |
          aws s3 ls s3://${S3_BUCKET}/opencue/${BUILD_ID}/