[CICD] Remove published whls from release (#2034) #67
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: OpenCue Release Pipeline | |
| # Trigger this pipeline when a commit is tagged with a version number, e.g. "v0.4.32". | |
| on: | |
| push: | |
| tags: | |
| - "v*" | |
| jobs: | |
| build_opencue_packages: | |
| name: Build Python Packages | |
| runs-on: ubuntu-22.04 | |
| container: python:3.7 | |
| outputs: | |
| opencue_proto_path: ${{ steps.package_outputs.outputs.opencue_proto_path }} | |
| opencue_rqd_path: ${{ steps.package_outputs.outputs.opencue_rqd_path }} | |
| opencue_pycue_path: ${{ steps.package_outputs.outputs.opencue_pycue_path }} | |
| opencue_pyoutline_path: ${{ steps.package_outputs.outputs.opencue_pyoutline_path }} | |
| opencue_cueadmin_path: ${{ steps.package_outputs.outputs.opencue_cueadmin_path }} | |
| opencue_cueman_path: ${{ steps.package_outputs.outputs.opencue_cueman_path }} | |
| opencue_cuesubmit_path: ${{ steps.package_outputs.outputs.opencue_cuesubmit_path }} | |
| opencue_cuegui_path: ${{ steps.package_outputs.outputs.opencue_cuegui_path }} | |
| opencue_cuenimby_path: ${{ steps.package_outputs.outputs.opencue_cuenimby_path }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-tags: true | |
| fetch-depth: 0 | |
| - name: Mark repository as safe (Fix for https://github.com/actions/checkout/issues/1048) | |
| run: git config --global --add safe.directory $GITHUB_WORKSPACE | |
| - name: Get current tag name | |
| run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> ${GITHUB_ENV} | |
| - name: Verify tag name and version match | |
| run: | | |
| set -e | |
| if [ "v$(ci/generate_version_number.py)" != "${TAG_NAME}" ]; then | |
| echo "Version check failed: code version v$(ci/generate_version_number.py) does not match tag name ${TAG_NAME}" | |
| echo "Original GITHUB_REF: ${GITHUB_REF}" | |
| exit 1 | |
| fi | |
| - uses: ./.github/actions/build-python-packages | |
| - name: Gather package paths | |
| id: package_outputs | |
| run: | | |
| echo "opencue_proto_path=$(find ./packages -name 'opencue_proto-*.whl' -print -quit)" >> $GITHUB_OUTPUT | |
| echo "opencue_rqd_path=$(find ./packages -name 'opencue_rqd-*.whl' -print -quit)" >> $GITHUB_OUTPUT | |
| echo "opencue_pycue_path=$(find ./packages -name 'opencue_pycue-*.whl' -print -quit)" >> $GITHUB_OUTPUT | |
| echo "opencue_pyoutline_path=$(find ./packages -name 'opencue_pyoutline-*.whl' -print -quit)" >> $GITHUB_OUTPUT | |
| echo "opencue_cuesubmit_path=$(find ./packages -name 'opencue_cuesubmit-*.whl' -print -quit)" >> $GITHUB_OUTPUT | |
| echo "opencue_cueadmin_path=$(find ./packages -name 'opencue_cueadmin-*.whl' -print -quit)" >> $GITHUB_OUTPUT | |
| echo "opencue_cueman_path=$(find ./packages -name 'opencue_cueman-*.whl' -print -quit)" >> $GITHUB_OUTPUT | |
| echo "opencue_cuegui_path=$(find ./packages -name 'opencue_cuegui-*.whl' -print -quit)" >> $GITHUB_OUTPUT | |
| echo "opencue_cuenimby_path=$(find ./packages -name 'opencue_cuenimby-*.whl' -print -quit)" >> $GITHUB_OUTPUT | |
| build_rust_binaries: | |
| name: Build Rust Binaries | |
| strategy: | |
| matrix: | |
| include: | |
| - target: x86_64-unknown-linux-gnu | |
| os: ubuntu-22.04 | |
| - target: x86_64-unknown-linux-musl | |
| os: ubuntu-22.04 | |
| - target: x86_64-apple-darwin | |
| os: macos-13 | |
| - target: aarch64-apple-darwin | |
| os: macos-13 | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-tags: true | |
| fetch-depth: 0 | |
| - name: Mark repository as safe | |
| run: git config --global --add safe.directory $GITHUB_WORKSPACE | |
| - name: Get current tag name | |
| run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> ${GITHUB_ENV} | |
| - name: Verify tag name and version match | |
| run: | | |
| set -e | |
| if [ "v$(ci/generate_version_number.py)" != "${TAG_NAME}" ]; then | |
| echo "Version check failed: code version v$(ci/generate_version_number.py) does not match tag name ${TAG_NAME}" | |
| echo "Original GITHUB_REF: ${GITHUB_REF}" | |
| exit 1 | |
| fi | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| targets: ${{ matrix.target }} | |
| - name: Install dependencies (Ubuntu) | |
| if: matrix.os == 'ubuntu-22.04' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libx11-dev protobuf-compiler | |
| if [ "${{ matrix.target }}" = "x86_64-unknown-linux-musl" ]; then | |
| sudo apt-get install -y musl-tools | |
| fi | |
| - name: Install dependencies (macOS) | |
| if: matrix.os == 'macos-13' | |
| run: | | |
| brew install protobuf | |
| - name: Set build ID | |
| run: | | |
| echo "BUILD_ID=$(ci/generate_version_number.py)" >> $GITHUB_ENV | |
| - name: Build Rust binaries | |
| run: | | |
| cd rust | |
| cargo build --release --target ${{ matrix.target }} --no-default-features | |
| - name: Create release directory | |
| run: mkdir -p release | |
| - name: Copy binaries (Linux) | |
| if: matrix.os == 'ubuntu-22.04' | |
| run: | | |
| cp rust/target/${{ matrix.target }}/release/openrqd release/openrqd-${{ env.BUILD_ID }}-${{ matrix.target }} | |
| - name: Copy binaries (macOS) | |
| if: matrix.os == 'macos-13' | |
| run: | | |
| cp rust/target/${{ matrix.target }}/release/openrqd release/openrqd-${{ env.BUILD_ID }}-${{ matrix.target }} | |
| - name: Upload binaries | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: rust-binaries-${{ matrix.target }} | |
| path: release/ | |
| upload_python_packages_pypi: | |
| needs: build_opencue_packages | |
| runs-on: ubuntu-22.04 | |
| container: python:3.11 | |
| name: "Upload python packages to pypi repo" | |
| environment: release | |
| env: | |
| TWINE_USERNAME: "__token__" | |
| TWINE_PASSWORD: "${{ secrets.PYPI_API_TOKEN }}" | |
| steps: | |
| - name: Download artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: opencue_packages | |
| path: packages | |
| - name: Upload packages | |
| run: | | |
| python3 -m pip install --upgrade twine | |
| python3 -m twine upload --verbose packages/opencue_* | |
| release_docker_images: | |
| # needs: preflight | |
| runs-on: ubuntu-22.04 | |
| environment: release | |
| strategy: | |
| matrix: | |
| component: [cuebot, rqd] | |
| name: Release ${{ matrix.component }} Docker image | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-tags: true | |
| fetch-depth: 0 | |
| - name: Mark repository as safe (Fix for https://github.com/actions/checkout/issues/1048) | |
| run: git config --global --add safe.directory $GITHUB_WORKSPACE | |
| - name: Set build ID | |
| run: | | |
| set -e | |
| echo "Build ID: $(ci/generate_version_number.py)" | |
| echo "BUILD_ID=$(ci/generate_version_number.py)" >> ${GITHUB_ENV} | |
| - name: Get current tag name | |
| run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> ${GITHUB_ENV} | |
| - name: Verify tag name and version match | |
| run: | | |
| set -e | |
| if [ "v$(ci/generate_version_number.py)" != "${TAG_NAME}" ]; then | |
| echo "Version check failed: code version v$(ci/generate_version_number.py) does not match tag name ${TAG_NAME}" | |
| echo "Original GITHUB_REF: ${GITHUB_REF}" | |
| exit 1 | |
| fi | |
| - name: Pull Docker image from staging | |
| run: | | |
| set -e | |
| docker pull opencuebuild/${{ matrix.component }}:${BUILD_ID} | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v2 | |
| with: | |
| username: ${{ secrets.DOCKER_USER }} | |
| password: ${{ secrets.DOCKER_PASS }} | |
| - name: Rebuild and push Docker image | |
| run: | | |
| set -e | |
| docker tag opencuebuild/${{ matrix.component }}:${BUILD_ID} opencue/${{ matrix.component }}:${{ env.BUILD_ID }} | |
| docker push opencue/${{ matrix.component }}:${{ env.BUILD_ID }} | |
| docker tag opencuebuild/${{ matrix.component }}:${BUILD_ID} opencue/${{ matrix.component }}:latest | |
| docker push opencue/${{ matrix.component }}:latest | |
| # This step has been failing with permission issues. | |
| # Commenting this out temporarily to unblock the release of v1.4 | |
| # - name: Docker Hub Description | |
| # uses: peter-evans/dockerhub-description@v4 | |
| # with: | |
| # username: ${{ secrets.DOCKER_USER }} | |
| # password: ${{ secrets.DOCKER_PASS }} | |
| # repository: opencue/${{ matrix.component }} | |
| # readme-filepath: ./${{ matrix.component }}/README.md | |
| create_release: | |
| needs: build_rust_binaries | |
| name: Create Release | |
| environment: release | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.S3_REGION }} | |
| role-to-assume: ${{ secrets.AWS_S3_ROLE }} | |
| role-duration-seconds: 1800 | |
| - name: Set build ID | |
| run: | | |
| set -e | |
| ci/generate_version_number.sh > VERSION | |
| echo "Build ID: $(ci/generate_version_number.py)" | |
| echo "BUILD_ID=$(ci/generate_version_number.py)" >> ${GITHUB_ENV} | |
| - name: Download Rust binaries artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: rust-binaries-* | |
| path: rust-binaries | |
| merge-multiple: true | |
| - name: Fetch artifacts | |
| id: fetch_artifacts | |
| env: | |
| S3_BUCKET: ${{ secrets.S3_BUCKET }} | |
| run: | | |
| mkdir -p "${GITHUB_WORKSPACE}/artifacts/" | |
| aws s3 sync "s3://${S3_BUCKET}/opencue/${BUILD_ID}/" "${GITHUB_WORKSPACE}/artifacts/" | |
| echo "filenames=$(ls "${GITHUB_WORKSPACE}/artifacts/" | xargs)" >> ${GITHUB_OUTPUT} | |
| - name: List artifacts | |
| run: | | |
| echo ${{ steps.fetch_artifacts.outputs.filenames }} | |
| - name: Generate release notes | |
| id: release_notes | |
| run: | | |
| last_tagged_version=$(git describe --tags --abbrev=0 $(git rev-list --tags --skip=1 --max-count=1)) | |
| commits_since_last_release=$(git log --reverse --pretty="* %H %s" ${last_tagged_version}..HEAD) | |
| # Use a delimiter to preserve the multiline string. | |
| # See https://github.qkg1.topmunity/t/set-output-truncates-multiline-strings/16852 | |
| delimiter="$(openssl rand -hex 8)" | |
| echo "commits<<${delimiter}" >> ${GITHUB_OUTPUT} | |
| echo "${commits_since_last_release}" >> ${GITHUB_OUTPUT} | |
| echo "${delimiter}" >> ${GITHUB_OUTPUT} | |
| - name: Create release | |
| id: create_release | |
| uses: actions/create-release@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| tag_name: v${{ env.BUILD_ID }} | |
| release_name: v${{ env.BUILD_ID }} | |
| body: | | |
| To learn how to install and configure OpenCue, see our [Getting Started guide](https://www.opencue.io/docs/getting-started/). | |
| ## Changes: | |
| ${{ steps.release_notes.outputs.commits }} | |
| draft: true | |
| prerelease: false | |
| - name: Upload License | |
| uses: actions/upload-release-asset@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| upload_url: ${{ steps.create_release.outputs.upload_url }} | |
| asset_path: ${{ github.workspace }}/artifacts/LICENSE | |
| asset_name: LICENSE | |
| asset_content_type: application/octet-stream | |
| - name: Upload Database Schema | |
| uses: actions/upload-release-asset@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| upload_url: ${{ steps.create_release.outputs.upload_url }} | |
| asset_path: ${{ github.workspace }}/artifacts/schema-${{ env.BUILD_ID }}.sql | |
| asset_name: schema-${{ env.BUILD_ID }}.sql | |
| asset_content_type: application/octet-stream | |
| - name: Upload Demo Data | |
| uses: actions/upload-release-asset@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| upload_url: ${{ steps.create_release.outputs.upload_url }} | |
| asset_path: ${{ github.workspace }}/artifacts/seed_data-${{ env.BUILD_ID }}.sql | |
| asset_name: seed_data-${{ env.BUILD_ID }}.sql | |
| asset_content_type: application/octet-stream | |
| - name: Upload Cuebot JAR | |
| uses: actions/upload-release-asset@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| upload_url: ${{ steps.create_release.outputs.upload_url }} | |
| asset_path: ${{ github.workspace }}/artifacts/cuebot-${{ env.BUILD_ID }}-all.jar | |
| asset_name: cuebot-${{ env.BUILD_ID }}-all.jar | |
| asset_content_type: application/octet-stream | |
| - name: Upload Cuebot RPM | |
| uses: actions/upload-release-asset@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| upload_url: ${{ steps.create_release.outputs.upload_url }} | |
| asset_path: ${{ github.workspace }}/artifacts/opencue-cuebot-${{ env.BUILD_ID }}-1.noarch.rpm | |
| asset_name: opencue-cuebot-${{ env.BUILD_ID }}-1.noarch.rpm | |
| asset_content_type: application/octet-stream | |
| # Pause python artifact releases for now while ASWF discusses the implications of publishing | |
| # binary artifacts | |
| # | |
| # - name: Upload RQD package | |
| # uses: actions/upload-release-asset@v1 | |
| # env: | |
| # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # with: | |
| # upload_url: ${{ steps.create_release.outputs.upload_url }} | |
| # asset_path: ${{ github.workspace }}/artifacts/opencue_rqd-${{ env.BUILD_ID }}-py2.py3-none-any.whl | |
| # asset_name: rqd-${{ env.BUILD_ID }}-py2.py3-none-any.whl | |
| # asset_content_type: application/octet-stream | |
| # - name: Upload CueGUI package | |
| # uses: actions/upload-release-asset@v1 | |
| # env: | |
| # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # with: | |
| # upload_url: ${{ steps.create_release.outputs.upload_url }} | |
| # asset_path: ${{ github.workspace }}/artifacts/opencue_cuegui-${{ env.BUILD_ID }}-py2.py3-none-any.whl | |
| # asset_name: cuegui-${{ env.BUILD_ID }}-py2.py3-none-any.whl | |
| # asset_content_type: application/octet-stream | |
| # - name: Upload PyCue package | |
| # uses: actions/upload-release-asset@v1 | |
| # env: | |
| # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # with: | |
| # upload_url: ${{ steps.create_release.outputs.upload_url }} | |
| # asset_path: ${{ github.workspace }}/artifacts/opencue_pycue-${{ env.BUILD_ID }}-py2.py3-none-any.whl | |
| # asset_name: pycue-${{ env.BUILD_ID }}-py2.py3-none-any.whl | |
| # asset_content_type: application/octet-stream | |
| # - name: Upload PyOutline package | |
| # uses: actions/upload-release-asset@v1 | |
| # env: | |
| # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # with: | |
| # upload_url: ${{ steps.create_release.outputs.upload_url }} | |
| # asset_path: ${{ github.workspace }}/artifacts/opencue_pyoutline-${{ env.BUILD_ID }}-py2.py3-none-any.whl | |
| # asset_name: pyoutline-${{ env.BUILD_ID }}-py2.py3-none-any.whl | |
| # asset_content_type: application/octet-stream | |
| # - name: Upload CueSubmit package | |
| # uses: actions/upload-release-asset@v1 | |
| # env: | |
| # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # with: | |
| # upload_url: ${{ steps.create_release.outputs.upload_url }} | |
| # asset_path: ${{ github.workspace }}/artifacts/opencue_cuesubmit-${{ env.BUILD_ID }}-py2.py3-none-any.whl | |
| # asset_name: cuesubmit-${{ env.BUILD_ID }}-py2.py3-none-any.whl | |
| # asset_content_type: application/octet-stream | |
| # - name: Upload CueAdmin package | |
| # uses: actions/upload-release-asset@v1 | |
| # env: | |
| # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # with: | |
| # upload_url: ${{ steps.create_release.outputs.upload_url }} | |
| # asset_path: ${{ github.workspace }}/artifacts/opencue_cueadmin-${{ env.BUILD_ID }}-py2.py3-none-any.whl | |
| # asset_name: cueadmin-${{ env.BUILD_ID }}-py2.py3-none-any.whl | |
| # asset_content_type: application/octet-stream | |
| # - name: Upload CueMan package | |
| # uses: actions/upload-release-asset@v1 | |
| # env: | |
| # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # with: | |
| # upload_url: ${{ steps.create_release.outputs.upload_url }} | |
| # asset_path: ${{ github.workspace }}/artifacts/opencue_cueman-${{ env.BUILD_ID }}-py2.py3-none-any.whl | |
| # asset_name: cueman-${{ env.BUILD_ID }}-py2.py3-none-any.whl | |
| # asset_content_type: application/octet-stream | |
| # - name: Upload CueNIMBY package | |
| # uses: actions/upload-release-asset@v1 | |
| # env: | |
| # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # with: | |
| # upload_url: ${{ steps.create_release.outputs.upload_url }} | |
| # asset_path: ${{ github.workspace }}/artifacts/opencue_cuenimby-${{ env.BUILD_ID }}-py2.py3-none-any.whl | |
| # asset_name: cuenimby-${{ env.BUILD_ID }}-py2.py3-none-any.whl | |
| # asset_content_type: application/octet-stream | |
| - name: Upload openrqd (Linux GNU) | |
| uses: actions/upload-release-asset@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| upload_url: ${{ steps.create_release.outputs.upload_url }} | |
| asset_path: ${{ github.workspace }}/artifacts/openrqd-${{ env.BUILD_ID }}-x86_64-unknown-linux-gnu | |
| asset_name: openrqd-${{ env.BUILD_ID }}-x86_64-unknown-linux-gnu | |
| asset_content_type: application/octet-stream | |
| - name: Upload openrqd (Linux MUSL) | |
| uses: actions/upload-release-asset@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| upload_url: ${{ steps.create_release.outputs.upload_url }} | |
| asset_path: ${{ github.workspace }}/artifacts/openrqd-${{ env.BUILD_ID }}-x86_64-unknown-linux-musl | |
| asset_name: openrqd-${{ env.BUILD_ID }}-x86_64-unknown-linux-musl | |
| asset_content_type: application/octet-stream | |
| - name: Upload openrqd (macOS Intel) | |
| uses: actions/upload-release-asset@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| upload_url: ${{ steps.create_release.outputs.upload_url }} | |
| asset_path: ${{ github.workspace }}/artifacts/openrqd-${{ env.BUILD_ID }}-x86_64-apple-darwin | |
| asset_name: openrqd-${{ env.BUILD_ID }}-x86_64-apple-darwin | |
| asset_content_type: application/octet-stream | |
| - name: Upload openrqd (macOS Apple Silicon) | |
| uses: actions/upload-release-asset@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| upload_url: ${{ steps.create_release.outputs.upload_url }} | |
| asset_path: ${{ github.workspace }}/artifacts/openrqd-${{ env.BUILD_ID }}-aarch64-apple-darwin | |
| asset_name: openrqd-${{ env.BUILD_ID }}-aarch64-apple-darwin | |
| asset_content_type: application/octet-stream |