Build #60647
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| name: Build | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| force_build: | |
| description: "Force run build job when manually triggered" | |
| required: false | |
| default: "true" | |
| schedule: | |
| - cron: '*/30 * * * *' | |
| jobs: | |
| prerelease: | |
| name: Prerelease | |
| runs-on: macos-14 | |
| env: | |
| GH_REPO: ${{ github.repository }} | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| permissions: | |
| contents: write | |
| outputs: | |
| should_release: ${{ steps.check_diff.outputs.should_release }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: 'apache/doris' | |
| fetch-depth: 0 | |
| - name: Check Diff | |
| id: check_diff | |
| run: | | |
| if [[ -z "$(gh release list)" ]] || | |
| ! gh release list | awk -F "\t" '{ print $3 }' | grep 'automation' >/dev/null; then | |
| gh release create -t 'Apache Doris Third Party Prebuilt' automation | |
| fi | |
| last_version="$(gh release view automation | sed -n -E 's/Doris Version: \*(.*)\*.*/\1/p')" | |
| current_version="$(git log -1 --format='%H')" | |
| echo "Last Version: ${last_version}" | |
| echo "Current Version: ${current_version}" | |
| should_release=false | |
| if [[ -z "${last_version}" ]]; then | |
| echo "The first release was detected." | |
| should_release=true | |
| elif [[ "${last_version}" != "${current_version}" ]]; then | |
| cmd="git diff --name-only ${last_version} ${current_version} | grep -E '^thirdparty/'" | |
| echo "Execute: ${cmd}" | |
| content="$(eval "${cmd}")" || true | |
| if [[ -n "${content}" ]]; then | |
| echo -e "Detect changes:\n${content}" | |
| should_release=true | |
| fi | |
| fi | |
| if "${should_release}"; then | |
| echo -ne "Update Time: *$(date)*\nDoris Version: *${current_version}*\nStatus: *BUILDING*" >release_note.md | |
| else | |
| gh release view automation | sed -n '/--/,$p' | awk '{ if (NR > 1) print $0 }' | sed "{ | |
| s/Update Time:.*/Update Time: *$(date)*/ | |
| s/Doris Version:.*/Doris Version: *${current_version}*/ | |
| }" >release_note.md | |
| fi | |
| gh release edit -F release_note.md automation | |
| echo "should_release=${should_release}" >> $GITHUB_OUTPUT | |
| - name: Download Source and Upload | |
| if: steps.check_diff.outputs.should_release == 'true' | |
| run: | | |
| cd thirdparty | |
| sed '/# unpacking thirdpart archives/,$d' download-thirdparty.sh | bash - | |
| tar -zcvf doris-thirdparty-source.tgz src | |
| gh release upload --clobber automation doris-thirdparty-source.tgz | |
| build: | |
| name: Build | |
| needs: prerelease | |
| if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: | |
| - name: macOS-x86_64 | |
| os: macos-15-intel | |
| continue-on-error: true | |
| packages: >- | |
| 'm4' | |
| 'automake' | |
| 'autoconf' | |
| 'libtool' | |
| 'pkg-config' | |
| 'texinfo' | |
| 'coreutils' | |
| 'gnu-getopt' | |
| 'python@3' | |
| 'ninja' | |
| 'ccache' | |
| 'bison' | |
| 'byacc' | |
| 'gettext' | |
| 'wget' | |
| 'pcre' | |
| 'openjdk@11' | |
| 'maven' | |
| 'node' | |
| 'llvm@20' | |
| - name: macOS-arm64 | |
| os: macos-14 | |
| packages: >- | |
| 'm4' | |
| 'automake' | |
| 'autoconf' | |
| 'libtool' | |
| 'pkg-config' | |
| 'texinfo' | |
| 'coreutils' | |
| 'gnu-getopt' | |
| 'python@3' | |
| 'ninja' | |
| 'ccache' | |
| 'bison' | |
| 'byacc' | |
| 'gettext' | |
| 'wget' | |
| 'pcre' | |
| 'openjdk@11' | |
| 'maven' | |
| 'node' | |
| 'llvm@20' | |
| - name: Linux | |
| os: ubuntu-22.04 | |
| packages: >- | |
| 'build-essential' | |
| 'automake' | |
| 'autoconf' | |
| 'libtool-bin' | |
| 'pkg-config' | |
| 'cmake=3.22.1-1ubuntu1.22.04.2' | |
| 'ninja-build' | |
| 'ccache' | |
| 'python-is-python3' | |
| 'bison' | |
| 'byacc' | |
| 'flex' | |
| 'binutils-dev' | |
| 'libiberty-dev' | |
| 'curl' | |
| 'git' | |
| 'zip' | |
| 'unzip' | |
| 'autopoint' | |
| 'openjdk-8-jdk' | |
| 'openjdk-8-jdk-headless' | |
| 'maven' | |
| runs-on: ${{ matrix.config.os }} | |
| continue-on-error: ${{ matrix.config.continue-on-error == true }} | |
| env: | |
| GH_REPO: ${{ github.repository }} | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Checkout easimon/maximize-build-space | |
| if: ${{ matrix.config.name == 'Linux' }} | |
| run: | | |
| git clone -b v7 https://github.com/easimon/maximize-build-space | |
| #- name: Maximize build space | |
| # if: ${{ matrix.config.name == 'Linux' }} | |
| # uses: ./maximize-build-space | |
| # with: | |
| # root-reserve-mb: 4096 | |
| # swap-size-mb: 8192 | |
| # remove-dotnet: 'true' | |
| # remove-android: 'true' | |
| # remove-haskell: 'true' | |
| # remove-codeql: 'true' | |
| # remove-docker-images: 'true' | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: 'apache/doris' | |
| - name: Download | |
| run: | | |
| cd thirdparty | |
| curl -L https://github.com/${{ github.repository }}/releases/download/automation/doris-thirdparty-source.tgz \ | |
| -o doris-thirdparty-source.tgz | |
| tar -zxvf doris-thirdparty-source.tgz | |
| - name: Prepare for ${{ matrix.config.os }} | |
| run: | | |
| # Doris's lance-c build prefers Rust 1.91.0 when it is installed. | |
| # Pin it here so hosted-runner toolchain updates do not break locked Rust dependencies. | |
| rustup toolchain install 1.91.0 --profile minimal | |
| rustup run 1.91.0 rustc --version | |
| if [[ "${{ matrix.config.name }}" =~ macOS-* ]]; then | |
| # Install packages except cmake | |
| brew install ${{ matrix.config.packages }} || true | |
| # Install specific version of cmake | |
| brew unlink cmake || true | |
| wget https://github.com/Kitware/CMake/releases/download/v3.22.1/cmake-3.22.1-macos-universal.tar.gz | |
| tar -xzf cmake-3.22.1-macos-universal.tar.gz | |
| sudo cp -r cmake-3.22.1-macos-universal/CMake.app/Contents/* /usr/local/ | |
| cmake --version | |
| else | |
| export DEFAULT_DIR='/opt/doris' | |
| export PATH="${DEFAULT_DIR}/ldb-toolchain/bin:${PATH}" | |
| sudo apt update | |
| sudo apt-cache policy cmake | |
| sudo DEBIAN_FRONTEND=noninteractive apt install --yes ${{ matrix.config.packages }} | |
| mkdir -p "${DEFAULT_DIR}" | |
| wget https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.25/ldb_toolchain_gen.sh \ | |
| -q -O /tmp/ldb_toolchain_gen.sh | |
| bash /tmp/ldb_toolchain_gen.sh "${DEFAULT_DIR}/ldb-toolchain" | |
| fi | |
| - name: Build and Upload | |
| run: | | |
| if [[ "${{ matrix.config.name }}" == 'Linux' ]]; then | |
| export DEFAULT_DIR='/opt/doris' | |
| export PATH="${DEFAULT_DIR}/ldb-toolchain/bin:${PATH}" | |
| export PATH="$(find /usr/lib/jvm/java-8-openjdk* -maxdepth 1 -type d -name 'bin'):${PATH}" | |
| export JAVA_HOME="$(find /usr/lib/jvm/java-8-openjdk* -maxdepth 0)" | |
| export DORIS_TOOLCHAIN=clang | |
| else | |
| export MACOSX_DEPLOYMENT_TARGET=12.0 | |
| fi | |
| export CMAKE_POLICY_VERSION_MINIMUM=3.10 | |
| export CUSTOM_CMAKE="/usr/local/bin/cmake" | |
| cd thirdparty | |
| ./build-thirdparty.sh -j "$(nproc)" | |
| kernel="$(uname -s | awk '{print tolower($0)}')" | |
| arch="$(uname -m)" | |
| rm -rf "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" | |
| tar -cf - installed | xz -z -T0 - >"doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" | |
| gh release upload --clobber automation "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" | |
| update-docker: | |
| name: Update Docker Image | |
| needs: [prerelease, build] | |
| if: | | |
| always() && ( | |
| (needs.prerelease.outputs.should_release == 'true' && needs.build.result == 'success') || | |
| (github.event_name == 'workflow_dispatch' && needs.build.result != 'failure') | |
| ) | |
| runs-on: ubuntu-latest | |
| env: | |
| GH_REPO: ${{ github.repository }} | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Checkout docker/setup-buildx-action | |
| run: | | |
| rm -rf ./.github/actions/setup-buildx-action | |
| git clone https://github.com/docker/setup-buildx-action .github/actions/setup-buildx-action | |
| pushd .github/actions/setup-buildx-action &>/dev/null | |
| git checkout 8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 | |
| popd &>/dev/null | |
| - name: Checkout docker/login-action | |
| run: | | |
| rm -rf ./.github/actions/login-action | |
| git clone https://github.com/docker/login-action .github/actions/login-action | |
| pushd .github/actions/login-action &>/dev/null | |
| git checkout c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 | |
| popd &>/dev/null | |
| - name: Checkout docker/build-push-action | |
| run: | | |
| rm -rf ./.github/actions/build-push-action | |
| git clone https://github.com/docker/build-push-action .github/actions/build-push-action | |
| pushd .github/actions/build-push-action &>/dev/null | |
| git checkout ca052bb54ab0790a636c9b5f226502c73d547a25 # v5 | |
| popd &>/dev/null | |
| - name: Set up Docker Buildx | |
| uses: ./.github/actions/setup-buildx-action | |
| - name: Login to Docker Hub | |
| uses: ./.github/actions/login-action | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Prepare Dockerfiles | |
| run: | | |
| # Pull the upstream Dockerfile from apache/doris to stay in sync with | |
| # any toolchain / dependency changes they make. | |
| curl -fsSL https://raw.githubusercontent.com/apache/doris/master/docker/compilation/Dockerfile \ | |
| -o Dockerfile.upstream | |
| python3 - << 'PYEOF' | |
| import sys, re | |
| with open('Dockerfile.upstream') as f: | |
| content = f.read() | |
| # Normalize line endings (raw.githubusercontent.com may serve \r\n) | |
| content = content.replace('\r\n', '\n') | |
| # Patch 1: replace "clone & build thirdparty" block with downloading | |
| # our prebuilt artifact. The block starts with "# clone lastest source | |
| # code" comment and ends with "rm -rf ${DEFAULT_DIR}/doris". | |
| prebuilt_block = ( | |
| '# Download prebuilt thirdparty from GitHub Release (built by doris-thirdparty automation)\n' | |
| 'ARG GITHUB_REPOSITORY\n' | |
| 'RUN mkdir -p /var/local/thirdparty \\\n' | |
| ' && wget -q "https://github.com/${GITHUB_REPOSITORY}/releases/download/automation/doris-thirdparty-prebuilt-linux-x86_64.tar.xz" \\\n' | |
| ' -O /tmp/prebuilt.tar.xz \\\n' | |
| ' && tar -xf /tmp/prebuilt.tar.xz -C /var/local/thirdparty \\\n' | |
| ' && rm /tmp/prebuilt.tar.xz\n' | |
| ) | |
| patched = re.sub( | |
| r'# clone lastest source code.*?rm -rf \$\{DEFAULT_DIR\}/doris\n', | |
| prebuilt_block, | |
| content, flags=re.DOTALL | |
| ) | |
| assert patched != content, "Patch 1 failed: 'clone lastest source code' block not found in upstream Dockerfile" | |
| # Write normal image Dockerfile | |
| with open('Dockerfile.patched', 'w') as f: | |
| f.write(patched) | |
| # Patch 2 (no-avx2): add USE_AVX2=0 to the ENV block in builder stage. | |
| # The ENV block ends with PATH=... just before "# install ccache". | |
| # The PATH now includes /root/.cargo/bin for the Rust toolchain. | |
| old_avx2 = 'PATH="/root/.cargo/bin:/var/local/ldb-toolchain/bin/:$PATH"\n # USE_AVX2=0' | |
| new_avx2 = 'PATH="/root/.cargo/bin:/var/local/ldb-toolchain/bin/:$PATH" \\\n USE_AVX2=0' | |
| assert old_avx2 in patched, "Patch 2 failed: '# USE_AVX2=0' comment not found; upstream Dockerfile may have changed" | |
| noavx2 = patched.replace(old_avx2, new_avx2) | |
| assert noavx2 != patched, "Patch 2 was a no-op: no-avx2 Dockerfile is identical to normal Dockerfile" | |
| with open('Dockerfile.patched-noavx2', 'w') as f: | |
| f.write(noavx2) | |
| print("=== normal: check ===") | |
| for line in open('Dockerfile.patched'): | |
| if any(k in line for k in ['COPY doris', 'build-thirdparty', 'ARG GITHUB', 'prebuilt', 'USE_AVX2']): | |
| print(line, end='') | |
| print("=== noavx2: check ===") | |
| for line in open('Dockerfile.patched-noavx2'): | |
| if any(k in line for k in ['COPY doris', 'build-thirdparty', 'ARG GITHUB', 'prebuilt', 'USE_AVX2']): | |
| print(line, end='') | |
| PYEOF | |
| - name: Build and Push Docker Image (normal) | |
| uses: ./.github/actions/build-push-action | |
| with: | |
| context: . | |
| file: Dockerfile.patched | |
| build-args: | | |
| GITHUB_REPOSITORY=${{ github.repository }} | |
| push: true | |
| tags: ${{ vars.DOCKER_TAGS || 'apache/doris:build-env-ldb-toolchain-latest' }} | |
| platforms: linux/amd64 | |
| - name: Build and Push Docker Image (no-avx2) | |
| uses: ./.github/actions/build-push-action | |
| with: | |
| context: . | |
| file: Dockerfile.patched-noavx2 | |
| build-args: | | |
| GITHUB_REPOSITORY=${{ github.repository }} | |
| push: true | |
| tags: ${{ vars.DOCKER_TAGS_NOAVX2 || 'apache/doris:build-env-ldb-toolchain-no-avx2-latest' }} | |
| platforms: linux/amd64 | |
| success: | |
| name: Success | |
| needs: [prerelease, build, update-docker] | |
| if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') | |
| runs-on: ubuntu-latest | |
| env: | |
| GH_REPO: ${{ github.repository }} | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Update Checksums | |
| run: | | |
| gh release download automation | |
| content="$(gh release view automation | sed -n '/Update Time:/,/Doris Version:/p')" | |
| echo -ne "${content}\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md | |
| gh release edit --latest -F release_note.md automation | |
| failure: | |
| name: Failure | |
| needs: [build, update-docker] | |
| if: always() && failure() | |
| runs-on: ubuntu-latest | |
| env: | |
| GH_REPO: ${{ github.repository }} | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Update Checksums | |
| run: | | |
| gh release download automation | |
| echo -ne "Status: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md | |
| gh release edit --latest -F release_note.md automation |