build.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: Build

on:
  workflow_dispatch:
    inputs:
      force_build:
        description: "Force run build job when manually triggered"
        required: false
        default: "true"
  schedule:
    - cron: '*/30 * * * *'

jobs:
  prerelease:
    name: Prerelease
    runs-on: macos-14
    env:
      GH_REPO: ${{ github.repository }}
      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
    permissions:
      contents: write
    outputs:
      should_release: ${{ steps.check_diff.outputs.should_release }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          repository: 'apache/doris'
          fetch-depth: 0

      - name: Check Diff
        id: check_diff
        run: |
          if [[ -z "$(gh release list)" ]] ||
              ! gh release list | awk -F "\t" '{ print $3 }' | grep 'automation' >/dev/null; then
            gh release create -t 'Apache Doris Third Party Prebuilt' automation
          fi
          last_version="$(gh release view automation | sed -n -E 's/Doris Version: \*(.*)\*.*/\1/p')"
          current_version="$(git log -1 --format='%H')"

          echo "Last Version: ${last_version}"
          echo "Current Version: ${current_version}"

          should_release=false
          if [[ -z "${last_version}" ]]; then
            echo "The first release was detected."
            should_release=true
          elif [[ "${last_version}" != "${current_version}" ]]; then
            cmd="git diff --name-only ${last_version} ${current_version} | grep -E '^thirdparty/'"
            echo "Execute: ${cmd}"
            content="$(eval "${cmd}")" || true
            if [[ -n "${content}" ]]; then
              echo -e "Detect changes:\n${content}"
              should_release=true
            fi
          fi

          if "${should_release}"; then
            echo -ne "Update Time: *$(date)*\nDoris Version: *${current_version}*\nStatus: *BUILDING*" >release_note.md
          else
            gh release view automation | sed -n '/--/,$p' | awk '{ if (NR > 1) print $0 }' | sed "{
              s/Update Time:.*/Update Time: *$(date)*/
              s/Doris Version:.*/Doris Version: *${current_version}*/
            }" >release_note.md
          fi
          gh release edit -F release_note.md automation

          echo "should_release=${should_release}" >> $GITHUB_OUTPUT

      - name: Download Source and Upload
        if: steps.check_diff.outputs.should_release == 'true'
        run: |
          cd thirdparty
          sed '/# unpacking thirdpart archives/,$d' download-thirdparty.sh | bash -

          tar -zcvf doris-thirdparty-source.tgz src

          gh release upload --clobber automation doris-thirdparty-source.tgz

  build:
    name: Build
    needs: prerelease
    if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true')
    strategy:
      fail-fast: false
      matrix:
        config:
          - name: macOS-x86_64
            os: macos-15-intel
            continue-on-error: true
            packages: >-
              'm4'
              'automake'
              'autoconf'
              'libtool'
              'pkg-config'
              'texinfo'
              'coreutils'
              'gnu-getopt'
              'python@3'
              'ninja'
              'ccache'
              'bison'
              'byacc'
              'gettext'
              'wget'
              'pcre'
              'openjdk@11'
              'maven'
              'node'
              'llvm@20'

          - name: macOS-arm64
            os: macos-14
            packages: >-
              'm4'
              'automake'
              'autoconf'
              'libtool'
              'pkg-config'
              'texinfo'
              'coreutils'
              'gnu-getopt'
              'python@3'
              'ninja'
              'ccache'
              'bison'
              'byacc'
              'gettext'
              'wget'
              'pcre'
              'openjdk@11'
              'maven'
              'node'
              'llvm@20'

          - name: Linux
            os: ubuntu-22.04
            packages: >-
              'build-essential'
              'automake'
              'autoconf'
              'libtool-bin'
              'pkg-config'
              'cmake=3.22.1-1ubuntu1.22.04.2'
              'ninja-build'
              'ccache'
              'python-is-python3'
              'bison'
              'byacc'
              'flex'
              'binutils-dev'
              'libiberty-dev'
              'curl'
              'git'
              'zip'
              'unzip'
              'autopoint'
              'openjdk-8-jdk'
              'openjdk-8-jdk-headless'
              'maven'

    runs-on: ${{ matrix.config.os }}
    continue-on-error: ${{ matrix.config.continue-on-error == true }}
    env:
      GH_REPO: ${{ github.repository }}
      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Checkout easimon/maximize-build-space
        if: ${{ matrix.config.name == 'Linux' }}
        run: |
          git clone -b v7 https://github.com/easimon/maximize-build-space

      #- name: Maximize build space
      #  if: ${{ matrix.config.name == 'Linux' }}
      #  uses: ./maximize-build-space
      #  with:
      #      root-reserve-mb: 4096
      #      swap-size-mb: 8192
      #      remove-dotnet: 'true'
      #      remove-android: 'true'
      #      remove-haskell: 'true'
      #      remove-codeql: 'true'
      #      remove-docker-images: 'true'

      - name: Checkout
        uses: actions/checkout@v4
        with:
          repository: 'apache/doris'

      - name: Download
        run: |
          cd thirdparty
          curl -L https://github.com/${{ github.repository }}/releases/download/automation/doris-thirdparty-source.tgz \
            -o doris-thirdparty-source.tgz
          tar -zxvf doris-thirdparty-source.tgz

      - name: Prepare for ${{ matrix.config.os }}
        run: |
          # Doris's lance-c build prefers Rust 1.91.0 when it is installed.
          # Pin it here so hosted-runner toolchain updates do not break locked Rust dependencies.
          rustup toolchain install 1.91.0 --profile minimal
          rustup run 1.91.0 rustc --version

          if [[ "${{ matrix.config.name }}" =~ macOS-* ]]; then
            # Install packages except cmake
            brew install ${{ matrix.config.packages }} || true
            # Install specific version of cmake
            brew unlink cmake || true
            wget https://github.com/Kitware/CMake/releases/download/v3.22.1/cmake-3.22.1-macos-universal.tar.gz
            tar -xzf cmake-3.22.1-macos-universal.tar.gz
            sudo cp -r cmake-3.22.1-macos-universal/CMake.app/Contents/* /usr/local/
            cmake --version
          else
            export DEFAULT_DIR='/opt/doris'
            export PATH="${DEFAULT_DIR}/ldb-toolchain/bin:${PATH}"

            sudo apt update
            sudo apt-cache policy cmake
            sudo DEBIAN_FRONTEND=noninteractive apt install --yes ${{ matrix.config.packages }}

            mkdir -p "${DEFAULT_DIR}"
            wget https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.25/ldb_toolchain_gen.sh \
              -q -O /tmp/ldb_toolchain_gen.sh
            bash /tmp/ldb_toolchain_gen.sh "${DEFAULT_DIR}/ldb-toolchain"
          fi

      - name: Build and Upload
        run: |
          if [[ "${{ matrix.config.name }}" == 'Linux' ]]; then
            export DEFAULT_DIR='/opt/doris'
            export PATH="${DEFAULT_DIR}/ldb-toolchain/bin:${PATH}"
            export PATH="$(find /usr/lib/jvm/java-8-openjdk* -maxdepth 1 -type d -name 'bin'):${PATH}"
            export JAVA_HOME="$(find /usr/lib/jvm/java-8-openjdk* -maxdepth 0)"
            export DORIS_TOOLCHAIN=clang
          else
            export MACOSX_DEPLOYMENT_TARGET=12.0
          fi

          export CMAKE_POLICY_VERSION_MINIMUM=3.10
          export CUSTOM_CMAKE="/usr/local/bin/cmake"
          cd thirdparty
          ./build-thirdparty.sh -j "$(nproc)"

          kernel="$(uname -s | awk '{print tolower($0)}')"
          arch="$(uname -m)"
          rm -rf "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz"
          tar -cf - installed | xz -z -T0 - >"doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz"
          gh release upload --clobber automation "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz"

  update-docker:
    name: Update Docker Image
    needs: [prerelease, build]
    if: |
      always() && (
        (needs.prerelease.outputs.should_release == 'true' && needs.build.result == 'success') ||
        (github.event_name == 'workflow_dispatch' && needs.build.result != 'failure')
      )
    runs-on: ubuntu-latest
    env:
      GH_REPO: ${{ github.repository }}
      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Checkout docker/setup-buildx-action
        run: |
          rm -rf ./.github/actions/setup-buildx-action
          git clone https://github.com/docker/setup-buildx-action .github/actions/setup-buildx-action

          pushd .github/actions/setup-buildx-action &>/dev/null
          git checkout 8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
          popd &>/dev/null

      - name: Checkout docker/login-action
        run: |
          rm -rf ./.github/actions/login-action
          git clone https://github.com/docker/login-action .github/actions/login-action

          pushd .github/actions/login-action &>/dev/null
          git checkout c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
          popd &>/dev/null

      - name: Checkout docker/build-push-action
        run: |
          rm -rf ./.github/actions/build-push-action
          git clone https://github.com/docker/build-push-action .github/actions/build-push-action

          pushd .github/actions/build-push-action &>/dev/null
          git checkout ca052bb54ab0790a636c9b5f226502c73d547a25 # v5
          popd &>/dev/null

      - name: Set up Docker Buildx
        uses: ./.github/actions/setup-buildx-action

      - name: Login to Docker Hub
        uses: ./.github/actions/login-action
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Prepare Dockerfiles
        run: |
          # Pull the upstream Dockerfile from apache/doris to stay in sync with
          # any toolchain / dependency changes they make.
          curl -fsSL https://raw.githubusercontent.com/apache/doris/master/docker/compilation/Dockerfile \
            -o Dockerfile.upstream

          python3 - << 'PYEOF'
          import sys, re

          with open('Dockerfile.upstream') as f:
              content = f.read()

          # Normalize line endings (raw.githubusercontent.com may serve \r\n)
          content = content.replace('\r\n', '\n')

          # Patch 1: replace "clone & build thirdparty" block with downloading
          # our prebuilt artifact. The block starts with "# clone lastest source
          # code" comment and ends with "rm -rf ${DEFAULT_DIR}/doris".
          prebuilt_block = (
              '# Download prebuilt thirdparty from GitHub Release (built by doris-thirdparty automation)\n'
              'ARG GITHUB_REPOSITORY\n'
              'RUN mkdir -p /var/local/thirdparty \\\n'
              '    && wget -q "https://github.com/${GITHUB_REPOSITORY}/releases/download/automation/doris-thirdparty-prebuilt-linux-x86_64.tar.xz" \\\n'
              '        -O /tmp/prebuilt.tar.xz \\\n'
              '    && tar -xf /tmp/prebuilt.tar.xz -C /var/local/thirdparty \\\n'
              '    && rm /tmp/prebuilt.tar.xz\n'
          )
          patched = re.sub(
              r'# clone lastest source code.*?rm -rf \$\{DEFAULT_DIR\}/doris\n',
              prebuilt_block,
              content, flags=re.DOTALL
          )
          assert patched != content, "Patch 1 failed: 'clone lastest source code' block not found in upstream Dockerfile"

          # Write normal image Dockerfile
          with open('Dockerfile.patched', 'w') as f:
              f.write(patched)

          # Patch 2 (no-avx2): add USE_AVX2=0 to the ENV block in builder stage.
          # The ENV block ends with PATH=... just before "# install ccache".
          # The PATH now includes /root/.cargo/bin for the Rust toolchain.
          old_avx2 = 'PATH="/root/.cargo/bin:/var/local/ldb-toolchain/bin/:$PATH"\n    # USE_AVX2=0'
          new_avx2 = 'PATH="/root/.cargo/bin:/var/local/ldb-toolchain/bin/:$PATH" \\\n    USE_AVX2=0'
          assert old_avx2 in patched, "Patch 2 failed: '# USE_AVX2=0' comment not found; upstream Dockerfile may have changed"
          noavx2 = patched.replace(old_avx2, new_avx2)
          assert noavx2 != patched, "Patch 2 was a no-op: no-avx2 Dockerfile is identical to normal Dockerfile"
          with open('Dockerfile.patched-noavx2', 'w') as f:
              f.write(noavx2)

          print("=== normal: check ===")
          for line in open('Dockerfile.patched'):
              if any(k in line for k in ['COPY doris', 'build-thirdparty', 'ARG GITHUB', 'prebuilt', 'USE_AVX2']):
                  print(line, end='')
          print("=== noavx2: check ===")
          for line in open('Dockerfile.patched-noavx2'):
              if any(k in line for k in ['COPY doris', 'build-thirdparty', 'ARG GITHUB', 'prebuilt', 'USE_AVX2']):
                  print(line, end='')
          PYEOF

      - name: Build and Push Docker Image (normal)
        uses: ./.github/actions/build-push-action
        with:
          context: .
          file: Dockerfile.patched
          build-args: |
            GITHUB_REPOSITORY=${{ github.repository }}
          push: true
          tags: ${{ vars.DOCKER_TAGS || 'apache/doris:build-env-ldb-toolchain-latest' }}
          platforms: linux/amd64

      - name: Build and Push Docker Image (no-avx2)
        uses: ./.github/actions/build-push-action
        with:
          context: .
          file: Dockerfile.patched-noavx2
          build-args: |
            GITHUB_REPOSITORY=${{ github.repository }}
          push: true
          tags: ${{ vars.DOCKER_TAGS_NOAVX2 || 'apache/doris:build-env-ldb-toolchain-no-avx2-latest' }}
          platforms: linux/amd64

  success:
    name: Success
    needs: [prerelease, build, update-docker]
    if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true')
    runs-on: ubuntu-latest
    env:
      GH_REPO: ${{ github.repository }}
      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
    permissions:
      contents: write
    steps:
      - name: Update Checksums
        run: |
          gh release download automation

          content="$(gh release view automation | sed -n '/Update Time:/,/Doris Version:/p')"
          echo -ne "${content}\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md
          gh release edit --latest -F release_note.md automation

  failure:
    name: Failure
    needs: [build, update-docker]
    if: always() && failure()
    runs-on: ubuntu-latest
    env:
      GH_REPO: ${{ github.repository }}
      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
    permissions:
      contents: write
    steps:
      - name: Update Checksums
        run: |
          gh release download automation

          echo -ne "Status: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md
          gh release edit --latest -F release_note.md automation