add IXUCA publish workflow #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: paddle ixuca build and publish | ||
|
Check failure on line 1 in .github/workflows/Publish_IXUCA.yaml
|
||
| on: | ||
| workflow_dispatch: | ||
| schedule: | ||
| - cron: '0 18 * * *' # 2:00 AM China Standard Time (UTC+8) | ||
| push: | ||
| tags: | ||
| - '*' | ||
| pull_request: | ||
| types: [opened, synchronize] | ||
| branches: [develop, release/**] | ||
| permissions: read-all | ||
| defaults: | ||
| run: | ||
| shell: bash | ||
| jobs: | ||
| code-checkout: | ||
| runs-on: | ||
| group: HK-Clone | ||
| env: | ||
| BRANCH: ${{ github.ref_name }} | ||
| COMMIT: ${{ github.sha }} | ||
| output: | ||
| repo_archive_url: ${{ steps.code_upload.outputs.repo_archive_url }} | ||
| steps: | ||
| - name: Clone PaddleCustomDevice | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| ref: ${{ github.ref_name }} | ||
| submodules: 'recursive' | ||
| fetch-depth: 1000 | ||
| - name: Download bos client | ||
| env: | ||
| home_path: "/home/paddle/actions-runner/" | ||
| bos_file: "/home/paddle/actions-runner/bos/BosClient.py" | ||
| run: | | ||
| if [ ! -f "${bos_file}" ]; then | ||
| wget -q --no-proxy -O ${home_path}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate | ||
| mkdir ${home_path}/bos | ||
| tar xf ${home_path}/bos_new.tar.gz -C ${home_path}/bos | ||
| fi | ||
| - name: Push paddle-action.tar.gz to bos | ||
| env: | ||
| AK: paddle | ||
| SK: paddle | ||
| bos_file: "/home/paddle/actions-runner/bos/BosClient.py" | ||
| id: code_upload | ||
| run: | | ||
| set -x | ||
| git reset --hard ${COMMIT} | ||
| git log -n 3 | ||
| cd .. | ||
| tar -I 'zstd -T0' -cf PaddleCustomDevice.tar.gz PaddleCustomDevice | ||
| echo "::group::Install bce-python-sdk" | ||
| python -m pip install bce-python-sdk==0.8.74 | ||
| echo "::endgroup::" | ||
| python ${bos_file} PaddleCustomDevice.tar.gz paddle-github-action/PaddleCustomDevice/COMMIT/${BRANCH}/${COMMIT} | ||
| rm PaddleCustomDevice.tar.gz | ||
| repo_archive_url=https://paddle-github-action.bj.bcebos.com/PaddleCustomDevice/COMMIT/${BRANCH}/${COMMIT}/PaddleCustomDevice.tar.gz | ||
| echo "repo_archive_url=${repo_archive_url}" >> $GITHUB_OUTPUT | ||
| ixuca-gpu-publish: | ||
| needs: code-checkout | ||
| environment: PublishUploadUse | ||
| env: | ||
| PaddleCustomDeviceVersion: ${{ env.PaddleCustomDeviceVersion }} | ||
| AK: ${{ secrets.BOS_AK }} | ||
| SK: ${{ secrets.BOS_SK }} | ||
| TASK: paddle-CI-${{ github.sha }}-ixuca | ||
| no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn" | ||
| REPO_ARCHIVE_URL: ${{ needs.code-checkout.outputs.repo_archive_url }} | ||
| BRANCH: ${{ github.ref_name }} | ||
| COMMIT: ${{ github.sha }} | ||
| REF_TYPE: ${{ github.ref_type }} | ||
| runs-on: | ||
| group: test2 | ||
| steps: | ||
| - name: Cleanup | ||
| run: | | ||
| rm -rf * .[^.]* | ||
| - name: Check docker image and run container | ||
| env: | ||
| GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
| RUN_ID: ${{ github.run_id }} | ||
| cache_dir: /root/.cache | ||
| ccache_dir: /root/.ccache | ||
| docker_image: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/device/paddle-ixuca:latest | ||
| run: | | ||
| container_name=${TASK}-$(date +%Y%m%d-%H%M%S) | ||
| echo "container_name=${container_name}" >> ${{ github.env }} | ||
| docker run --shm-size=128G \ | ||
| -d -t --name ${container_name} \ | ||
| --privileged --network=host \ | ||
| -e PR_ID \ | ||
| -e COMMIT_ID \ | ||
| -e BRANCH \ | ||
| -e no_proxy \ | ||
| -e PaddleCustomDeviceVersion \ | ||
| -e AK \ | ||
| -e SK \ | ||
| -e LD_LIBRARY_PATH=/usr/local/corex-4.3.0/lib \ | ||
| -e LIBRARY_PATH=/usr/local/corex-4.3.0/lib \ | ||
| -v ${cache_dir}:/root/.cache \ | ||
| -v ${ccache_dir}:/root/.ccache \ | ||
| -v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \ | ||
| -v ${{ github.workspace }}:/workspace -w /workspace \ | ||
| ${docker_image} /bin/bash | ||
| - name: Download Code | ||
| run: | | ||
| docker exec -t ${{ env.container_name }} /bin/bash -c ' | ||
| source ${{ github.workspace }}/../../../proxy | ||
| set -x | ||
| wget -q --tries=5 --no-proxy ${REPO_ARCHIVE_URL} --no-check-certificate | ||
| echo "Extracting PaddleCustomDevice.tar.gz" | ||
| tar -xf PaddleCustomDevice.tar.gz | ||
| cd PaddleCustomDevice | ||
| git config --global --add safe.directory "*" | ||
| git --no-pager log --pretty=oneline -5' | ||
| - name: RUN IXUCA | ||
| id: run-ixuca | ||
| if: steps.check-bypass.outputs.can-skip != 'true' | ||
| run: | | ||
| docker exec -t ${{ env.container_name }} /bin/bash -c ' | ||
| cd /workspace/PaddleCustomDevice | ||
| export PATH=/usr/local/corex-4.3.0/bin:$PATH | ||
| mkdir -p /workspace/PaddleCustomDevice/Paddle/third_party/mklml/Linux | ||
| curl -o /workspace/PaddleCustomDevice/Paddle/third_party/mklml/Linux/csrmm_mklml_lnx_2019.0.5.tgz http://paddlepaddledeps.bj.bcebos.com/csrmm_mklml_lnx_2019.0.5.tgz | ||
| tar xf /workspace/PaddleCustomDevice/Paddle/third_party/mklml/Linux/csrmm_mklml_lnx_2019.0.5.tgz | ||
| python3 -m pip install --pre paddlepaddle -i https://www.paddlepaddle.org.cn/packages/nightly/cpu/ | ||
| python3 -m pip install parameterized' | ||
| - name: Build and Install paddle_iluvatar_gpu | ||
| run: | | ||
| docker exec -t ${{ env.container_name }} /bin/bash -c ' | ||
| cd /workspace/PaddleCustomDevice/ | ||
| if [[ "${BRANCH}" == "develop" ]];then | ||
| GIT_COMMIT_TIME=$(git --no-pager show -s --format=%ci HEAD) | ||
| DATE_ONLY=$(echo "$GIT_COMMIT_TIME" | sed "s/ .*//;s/-//g") | ||
| echo "Git Commit Time: $GIT_COMMIT_TIME" | ||
| echo "Date Only: $DATE_ONLY" | ||
| export PADDLE_VERSION=${PaddleCustomDeviceVersion}.dev${DATE_ONLY} | ||
| export PLUGIN_VERSION=${PaddleCustomDeviceVersion}.dev${DATE_ONLY} | ||
| elif [[ "${REF_TYPE}" == "tag" ]]; then | ||
| VERSION="${BRANCH#v}" | ||
| echo "VERSION=$VERSION" | ||
| export PADDLE_VERSION="${VERSION}" | ||
| export PLUGIN_VERSION="${VERSION}" | ||
| else | ||
| echo "Not develop or tag, do nothing" | ||
| fi | ||
| cd /workspace/PaddleCustomDevice/backends/iluvatar_gpu | ||
| # build | ||
| bash build_paddle.sh | ||
| # Install | ||
| bash install_paddle.sh' | ||
| - name: push whl | ||
| env: | ||
| AK: ${{ secrets.BOS_AK }} | ||
| SK: ${{ secrets.BOS_SK }} | ||
| run: | | ||
| docker exec -t ${{ env.container_name }} /bin/bash -c ' | ||
| if [[ "${BRANCH}" == "develop" ]];then | ||
| target_path="paddle-whl/nightly/ixuca/paddle-iluvatar-gpu" | ||
| elif [[ "${REF_TYPE}" == "tag" ]]; then | ||
| target_path="paddle-whl/stable/ixuca/paddle-iluvatar-gpu" | ||
| else | ||
| echo "Not develop or tag, do nothing" | ||
| fi | ||
| pip install bce-python-sdk==0.8.74 | ||
| if [ ! -f "BosClient.py}" ]; then | ||
| wget -q --no-proxy https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate | ||
| tar xf bos_retry.tar.gz | ||
| fi | ||
| cp backends/iluvatar_gpu/build/dist/paddle_iluvatar_gpu*.whl . | ||
| python BosClient.py paddle_iluvatar_gpu*.whl ${target_path} | ||
| ' | ||
| - name: Terminate and delete the container | ||
| if: always() | ||
| run: | | ||
| docker exec -t ${{ env.container_name }} /bin/bash -c ' | ||
| rm -rf * .[^.]* | ||
| ' | ||
| docker stop ${container_name} | ||
| docker rm ${container_name} | ||
