esp/ci: Reorginize pipeline to minimize jobs running for MRs

Toolchain tests are ran on default branch only or when forced by
'toolchain_tests' label in MR.

Author: gerekon

.gitlab-ci.yml

@@ -31,6 +31,7 @@ variables:
   ESP_RELEASE_BRANCH: origin/$CI_DEFAULT_BRANCH
   CCACHE_DIR: ${CI_PROJECT_DIR}/.ccache
   CCACHE_MAXSIZE: 10G
+  CCACHE_SLOPPINESS: "time_macros,file_macro,include_file_mtime,include_file_ctime"
 
 workflow:
   rules:
@@ -169,7 +170,7 @@ rebase_on_main:
       else
         rc=$?
         echo "Rebasing branch '$BRANCH_NAME' on 'origin/$SCHED_REBASE_ONTO_BR' failed!"
-        echo “Please rebase '$BRANCH_NAME' onto 'origin/$SCHED_REBASE_ONTO_BR' branch manually.”
+        echo "Please rebase '$BRANCH_NAME' onto 'origin/$SCHED_REBASE_ONTO_BR' branch manually."
         if [ -d "${GIT_DIR}/rebase-merge" ] || [ -d "${GIT_DIR}/rebase-apply" ] ; then
             echo "Abort rebase"
             git rebase --abort
@@ -219,12 +220,14 @@ check_commit_message:
     USE_LINKER: "ld"
     CROSS_BUILD_MINGW: "OFF"
     BUILD_TARGET_LIBS: "OFF"
+    BUILD_GNU_BINUTILS: "OFF"
     RUN_CORE_TESTS: "OFF"
     RUN_PKG_TESTS: "OFF"
     RUN_TOOLCHAIN_TESTS: "OFF"
     PACK_TOOLCHAIN: "ON"
-    PACK_STANDALONE_LIBS: "ON"
+    PACK_STANDALONE_LIBS: "OFF"
     PACK_TARGET_LIBS: "OFF"
+    PACK_EXTRA_TOOLS: "OFF"
   rules:
     - <<: *if-esp-main-workflow
       when: never
@@ -283,8 +286,8 @@ check_commit_message:
       fi
     - AUX_BUILD_ARGS="";
     - >
-      if [ "${ESP_FORCE_LLVM_PARALLEL_COMPILE_JOBS:-}" != "" ]; then
-        AUX_BUILD_ARGS="${AUX_BUILD_ARGS} -DLLVM_PARALLEL_COMPILE_JOBS=${ESP_FORCE_LLVM_PARALLEL_COMPILE_JOBS}";
+      if [ "${ESP_LLVM_PARALLEL_COMPILE_JOBS:-}" != "" ]; then
+        AUX_BUILD_ARGS="${AUX_BUILD_ARGS} -DLLVM_PARALLEL_COMPILE_JOBS=${ESP_LLVM_PARALLEL_COMPILE_JOBS}";
       fi
     # build toolchain core w/o any libs and GNU components
     - time cmake $PWD/esp-llvm-embedded-toolchain -GNinja
@@ -298,7 +301,7 @@ check_commit_message:
       -DLLVM_TOOLCHAIN_C_LIBRARY=${USE_LIBC}
       -DLLVM_TOOLCHAIN_CXX_LIBRARIES=${USE_LIBCXX}
       -DLLVM_TOOLCHAIN_RT_LIBRARIES=${USE_RTLIB}
-      -DLLVM_TOOLCHAIN_INCLUDE_GNU_BINUTILS=ON
+      -DLLVM_TOOLCHAIN_INCLUDE_GNU_BINUTILS=${BUILD_GNU_BINUTILS}
       -DLLVM_TOOLCHAIN_ESPRESSIF=ON
       -DLLVM_TOOLCHAIN_HOST_TRIPLE=${CONF_HOST}
       -DLLVM_TOOLCHAIN_ENABLED_TARGETS="${TARGET}"
@@ -357,9 +360,20 @@ check_commit_message:
     - mkdir -p ${PWD}/${DIST_DIR}
     - >
       if [[ "${PACK_TOOLCHAIN}" == "ON" ]]; then
-        echo "Build/Unpack toolchain";
+        echo "Pack toolchain";
+        time ninja -C ${BUILD_PATH} package-llvm-toolchain 2>&1  >> ${BUILD_PATH}/build.log
+        echo "Pack toolchain finished";
+        DISTRO_PACK_PATH=$(ninja -C ${BUILD_PATH} print-llvm-toolchain-package-path | tail -n 1)
+        echo "DISTRO_PACK_PATH=${DISTRO_PACK_PATH}"
+        mv ${DISTRO_PACK_PATH} ${PWD}/${DIST_DIR}/
+        ARCHIVE_NAME=$(basename ${DISTRO_PACK_PATH})
+        echo "${ARCHIVE_NAME}" > ${PWD}/${DIST_DIR}/dist_name_${CONF_HOST}
+      fi
+    - >
+      if [[ "${PACK_EXTRA_TOOLS}" == "ON" ]]; then
+        echo "Unpack toolchain";
         time ninja -C ${BUILD_PATH} unpack-llvm-toolchain 2>&1  >> ${BUILD_PATH}/build.log
-        echo "Build/Unpack toolchain finished";
+        echo "Unpack toolchain finished";
         # pack clangd
         echo "Pack clangd";
         time ninja -C ${BUILD_PATH} package-clangd 2>&1  >> ${BUILD_PATH}/build.log
@@ -373,11 +387,6 @@ check_commit_message:
         echo "Re-pack toolchain";
         time ninja -C ${BUILD_PATH} repack-llvm-toolchain 2>&1  >> ${BUILD_PATH}/build.log
         echo "Re-pack toolchain finished";
-        DISTRO_PACK_PATH=$(ninja -C ${BUILD_PATH} print-llvm-toolchain-package-path | tail -n 1)
-        echo "DISTRO_PACK_PATH=${DISTRO_PACK_PATH}"
-        mv ${DISTRO_PACK_PATH} ${PWD}/${DIST_DIR}/
-        ARCHIVE_NAME=$(basename ${DISTRO_PACK_PATH})
-        echo "${ARCHIVE_NAME}" > ${PWD}/${DIST_DIR}/dist_name_${CONF_HOST}
       fi
     # pack distro with standalone libs
     - >
@@ -432,7 +441,7 @@ build_and_test:
     - if: '$CI_COMMIT_REF_NAME == $ESP_MAIN_BRANCH || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $ESP_MAIN_BRANCH'
       variables:
         CACHE_KEY: ccache-esp_main-$CI_JOB_NAME
-    - when: on_success
+    - when: always
   cache:
     key: $CACHE_KEY
     paths:
@@ -456,8 +465,8 @@ build_and_test:
     - mkdir -p ${BUILD_PATH}
     - AUX_BUILD_ARGS="";
     - >
-      if [ "${ESP_FORCE_LLVM_PARALLEL_COMPILE_JOBS:-}" != "" ]; then
-        AUX_BUILD_ARGS="${AUX_BUILD_ARGS} -DLLVM_PARALLEL_COMPILE_JOBS=${ESP_FORCE_LLVM_PARALLEL_COMPILE_JOBS}";
+      if [ "${ESP_LLVM_PARALLEL_COMPILE_JOBS:-}" != "" ]; then
+        AUX_BUILD_ARGS="${AUX_BUILD_ARGS} -DLLVM_PARALLEL_COMPILE_JOBS=${ESP_LLVM_PARALLEL_COMPILE_JOBS}";
       fi
     - time cmake -G Ninja
       -S llvm
@@ -518,10 +527,24 @@ build_x86_64-linux-gnu:
   variables:
     CONF_HOST: "x86_64-linux-gnu"
     USE_LINKER: "lld"
-    # Build complete distro it is necessary for running package tests
-    BUILD_TARGET_LIBS: "ON"
-    RUN_TOOLCHAIN_TESTS: "ON"
-    RUN_PKG_TESTS: "ON"
+  rules:
+    # Build complete distro and run all tests on default branch
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_MERGE_REQUEST_LABELS =~ /(^|,)toolchain_tests(,|$)/'
+      variables:
+        BUILD_TARGET_LIBS: "ON"
+        BUILD_GNU_BINUTILS: "ON"
+        RUN_TOOLCHAIN_TESTS: "ON"
+        RUN_PKG_TESTS: "ON"
+    # Build all distros and run all tests on tag
+    - if: '$CI_COMMIT_TAG'
+      variables:
+        BUILD_TARGET_LIBS: "ON"
+        BUILD_GNU_BINUTILS: "ON"
+        RUN_TOOLCHAIN_TESTS: "ON"
+        RUN_PKG_TESTS: "ON"
+        PACK_STANDALONE_LIBS: "ON"
+        PACK_EXTRA_TOOLS: "ON"
+    - when: always
 
 # Target libs are built in build_x86_64-linux-gnu, but due to artifacts
 # size limit we have to produce target libs archive in this job too.
@@ -535,7 +558,10 @@ build_target_libs:
     CONF_HOST: "x86_64-linux-gnu"
     PACK_TARGET_LIBS: "ON"
     PACK_TOOLCHAIN: "OFF"
-    PACK_STANDALONE_LIBS: "OFF"
+  rules:
+    # Build target libs only on default branch or tag
+    - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+      when: on_success
 
 build_x86_64-w64-mingw32:
   extends: .build_toolchain_template
@@ -545,34 +571,59 @@ build_x86_64-w64-mingw32:
     USE_LINKER: ""
     CONF_HOST: "x86_64-w64-mingw32"
     CROSS_BUILD_MINGW: "ON"
+  rules:
+    - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+      variables:
+        BUILD_GNU_BINUTILS: "ON"
+    - when: on_success
 
 build_arm-linux-gnueabihf:
   extends: .build_linux-gnu_template
   stage: build
   image: ${CROSS_ARM_IMAGE}
   variables:
     CONF_HOST: "arm-linux-gnueabihf"
+  rules:
+    - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+      variables:
+        BUILD_GNU_BINUTILS: "ON"
+    - when: on_success
 
 build_aarch64-linux-gnu:
   extends: .build_linux-gnu_template
   stage: build
   image: ${CROSS_ARM_IMAGE}
   variables:
     CONF_HOST: "aarch64-linux-gnu"
+  rules:
+    - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+      variables:
+        BUILD_GNU_BINUTILS: "ON"
+    - when: on_success
 
 build_x86_64-apple-darwin:
   extends: .build_toolchain_template
   stage: build
   image: ${CROSS_MACOS_IMAGE}
   variables:
     CONF_HOST: "x86_64-apple-darwin21.1"
+  rules:
+    - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+      variables:
+        BUILD_GNU_BINUTILS: "ON"
+    - when: on_success
 
 build_aarch64-apple-darwin:
   extends: .build_toolchain_template
   stage: build
   image: ${CROSS_MACOS_IMAGE}
   variables:
     CONF_HOST: "aarch64-apple-darwin21.1"
+  rules:
+    - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+      variables:
+        BUILD_GNU_BINUTILS: "ON"
+    - when: on_success
 
 .unpack_distro: &unpack_distro |
     pushd ${DIST_DIR}
@@ -601,7 +652,9 @@ build_aarch64-apple-darwin:
       when: never
     - <<: *if-scheduled-sync
       when: never
-    - when: on_success
+    # Pack distros only on default branch or tag
+    - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+      when: on_success
   script:
     # update distro
     - *unpack_distro
@@ -674,7 +727,8 @@ pack_aarch64-apple-darwin:
     - if: $CI_COMMIT_TAG
       when: on_success
       allow_failure: false
-    - when: manual
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+      when: manual
       allow_failure: true
   resource_group: macos_codesign
   tags: [ "darwin", "codesign" ]
@@ -752,7 +806,9 @@ sign_aarch64-apple-darwin:
       when: never
     - <<: *if-scheduled-sync
       when: never
-    - when: on_success
+    # job "build_x86_64-linux-gnu" produces full distro only on default branch or tag or when toolchain_tests label is present
+    - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_MERGE_REQUEST_LABELS =~ /(^|,)toolchain_tests(,|$)/'
+      when: on_success
   script:
     - mkdir -p $PWD/${BUILD_DIR}
     - export BUILD_LOG=$PWD/${BUILD_DIR}/build.log