DAOS-19111 build: Enable code coverage

ci/provisioning/post_provision_config.sh

@@ -143,6 +143,7 @@ if ! retry_cmd 2400 clush -B -S -l root -w "$NODESTRING" \
            DAOS_CI_INFO_DIR=\"${DAOS_CI_INFO_DIR:?DAOS_CI_INFO_DIR is missing. Can not continue with node(s) provisioning process}\"
            CI_SCONS_ARGS=\"${CI_SCONS_ARGS:-}\"
            PYTHON_VERSION=\"${PYTHON_VERSION}\"
+           CI_BULLSEYE=\"${CI_BULLSEYE:-false}\"
            $(cat ci/stacktrace.sh)
            $(cat ci/junit.sh)
            $(cat ci/provisioning/post_provision_config_common_functions.sh)

ci/provisioning/post_provision_config_common_functions.sh

@@ -60,13 +60,41 @@ add_repo() {
     fi
 }
 
+add_inst_repo() {
+    local repo="$1"
+    local branch="$2"
+    local build_number="$3"
+    local distro_name="$4"
+    local bullseye="${5:-false}"
+    local sudo="${6:-false}"
+    local dnf_cmd=("dnf" "-y")
+    local repo_base="${ARTIFACTS_URL:-${JENKINS_URL}job/}daos-stack/job/${repo}/job/${branch//\//%252F}/${build_number}/artifact/artifacts"
+    local repo_url="${repo_base}/$distro_name/"
+    if [[ "$bullseye" == "true" ]]; then
+        repo_url="${repo_base}/${distro_name}-bullseye/"
+    fi
+    if [[ "$sudo" == "true" ]]; then
+        dnf_cmd=("sudo" "${dnf_cmd[@]}")
+    fi
+    "${dnf_cmd[@]}" config-manager --add-repo="$repo_url"
+    repo="$(url_to_repo "$repo_url")"
+    # PR-repos: should always be able to upgrade modular packages
+    "${dnf_cmd[@]}" config-manager --save --setopt "$repo.module_hotfixes=true" "$repo"
+    disable_gpg_check "$repo_url" "$sudo"
+}
+
 disable_gpg_check() {
     local url="$1"
+    local sudo="${2:-false}"
+    local dnf_cmd=("dnf" "-y")
+    if [[ "$sudo" == "true" ]]; then
+        dnf_cmd=("sudo" "${dnf_cmd[@]}")
+    fi
 
     repo="$(url_to_repo "$url")"
     # bug in EL7 DNF: this needs to be enabled before it can be disabled
-    dnf -y config-manager --save --setopt="$repo".gpgcheck=1
-    dnf -y config-manager --save --setopt="$repo".gpgcheck=0
+    "${dnf_cmd[@]}" config-manager --save --setopt="$repo".gpgcheck=1
+    "${dnf_cmd[@]}" config-manager --save --setopt="$repo".gpgcheck=0
     # but even that seems to be not enough, so just brute-force it
     if [ -d /etc/yum.repos.d ] &&
        ! grep gpgcheck /etc/yum.repos.d/"$repo".repo; then
@@ -304,6 +332,8 @@ update_repos() {
 }
 
 post_provision_config_nodes() {
+    local bullseye="${1:-false}"
+
     # shellcheck disable=SC2154
     if ! update_repos "$DISTRO_NAME"; then
         # need to use the image supplied repos
@@ -379,16 +409,7 @@ post_provision_config_nodes() {
                 branch="${branch%:*}"
             fi
         fi
-        local subdir
-        if ! $COVFN_DISABLED; then
-            subdir="bullseye/"
-        fi
-        local repo_url="${ARTIFACTS_URL:-${JENKINS_URL}job/}"daos-stack/job/"$repo"/job/"${branch//\//%252F}"/"$build_number"/artifact/artifacts/"${subdir:-}"$DISTRO_NAME/
-        dnf -y config-manager --add-repo="$repo_url"
-        repo="$(url_to_repo "$repo_url")"
-        # PR-repos: should always be able to upgrade modular packages
-        dnf -y config-manager --save --setopt "$repo.module_hotfixes=true" "$repo"
-        disable_gpg_check "$repo_url"
+        add_inst_repo "${repo}" "${branch}" "${build_number}" "${DISTRO_NAME}" "${bullseye}"
     done
 
     # start with everything fully up-to-date

ci/provisioning/post_provision_config_nodes.sh

@@ -63,7 +63,7 @@ fi
 
 # defined in ci/functional/post_provision_config_nodes_<distro>.sh
 # and catted to the remote node along with this script
-if ! post_provision_config_nodes; then
+if ! post_provision_config_nodes "${CI_BULLSEYE}"; then
     rc=${PIPESTATUS[0]}
     echo "post_provision_config_nodes failed with rc=$rc"
     exit "$rc"

ci/rpm/build_deps.sh

@@ -1,3 +1,17 @@
 #!/bin/bash
+#
+#  Copyright 2025-2026 Hewlett Packard Enterprise Development LP
+#
+# Build DAOS dependencies
+code_coverage="${1:-false}"
+bullseye_key="${2:-}"
+mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+
 cd /home/daos/pre || exit 1
 scons install --build-deps=only USE_INSTALLED=all PREFIX=/opt/daos TARGET_TYPE=release -j 32
+
+if [[ "${code_coverage}" == "true" ]] ; then
+    pushd "${mydir}/../.." || exit 1
+    utils/rpms/bullseye_build.sh "${bullseye_key}"
+    popd || exit 1
+fi

ci/rpm/build_success.sh

@@ -43,11 +43,13 @@ elif [ -d /var/cache/pbuilder/ ]; then
 fi
 
 if [ -d /home/daos/rpms/ ]; then
-  if [ -d /home/daos/rpms/deps ]; then
-    mkdir -p "$artdir/deps"
-    cp /home/daos/rpms/deps/*.rpm "${artdir}/deps"
-  fi
-  cp /home/daos/rpms/daos/*.rpm "${artdir}/daos"
+  # shellcheck disable=SC2044
+  for dir in $(find /home/daos/rpms/ -maxdepth 1 -mindepth 1 -type d -exec basename {} \;); do
+    if [ -d "/home/daos/rpms/${dir}" ]; then
+      mkdir -p "${artdir}/${dir}"
+      cp "/home/daos/rpms/${dir}"/*.rpm "${artdir}/${dir}"
+    fi
+  done
 else
   mockroot="/var/lib/mock/${CHROOT_NAME}"
   cat "$mockroot"/result/{root,build}.log 2>/dev/null || true

ci/rpm/gen_rpms.sh

@@ -16,20 +16,29 @@ if [ -e "${ci_envs}" ]; then
   source "${ci_envs}"
 fi
 
-env
+function mv_rpms() {
+  local dir="${1}"
+  if ls -1 ./*.rpm; then
+    mkdir -p "/home/daos/rpms/${dir}"
+    cp ./*.rpm "/home/daos/rpms/${dir}"
+    rm -f ./*.rpm
+  fi
+}
+
+env | sort -n
 
 pushd "${mydir}/../.." || exit 1
 export DISTRO="${1}"
 export DAOS_RELVAL="${2}"
+code_coverage="${3:-false}"
+build_types="deps daos"
+if [[ "${code_coverage}" == "true" ]]; then
+  build_types="deps bullseye"
+fi
 rm -f ./*.rpm
 rm -rf /home/daos/rpms/*
-utils/rpms/build_packages.sh deps
-if ls -1 ./*.rpm; then
-  mkdir -p /home/daos/rpms/deps
-  cp ./*.rpm /home/daos/rpms/deps
-  rm -f ./*.rpm
-fi
-utils/rpms/build_packages.sh daos
-mkdir -p /home/daos/rpms/daos
-cp ./*.rpm /home/daos/rpms/daos
+for build_type in ${build_types}; do
+  utils/rpms/build_packages.sh "${build_type}"
+  mv_rpms "${build_type}"
+done
 popd || exit 1

ci/rpm/install_deps.sh

@@ -21,6 +21,7 @@ env
 pushd "${mydir}/../.." || exit 1
 export DISTRO="${1}"
 export DAOS_RELVAL="${2}"
+code_coverage="${3:-false}"
 libfabric_pkg="$(utils/rpms/package_version.sh libfabric dev)"
 mercury_pkg="$(utils/rpms/package_version.sh mercury dev)"
 argobots_pkg="$(utils/rpms/package_version.sh argobots dev)"
@@ -38,4 +39,10 @@ sudo dnf install --allowerasing -y "${fused_pkg}" || echo "${fused_pkg} not avai
 sudo dnf install --allowerasing -y "${pmdk_pkg}" || echo "${pmdk_pkg} not available"
 sudo dnf install --allowerasing -y "${isal_pkg}" || echo "${isal_pkg} not available"
 sudo dnf install --allowerasing -y "${isal_crypto_pkg}" || echo "${isal_crypto_pkg} not available"
+
+if [[ "${code_coverage}" == "true" ]] ; then
+  bullseye_pkg="$(utils/rpms/package_version.sh bullseye normal)"
+  sudo dnf install --allowerasing -y "${bullseye_pkg}" || echo "${bullseye_pkg} not available"
+fi
+
 popd || exit 1

ci/summary/bullseye_report.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+#
+#  Copyright 2026 Hewlett Packard Enterprise Development LP
+#
+# Script for generating a bullseye code coverage report summary
+set -uex
+
+if [ ! -d '/opt/BullseyeCoverage/bin' ]; then
+  echo 'Bullseye not found.'
+  exit 1
+fi
+export COVFILE="${WORKSPACE:-/tmp}/test.cov"
+export PATH="/opt/BullseyeCoverage/bin:$PATH"
+
+# Merge all coverage files
+cp /opt/BullseyeCoverage/daos/test.cov "${COVFILE}"
+readarray -t cov_files < <(find "${WORKSPACE}" -name test.cov)
+if [ ${#cov_files[@]} -gt 0 ]; then
+  covmerge --no-banner --file "${COVFILE}" "${cov_files[@]}"
+fi
+
+if [ ! -e "$COVFILE" ]; then
+  echo "Coverage file ${COVFILE} is missing"
+  exit 1
+fi
+
+# Generate the html report
+rm -fr bullseye_code_coverage_report || true
+mkdir bullseye_code_coverage_report
+cd bullseye_code_coverage_report
+mkdir sources
+tar -xf /opt/BullseyeCoverage/daos/bullseye_sources.tar.gz -C sources/
+covhtml --srcdir sources --file "${COVFILE}" .

ci/summary/install_pkgs.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+#
+#  Copyright 2026 Hewlett Packard Enterprise Development LP
+#
+# Script for installing packages used for CI summary steps
+set -uex
+
+id
+if [ "$(id -u)" = "0" ]; then
+    echo "Should not be run as root"
+    exit 1
+fi
+
+# Distro name for the repository path for accessing packages built by CI
+distro_name="${1:-el9}"
+code_coverage="${2:-false}"
+
+# Import provisioning functions to add the repo
+mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+files=("$mydir/../parse_ci_envs.sh")
+files+=("$mydir/../provisioning/post_provision_config_common_functions.sh")
+for src_file in "${files[@]}"; do
+    if [ -e "${src_file}" ]; then
+        # shellcheck source=parse_ci_envs.sh disable=SC1091
+        source "${src_file}"
+    fi
+done
+
+env | sort -n
+
+# Add the repo for packages built by CI
+add_inst_repo "daos" "${BRANCH_NAME}" "${BUILD_NUMBER}" "${distro_name}" "${code_coverage}" "true"
+
+# Install bullseye
+export DISTRO="${distro_name}"
+bullseye_pkg="$(utils/rpms/package_version.sh bullseye normal)"
+sudo dnf install --allowerasing -y "${bullseye_pkg}"

ci/unit/required_packages.sh

@@ -7,47 +7,50 @@
 #
 set -eu
 
-# Provided by pipeline-lib
-# distro="$1"
-# quick_build="${2:-false}"
+distro="${1:?ERROR: Missing distro argument. Usage: $0 <distro>}"
+code_coverage="${2:-false}"
 
 OPENMPI_VER=""
 PY_MINOR_VER=""
 
-DISTRO="${1:?ERROR: Missing distro argument. Usage: $0 <distro>}"
-export DISTRO="${DISTRO%%.*}"
+export DISTRO="${distro%%.*}"
 
-pkgs="boost-python3$PY_MINOR_VER-devel                               \
-      capstone                                                       \
-      $(utils/rpms/package_version.sh argobots lib)                  \
-      $(utils/rpms/package_version.sh argobots debug)                \
-      $(utils/rpms/package_version.sh daos_spdk dev)                 \
-      $(utils/rpms/package_version.sh daos_spdk debug)               \
-      $(utils/rpms/package_version.sh isal dev)                      \
-      $(utils/rpms/package_version.sh isal_crypto lib)               \
-      $(utils/rpms/package_version.sh isal_crypto debug)             \
-      $(utils/rpms/package_version.sh libfabric dev)                 \
-      $(utils/rpms/package_version.sh libfabric debug)               \
-      $(utils/rpms/package_version.sh mercury dev)                   \
-      $(utils/rpms/package_version.sh mercury debug)                 \
-      $(utils/rpms/package_version.sh mercury lib mercury_libfabric) \
-      $(utils/rpms/package_version.sh pmdk lib pmemobj)              \
-      $(utils/rpms/package_version.sh pmdk debug pmemobj)            \
-      $(utils/rpms/package_version.sh pmdk debug pmem)               \
-      fuse3                                                          \
-      gotestsum                                                      \
-      hwloc-devel                                                    \
-      libasan                                                        \
-      libipmctl-devel                                                \
-      libyaml-devel                                                  \
-      numactl                                                        \
-      numactl-devel                                                  \
-      openmpi$OPENMPI_VER                                            \
-      patchelf                                                       \
-      pciutils-devel                                                 \
-      protobuf-c                                                     \
-      valgrind-devel"
+pkgs=("boost-python3${PY_MINOR_VER}-devel")
+pkgs+=("capstone")
+pkgs+=("$(utils/rpms/package_version.sh argobots lib)")
+pkgs+=("$(utils/rpms/package_version.sh argobots debug)")
+pkgs+=("$(utils/rpms/package_version.sh daos_spdk dev)")
+pkgs+=("$(utils/rpms/package_version.sh daos_spdk debug)")
+pkgs+=("$(utils/rpms/package_version.sh isal dev)")
+pkgs+=("$(utils/rpms/package_version.sh isal_crypto lib)")
+pkgs+=("$(utils/rpms/package_version.sh isal_crypto debug)")
+pkgs+=("$(utils/rpms/package_version.sh libfabric dev)")
+pkgs+=("$(utils/rpms/package_version.sh libfabric debug)")
+pkgs+=("$(utils/rpms/package_version.sh mercury dev)")
+pkgs+=("$(utils/rpms/package_version.sh mercury debug)")
+pkgs+=("$(utils/rpms/package_version.sh mercury lib mercury_libfabric)")
+pkgs+=("$(utils/rpms/package_version.sh pmdk lib pmemobj)")
+pkgs+=("$(utils/rpms/package_version.sh pmdk debug pmemobj)")
+pkgs+=("$(utils/rpms/package_version.sh pmdk debug pmem)")
+pkgs+=("fuse3")
+pkgs+=("gotestsum")
+pkgs+=("gperftools-devel")
+pkgs+=("hwloc-devel")
+pkgs+=("libasan")
+pkgs+=("libipmctl-devel")
+pkgs+=("libyaml-devel")
+pkgs+=("numactl")
+pkgs+=("numactl-devel")
+pkgs+=("openmpi${OPENMPI_VER}")
+pkgs+=("patchelf")
+pkgs+=("pciutils-devel")
+pkgs+=("protobuf-c")
+pkgs+=("valgrind-devel")
+
+if [ "${code_coverage}" == "true" ] ; then
+    pkgs+=("$(utils/rpms/package_version.sh bullseye normal)")
+fi
 
 # output with trailing newline suppressed
-echo  -e "$pkgs\c"
+printf '%s' "${pkgs[*]}"
 exit 0

ci/unit/test_main.sh

@@ -20,23 +20,13 @@ mkdir test_results
 chmod 777 test_results
 
 # Check if this is a Bulleye stage
-USE_BULLSEYE=false
 BDEV_TEST=false
 case $STAGE_NAME in
-  *Bullseye**)
-  USE_BULLSEYE=true
-  ;;
   *bdev**)
   BDEV_TEST=true
   ;;
 esac
 
-if $USE_BULLSEYE; then
-  rm -rf bullseye
-  mkdir -p bullseye
-  tar -C bullseye --strip-components=1 -xf bullseye.tar
-fi
-
 NODE=${NODELIST%%,*}
 : "${PYTHON_VERSION:=3.11}"