Skip to content

extend ci oneDAL #3584

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Alexandr-Solovev wants to merge 20 commits into
base: main
Choose a base branch
from
Draft

extend ci oneDAL #3584

Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
bfa4693
init restore
Alexandr-Solovev Mar 28, 2026
ee2814d
fixes
Alexandr-Solovev Mar 30, 2026
8a42e6a
fixes and additional rpath
Alexandr-Solovev Mar 30, 2026
2edb9d6
Merge branch 'main' into dev/asolovev_extend_ci_opt
Alexandr-Solovev Mar 30, 2026
f0388b6
fixes
Alexandr-Solovev Mar 31, 2026
7758f8e
fixes
Alexandr-Solovev Mar 31, 2026
cea3669
Merge branch 'main' into dev/asolovev_extend_ci_opt
Alexandr-Solovev Mar 31, 2026
af07f1c
Merge branch 'main' into dev/asolovev_extend_ci_opt
Alexandr-Solovev Apr 7, 2026
9da9016
fixes
Alexandr-Solovev Apr 7, 2026
275cc3d
fixes
Alexandr-Solovev Apr 13, 2026
92a63c9
Merge branch 'main' into dev/asolovev_extend_ci_opt
Alexandr-Solovev Apr 13, 2026
9c937ac
fixes
Alexandr-Solovev Apr 14, 2026
6e0b498
experiment: using internal script for coverage
Alexandr-Solovev Apr 17, 2026
0365eb9
fixes
Alexandr-Solovev Apr 17, 2026
8521af2
fixes
Alexandr-Solovev Apr 20, 2026
03e9297
Merge branch 'main' into dev/asolovev_extend_ci_opt
Alexandr-Solovev Apr 20, 2026
5078a3b
fixes for azure
Alexandr-Solovev Apr 20, 2026
553400f
fixes for CI
Alexandr-Solovev Apr 20, 2026
c33f75b
fixes
Alexandr-Solovev Apr 21, 2026
778fcc9
fixes
Alexandr-Solovev Apr 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .ci/env/apt.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ function install_clang-format {
}

function install_dev-base {
sudo apt-get install -y gcc-multilib g++-multilib tree
sudo apt-get install -y gcc-multilib g++-multilib tree lld
}

function install_dev-base-conda {
Expand Down
164 changes: 103 additions & 61 deletions .ci/pipeline/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,6 @@ variables:
VM_IMAGE : 'ubuntu-24.04'
WIN_VM_IMAGE : 'windows-2022'
SYSROOT_OS: 'noble'
PY_VERSION: '3.14'
SKL_VERSION: '1.8'
# Link to latest version can be taken from basekit download page:
# https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit-download.html
# Check section 'Install through a Command Line':
Expand Down Expand Up @@ -97,24 +95,12 @@ jobs:
displayName: 'System info'
- script: |
source /opt/intel/oneapi/setvars.sh
.ci/scripts/build.sh --compiler gnu --optimizations avx2 --target daal
.ci/scripts/build.sh --compiler gnu --optimizations avx2 --sycl-split off --target daal
displayName: 'make daal'
- script: |
source /opt/intel/oneapi/setvars.sh
.ci/scripts/build.sh --compiler gnu --optimizations avx2 --target onedal_c
.ci/scripts/build.sh --compiler gnu --optimizations avx2 --sycl-split off --target onedal_c
displayName: 'make onedal_c'
- task: PublishPipelineArtifact@1
inputs:
artifactName: '$(platform.type) build'
targetPath: '$(Build.Repository.LocalPath)/$(release.dir)'
displayName: 'Upload build artifacts'
continueOnError: true
- task: PublishPipelineArtifact@1
inputs:
artifactName: 'oneDAL environment'
targetPath: '$(Build.Repository.LocalPath)/.ci/env'
displayName: 'Upload environment artifacts'
continueOnError: true
- script: |
source /opt/intel/oneapi/setvars.sh
.ci/scripts/test.sh --test-kind examples --build-dir $(release.dir) --compiler gnu --interface daal/cpp --build-system cmake
Expand All @@ -130,18 +116,6 @@ jobs:
source /opt/intel/oneapi/setvars.sh
.ci/scripts/test.sh --test-kind samples --build-dir $(release.dir) --compiler gnu --interface daal/cpp/mpi --conda-env ci-env --build-system cmake
displayName: 'daal/cpp/mpi samples'
- script: |
deploy/nuget/prepare_dal_nuget.sh --release-dir $(release.dir) --build-nupkg yes
tree -h -I include __nuget/inteldal*/
ls -lh __nuget/inteldal*.nupkg
displayName: 'nuget pkg'
- task: PublishPipelineArtifact@1
inputs:
artifactName: '$(platform.type) fail'
targetPath: '$(Build.Repository.LocalPath)/$(release.dir)'
displayName: 'Uploading on fail'
condition: failed()
continueOnError: true

- job: 'LinuxMakeIntel_MKL'
timeoutInMinutes: 0
Expand Down Expand Up @@ -174,12 +148,24 @@ jobs:
displayName: 'System info'
- script: |
source /opt/intel/oneapi/setvars.sh
.ci/scripts/build.sh --compiler icx --optimizations avx2 --target daal
.ci/scripts/build.sh --compiler icx --optimizations avx2 --sycl-split off --target daal
displayName: 'make daal'
- script: |
source /opt/intel/oneapi/setvars.sh
.ci/scripts/build.sh --compiler icx --optimizations avx2 --target onedal_dpc
.ci/scripts/build.sh --compiler icx --optimizations avx2 --sycl-split off --linker lld --target onedal_dpc
displayName: 'make onedal_dpc'
- task: PublishPipelineArtifact@1
inputs:
artifactName: '$(platform.type) build'
targetPath: '$(Build.Repository.LocalPath)/$(release.dir)'
displayName: 'Upload build artifacts'
continueOnError: true
- task: PublishPipelineArtifact@1
inputs:
artifactName: 'oneDAL environment'
targetPath: '$(Build.Repository.LocalPath)/.ci/env'
displayName: 'Upload environment artifacts'
continueOnError: true
- script: |
source /opt/intel/oneapi/setvars.sh
.ci/scripts/test.sh --test-kind examples --build-dir $(release.dir) --compiler icx --interface daal/cpp --build-system cmake
Expand All @@ -199,6 +185,18 @@ jobs:
source /opt/intel/oneapi/setvars.sh
.ci/scripts/test.sh --test-kind samples --build-dir $(release.dir) --compiler gnu --interface daal/cpp/mpi --conda-env ci-env --build-system cmake
displayName: 'daal/cpp/mpi samples'
- script: |
deploy/nuget/prepare_dal_nuget.sh --release-dir $(release.dir) --build-nupkg yes
tree -h -I include __nuget/inteldal*/
ls -lh __nuget/inteldal*.nupkg
displayName: 'nuget pkg'
- task: PublishPipelineArtifact@1
inputs:
artifactName: '$(platform.type) fail'
targetPath: '$(Build.Repository.LocalPath)/$(release.dir)'
displayName: 'Uploading on fail'
condition: failed()
continueOnError: true

- job: 'LinuxMakeGNU_MKL_Conda'
timeoutInMinutes: 300
Expand All @@ -217,7 +215,7 @@ jobs:
conda create -y -n $(CONDA_ENV)
conda activate $(CONDA_ENV)
conda install -y -c conda-forge \
make \
make lld \
tbb tbb-devel \
mkl mkl-devel mkl-static \
cmake \
Expand All @@ -232,13 +230,13 @@ jobs:
source $(conda info --base)/etc/profile.d/conda.sh
conda activate $(CONDA_ENV)
export TBBROOT="${CONDA_PREFIX}"
.ci/scripts/build.sh --compiler gnu --optimizations avx2 --backend-config mkl --target daal
.ci/scripts/build.sh --compiler gnu --optimizations avx2 --backend-config mkl --target daal
displayName: 'make daal'
- script: |
source $(conda info --base)/etc/profile.d/conda.sh
conda activate $(CONDA_ENV)
export TBBROOT="${CONDA_PREFIX}"
.ci/scripts/build.sh --compiler gnu --optimizations avx2 --backend-config mkl --target onedal_c
.ci/scripts/build.sh --compiler gnu --optimizations avx2 --backend-config mkl --target onedal_c
displayName: 'make onedal_c'
- script: |
source $(conda info --base)/etc/profile.d/conda.sh
Expand All @@ -248,7 +246,7 @@ jobs:
dpcpp-cpp-rt dpcpp_linux-64 intel-sycl-rt \
onedpl-devel
export TBBROOT="${CONDA_PREFIX}"
.ci/scripts/build.sh --compiler gnu --optimizations avx2 --backend-config mkl --target onedal_dpc
.ci/scripts/build.sh --compiler gnu --optimizations avx2 --backend-config mkl --target onedal_dpc
mv __release_lnx_gnu __release_lnx_gnu_conda
displayName: 'make onedal_dpc'
- script: |
Expand Down Expand Up @@ -329,7 +327,7 @@ jobs:
path: $(OPENBLAS_CACHE_DIR)
cacheHitVar: OPENBLAS_RESTORED
- script: |
.ci/env/openblas.sh --target RISCV64_ZVL128B --host-compiler gcc --compiler clang --target-arch riscv64 --cross-compile --prefix $(OPENBLAS_CACHE_DIR) --sysroot $(SYSROOT_CACHE_DIR) --version $(OPENBLAS_VERSION)
.ci/env/openblas.sh --target RISCV64_ZVL128B --host-compiler gcc --compiler clang --target-arch riscv64 --cross-compile --prefix $(OPENBLAS_CACHE_DIR) --sysroot $(SYSROOT_CACHE_DIR) --version $(OPENBLAS_VERSION)
displayName: 'Build OpenBLAS'
condition: ne(variables.OPENBLAS_RESTORED, 'true')
- task: Cache@2
Expand All @@ -343,10 +341,10 @@ jobs:
displayName: 'Build oneTBB'
condition: ne(variables.TBB_RESTORED, 'true')
- script: |
.ci/scripts/build.sh --compiler clang --optimizations rv64 --target daal --backend-config ref --cross-compile --plat lnxriscv64 --sysroot $(SYSROOT_CACHE_DIR) --blas-dir $(OPENBLAS_CACHE_DIR) --tbb-dir $(TBB_CACHE_DIR)
.ci/scripts/build.sh --compiler clang --optimizations rv64 --target daal --backend-config ref --cross-compile --plat lnxriscv64 --sysroot $(SYSROOT_CACHE_DIR) --blas-dir $(OPENBLAS_CACHE_DIR) --tbb-dir $(TBB_CACHE_DIR)
displayName: 'make daal'
- script: |
.ci/scripts/build.sh --compiler clang --optimizations rv64 --target onedal_c --backend-config ref --cross-compile --plat lnxriscv64 --sysroot $(SYSROOT_CACHE_DIR) --blas-dir $(OPENBLAS_CACHE_DIR) --tbb-dir $(TBB_CACHE_DIR)
.ci/scripts/build.sh --compiler clang --optimizations rv64 --target onedal_c --backend-config ref --cross-compile --plat lnxriscv64 --sysroot $(SYSROOT_CACHE_DIR) --blas-dir $(OPENBLAS_CACHE_DIR) --tbb-dir $(TBB_CACHE_DIR)
displayName: 'make onedal_c'
- task: PublishPipelineArtifact@1
inputs:
Expand Down Expand Up @@ -585,23 +583,31 @@ jobs:
displayName: 'bazel-cache-limit'

- job: LinuxSklearnex
dependsOn: LinuxMakeGNU_MKL
dependsOn: LinuxMakeIntel_MKL
timeoutInMinutes: 0
variables:
release.dir: '__release_lnx_gnu'
release.dir: '__release_lnx'
platform.type: 'lnx32e'
DALROOT: '$(Pipeline.Workspace)/daal/latest'
TBBROOT: '/opt/intel/oneapi/tbb/latest'
NO_DPC: 1
NO_DIST: 1
pool:
vmImage: '$(VM_IMAGE)'
maxParallel: 2
strategy:
matrix:
Python311:
PYTHON_VERSION: '$(PY_VERSION)'
SKLEARN_VERSION: '$(SKL_VERSION)'
Python314_Sklearn18:
PYTHON_VERSION: '3.14'
SKLEARN_VERSION: '1.8'
Python313_Sklearn18:
PYTHON_VERSION: '3.13'
SKLEARN_VERSION: '1.8'
Python312_Sklearn18:
PYTHON_VERSION: '3.12'
SKLEARN_VERSION: '1.8'
Python311_Sklearn18:
PYTHON_VERSION: '3.11'
SKLEARN_VERSION: '1.8'
steps:
- checkout: none
- script: |
Expand All @@ -619,7 +625,20 @@ jobs:
chmod -R 755 .ci/env
.ci/env/apt.sh tbb
displayName: 'tbb installation'
- script: |
.ci/env/apt.sh mkl
displayName: 'mkl installation'
- template: .ci/pipeline/build-and-test-lnx.yml@sklearnex
- script: |
conda run -n CB python -c "import onedal._onedal_py_dpc; print('OK')"
displayName: 'Check DPC module import no set MKL'
workingDirectory: /tmp
- script: |
source /opt/intel/oneapi/mkl/latest/env/vars.sh
conda run -n CB python -c "import onedal._onedal_py_dpc; print('OK')"
displayName: 'Check DPC module import'
workingDirectory: /tmp
condition: always()
- task: PublishPipelineArtifact@1
inputs:
artifactName: '$(platform.type) sklearnex build'
Expand Down Expand Up @@ -652,18 +671,6 @@ jobs:
call %TEMP%\oneapi\setvars.bat --force
.\.ci\scripts\build.bat onedal_c vc avx2
displayName: 'make onedal_c'
- task: PublishPipelineArtifact@1
inputs:
artifactName: '$(platform.type) build'
targetPath: '$(Build.Repository.LocalPath)/$(release.dir)'
displayName: 'Upload build artifacts'
continueOnError: true
- task: PublishPipelineArtifact@1
inputs:
artifactName: 'oneDAL scripts'
targetPath: '$(Build.Repository.LocalPath)/.ci/scripts'
displayName: 'Upload environment artifacts'
continueOnError: true
- script: |
call %TEMP%\oneapi\setvars.bat --force
.\.ci\scripts\test.bat daal\cpp lib msvs cmake
Expand Down Expand Up @@ -715,6 +722,18 @@ jobs:
call %TEMP%\oneapi\setvars.bat --force
.\.ci\scripts\build.bat onedal_dpc icx avx2
displayName: 'make onedal_dpc'
- task: PublishPipelineArtifact@1
inputs:
artifactName: '$(platform.type) build'
targetPath: '$(Build.Repository.LocalPath)/$(release.dir)'
displayName: 'Upload build artifacts'
continueOnError: true
- task: PublishPipelineArtifact@1
inputs:
artifactName: 'oneDAL scripts'
targetPath: '$(Build.Repository.LocalPath)/.ci/scripts'
displayName: 'Upload environment artifacts'
continueOnError: true
- script: |
call %TEMP%\oneapi\setvars.bat --force
.\.ci\scripts\test.bat daal\cpp lib icx cmake
Expand All @@ -731,13 +750,20 @@ jobs:
tree -h -I include __nuget/inteldal*/
ls -lh __nuget/inteldal*.nupkg
displayName: 'nuget pkg'
- task: PublishPipelineArtifact@1
inputs:
artifactName: '$(platform.type) fail'
targetPath: '$(Build.Repository.LocalPath)/$(release.dir)'
displayName: 'Uploading on fail'
condition: failed()
continueOnError: true


- job: WindowsSklearnex
dependsOn: WindowsMakeVC
dependsOn: WindowsMakeIntel
timeoutInMinutes: 0
variables:
release.dir: '__release_win_vc'
release.dir: '__release_win'
platform.type: 'win32e'
DALROOT: '$(Pipeline.Workspace)\daal\latest'
TBBROOT: '%TEMP%\oneapi\tbb\latest'
Expand All @@ -746,9 +772,18 @@ jobs:
maxParallel: 2
strategy:
matrix:
Python311:
PYTHON_VERSION: '$(PY_VERSION)'
SKLEARN_VERSION: '$(SKL_VERSION)'
Python314_Sklearn18:
PYTHON_VERSION: '3.14'
SKLEARN_VERSION: '1.8'
Python313_Sklearn18:
PYTHON_VERSION: '3.13'
SKLEARN_VERSION: '1.8'
Python312_Sklearn18:
PYTHON_VERSION: '3.12'
SKLEARN_VERSION: '1.8'
Python311_Sklearn18:
PYTHON_VERSION: '3.11'
SKLEARN_VERSION: '1.8'
steps:
- checkout: none
- script: |
Expand All @@ -763,8 +798,15 @@ jobs:
artifact: '$(platform.type) build'
path: $(Pipeline.Workspace)
- script: .ci\env\install_basekit.bat $(WINDOWS_BASEKIT_URL) $(WINDOWS_DPCPP_COMPONENTS)
displayName: 'TBB installation'
displayName: 'Toolkit installation'
- template: .ci/pipeline/build-and-test-win.yml@sklearnex
- script: |
call "%TEMP%\oneapi\setvars.bat" --force
call conda activate CB && python -c "import onedal._onedal_py_dpc; print('OK')"
Comment thread
Alexandr-Solovev marked this conversation as resolved.
Outdated
displayName: 'Check DPC module import'
workingDirectory: $(Agent.TempDirectory)
continueOnError: true
condition: always()
- task: PublishPipelineArtifact@1
inputs:
artifactName: '$(platform.type) sklearnex build'
Expand Down
15 changes: 15 additions & 0 deletions .ci/scripts/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ show_help() {
column -t -s":" <<< '--help:Show this help message
--compiler:The compiler toolchain to use. This is a value that is recognised by the oneDAL top level Makefile, and must be one of [gnu, clang, icx]
--optimizations:The microarchitecture to optimize the build for. This is a value that is recognised by the oneDAL top level Makefile
--sycl-split:The SYCL device code split mode to use. This is a value that is recognised by the oneDAL top level Makefile and must be one of [off, per_device, per_kernel]
--target:The oneDAL target to build. This is passed directly to the oneDAL top level Makefile. Multiple targets can be passed by supplying a space-separated string as an argument
--backend-config:The optimised backend CPU library to use. Must be one of [mkl, ref]
--conda-env:The name of the conda environment to load
Expand All @@ -52,6 +53,15 @@ while [[ $# -gt 0 ]]; do
--optimizations)
optimizations="$2"
shift;;
--optflag)
optflag="$2"
shift;;
--linker)
linker="$2"
shift;;
--sycl-split)
sycl_split="$2"
shift;;
--target)
target="$2"
shift;;
Expand Down Expand Up @@ -105,6 +115,8 @@ OS=${PLAT::3}
ARCH=${PLAT:3}

backend_config=${backend_config:-mkl}
sycl_split=${sycl_split:-off}
optflag=${optflag:-O2}

if [ "${OS}" == "lnx" ]; then
if [ "${conda_env}" != "" ]; then
Expand Down Expand Up @@ -264,6 +276,9 @@ make_options=("${target:-onedal_c}"
REQCPU="${optimizations}"
BACKEND_CONFIG="${backend_config}"
PLAT="${PLAT}"
OPTFLAG="${optflag}"
SYCLSPLIT="${sycl_split}"
LINKER="${linker}"
)

if [ "${cross_compile}" == "yes" ] && [ "${compiler}" == "clang" ] ; then
Expand Down
Loading
Loading