Skip to content

Commit 4fb19b3

Browse files
rljacobrljacob
authored
Merge branch 'tcclevenger/chrysalis_oneapi_default' (PR #8079)
Make "intel" now refer to oneapi. Intel-classic compiler on chrysalis will be incompatible with Kokkos 5/C++20. Moving the default to oneapi before updating kokkos to allow users to switch. Keep old intel compiler around for now as intel-classic. [non-BFB] for all "intel" tests on Chrysalis. Compiler is changing but "intel" test names aren't changing.
2 parents c2a18cd + c368abf commit 4fb19b3

6 files changed

Lines changed: 122 additions & 47 deletions

File tree

‎cime_config/machines/cmake_macros/chrysalis_intel-classic.cmake‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
if (compile_threaded)
2+
string(APPEND CMAKE_C_FLAGS " -qopenmp")
3+
string(APPEND CMAKE_CXX_FLAGS " -qopenmp")
4+
string(APPEND CMAKE_Fortran_FLAGS " -qopenmp")
5+
string(APPEND CMAKE_EXE_LINKER_FLAGS " -qopenmp")
6+
7+
string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_AGGRESSIVE_VECTORIZATION=On")
8+
endif()
9+
10+
string(APPEND CMAKE_C_FLAGS " -fp-model precise -std=gnu99 -static-intel -march=core-avx2")
11+
string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g")
12+
string(APPEND CMAKE_C_FLAGS_RELEASE " -O3 -debug minimal")
13+
string(APPEND CMAKE_CXX_FLAGS " -fp-model source -static-intel -axCORE-AVX2")
14+
string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g")
15+
string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O3")
16+
string(APPEND CMAKE_EXE_LINKER_FLAGS " -static-intel")
17+
string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRINTEL")
18+
19+
string(APPEND CMAKE_Fortran_FLAGS " -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model source -static-intel -axCORE-AVX2")
20+
string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -init=snan,arrays")
21+
string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O3 -qno-opt-dynamic-align -debug minimal")
22+
string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -fixed -132")
23+
string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -free")
24+
25+
string(APPEND KOKKOS_OPTIONS " -DKokkos_ARCH_ZEN2=On")
26+
27+
set(HAS_F2008_CONTIGUOUS "TRUE")
28+
set(E3SM_LINK_WITH_FORTRAN "TRUE")
29+
set(PIO_FILESYSTEM_HINTS "gpfs")
30+
31+
if (COMP_NAME STREQUAL gptl)
32+
string(APPEND CPPDEFS " -DHAVE_SLASHPROC")
33+
endif()
34+
if (COMP_NAME STREQUAL cice)
35+
string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -init=nosnan,arrays")
36+
endif()
37+
if (COMP_NAME STREQUAL gcam)
38+
string(APPEND CMAKE_C_FLAGS " -mcmodel=medium -shared-intel")
39+
string(APPEND CMAKE_Fortran_FLAGS " -mcmodel=medium -shared-intel")
40+
string(APPEND CMAKE_CXX_FLAGS " -shared-intel")
41+
string(APPEND CMAKE_EXE_LINKER_FLAGS " -shared-intel")
42+
string(APPEND CMAKE_CXX_FLAGS " -DNDEBUG")
43+
endif()
44+
45+
set(MPICC "mpicc")
46+
set(MPICXX "mpicxx")
47+
set(MPIFC "mpif90")
48+
set(SCC "icc")
49+
set(SCXX "icpc")
50+
set(SFC "ifort")
51+
if (MPILIB STREQUAL impi)
52+
set(MPICC "mpiicc")
53+
set(MPICXX "mpiicpc")
54+
set(MPIFC "mpiifort")
55+
endif()

‎cime_config/machines/cmake_macros/chrysalis_intel.cmake‎

Lines changed: 24 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,27 @@
1-
if (COMP_NAME STREQUAL gptl)
2-
string(APPEND CPPDEFS " -DHAVE_SLASHPROC")
1+
string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/gpfs/fs1/soft/chrysalis/spack-latest/opt/spack/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.3.0-jkpmtgq/lib64 -lstdc++")
2+
3+
# Workaround for oneapi ifx v2025.2.0 (and earlier than 2025.3.0):
4+
# use -mllvm -disable-hir-temp-cleanup to avoid ICE
5+
# See: https://github.com/argonne-lcf/AuroraBugTracking/issues/64
6+
if (CMAKE_Fortran_COMPILER_ID STREQUAL "IntelLLVM"
7+
AND CMAKE_Fortran_COMPILER_VERSION VERSION_LESS "2025.3.0")
8+
string(APPEND CMAKE_Fortran_FLAGS " -mllvm -disable-hir-temp-cleanup")
39
endif()
4-
string(APPEND CMAKE_C_FLAGS " -static-intel")
5-
string(APPEND CMAKE_C_FLAGS " -march=core-avx2")
6-
string(APPEND CMAKE_C_FLAGS_RELEASE " -O3")
7-
string(APPEND CMAKE_CXX_FLAGS " -static-intel")
8-
string(APPEND CMAKE_CXX_FLAGS " -axCORE-AVX2")
9-
string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O3")
10-
string(APPEND CMAKE_Fortran_FLAGS " -static-intel")
11-
string(APPEND CMAKE_Fortran_FLAGS " -axCORE-AVX2")
12-
string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O3 -qno-opt-dynamic-align")
13-
set(PIO_FILESYSTEM_HINTS "gpfs")
14-
string(APPEND CMAKE_EXE_LINKER_FLAGS " -static-intel")
15-
if (MPILIB STREQUAL impi)
16-
set(MPICC "mpiicc")
17-
set(MPICXX "mpiicpc")
18-
set(MPIFC "mpiifort")
19-
endif()
20-
string(APPEND KOKKOS_OPTIONS " -DKokkos_ARCH_ZEN2=On")
10+
11+
string(APPEND CPPDEFS " -DHAVE_SLASHPROC -DHIDE_MPI")
12+
13+
# Hack similar to pm-cpu_intel.cmake. Reset the following flags and load
14+
# with oneapi supported options.
15+
set(CMAKE_CXX_FLAGS " ")
16+
set(CMAKE_Fortran_FLAGS_DEBUG " ")
17+
2118
if (compile_threaded)
22-
string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_AGGRESSIVE_VECTORIZATION=On")
23-
endif()
24-
if (COMP_NAME STREQUAL gcam)
25-
string(APPEND CMAKE_C_FLAGS " -mcmodel=medium -shared-intel")
26-
string(APPEND CMAKE_Fortran_FLAGS " -mcmodel=medium -shared-intel")
27-
string(APPEND CMAKE_CXX_FLAGS " -shared-intel")
28-
string(APPEND CMAKE_EXE_LINKER_FLAGS " -shared-intel")
29-
string(APPEND CMAKE_CXX_FLAGS " -DNDEBUG")
19+
string(APPEND CMAKE_CXX_FLAGS " -qopenmp")
3020
endif()
21+
string(APPEND CMAKE_CXX_FLAGS " -fp-model precise")
22+
string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g ")
23+
24+
set(MPIFC "mpifort")
25+
set(SCC "icx")
26+
set(SCXX "icpx")
27+
set(SFC "ifx")

‎cime_config/machines/cmake_macros/chrysalis_oneapi-ifx.cmake‎

Lines changed: 0 additions & 11 deletions
This file was deleted.

‎cime_config/machines/config_machines.xml‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2487,7 +2487,7 @@
24872487
<DESC>ANL LCRC cluster 512-node AMD Epyc 7532 2-sockets 64-cores per node</DESC>
24882488
<NODENAME_REGEX>chr.*</NODENAME_REGEX>
24892489
<OS>LINUX</OS>
2490-
<COMPILERS>intel,gnu,oneapi-ifx</COMPILERS>
2490+
<COMPILERS>intel,intel-classic,gnu</COMPILERS>
24912491
<MPILIBS>openmpi,impi</MPILIBS>
24922492
<PROJECT>e3sm</PROJECT>
24932493
<SAVE_TIMING_DIR>/lcrc/group/e3sm/PERF_Chrysalis</SAVE_TIMING_DIR>
@@ -2529,18 +2529,18 @@
25292529
<command name="load">perl/5.32.0-bsnc6lt</command>
25302530
<command name="load">cmake/3.24.2-whgdv7y</command>
25312531
</modules>
2532-
<modules compiler="intel">
2532+
<modules compiler="intel-classic">
25332533
<command name="load">intel/20.0.4-kodw73g</command>
25342534
<command name="load">intel-mkl/2020.4.304-g2qaxzf</command>
25352535
</modules>
2536-
<modules compiler="intel" mpilib="openmpi">
2536+
<modules compiler="intel-classic" mpilib="openmpi">
25372537
<command name="load">openmpi/4.1.6-2mm63n2</command>
25382538
<command name="load">hdf5/1.10.7-4cghwvq</command>
25392539
<command name="load">netcdf-c/4.7.4-4qjdadt</command>
25402540
<command name="load">netcdf-fortran/4.5.3-qozrykr</command>
25412541
<command name="load">parallel-netcdf/1.11.0-icrpxty</command>
25422542
</modules>
2543-
<modules compiler="intel" mpilib="impi">
2543+
<modules compiler="intel-classic" mpilib="impi">
25442544
<command name="load">intel-mpi/2019.9.304-tkzvizk</command>
25452545
<command name="load">hdf5/1.10.7-wczt56s</command>
25462546
<command name="load">netcdf-c/4.7.4-ba6agmb</command>
@@ -2569,10 +2569,10 @@
25692569
<command name="load">netcdf-fortran/4.5.3-ova6t37</command>
25702570
<command name="load">parallel-netcdf/1.11.0-ifdodru</command>
25712571
</modules>
2572-
<modules compiler="oneapi-ifx">
2572+
<modules compiler="intel">
25732573
<command name="load">intel-oneapi-compilers/2025.2.0-mdlxe55</command>
25742574
</modules>
2575-
<modules compiler="oneapi-ifx" mpilib="openmpi">
2575+
<modules compiler="intel" mpilib="openmpi">
25762576
<command name="load">openmpi/4.1.8-nygtpa3</command>
25772577
<command name="load">intel-oneapi-mkl/2025.2.0-bcimxay</command>
25782578
<command name="load">hdf5/1.14.6-5cgownf</command>
@@ -2598,16 +2598,22 @@
25982598
<environment_variables BUILD_THREADED="TRUE">
25992599
<env name="OMP_STACKSIZE">128M</env>
26002600
</environment_variables>
2601-
<environment_variables BUILD_THREADED="TRUE" compiler="intel" MAX_TASKS_PER_NODE="!128">
2601+
<environment_variables BUILD_THREADED="TRUE" compiler="intel-classic" MAX_TASKS_PER_NODE="!128">
26022602
<env name="KMP_AFFINITY">granularity=core,balanced</env>
26032603
</environment_variables>
2604-
<environment_variables BUILD_THREADED="TRUE" compiler="intel" MAX_TASKS_PER_NODE="128">
2604+
<environment_variables BUILD_THREADED="TRUE" compiler="intel-classic" MAX_TASKS_PER_NODE="128">
26052605
<env name="KMP_AFFINITY">granularity=thread,balanced</env>
26062606
</environment_variables>
26072607
<environment_variables BUILD_THREADED="TRUE" compiler="gnu">
26082608
<env name="OMP_PLACES">cores</env>
26092609
</environment_variables>
26102610
<environment_variables compiler="intel" mpilib="openmpi">
2611+
<env name="UCX_RCACHE_MAX_REGIONS">1000</env>
2612+
</environment_variables>
2613+
<environment_variables compiler="intel" mpilib="openmpi">
2614+
<env name="MOAB_ROOT">$SHELL{if [ -z "$MOAB_ROOT" ]; then echo /lcrc/soft/climate/moab/chrysalis/intel-oneapi-compilers-2025.2.0; else echo "$MOAB_ROOT"; fi}</env>
2615+
</environment_variables>
2616+
<environment_variables compiler="intel-classic" mpilib="openmpi">
26112617
<env name="MOAB_ROOT">$SHELL{if [ -z "$MOAB_ROOT" ]; then echo /lcrc/soft/climate/moab/chrysalis/intel; else echo "$MOAB_ROOT"; fi}</env>
26122618
<env name="Albany_ROOT">$SHELL{if [ -z "$Albany_ROOT" ]; then echo /lcrc/group/e3sm/soft/albany/2024.03.26/intel/20.0.4; else echo "$Albany_ROOT"; fi}</env>
26132619
<env name="Trilinos_ROOT">$SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /lcrc/group/e3sm/soft/trilinos/15.1.1/intel/20.0.4; else echo "$Trilinos_ROOT"; fi}</env>
@@ -2948,7 +2954,7 @@
29482954
<command name="load">cmake/3.26.3</command>
29492955
<command name="load">mkl/2022.1.0</command>
29502956
</modules>
2951-
<modules compiler="oneapi-ifx">
2957+
<modules compiler="intel">
29522958
<command name="load">intel/2023.2.1-magic</command>
29532959
<command name="use">/usr/workspace/e3sm/spack/dane/intel/modules/Core</command>
29542960
<command name="load">intel-oneapi-runtime/2023.2.0</command>

‎components/homme/cmake/machineFiles/chrysalis-bfb.cmake‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,14 @@
22
#
33
EXECUTE_PROCESS(COMMAND which mpiifort RESULT_VARIABLE MPIIFORT_RESULT)
44
EXECUTE_PROCESS(COMMAND which ifort RESULT_VARIABLE IFORT_RESULT)
5+
EXECUTE_PROCESS(COMMAND which ifx RESULT_VARIABLE IFX_RESULT)
56
IF (${MPIIFORT_RESULT} EQUAL 0 AND ${IFORT_RESULT} EQUAL 0)
67
MESSAGE(STATUS "Found Intel compiler and Intel MPI: building with mpiifort/mpiicc/mpiicpc")
78
SET (CMAKE_Fortran_COMPILER mpiifort CACHE FILEPATH "")
89
SET (CMAKE_C_COMPILER mpiicc CACHE FILEPATH "")
910
SET (CMAKE_CXX_COMPILER mpiicpc CACHE FILEPATH "")
1011
ELSE()
12+
# Use mpif90/mpicc/mpicxx for oneapi-ifx and gnu compiler
1113
MESSAGE(STATUS "Did not detect ifort or mpiifort: building with mpif90/mpicc/mpicxx")
1214
SET (CMAKE_Fortran_COMPILER mpif90 CACHE FILEPATH "")
1315
SET (CMAKE_C_COMPILER mpicc CACHE FILEPATH "")
@@ -58,6 +60,17 @@ IF (${IFORT_RESULT} EQUAL 0)
5860
SET (HOMMEXX_BFB_TESTING TRUE CACHE BOOL "")
5961
SET (HOMME_TESTING_PROFILE "short" CACHE STRING "")
6062
SET (BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "")
63+
ELSEIF (${IFX_RESULT} EQUAL 0)
64+
SET (ADD_C_FLAGS " -traceback -fp-model strict -qopenmp -O1" CACHE STRING "")
65+
SET (ADD_CXX_FLAGS " -traceback -fp-model strict -qopenmp -O1" CACHE STRING "")
66+
SET (ADD_Fortran_FLAGS " -traceback -DCPRINTEL -fp-model strict -qopenmp -O1" CACHE STRING "")
67+
SET (ADD_LINKER_FLAGS " -L/gpfs/fs1/soft/chrysalis/spack-latest/opt/spack/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.3.0-jkpmtgq/lib64 -lstdc++" CACHE STRING "")
68+
SET (BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "")
69+
SET (HOMMEXX_BFB_TESTING TRUE CACHE BOOL "")
70+
SET (HOMME_TESTING_PROFILE "short" CACHE STRING "")
71+
SET (BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "")
72+
SET (MKLROOT $ENV{MKLROOT} CACHE FILEPATH "")
73+
SET (HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "")
6174
ELSE()
6275
SET (MKLROOT $ENV{MKLROOT} CACHE FILEPATH "")
6376
SET (HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "")

‎components/homme/cmake/machineFiles/chrysalis.cmake‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,14 @@
22
#
33
EXECUTE_PROCESS(COMMAND which mpiifort RESULT_VARIABLE MPIIFORT_RESULT)
44
EXECUTE_PROCESS(COMMAND which ifort RESULT_VARIABLE IFORT_RESULT)
5+
EXECUTE_PROCESS(COMMAND which ifx RESULT_VARIABLE IFX_RESULT)
56
IF (${MPIIFORT_RESULT} EQUAL 0 AND ${IFORT_RESULT} EQUAL 0)
67
MESSAGE(STATUS "Found Intel compiler and Intel MPI: building with mpiifort/mpiicc/mpiicpc")
78
SET (CMAKE_Fortran_COMPILER mpiifort CACHE FILEPATH "")
89
SET (CMAKE_C_COMPILER mpiicc CACHE FILEPATH "")
910
SET (CMAKE_CXX_COMPILER mpiicpc CACHE FILEPATH "")
1011
ELSE()
12+
# Use mpif90/mpicc/mpicxx for oneapi-ifx and gnu compiler
1113
MESSAGE(STATUS "Did not detect ifort or mpiifort: building with mpif90/mpicc/mpicxx")
1214
SET (CMAKE_Fortran_COMPILER mpif90 CACHE FILEPATH "")
1315
SET (CMAKE_C_COMPILER mpicc CACHE FILEPATH "")
@@ -55,6 +57,19 @@ IF (${IFORT_RESULT} EQUAL 0)
5557
SET (ADD_C_FLAGS "-traceback" CACHE STRING "")
5658
SET (ADD_CXX_FLAGS "-traceback" CACHE STRING "")
5759
SET (BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "")
60+
ELSEIF (${IFX_RESULT} EQUAL 0)
61+
SET (ADD_C_FLAGS "-traceback" CACHE STRING "")
62+
SET (ADD_CXX_FLAGS "-traceback" CACHE STRING "")
63+
64+
# Workaround for oneapi ifx v2025.2.0 (and earlier than 2025.3.0):
65+
# use -mllvm -disable-hir-temp-cleanup to avoid ICE
66+
# See: argonne-lcf/AuroraBugTracking#64
67+
SET (ADD_Fortran_FLAGS " -traceback -DCPRINTEL -mllvm -disable-hir-temp-cleanup" CACHE STRING "")
68+
69+
SET (ADD_LINKER_FLAGS " -L/gpfs/fs1/soft/chrysalis/spack-latest/opt/spack/linux-rhel8-x86_64/gcc-8.5.0/gcc-11.3.0-jkpmtgq/lib64 -lstdc++" CACHE STRING "")
70+
SET (BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "")
71+
SET (MKLROOT $ENV{MKLROOT} CACHE FILEPATH "")
72+
SET (HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "")
5873
ELSE()
5974
SET (MKLROOT $ENV{MKLROOT} CACHE FILEPATH "")
6075
SET (HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "")

0 commit comments

Comments
 (0)