Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,7 @@ subroutine ice_write_hist (ns)
else
ncfile = trim(history_dir)//ncfile
endif
write(nu_diag,*) subname,' Writing ',trim(ncfile)

! create file
if (history_format == 'cdf1') then
Expand Down Expand Up @@ -289,7 +290,8 @@ subroutine ice_write_hist (ns)
endif

! Define coord time_bounds if hist_avg is true
if (hist_avg(ns) .and. .not. write_ic) then
! History restarts are snapshots of accumulated data at current time, do not write time bounds
if (hist_avg(ns) .and. .not. write_ic .and. .not. write_histrest_now) then
time_coord = coord_attributes('time_bounds', 'time interval endpoints', trim(cal_units), 'undefined')

dimid(1) = boundid
Expand Down Expand Up @@ -741,7 +743,8 @@ subroutine ice_write_hist (ns)

! Some coupled models require the time axis "stamp" to be in the middle
! or even beginning of averaging interval.
if (hist_avg(ns)) then
! history restarts are snapshots of accumulated data at current time
if (hist_avg(ns) .and. .not. write_histrest_now) then
if (trim(hist_time_axis) == "begin" ) ltime2 = time_beg(ns)
if (trim(hist_time_axis) == "middle") ltime2 = p5*(time_beg(ns)+time_end(ns))
endif
Expand All @@ -753,7 +756,8 @@ subroutine ice_write_hist (ns)
call ice_check_nc(status, subname// ' ERROR: writing time variable', &
file=__FILE__, line=__LINE__)

if (hist_avg(ns) .and. .not. write_ic) then
! History restarts are snapshots of accumulated data at current time, do not write time bounds
if (hist_avg(ns) .and. .not. write_ic .and. .not. write_histrest_now) then
status = nf90_inq_varid(ncid,'time_bounds',varid)
call ice_check_nc(status, subname// ' ERROR: getting time_bounds id', &
file=__FILE__, line=__LINE__)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -281,7 +281,8 @@ subroutine ice_write_hist (ns)
call ice_hist_coord_def(File, time_coord, pio_double, (/timid/), varid)
call ice_pio_check(pio_put_att(File,varid,'calendar',cal_att), &
subname//' ERROR: defining att calendar: '//cal_att,file=__FILE__,line=__LINE__)
if (hist_avg(ns) .and. .not. write_ic) then
! History restarts are snapshots of accumulated data at current time, do not write time bounds
if (hist_avg(ns) .and. .not. write_ic .and. .not. write_histrest_now) then
call ice_pio_check(pio_put_att(File,varid,'bounds','time_bounds'), &
subname//' ERROR: defining att bounds time_bounds',file=__FILE__,line=__LINE__)
endif
Expand Down Expand Up @@ -723,7 +724,8 @@ subroutine ice_write_hist (ns)

! Some coupled models require the time axis "stamp" to be in the middle
! or even beginning of averaging interval.
if (hist_avg(ns)) then
! history restarts are snapshots of accumulated data at current time
if (hist_avg(ns) .and. .not. write_histrest_now) then
if (trim(hist_time_axis) == "begin" ) ltime2 = time_beg(ns)
if (trim(hist_time_axis) == "middle") ltime2 = p5*(time_beg(ns)+time_end(ns))
endif
Expand All @@ -733,7 +735,8 @@ subroutine ice_write_hist (ns)
call ice_pio_check(pio_put_var(File,varid,(/1/),ltime2), &
subname//' ERROR: setting var time',file=__FILE__,line=__LINE__)

if (hist_avg(ns) .and. .not. write_ic) then
! History restarts are snapshots of accumulated data at current time, do not write time bounds
if (hist_avg(ns) .and. .not. write_ic .and. .not. write_histrest_now) then
call ice_pio_check(pio_inq_varid(File,'time_bounds',varid), &
subname//' ERROR: getting time_bounds' ,file=__FILE__,line=__LINE__)
time_bounds=(/time_beg(ns),time_end(ns)/)
Expand Down
205 changes: 205 additions & 0 deletions cicecore/drivers/unittest/mpif08/mpif08.F90
Original file line number Diff line number Diff line change
@@ -0,0 +1,205 @@

program mpif08

! This tests the availability of mpi_f08 on porting platforms

#define USE_MPI_F08
#ifdef USE_MPI
use mpi
#else
use mpi_f08
#endif

implicit none

integer, parameter :: &
int_kind = selected_int_kind(6), &
real_kind = selected_real_kind(6), &
dbl_kind = selected_real_kind(13)

integer(int_kind) :: i,j,k,n

integer(int_kind), parameter :: dsize = 8
integer(int_kind) :: arrayI(dsize,dsize)
real(real_kind) :: array4(dsize,dsize)
real(dbl_kind) :: array8(dsize,dsize)
real(dbl_kind) :: carray8(dsize,dsize),rarray8(dsize,dsize)

integer(int_kind) :: my_task,master_task,npes
integer(int_kind) :: ierr,tag
integer(int_kind) :: sro ! send recv pe offset
#ifdef USE_MPI
integer :: MPI_COMM_ICE
integer :: rrequest
integer :: rstatus(MPI_STATUS_SIZE)
#else
type(MPI_COMM) :: MPI_COMM_ICE
type(MPI_REQUEST) :: rrequest
type(MPI_STATUS) :: rstatus
#endif

integer(int_kind), parameter :: ntests1 = 4
integer(int_kind) :: iflag, gflag
integer(int_kind) :: cval,ival,sumi
real(real_kind) :: sum4
real(dbl_kind) :: sum8
character(len=8) :: errorflag0
character(len=8) :: errorflag1(ntests1)
character(len=32) :: stringflag1(ntests1)

character(len=*), parameter :: &
passflag = 'PASS', &
failflag = 'FAIL'

character(len=*), parameter :: subname = '(mpif08)'

! ---------------------------

call MPI_INIT(ierr)
if (ierr /= MPI_SUCCESS) write(6,*) ' ERROR MPI_INIT'
call MPI_COMM_DUP(MPI_COMM_WORLD, MPI_COMM_ICE, ierr)
if (ierr /= MPI_SUCCESS) write(6,*) ' ERROR MPI_COMM_DUP'

master_task = 0
call MPI_COMM_SIZE(MPI_COMM_ICE, npes, ierr)
if (ierr /= MPI_SUCCESS) write(6,*) ' ERROR MPI_COMM_SIZE'
call MPI_COMM_RANK(MPI_COMM_ICE, my_task, ierr)
if (ierr /= MPI_SUCCESS) write(6,*) ' ERROR MPI_COMM_RANK'
sro = max(npes/2,1)

if (my_task == master_task) then
write(6,*) ' '
write(6,*) '=========================================================='
write(6,*) ' '
write(6,*) 'RunningUnitTest MPIF08'
#ifdef USE_MPI
write(6,*) ' with use mpi'
#else
write(6,*) ' with use mpi_f08'
#endif
write(6,*) ' '
write(6,*) ' npes = ',npes
write(6,*) ' my_task = ',my_task
write(6,*) ' sr pe offset = ',sro
write(6,*) ' '
flush(6)
endif

! ---------------------------

errorflag0 = passflag
errorflag1 = passflag
stringflag1 = ' '

ival = 0
do k = 0,npes-1
do j = 1,dsize
do i = 1,dsize
cval = (k*100 + j*10 + i)
ival = ival + cval
enddo
enddo
enddo

do j = 1,dsize
do i = 1,dsize
cval = (my_task*100 + j*10 + i)
arrayI(i,j) = cval
array4(i,j) = real(cval,real_kind)
array8(i,j) = real(cval,dbl_kind)
enddo
enddo

! for test 4, communicate array to neighbor task
do j = 1,dsize
do i = 1,dsize
cval = (mod(my_task+sro,npes)*100 + j*10 + i)
carray8(i,j) = real(cval,dbl_kind)
enddo
enddo

! if (my_task == master_task) write(6,*) 'ival = ',ival

do n = 1,ntests1
iflag = 0
gflag = 0
if (n == 1) then
stringflag1(n) = 'allreduce_int'
call MPI_ALLREDUCE(sum(arrayI), sumi, 1, MPI_INTEGER, MPI_SUM, MPI_COMM_ICE, ierr)
if (ierr /= MPI_SUCCESS) write(6,*) ' ERROR MPI_ALLREDUCE arrayI'
if (sumi /= ival) iflag=1
if (my_task == master_task) write(6,*) ' Test ',n,ival,sumi
flush(6)
elseif (n == 2) then
stringflag1(n) = 'allreduce_r4'
call MPI_ALLREDUCE(sum(array4), sum4, 1, MPI_REAL, MPI_SUM, MPI_COMM_ICE, ierr)
if (ierr /= MPI_SUCCESS) write(6,*) ' ERROR MPI_ALLREDUCE array4'
if (sum4 /= real(ival,real_kind)) iflag=1
if (my_task == master_task) write(6,*) ' Test ',n,ival,sum4
flush(6)
elseif (n == 3) then
stringflag1(n) = 'allreduce_r8'
call MPI_ALLREDUCE(sum(array8), sum8, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_ICE, ierr)
if (ierr /= MPI_SUCCESS) write(6,*) ' ERROR MPI_ALLREDUCE array8'
if (sum8 /= real(ival,dbl_kind)) iflag=1
if (my_task == master_task) write(6,*) ' Test ',n,ival,sum8
flush(6)
elseif (n == 4) then
stringflag1(n) = 'irecv_send_r8'
tag = 1001
call MPI_IRECV(rarray8, dsize*dsize, MPI_DOUBLE, mod(my_task+sro,npes), tag, MPI_COMM_ICE, rrequest, ierr)
if (ierr /= MPI_SUCCESS) write(6,*) ' ERROR MPI_IRECV'
call MPI_SEND(array8, dsize*dsize, MPI_DOUBLE, mod(my_task-sro+npes,npes), tag, MPI_COMM_ICE, ierr)
if (ierr /= MPI_SUCCESS) write(6,*) ' ERROR MPI_SEND'
call MPI_WAIT(rrequest, rstatus, ierr)
if (ierr /= MPI_SUCCESS) write(6,*) ' ERROR MPI_WAIT'
do j = 1,dsize
do i = 1,dsize
if (rarray8(i,j) /= carray8(i,j)) iflag=1
enddo
enddo
if (my_task == master_task) write(6,*) ' Test ',n,sum(carray8),sum(rarray8)
flush(6)
endif
call MPI_REDUCE(iflag, gflag, 1, MPI_INTEGER, MPI_MAX, master_task, MPI_COMM_ICE, ierr)
if (ierr /= MPI_SUCCESS) write(6,*) ' ERROR MPI_REDUCE iflag'
if (my_task == master_task) then
if (gflag /= 0) then
write(6,*) ' **** ERROR test ',n
flush(6)
errorflag1(n) = failflag
errorflag0 = failflag
endif
endif
enddo

call MPI_BARRIER(MPI_COMM_ICE, ierr)
if (ierr /= MPI_SUCCESS) write(6,*) ' ERROR MPI_BARRIER 2'

! ---------------------------

if (my_task == master_task) then
write(6,*) ' '
do k = 1,ntests1
write(6,*) errorflag1(k),stringflag1(k)
enddo
write(6,*) ' '
write(6,*) 'MPIF08 COMPLETED SUCCESSFULLY'
if (errorflag0 == passflag) then
write(6,*) 'MPIF08 TEST COMPLETED SUCCESSFULLY'
else
write(6,*) 'MPIF08 TEST FAILED'
endif
endif

flush(6)

! ---------------------------
! exit gracefully

call MPI_FINALIZE(ierr)
if (ierr /= MPI_SUCCESS) write(6,*) ' ERROR MPI_FINALIZE'

end program mpif08

!=======================================================================
8 changes: 6 additions & 2 deletions configuration/scripts/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ AR := ar

.SUFFIXES:

.PHONY: all cice libcice targets target db_files db_flags clean realclean helloworld calchk sumchk bcstchk gridavgchk halochk optargs opticep
.PHONY: all cice libcice targets target db_files db_flags clean realclean helloworld calchk sumchk bcstchk gridavgchk halochk optargs opticep mpif08
all: $(EXEC)

cice: $(EXEC)
Expand All @@ -93,7 +93,7 @@ targets:
@echo " "
@echo "Supported Makefile Targets are: cice, libcice, makdep, depends, clean, realclean"
@echo " Diagnostics: targets, db_files, db_flags"
@echo " Unit Tests : helloworld, calchk, sumchk, bcstchk, gridavgchk, halochk, optargs, opticep"
@echo " Unit Tests : helloworld, calchk, sumchk, bcstchk, gridavgchk, halochk, optargs, opticep mpif08"
target: targets

db_files:
Expand Down Expand Up @@ -165,6 +165,10 @@ OAOBJS := optargs.o optargs_subs.o
optargs: $(OAOBJS)
$(LD) -o $(EXEC) $(LDFLAGS) $(OAOBJS) $(ULIBS) $(SLIBS)

M8OBJS := mpif08.o
mpif08: $(M8OBJS)
$(LD) -o $(EXEC) $(LDFLAGS) $(M8OBJS) $(ULIBS) $(SLIBS)

#-------------------------------------------------------------------------------
# build rules: MACFILE, cmd-line, or env vars must provide the needed macros
#-------------------------------------------------------------------------------
Expand Down
3 changes: 2 additions & 1 deletion configuration/scripts/machines/Macros.derecho_intel
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -trace
FFLAGS_NOOPT:= -O0

ifeq ($(ICE_BLDDEBUG), true)
FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg
# FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg
FFLAGS += -O0 -check all -check noarg_temp_created -warn -warn noerrors -fp-stack-check -fstack-protector-all -fpe0 -debug -ftrapuv -init=snan,arrays -link_mpi=dbg
# FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -link_mpi=dbg -stand f08
# FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays -link_mpi=dbg
else
Expand Down
4 changes: 3 additions & 1 deletion configuration/scripts/machines/Macros.derecho_inteloneapi
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@ FFLAGS_NOOPT:= -O0

ifeq ($(ICE_BLDDEBUG), true)
# -check uninit is needed on the ld step but it still throws errors in 2023.* and 2024.0.*, likely compiler bug

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Re this note

I doesn't look like intel will change this -check uninit behaviour

See https://www.intel.com/content/www/us/en/docs/fortran-compiler/developer-guide-reference/2025-1/check.html:

-check all now behaves as -check all -check nouninit. Setting -check uninit and linking with libraries not built with this option, such as Intel® oneAPI Math Kernel Library (oneMKL) or Intel® MPI Library, causes runtime failures. -check uninit is no longer set by -check all.

Which basically makes -check uninit unusable for us, as it's very hard to build all the dependencies with -check uninit (unless you rebuild the compiler) and then the resulting executable needs to be run with the LLVM Memory Sanitizer path variable set as well

FFLAGS += -O0 -g -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg
# FFLAGS += -O0 -g -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg
FFLAGS += -O0 -check all -check noarg_temp_created -warn -warn noerrors -fp-stack-check -fstack-protector-all -fpe0 -debug -ftrapuv -init=snan,arrays -link_mpi=dbg
# LDFLAGS += -check all
# FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg
# LDFLAGS += -check uninit
else
Expand Down
20 changes: 10 additions & 10 deletions configuration/scripts/machines/env.derecho_inteloneapi
Original file line number Diff line number Diff line change
Expand Up @@ -10,28 +10,28 @@ if ("$inp" != "-nomodules") then
source ${MODULESHOME}/init/csh

module --force purge
module load ncarenv/23.09
module load ncarenv/25.10
module load craype
module load intel-oneapi/2023.2.1
module load intel/2025.2.1
#module load mkl/2023.3.0
module load ncarcompilers/1.0.0
module load cray-mpich/8.1.27
module load netcdf/4.9.2
module load ncarcompilers/1.2.0
module load cray-mpich/8.1.32
module load netcdf/4.9.3
#module load hdf5/1.12.2
#module load netcdf-mpi/4.9.2

module load cray-libsci/23.09.1.1
module load cray-libsci/25.03.0
module load nco

if ($?ICE_IOTYPE) then
if ($ICE_IOTYPE =~ pio*) then
module unload netcdf
module load netcdf-mpi/4.9.2
module load parallel-netcdf/1.12.3
module load netcdf-mpi/4.9.3
module load parallel-netcdf/1.14.1
if ($ICE_IOTYPE == "pio1") then
module load parallelio/1.10.1
else
module load parallelio/2.6.2
module load parallelio/2.6.8
endif
endif
endif
Expand Down Expand Up @@ -63,7 +63,7 @@ setenv OMP_STACKSIZE 64M
setenv ICE_MACHINE_MACHNAME derecho
setenv ICE_MACHINE_MACHINFO "HPE Cray EX Milan Slingshot 11"
setenv ICE_MACHINE_ENVNAME inteloneapi
setenv ICE_MACHINE_ENVINFO "oneAPI DPC++/C++/ifx 2023.2.0 20230721, cray-mpich 8.1.27, netcdf4.9.2, pnetcdf1.12.3, pio1.10.1, pio2.6.2"
setenv ICE_MACHINE_ENVINFO "oneAPI DPC++/C++/ifx 2025.2.1 20250806, cray-mpich 8.1.32, netcdf4.9.3, pnetcdf1.14.1, pio1.10.1, pio2.6.8"
setenv ICE_MACHINE_MAKE gmake
setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/CICE_RUNS
setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg
Expand Down
2 changes: 2 additions & 0 deletions configuration/scripts/options/set_env.mpif08
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
setenv ICE_DRVOPT unittest/mpif08
setenv ICE_TARGET mpif08
1 change: 1 addition & 0 deletions configuration/scripts/tests/unittest_suite.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ smoke gx3 8x2 diag1,run5day
smoke gx3 4x2x25x29x4 debug,run2day,dslenderX2
unittest gx3 1x1 helloworld
unittest gx3 1x1 calchk,short
unittest gx3 8x1 mpif08,debug
unittest gx3 4x1x25x29x4 sumchk
unittest gx3 1x1x25x29x16 sumchk
unittest tx1 8x1 sumchk
Expand Down
1 change: 1 addition & 0 deletions doc/source/user_guide/ug_testing.rst
Original file line number Diff line number Diff line change
Expand Up @@ -742,6 +742,7 @@ The following are brief descriptions of some of the current unit tests,
- **helloworld** is a simple test that writes out helloworld and uses no CICE infrastructure.
This tests exists to demonstrate how to build a unit test by specifying the object files directly
in the Makefile
- **mpif08** is a simple standalone unit test that checks whether the mpi_f08 module is available.
- **optargs** is a unit test that tests passing optional arguments down a calling tree and verifying
that the optional attribute is preserved correctly.
- **opticep** is a cice test that turns off the icepack optional arguments passed into icepack. This
Expand Down
Loading