Initial commits for MAPL3

Signed-off-by: Lizzie Lundgren <elundgren@seas.harvard.edu>

Author: lizziel

CMakeLists.txt

@@ -34,9 +34,9 @@ message("=================================================================")
 #
 # Optionally, super project can define and configure this target.
 #-----------------------------------------------------------------------------
-if(NOT TARGET GEOSChemBuildProperties)
+#if(NOT TARGET GEOSChemBuildProperties)
     add_library(GEOSChemBuildProperties INTERFACE)
-endif()
+#endif()
 
 #-----------------------------------------------------------------------------
 # Add all the subdirectories.
@@ -54,15 +54,193 @@ add_subdirectory(GTMM)
 add_subdirectory(GeosCore)
 
 #-----------------------------------------------------------------------------
-# Build GEOS-Chem Classic driver program (if MODEL_CLASSIC=T)
+# Build GEOS-Chem Classic driver program if using GC-Classic
 #-----------------------------------------------------------------------------
 if(MODEL_CLASSIC)
   add_subdirectory(Interfaces/GCClassic)
 endif()
 
 #-----------------------------------------------------------------------------
-# Build GCHPctm driver program (if MODEL_GCHPCTM=T)
+# Build GCHP driver program if using GCHP
 #-----------------------------------------------------------------------------
-if(MODEL_GCHPCTM)
+if(MODEL_GCHP)
   add_subdirectory(Interfaces/GCHP)
 endif()
+
+#-----------------------------------------------------------------------------
+# Additional GEOS-Chem settings if using GCHP (all necessary?)
+#-----------------------------------------------------------------------------
+if(MODEL_GCHP)
+
+  set(RRTMG OFF CACHE BOOL
+    "Switch to build RRTMG as a component of GEOS-Chem"
+  )
+  set(TOMAS OFF CACHE BOOL
+    "Switch to build TOMAS as a component of GEOS-Chem"
+  )
+  set(LUO_WETDEP OFF CACHE BOOL
+    "Switch to build the Luo et al (2020) wetdep scheme into GEOS-Chem"
+  )
+  set(FASTJX OFF CACHE BOOL
+    "Switch to use legacy FAST-JX photolysis in GEOS-Chem"
+  )
+  set(KPPSA OFF CACHE BOOL
+    "Switch to build the KPP-Standalone Box Model"
+  )
+  
+  # Local variables
+  set(GTMM                    FALSE)
+  set(MECH "fullchem" CACHE STRING "Name of the chemistry mechanism to use")
+  
+  # Configure build properties for GEOS-Chem
+  target_compile_definitions(GEOSChemBuildProperties INTERFACE
+    MODEL_GCHPCTM
+    MODEL_GCHP
+    USE_ESMF
+    MAPL_ESMF
+    MAPL3
+    EXTERNAL_GRID
+    NC_HAS_COMPRESSION
+    $<$<BOOL:${USE_REAL8}>:USE_REAL8> $<$<BOOL:${RRTMG}>:RRTMG>
+    $<$<BOOL:${ADJOINT}>:ADJOINT>
+    $<$<BOOL:${REVERSE_OPERATORS}>:REVERSE_OPERATORS> 
+    $<$<BOOL:${TOMAS}>:TOMAS>
+    $<$<STREQUAL:${TOMAS_BINS},15>:TOMAS15>
+    $<$<STREQUAL:${TOMAS_BINS},40>:TOMAS40>
+    $<$<BOOL:${LUO_WETDEP}>:LUO_WETDEP>
+    $<$<BOOL:${FASTJX}>:FASTJX>
+  )
+  
+  if(NOT MAPL3)
+    target_link_libraries(GEOSChemBuildProperties INTERFACE
+      $<LINK_ONLY:MAPL.base>
+      $<LINK_ONLY:FVdycoreCubed_GridComp>
+      $<LINK_ONLY:HCOI_MAPL_ESMF>
+      $<$<BOOL:${OMP}>:OpenMP::OpenMP_Fortran>
+      MPI::MPI_C MPI::MPI_Fortran
+    )
+    target_include_directories(GEOSChemBuildProperties INTERFACE
+      $<TARGET_PROPERTY:MAPL.base,INTERFACE_INCLUDE_DIRECTORIES>
+      $<TARGET_PROPERTY:FVdycoreCubed_GridComp,INTERFACE_INCLUDE_DIRECTORIES>
+      $<TARGET_PROPERTY:HCOI_MAPL_ESMF,INTERFACE_INCLUDE_DIRECTORIES>
+      ${CMAKE_Fortran_MODULE_DIRECTORY}
+    )
+#   GeosCore
+  endif()
+  if(MAPL3)
+    target_link_libraries(GEOSChemBuildProperties INTERFACE
+      $<LINK_ONLY:MAPL.base>
+      $<LINK_ONLY:FVdycoreCubed_GridComp>
+      $<LINK_ONLY:HCOI_MAPL_ESMF>
+      $<$<BOOL:${OMP}>:OpenMP::OpenMP_Fortran>
+      MPI::MPI_C MPI::MPI_Fortran
+      $<LINK_ONLY:mapl3g>
+      $<LINK_ONLY:MAPL.generic3g>
+      $<LINK_ONLY:MAPL.state>
+    )
+    target_include_directories(GEOSChemBuildProperties INTERFACE
+      $<TARGET_PROPERTY:mapl3g,INTERFACE_INCLUDE_DIRECTORIES>
+      $<TARGET_PROPERTY:MAPL.generic3g,INTERFACE_INCLUDE_DIRECTORIES>
+      $<TARGET_PROPERTY:MAPL.state,INTERFACE_INCLUDE_DIRECTORIES>
+      $<TARGET_PROPERTY:MAPL.base,INTERFACE_INCLUDE_DIRECTORIES>
+      $<TARGET_PROPERTY:FVdycoreCubed_GridComp,INTERFACE_INCLUDE_DIRECTORIES>
+      $<TARGET_PROPERTY:HCOI_MAPL_ESMF,INTERFACE_INCLUDE_DIRECTORIES>
+      ${CMAKE_Fortran_MODULE_DIRECTORY}
+    )
+    # Turn off acg for now
+    #mapl_acg(GEOSchem_GridComp GEOSchem_StateSpecs.rc
+    #  IMPORT_SPECS EXPORT_SPECS INTERNAL_SPECS
+    #  GET_POINTERS DECLARE_POINTERS
+    #  3g
+    #)
+  endif()
+
+  set(GEOSChem_Fortran_FLAGS_Intel
+      -cpp -w -auto -noalign "SHELL:-convert big_endian" "SHELL:-fp-model source"
+      -mcmodel=medium -shared-intel -traceback -DLINUX_IFORT
+      CACHE STRING "GEOSChem compiler flags for all build types with Intel compilers"
+  )
+  set(GEOSChem_Fortran_FLAGS_RELEASE_Intel
+     -O2
+      CACHE STRING "GEOSChem compiler flags for build type release with Intel compilers"
+  )
+  set(GEOSChem_Fortran_FLAGS_RELWITHDEBINFO_Intel
+      -O2
+      CACHE STRING "GEOSChem compiler flags for build type relwithdebinfo with Intel compilers"
+  )
+  set(GEOSChem_Fortran_FLAGS_DEBUG_Intel
+      -g -O0 "SHELL:-check arg_temp_created" "SHELL:-debug all" -fpe0 -ftrapuv -check,bounds
+      CACHE STRING "GEOSChem compiler flags for build type debug with Intel compilers"
+  )
+  
+  set(GEOSChem_Fortran_FLAGS_GNU
+      -cpp -w -std=legacy -fautomatic -fno-align-commons
+      -fconvert=big-endian -fno-range-check -mcmodel=medium
+      -fbacktrace -g -DLINUX_GFORTRAN -ffree-line-length-none
+      CACHE STRING "GEOSChem compiler flags for all build types with GNU compilers"
+  )
+  set(GEOSChem_Fortran_FLAGS_RELEASE_GNU
+     -O3 -funroll-loops
+     CACHE STRING "GEOSChem compiler flags for build type release with GNU compilers"
+  )
+  set(GEOSChem_Fortran_FLAGS_RELWITHDEBINFO_GNU
+     -O3 -funroll-loops
+     CACHE STRING "GEOSChem compiler flags for build type relwithdebinfo with GNU compilers"
+  )
+  set(GEOSChem_Fortran_FLAGS_DEBUG_GNU
+      -g -gdwarf-2 -gstrict-dwarf -O0 -Wall -Wextra -Wconversion -Warray-temporaries
+      -fcheck=array-temps -ffpe-trap=invalid,zero,overflow -finit-real=snan
+      -fcheck=bounds -fcheck=pointer
+      CACHE STRING "GEOSChem compiler flags for build type debug with GNU compilers"
+  )
+  set(CMAKE_Fortran_FLAGS "")
+  set(CMAKE_Fortran_FLAGS_RELEASE "")
+  set(CMAKE_Fortran_FLAGS_Debug "")
+  set(CMAKE_Fortran_FLAGS_RelWithDebInfo "")
+  
+  set(GEOSChem_SUPPORTED_COMPILER_IDS "Intel" "GNU")
+  if(NOT CMAKE_Fortran_COMPILER_ID IN_LIST GEOSChem_SUPPORTED_COMPILER_IDS)
+     message(FATAL_ERROR "GEOSChem does not support ${CMAKE_Fortran_COMPILER_ID} compilers")
+  endif()
+  
+  #---------------------------------------------------------------------
+  # Assign comiler options to build properties
+  #---------------------------------------------------------------------
+  target_compile_options(GEOSChemBuildProperties
+     INTERFACE
+     $<$<STREQUAL:${CMAKE_Fortran_COMPILER_ID},Intel>:
+        ${GEOSChem_Fortran_FLAGS_Intel}
+        $<$<CONFIG:Debug>:${GEOSChem_Fortran_FLAGS_DEBUG_Intel}>
+        $<$<CONFIG:RelWithDebInfo>:${GEOSChem_Fortran_FLAGS_RELWITHDEBINFO_Intel}>
+        $<$<CONFIG:Release>:${GEOSChem_Fortran_FLAGS_RELEASE_Intel}>
+     >
+     $<$<STREQUAL:${CMAKE_Fortran_COMPILER_ID},GNU>:
+        ${GEOSChem_Fortran_FLAGS_GNU}
+        $<$<CONFIG:Debug>:${GEOSChem_Fortran_FLAGS_DEBUG_GNU}>
+        $<$<CONFIG:RelWithDebInfo>:${GEOSChem_Fortran_FLAGS_RELWITHDEBINFO_GNU}>
+        $<$<CONFIG:Release>:${GEOSChem_Fortran_FLAGS_RELEASE_GNU}>
+     >
+  )
+  
+  target_compile_definitions(GEOSChemBuildProperties
+      INTERFACE
+      $<$<STREQUAL:${CMAKE_Fortran_COMPILER_ID},Intel>:LINUX_IFORT>
+      $<$<STREQUAL:${CMAKE_Fortran_COMPILER_ID},GNU>:LINUX_GFORTRAN>
+  )
+  
+  # Print the options that are turned on in GEOS-Chem
+  list(INSERT CMAKE_MODULE_PATH 0 ${CMAKE_CURRENT_SOURCE_DIR}/CMakeScripts)
+  include(GC-Helpers)
+  gc_pretty_print(SECTION "Settings")
+  gc_pretty_print(VARIABLE MECH OPTIONS "fullchem" "carbon" "custom")
+  gc_pretty_print(VARIABLE OMP IS_BOOLEAN)
+  gc_pretty_print(VARIABLE USE_REAL8 IS_BOOLEAN)
+  gc_pretty_print(VARIABLE APM IS_BOOLEAN)
+  gc_pretty_print(VARIABLE TOMAS IS_BOOLEAN)
+  gc_pretty_print(VARIABLE RRTMG IS_BOOLEAN)
+  gc_pretty_print(VARIABLE GTMM IS_BOOLEAN)
+  gc_pretty_print(VARIABLE LUO_WETDEP IS_BOOLEAN)
+  gc_pretty_print(VARIABLE FASTJX IS_BOOLEAN)
+  gc_pretty_print(VARIABLE KPPSA IS_BOOLEAN)
+endif()
+

GeosCore/CMakeLists.txt

@@ -99,6 +99,7 @@ endif()
 
 # Define dependencies for libGeosCore.a
 target_link_libraries(GeosCore PUBLIC
+  GeosUtil
   ObsPack 
   History
   KPP

GeosCore/calc_met_mod.F90

@@ -30,7 +30,7 @@ MODULE CALC_MET_MOD
   PUBLIC  :: INTERP
   PUBLIC  :: SET_DRY_SURFACE_PRESSURE
   PUBLIC  :: Set_Clock_Tracer
-#if defined( ESMF_ ) || defined( EXTERNAL_GRID )
+#if defined( USE_ESMF ) || defined( EXTERNAL_GRID )
   PUBLIC  :: GCHP_Cap_Tropopause_Prs
 #endif
 !
@@ -845,7 +845,7 @@ SUBROUTINE INTERP( NTIME0, NTIME1, NTDT, Input_Opt, State_Grid, State_Met )
 
   END SUBROUTINE INTERP
 !EOC
-#if defined( ESMF_ ) || defined( EXTERNAL_GRID )
+#if defined( USE_ESMF ) || defined( EXTERNAL_GRID )
 !------------------------------------------------------------------------------
 !                  GEOS-Chem Global Chemical Transport Model                  !
 !------------------------------------------------------------------------------
@@ -858,6 +858,8 @@ END SUBROUTINE INTERP
 !  troposheric chemistry too high over the poles.  This is done in the
 !  standalone GEOS-Chem, and we also need to apply this when running
 !  GEOS-Chem within the GEOS-5 GCM.
+!
+! ewl question: do we do this in CESM too? If yes, update this description.
 !\\
 !\\
 ! !INTERFACE:

GeosCore/drydep_mod.F90

@@ -3643,7 +3643,7 @@ FUNCTION AERO_SFCRSII( K, II, PRESS, TEMP, USTAR, RHB, W10, VTSout, Input_Opt, S
     ! Use the formulation of Slinn and Slinn (1980) for the impaction over
     ! water surfaces (jaegle 5/11/11)
     IF (LUC == 14) THEN
-#ifdef MODEL_GCHPCTM
+#ifdef MODEL_GCHP
        ! Include check that winds are non-zero to avoid div by 0 error
        IF ( IS_SAFE_DIV(1.e+0_f8, W10) ) THEN
           RS = 1.e+0_f8 / (USTAR**2.e+0_f8/ (W10*VON_KARMAN) * &

GeosCore/dust_mod.F90

@@ -59,6 +59,7 @@ MODULE DUST_MOD
   ! Arrays
   REAL(fp), ALLOCATABLE :: FRAC_S(:)
   REAL(fp), ALLOCATABLE :: SRCE_FUNC(:,:,:)
+  ! ewl: it seems srce_func isn't used anywhere in the model. Can we delete?
 
 #ifdef TOMAS
   ! To replicate the obsolete Input_Opt%IDDEP field
@@ -2105,10 +2106,10 @@ SUBROUTINE INIT_DUST( Input_Opt,  State_Chm, State_Diag, State_Grid, RC )
     ENDIF
 #endif
 
-#if defined( ESMF_ ) || defined( TOMAS )
+#if defined( USE_ESMF ) || defined( TOMAS )
     ! EXPERIMENTAL: For archiving the dust source for GCHP
     !
-    ! Changed to use the ESMF_ flag and not EXTERNAL_GRID/EXTERNAL_FORCING, as
+    ! Changed to use the ESMF flag and not EXTERNAL_GRID/EXTERNAL_FORCING, as
     ! WRF-GC which uses these flags does not require SRCE_FUNC (hplin, 1/22/19)
     ALLOCATE( SRCE_FUNC(State_Grid%NX,State_Grid%NY,3), STAT=AS )
     IF ( AS /= 0 ) CALL ALLOC_ERR( 'SRCE_FUNC' )

GeosCore/hco_interface_gc_mod.F90

@@ -577,7 +577,7 @@ SUBROUTINE HCOI_GC_Init( Input_Opt, State_Chm, State_Grid, &
     ! compared to a stand-alone version: in ESMF, the source file name
     ! is set to the container name since this is the identifying name
     ! used by ExtData.
-#ifdef ESMF_
+#ifdef USE_ESMF
     HcoState%Options%isESMF = .TRUE.
 #else
     HcoState%Options%isESMF = .FALSE.
@@ -968,7 +968,7 @@ SUBROUTINE HCOI_GC_Run( Input_Opt, State_Chm, State_Grid,  &
        ENDIF
     ENDIF
 
-#if !defined( ESMF_ ) && !defined( MODEL_WRF )
+#if !defined( USE_ESMF ) && !defined( MODEL_WRF )
     ! Check if HEMCO has already been called for this timestep
     IF ( ( Phase == 1 ) .and. ( GET_TAU() == PrevTAU ) .and. Input_Opt%amIRoot ) THEN
        Print*, 'HEMCO already called for this timestep. Returning.'
@@ -1700,7 +1700,7 @@ SUBROUTINE ExtState_SetFields( Input_Opt, State_Chm, State_Grid,           &
     USE State_Met_Mod,  ONLY : MetState
     USE State_Chm_Mod,  ONLY : ChmState
     USE Drydep_Mod,     ONLY : DryCoeff
-#ifdef ESMF_
+#ifdef USE_ESMF
     USE HCOI_Esmf_Mod,  ONLY : HCO_SetExtState_ESMF
 #endif
 !
@@ -2814,7 +2814,7 @@ SUBROUTINE ExtState_SetFields( Input_Opt, State_Chm, State_Grid,           &
        ExtState%PBL_MAX => HCO_PBL_MAX
     ENDIF
 
-#ifdef ESMF_
+#ifdef USE_ESMF
     !=======================================================================
     ! ESMF environment: add some additional variables to ExtState.
     ! These values must be defined here and not in the initialization
@@ -2874,7 +2874,7 @@ SUBROUTINE ExtState_UpdateFields( Input_Opt,  State_Chm,             &
     USE State_Chm_Mod,        ONLY : ChmState
     USE State_Grid_Mod,       ONLY : GrdState
     USE State_Met_Mod,        ONLY : MetState
-#ifdef ESMF_
+#ifdef USE_ESMF
     USE HCOI_ESMF_MOD,        ONLY : HCO_SetExtState_ESMF
 #endif
 #if defined( MODEL_CLASSIC )
@@ -4140,7 +4140,7 @@ SUBROUTINE CheckSettings( HcoConfig, Input_Opt, State_Met, State_Chm, RC )
 
     ENDIF
 
-#ifdef ESMF_
+#ifdef USE_ESMF
     !-----------------------------------------------------------------------
     ! Also check that HEMCO_RESTART is not set in ESMF
     !-----------------------------------------------------------------------

GeosCore/hco_utilities_gc_mod.F90

@@ -2496,7 +2496,7 @@ SUBROUTINE Get_GC_Restart( Input_Opt, State_Chm, State_Grid, &
          ENDIF
       ELSE
          IF ( Input_Opt%amIRoot ) THEN
-#ifdef ESMF_
+#ifdef MAPL_ESMF
             ! ExtData and HEMCO behave ambiguously - if the file was found
             ! but was full of zeros throughout the domain of interest, it
             ! will result in the same output from ExtData as if the field

GeosCore/input_mod.F90

@@ -373,7 +373,7 @@ SUBROUTINE Config_Simulation( Config, Input_Opt, RC )
     LOGICAL                      :: v_bool
     INTEGER                      :: N,                C
     REAL(fp)                     :: JulianDateStart,  JulianDateEnd
-#if defined( ESMF_ ) || defined( MODEL_ )
+#if defined( MODEL_GCHP ) || defined( MODEL_GEOS )
     INTEGER                      :: H,       M,       S
     REAL(f4)                     :: init_UTC
 #endif
@@ -597,10 +597,10 @@ SUBROUTINE Config_Simulation( Config, Input_Opt, RC )
     ENDIF
     Input_Opt%CHEM_INPUTS_DIR = TRIM( v_str )
 
-#if defined( MODEL_GCHP )
     !------------------------------------------------------------------------
     ! Meteorology field
     !------------------------------------------------------------------------
+#ifdef MODEL_GCHP
     key   = "simulation%met_field"
     v_str = MISSING_STR
     CALL QFYAML_Add_Get( Config, TRIM( key ), v_str, "", RC )
@@ -612,7 +612,7 @@ SUBROUTINE Config_Simulation( Config, Input_Opt, RC )
     Input_Opt%MetField = TRIM( v_str )
 #endif
 
-#if defined( MODEL_GEOS )
+#ifdef MODEL_GEOS
     Input_Opt%MetField        = 'See ExtData.rc'
 #endif
 
@@ -2702,7 +2702,7 @@ SUBROUTINE Config_RRTMG( Config, Input_Opt, RC )
        ENDIF
     ENDIF
 
-#ifndef MODEL_GCHPCTM
+#ifndef MODEL_GCHP
     If (Input_Opt%RRTMG_FDH) Then
        errMsg = 'Fixed dynamical heating in RRTMG is currently only available in GCHP'
        CALL GC_Error( errMsg, RC, thisLoc )

GeosCore/linoz_mod.F90

@@ -352,7 +352,7 @@ SUBROUTINE LINOZ_CHEM3( DTCHEM, Input_Opt, State_Chm, State_Grid, &
 
           !IF ( Input_Opt%Verbose ) CALL DEBUG_MSG('DONE GET_PEDGE')
 
-#if defined( ESMF_ ) || defined( EXTERNAL_GRID ) || defined( EXTERNAL_FORCING )
+#if defined( USE_ESMF ) || defined( EXTERNAL_GRID ) || defined( EXTERNAL_FORCING )
           !-----------------------------------------------------------
           !       %%%%%%% GEOS-Chem HP (with ESMF & MPI) %%%%%%%
           !