Remove module scope access from LFRic Core driver/applications (#324)

Co-authored-by: Thomas Bendall <14180399+tommbendall@users.noreply.github.com>
Co-authored-by: Steve Mullerworth <26165365+stevemullerworth@users.noreply.github.com>
Co-authored-by: Andrew Coughtrie <24609575+andrewcoughtrie@users.noreply.github.com>
Co-authored-by: Matthew Hambley <MatthewHambley@users.noreply.github.com>
Co-authored-by: James Bruten <109733895+james-bruten-mo@users.noreply.github.com>

Author: 6 people

Makefile

@@ -22,6 +22,7 @@ OPERATE_ON ?= infrastructure                \
               mesh_tools                    \
               applications/skeleton         \
               applications/simple_diffusion \
+              applications/coupled          \
               applications/lbc_demo         \
               applications/io_demo
 

applications/coupled/example/configuration_glo.nml

@@ -60,6 +60,9 @@ coord_system='native'
 &partitioning
   partitioner='cubedsphere',
   panel_decomposition = 'auto',
+  tile_size_x=1,
+  tile_size_y=1,
+  inner_halo_tiles=.false.
 /
 
 &planet

applications/coupled/example/configuration_lam.nml

@@ -59,6 +59,9 @@ coord_system='native'
 &partitioning
   partitioner='planar',
   panel_decomposition = 'auto',
+  tile_size_x=1,
+  tile_size_y=1,
+  inner_halo_tiles=.false.
 /
 
 &planet

applications/coupled/rose-meta/lfric-coupled/versions.py

@@ -55,3 +55,37 @@ def upgrade(self, config, meta_config=None):
         )
 
         return config, self.reports
+
+
+class vn31_t324(MacroUpgrade):
+    """Upgrade macro for ticket #324 by Ricky Wong."""
+
+    BEFORE_TAG = "vn3.1_t238"
+    AFTER_TAG = "vn3.1_t324"
+
+    def upgrade(self, config, meta_config=None):
+        # Commands From: rose-meta/lfric-driver
+        # Only add in new configuration settings if the namelists
+        # are already present
+        #
+        if config.get(["namelist:partitioning"]) is not None:
+            self.add_setting(
+                config, ["namelist:partitioning", "inner_halo_tiles"], ".false."
+            )
+            self.add_setting(
+                config, ["namelist:partitioning", "tile_size_x"], "1"
+            )
+            self.add_setting(
+                config, ["namelist:partitioning", "tile_size_y"], "1"
+            )
+        if config.get(["namelist:multigrid"]) is not None:
+            self.add_setting(
+                config,
+                ["namelist:multigrid", "coarsen_multigrid_tiles"],
+                ".false.",
+            )
+            self.add_setting(
+                config, ["namelist:multigrid", "max_tiled_multigrid_level"], "1"
+            )
+
+        return config, self.reports

applications/coupled/source/coupled.f90

@@ -50,7 +50,8 @@ program coupled
   call init_config( filename, coupled_required_namelists, &
                     config=modeldb%config )
 
-  call init_logger( modeldb%mpi%get_comm(), &
+  call init_logger( modeldb%config, &
+                    modeldb%mpi%get_comm(), &
                     program_name//"_"//cpl_component_name )
 
   write(log_scratch_space,'(A)')                          &

applications/coupled/source/driver/coupled_driver_mod.f90

@@ -75,11 +75,17 @@ subroutine initialise( program_name, modeldb, calendar )
     integer(i_def) :: geometry
     integer(i_def) :: method
     integer(i_def) :: number_of_layers
+    integer(i_def) :: tile_size_x
+    integer(i_def) :: tile_size_y
     real(r_def)    :: domain_bottom
     real(r_def)    :: domain_height
     real(r_def)    :: scaled_radius
     logical        :: check_partitions
 
+    logical :: inner_halo_tiles
+    logical :: prepartitioned
+
+    integer(i_def), allocatable :: tile_size(:,:)
     integer(i_def) :: i
     integer(i_def), parameter :: one_layer = 1_i_def
 
@@ -91,6 +97,17 @@ subroutine initialise( program_name, modeldb, calendar )
     domain_height    = modeldb%config%extrusion%domain_height()
     number_of_layers = modeldb%config%extrusion%number_of_layers()
     scaled_radius    = modeldb%config%planet%scaled_radius()
+    prepartitioned   = modeldb%config%base_mesh%prepartitioned()
+
+    if (prepartitioned) then
+      tile_size_x = 1
+      tile_size_y = 1
+      inner_halo_tiles = .false.
+    else
+      tile_size_x = maxval([1,modeldb%config%partitioning%tile_size_x()])
+      tile_size_y = maxval([1,modeldb%config%partitioning%tile_size_y()])
+      inner_halo_tiles = modeldb%config%partitioning%inner_halo_tiles()
+    end if
 
     ! Initialise mesh
     ! Determine the required meshes
@@ -120,23 +137,29 @@ subroutine initialise( program_name, modeldb, calendar )
     ! Create the required meshes
     stencil_depth = 1
     check_partitions = .false.
+    allocate(tile_size(2,size(base_mesh_names)))
+    tile_size(1,:) = tile_size_x
+    tile_size(2,:) = tile_size_y
     call init_mesh( modeldb%config,              &
                     modeldb%mpi%get_comm_rank(), &
                     modeldb%mpi%get_comm_size(), &
                     base_mesh_names, extrusion,  &
+                    inner_halo_tiles, tile_size, &
                     stencil_depth, check_partitions )
 
     allocate( twod_names, source=base_mesh_names )
     do i=1, size(twod_names)
       twod_names(i) = trim(twod_names(i))//'_2d'
     end do
+
     call create_mesh( base_mesh_names, extrusion_2d, &
+                      inner_halo_tiles, tile_size,   &
                       alt_name=twod_names )
     call assign_mesh_maps( twod_names )
 
 
     ! Build the FEM function spaces and coordinate fields
-    call init_fem( mesh_collection, chi_inventory, panel_id_inventory )
+    call init_fem( modeldb%config, chi_inventory, panel_id_inventory )
 
     ! Create and initialise prognostic fields
     mesh => mesh_collection%get_mesh(prime_mesh_name)

applications/coupled/source/driver/init_coupled_mod.X90

@@ -76,15 +76,24 @@ module init_coupled_mod
     procedure(write_interface), pointer :: tmp_ptr
 
     integer(i_def) :: order_h, order_v
+    integer(i_def) :: coord_system
+    integer(i_def) :: geometry
+    integer(i_def) :: topology
+    real(r_def)    :: scaled_radius
+
     type(function_space_type), pointer :: fs
 
     call log_event( 'coupled: Initialising app ...', LOG_LEVEL_INFO )
 
     ! Get the name of the coupling component
     call modeldb%values%get_value("cpl_name", cpl_component_name)
 
-    order_h = modeldb%config%finite_element%element_order_h()
-    order_v = modeldb%config%finite_element%element_order_v()
+    order_h       = modeldb%config%finite_element%element_order_h()
+    order_v       = modeldb%config%finite_element%element_order_v()
+    coord_system  = modeldb%config%finite_element%coord_system()
+    geometry      = modeldb%config%base_mesh%geometry()
+    topology      = modeldb%config%base_mesh%topology()
+    scaled_radius = modeldb%config%planet%scaled_radius()
 
     fs => function_space_collection%get_fs(mesh, order_h, order_v, W3)
 
@@ -122,8 +131,10 @@ module init_coupled_mod
         call depository%get_field( trim(name),  field_1_ptr)
         ! Initialise the values in the field that will be sent to the coupler.
         ! Set them to the longitude of the cell-centre (converted to degrees)
-        call invoke(compute_latlon_kernel_type(field_2, field_1_ptr, &
-                                               chi, panel_id),       &
+        call invoke(compute_latlon_kernel_type(field_2, field_1_ptr,    &
+                                               chi, panel_id, geometry, &
+                                               topology, coord_system,  &
+                                               scaled_radius),          &
                     inc_a_times_X(radians_to_degrees, field_1_ptr))
         ! Add that field to the coupling 2d "send" field collection
         abs_field_ptr => field_1_ptr

applications/io_demo/example/configuration.nml

@@ -87,6 +87,9 @@ coord_system='native'
 &partitioning
   partitioner='cubedsphere',
   panel_decomposition = 'auto',
+  tile_size_x=1,
+  tile_size_y=1,
+  inner_halo_tiles=.false.
 /
 
 &planet

applications/io_demo/rose-meta/lfric-io_demo/versions.py

@@ -126,3 +126,37 @@ def upgrade(self, config, meta_config=None):
         )
 
         return config, self.reports
+
+
+class vn31_t324(MacroUpgrade):
+    """Upgrade macro for ticket #324 by Ricky Wong."""
+
+    BEFORE_TAG = "vn3.1_t287"
+    AFTER_TAG = "vn3.1_t324"
+
+    def upgrade(self, config, meta_config=None):
+        # Commands From: rose-meta/lfric-driver
+        # Only add in new configuration settings if the namelists
+        # are already present
+        #
+        if config.get(["namelist:partitioning"]) is not None:
+            self.add_setting(
+                config, ["namelist:partitioning", "inner_halo_tiles"], ".false."
+            )
+            self.add_setting(
+                config, ["namelist:partitioning", "tile_size_x"], "1"
+            )
+            self.add_setting(
+                config, ["namelist:partitioning", "tile_size_y"], "1"
+            )
+        if config.get(["namelist:multigrid"]) is not None:
+            self.add_setting(
+                config,
+                ["namelist:multigrid", "coarsen_multigrid_tiles"],
+                ".false.",
+            )
+            self.add_setting(
+                config, ["namelist:multigrid", "max_tiled_multigrid_level"], "1"
+            )
+
+        return config, self.reports

applications/io_demo/source/driver/io_demo_checkpoint_mod.f90

@@ -8,7 +8,8 @@
 !!          files and fields to it
 module io_demo_checkpoint_mod
 
-  use constants_mod,          only: i_def, str_max_filename, str_def, r_second
+  use constants_mod,          only: i_def, r_def, str_max_filename, &
+                                    str_def, r_second
   use driver_modeldb_mod,     only: modeldb_type
   use event_mod,              only: event_action
   use event_actor_mod,        only: event_actor_type
@@ -22,6 +23,12 @@ module io_demo_checkpoint_mod
   use lfric_xios_file_mod,    only: lfric_xios_file_type, OPERATION_ONCE
   use log_mod,                only: log_event, log_scratch_space, &
                                     LOG_LEVEL_DEBUG, LOG_LEVEL_ERROR
+  use mesh_mod,               only: mesh_type
+
+  use base_mesh_config_mod, only: geometry_spherical,      &
+                                  geometry_planar,         &
+                                  topology_fully_periodic, &
+                                  topology_non_periodic
 
   implicit none
 
@@ -54,8 +61,34 @@ subroutine setup_checkpoint_io(modeldb, chi, panel_id)
     character(len=str_def)          :: checkpoint_id
     integer(i_def) :: ts_start, ts_end, t_cp, freq_ts
 
+    type(mesh_type), pointer :: mesh
+
+    integer(i_def) :: geometry
+    integer(i_def) :: topology
+    integer(i_def) :: coord_system
+    real(r_def)    :: scaled_radius
+
     call log_event( 'io_demo: Setting up checkpoint I/O', LOG_LEVEL_DEBUG )
 
+    mesh => chi(1)%get_mesh()
+    if (mesh%is_geometry_spherical()) then
+      geometry = geometry_spherical
+    else
+      geometry = geometry_planar
+    end if
+
+    if (mesh%is_topology_periodic()) then
+      topology = topology_fully_periodic
+    else if (mesh%is_topology_non_periodic()) then
+      topology = topology_non_periodic
+    else
+      call log_event( 'Unsupported mesh topology', &
+                      log_level_error )
+    end if
+
+    coord_system  = modeldb%config%finite_element%coord_system()
+    scaled_radius = modeldb%config%planet%scaled_radius()
+
     ts_start = modeldb%calendar%parse_instance(modeldb%config%time%timestep_start())
     ts_end   = modeldb%calendar%parse_instance(modeldb%config%time%timestep_end())
     checkpoint_fields => modeldb%fields%get_field_collection("depository")
@@ -125,8 +158,10 @@ subroutine setup_checkpoint_io(modeldb, chi, panel_id)
     ! Add checkpoint context to clock events so that it is advanced at each timestep
     event_actor_ptr => cp_context
     context_advance => advance
-    call cp_context%initialise_xios_context( modeldb%mpi%get_comm(), chi, panel_id, &
-                                             modeldb%clock, modeldb%calendar )
+    call cp_context%initialise_xios_context(                   &
+                        modeldb%mpi%get_comm(), chi, panel_id, &
+                        modeldb%clock, modeldb%calendar,       &
+                        geometry, topology, coord_system, scaled_radius )
 
     call modeldb%clock%add_event(context_advance, event_actor_ptr)
     call cp_context%set_active(.true.)