Lfric core git test

dependencies.yaml

@@ -0,0 +1,26 @@
+##############################################################################
+# (c) Crown copyright 2024 Met Office. All rights reserved.
+# The file LICENCE, distributed with this code, contains details of the terms
+# under which the code may be used.
+##############################################################################
+
+# This file contains a list of all linked external repositories containing code
+# and data required to build lfric core and run the entire test suite
+#
+# Each section is the name of the repository
+# source: Is either the url to the repository (the upstream or
+#         a fork) or a path to a local clone with format `host:/path/to/clone`
+# ref: Is a valid git reference. This can either be a commit hash, a tag or a
+#      branch name
+#
+# The entry for this repository will have blank values for both source and ref
+# by default, which will cause the test suite to use the source code from the
+# local clone. These can be filled to set a different source if desired.
+
+lfric_core:
+    source:
+    ref:
+
+SimSys_Scripts:
+    source: git@github.com:james-bruten-mo/SimSys_Scripts.git
+    ref: main

rose-stem/app/check_config_dump/rose-app.conf

@@ -1,2 +1,2 @@
 [command]
-default=$CYLC_WORKFLOW_RUN_DIR/bin/check_config_dump.py -s $SOURCE_ROOT
+default=$CYLC_WORKFLOW_RUN_DIR/bin/check_config_dump.py -s $SOURCE_ROOT/lfric_core

rose-stem/app/check_global_variables/rose-app.conf

@@ -1,2 +1,2 @@
 [command]
-default=$CYLC_WORKFLOW_RUN_DIR/bin/occupyfortran -directory $SOURCE_ROOT -dirtylist dirtylist.txt source
+default=$CYLC_WORKFLOW_RUN_DIR/bin/occupyfortran -directory $SOURCE_ROOT/lfric_core -dirtylist dirtylist.txt source

rose-stem/app/check_style/rose-app.conf

@@ -1,2 +1,2 @@
 [command]
-default=$CYLC_WORKFLOW_RUN_DIR/bin/stylist_launcher.py -s $SOURCE_ROOT
+default=$CYLC_WORKFLOW_RUN_DIR/bin/stylist_launcher.py -s $SOURCE_ROOT/lfric_core

rose-stem/app/extract_source/rose-app.conf

@@ -0,0 +1,8 @@
+[command]
+default=python rose_stem_extract_source.py
+
+[file:get_git_sources.py]
+source=git:https://github.com/MetOffice/SimSys_Scripts.git::github_scripts/get_git_sources.py::main
+
+[file:rose_stem_extract_source.py]
+source=git:https://github.com/MetOffice/SimSys_Scripts.git::github_scripts/rose_stem_extract_source.py::main

rose-stem/app/rose-stem_lint_check/rose-app.conf

@@ -1,2 +1,2 @@
 [command]
-default=$CYLC_WORKFLOW_RUN_DIR/bin/rose-stem_lint_check.py -s $SOURCE_ROOT/rose-stem
+default=$CYLC_WORKFLOW_RUN_DIR/bin/rose-stem_lint_check.py -s $SOURCE_ROOT/lfric_core/rose-stem

rose-stem/app/site_validator/rose-app.conf

@@ -1,2 +1,2 @@
 [command]
-default=$CYLC_WORKFLOW_RUN_DIR/bin/site_validator.py -s $SOURCE_ROOT
+default=$CYLC_WORKFLOW_RUN_DIR/bin/site_validator.py -s $SOURCE_ROOT/lfric_core

rose-stem/app/validate_rose_meta/rose-app.conf

@@ -1,2 +1,2 @@
 [command]
-default=python3 $CYLC_WORKFLOW_SHARE_DIR/imported_github_scripts/lfric_macros/validate_rose_meta.py -c $SOURCE_ROOT
+default=python3 $SOURCE_ROOT/SimSys_Scripts/lfric_macros/validate_rose_meta.py -c $SOURCE_ROOT/lfric_core

rose-stem/flow.cylc

@@ -5,6 +5,28 @@
 {# under which the code may be used. #}
 {# ########################################################################### #}
 {% from "cylc.flow" import LOG %}
+{% from "utils" import get_site %}
+{% from "read_sources" import read_sources  %}
+
+{% if SITE is not defined %}
+    {% set SITE = get_site() %}
+{% endif %}
+{% do LOG.info("Site: " + SITE) %}
+
+{% set SOURCE_DIRECTORY = ROSE_ORIG_HOST~":"~CYLC_WORKFLOW_SRC_DIR~"/.." %}
+{# Include log message for suite_report #}
+{% do LOG.info("LFRIC_CORE SOURCE CLONE="~SOURCE_DIRECTORY) %}
+
+{# Read in groups to run #}
+{% if group is defined %}
+    {% set RUN_NAMES = group %}
+{% elif g is defined %}
+    {% set RUN_NAMES = g %}
+{% else %}
+    {% set RUN_NAMES = ["developer"] %}
+{% endif %}
+{% do LOG.info("Running groups: "~RUN_NAMES|join('
+                   ')) %}
 
 {# Include the site variables file #}
 {# Contains some settings used for the workflow #}
@@ -32,7 +54,8 @@
 {% set site_groups = {} %}
 {% include "site/"+SITE+"/groups.cylc" %}
 
-{% set lfric_source = SOURCE_LFRIC %}
+{# Read the sources of dependencies from the dependencies.yaml file #}
+{% set dependencies = read_sources(SOURCE_DIRECTORY, "lfric_core", USE_HEADS) %}
 
 {# Generate list of tasks given by --group= #}
 {# Expand any groups included so list is only tasks #}
@@ -42,7 +65,6 @@
 {# Generate a list of all scripts - used to determine if a task is a script #}
 {% set scripts_list = [] %}
 {{ expand_task_list(site_groups, ["scripts"], false, scripts_list) }}
-{% do set_add(scripts_list, "export_simsys_scripts") %}
 
 {# Load the task definitions for each application requested at current site #}
 {% set requested_tasks = {} %}
@@ -59,11 +81,11 @@
     UTC mode = True
     install = site/, templates/
     [[events]]
-        mail events = timeout
+        mail events = stall timeout
         abort on stall timeout = True
         stall timeout = PT3H
-        shutdown handlers = "suite_report.py"
-        stall handlers = "suite_report.py"
+        shutdown handlers = "suite_report_git.py -S $CYLC_WORKFLOW_RUN_DIR"
+        stall handlers = "suite_report_git.py -S $CYLC_WORKFLOW_RUN_DIR"
 
 [scheduling]
     [[graph]]
@@ -92,8 +114,7 @@
         [[[environment]]]
             SOURCE_ROOT    = $CYLC_WORKFLOW_SHARE_DIR/source
             OUTPUT_ROOT    = $CYLC_WORKFLOW_SHARE_DIR/output
-            LAUNCH_SCRIPT  = $SOURCE_ROOT/rose-stem/site/{{SITE}}/common/bin
-            INFRASTRUCTURE_ROSESTEM = $SOURCE_ROOT/infrastructure/rose-stem/
+            LAUNCH_SCRIPT  = $SOURCE_ROOT/lfric_core/rose-stem/site/{{SITE}}/common/bin
 
         [[[events]]]
             submission timeout = PT5H

rose-stem/lib/python/read_sources.py

@@ -0,0 +1,66 @@
+##############################################################################
+# (c) Crown copyright 2025 Met Office. All rights reserved.
+# The file LICENCE, distributed with this code, contains details of the terms
+# under which the code may be used.
+##############################################################################
+
+import yaml
+import tempfile
+import os
+from subprocess import run
+from shutil import rmtree
+
+def get_dependencies_file(wc_loc):
+    """
+    Copy the dependencies file to a temporary directory on the local machine that can be
+    read.
+    """
+
+    tempdir = tempfile.mkdtemp()
+
+    try:
+        host, path = wc_loc.split(":", 1)
+        path = os.path.join(path, "dependencies.yaml")
+        copy_command = f"scp -o StrictHostKeyChecking=no {host}:"
+    except ValueError:
+        path = os.path.join(wc_loc, "dependencies.yaml")
+        copy_command = "cp "
+    copy_command += f"{path} {tempdir}"
+
+    result = run(
+        copy_command.split(), capture_output=True, text=True, timeout=120
+    )
+
+    # Raise an error if the returncode is positive
+    if result.returncode:
+        raise RuntimeError(
+            f"An error occured while running the command '{copy_command}' "
+            "in order to read the dependencies file. The error message is:\n\n"
+            f"'{result.stderr}'"
+        )
+
+    return tempdir
+
+def read_sources(clone_source, repo, use_heads):
+    """
+    Load the dependencies.yaml file as a dictionary
+    """
+
+    dependencies_file = get_dependencies_file(clone_source)
+
+    with open(os.path.join(dependencies_file, "dependencies.yaml")) as stream:
+        dependencies = yaml.safe_load(stream)
+
+    if not dependencies[repo]["source"]:
+        dependencies[repo]["source"] = clone_source
+
+    # Populate parent, assume MetOffice is owner if not set
+    for dependency, values in dependencies.items():
+        if "parent" not in values:
+            dependencies[dependency]["parent"] = f"MetOffice/{dependency}.git"
+        if use_heads:
+            dependencies[dependency]["ref"] = ""
+
+    rmtree(dependencies_file)
+
+    return dependencies

rose-stem/lib/python/utils.py

@@ -0,0 +1,20 @@
+##############################################################################
+# (c) Crown copyright 2025 Met Office. All rights reserved.
+# The file LICENCE, distributed with this code, contains details of the terms
+# under which the code may be used.
+##############################################################################
+
+from subprocess import Popen, PIPE
+
+
+def get_site():
+    proc = Popen(
+        ["rose", "config", "rose-stem", "automatic-options"], stdout=PIPE, text=True
+    )
+    out, _ = proc.communicate()
+    if proc.returncode or "SITE" not in out:
+        raise Exception('Could not determine the rose-stem "SITE"')
+    # At some sites there may be many variables that are returned by rose config rose-stem
+    # Try to just grab the thing after SITE= and then ignore anything afterwards
+    site = out.split("SITE=")[1].split(' ')[0].strip()
+    return site

rose-stem/rose-suite.conf

@@ -1,8 +1,8 @@
-ROSE_STEM_VERSION=1
-
 [template variables]
 FFLAGS=''
 HOUSEKEEPING=true
 LDFLAGS=''
 OVERRIDE_LOG_LEVEL=''
-SIMSYS_BRANCH='main'
+USE_HEADS=false
+USE_MIRRORS=false
+USE_TOKENS=false

rose-stem/site/common/coupled/tasks_coupled.cylc

@@ -35,7 +35,7 @@
 
 {# Set default values for Coupled app tasks #}
 {% if task_dict %}
-    {% do task_dict.update({"application_dir": "applications/coupled"}) %}
+    {% do task_dict.update({"application_dir": "lfric_core/applications/coupled"}) %}
     {% do task_dict.update({"build_families": [task_ns.platform|upper~"_BUILD_COUPLED_"~task_ns.compiler|upper]}) %}
     {% do task_dict.update({"run_families": [task_ns.platform|upper~"_RUN_COUPLED_"~task_ns.compiler|upper]}) %}
 

rose-stem/site/common/coupling/tasks_coupling.cylc

@@ -15,7 +15,7 @@
 
 {# Set default values for coupling tasks #}
 {% if task_dict %}
-    {% do task_dict.update({"application_dir": "components/coupling"}) %}
+    {% do task_dict.update({"application_dir": "lfric_core/components/coupling"}) %}
     {% if "app_name" not in task_dict %}
         {% do task_dict.update({"app_name": "coupling"}) %}
     {% endif %}

rose-stem/site/common/driver/tasks_driver.cylc

@@ -15,7 +15,7 @@
 
 {# Set default values for driver tasks #}
 {% if task_dict %}
-    {% do task_dict.update({"application_dir": "components/driver"}) %}
+    {% do task_dict.update({"application_dir": "lfric_core/components/driver"}) %}
     {% if "app_name" not in task_dict %}
         {% do task_dict.update({"app_name": "driver"}) %}
     {% endif %}

rose-stem/site/common/infrastructure/tasks_infrastructure.cylc

@@ -21,7 +21,7 @@
 
 {# Set default values for Infrastructure tasks #}
 {% if task_dict %}
-    {% do task_dict.update({"application_dir": "infrastructure"}) %}
+    {% do task_dict.update({"application_dir": "lfric_core/infrastructure"}) %}
     {% if "app_name" not in task_dict %}
         {% do task_dict.update({"app_name": "infrastructure"}) %}
     {% endif %}

rose-stem/site/common/io_demo/tasks_io_demo.cylc

@@ -44,7 +44,7 @@
 
 {# Set default values for io_demo tasks #}
 {% if task_dict %}
-    {% do task_dict.update({"application_dir": "applications/io_demo"}) %}
+    {% do task_dict.update({"application_dir": "lfric_core/applications/io_demo"}) %}
     {% if "app_name" not in task_dict %}
         {% do task_dict.update({"app_name": "io_demo"}) %}
     {% endif %}

rose-stem/site/common/lbc_demo/tasks_lbc_demo.cylc

@@ -183,7 +183,7 @@
 
 {# Set default values for LBC demo tasks #}
 {% if task_dict %}
-    {% do task_dict.update({"application_dir": "applications/lbc_demo"}) %}
+    {% do task_dict.update({"application_dir": "lfric_core/applications/lbc_demo"}) %}
     {% if "app_name" not in task_dict %}
         {% do task_dict.update({"app_name": "lbc_demo"}) %}
     {% endif %}

rose-stem/site/common/lfric_xios/tasks_lfric_xios.cylc

@@ -19,7 +19,7 @@
 
 {# Set default values for lfric_xios tasks #}
 {% if task_dict %}
-    {% do task_dict.update({"application_dir": "components/lfric-xios"}) %}
+    {% do task_dict.update({"application_dir": "lfric_core/components/lfric-xios"}) %}
     {% if "app_name" not in task_dict %}
         {% do task_dict.update({"app_name": "lfric-xios"}) %}
     {% endif %}