Merge branch 'main' into 558_remove_update_recipe_file_KGOs

Author: ehogan

CMEW/app/configure_standardise/bin/command_line.py

@@ -2,6 +2,7 @@
 # The LICENSE.md file contains full licensing details.
 import argparse
 from create_variables_file import create_variables_file
+from create_request_file import create_request_file
 
 
 def parse_args_for_create_variables_file(arguments):
@@ -70,3 +71,119 @@ def main_for_create_variables_file(arguments=None):
         args.stream_config_fp,
         args.output_filepath,
     )
+
+
+def parse_args_for_create_request_file(arguments):
+    """
+    Return the names and values of the command line arguments for
+    :func:`main_for_create_request_file`.
+
+    Parameters
+    ----------
+    arguments : :obj:`list` of :obj:`str`
+        The command line arguments to be parsed.
+
+    Returns
+    -------
+    :class:`argparse.Namespace`
+        The names and values of the command line arguments.
+    """
+    parser = argparse.ArgumentParser(
+        description=(
+            "Create a request file to standardise model data with CDDS."
+        ),
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "--dataset",
+        help="The model run to be extracted from MASS and processed.",
+    )
+    parser.add_argument(
+        "--output_filepath",
+        help=(
+            "The full path to the file where the "
+            "request file will be written."
+        ),
+    )
+    parser.add_argument(
+        "--defaults_path",
+        help=(
+            "The full path to the file where the "
+            "default values for a CDDS request are written."
+        ),
+    )
+    parser.add_argument(
+        "--mip_table_dir",
+        help=(
+            "The full path to the directory containing the "
+            "MIP tables for CDDS to use."
+        ),
+    )
+    parser.add_argument(
+        "--model_runs_yml_fp",
+        help=(
+            "The full path to the YAML file "
+            "containing details of the model runs."
+        ),
+    )
+    parser.add_argument(
+        "--root_proc_dir",
+        help=(
+            "The full path to the directory where CDDS "
+            "should store data for the processing workflow."
+        ),
+    )
+    parser.add_argument(
+        "--root_data_dir",
+        help=(
+            "The full path to the directory where CDDS "
+            "should store raw and standardised data."
+        ),
+    )
+    parser.add_argument(
+        "--variables_file",
+        help=(
+            "The full path to the file where the "
+            "variables to be retrieved by CDDS are written."
+        ),
+    )
+    parser.add_argument(
+        "--raw_data_dir_mode",
+        help=("Whether to save or reuse raw CDDS data files."),
+    )
+    return parser.parse_args(arguments)
+
+
+def main_for_create_request_file(arguments=None):
+    """
+    Create a request file to standardise model data with CDDS.
+
+    Parameters
+    ----------
+    arguments : :obj:`list` of :obj:`str`
+        The command line arguments to be parsed.
+    """
+    # Parse the arguments.
+    args = parse_args_for_create_request_file(arguments)
+
+    # Run the code.
+    print(f"dataset: {args.dataset}"),
+    print(f"output_filepath: {args.output_filepath}"),
+    print(f"defaults_path: {args.defaults_path}"),
+    print(f"mip_table_dir: {args.mip_table_dir}"),
+    print(f"model_runs_yml_fp: {args.model_runs_yml_fp}"),
+    print(f"root_proc_dir: {args.root_proc_dir}"),
+    print(f"root_data_dir: {args.root_data_dir}"),
+    print(f"variables_file: {args.variables_file}"),
+    print(f"raw_data_dir_mode: {args.raw_data_dir_mode}"),
+    create_request_file(
+        args.dataset,
+        args.output_filepath,
+        args.defaults_path,
+        args.mip_table_dir,
+        args.model_runs_yml_fp,
+        args.root_proc_dir,
+        args.root_data_dir,
+        args.variables_file,
+        args.raw_data_dir_mode,
+    )

CMEW/app/configure_standardise/bin/configure_standardise.sh

@@ -7,9 +7,6 @@ set -eux
 
 echo "Running configure_standardise"
 
-# Create request configuration file.
-cmew-esmvaltool-env create_request_file.py
-
 # Create CDDS directory structure and variables list.
 cmew-standardise-env create_cdds_directory_structure "${REQUEST_PATH}"
 cmew-standardise-env prepare_generate_variable_list "${REQUEST_PATH}"

CMEW/app/configure_standardise/bin/configure_standardise_conftest.py

@@ -8,10 +8,18 @@ def mock_data_dir():
     return Path(__file__).parent.parent.parent / "unittest" / "mock_data"
 
 
+def model_runs_yml_fp():
+    return mock_data_dir() / "model_runs.yml"
+
+
 def kgo_dir():
     return Path(__file__).parent.parent.parent / "unittest" / "kgo"
 
 
+def request_u_cw673_cfg_fp():
+    return kgo_dir() / "request_u-cw673.cfg"
+
+
 def variables_txt_fp():
     return kgo_dir() / "variables.txt"
 
@@ -20,5 +28,9 @@ def etc_dir():
     return Path(__file__).parent.parent / "etc"
 
 
+def request_defaults_yml_fp():
+    return etc_dir() / "request_defaults.yml"
+
+
 def streams_yml_fp():
     return etc_dir() / "streams.yml"

CMEW/app/configure_standardise/bin/create_request_file

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+# (C) Crown Copyright 2026, Met Office.
+# The LICENSE.md file contains full licensing details.
+from command_line import main_for_create_request_file
+
+
+if __name__ == "__main__":
+    main_for_create_request_file()

CMEW/app/configure_standardise/bin/create_request_file.py

@@ -7,7 +7,6 @@
 import configparser
 import os
 import sys
-from pathlib import Path
 import yaml
 import logging
 
@@ -16,20 +15,22 @@
 logger = logging.getLogger(filename)
 
 
-def load_request_defaults():
+def load_request_defaults(defaults_path):
     """
     Load default values for request file.
 
+    Parameters
+    ----------
+    defaults_path : str
+        Path to the default configuration file.
+
     Returns
     -------
     dict
         CDDS request configuration default settings.
     """
-    # Get path to default settings
-    defaults = os.environ["REQUEST_DEFAULTS_PATH"]
-
     # Read the defaults
-    with open(defaults, "r") as f:
+    with open(defaults_path, "r") as f:
         config = yaml.safe_load(f)
 
     logger.debug(
@@ -84,28 +85,54 @@ def list_streams(variables_file):
     return stream_str
 
 
-def create_request(model_run):
+def create_request(
+    defaults_path,
+    dataset,
+    mip_table_dir,
+    model_runs_yml_fp,
+    root_proc_dir,
+    root_data_dir,
+    variables_file,
+    raw_data_dir_mode,
+):
     """
     Build a CDDS request configuration for a run identified by a suite_id.
 
     Uses information from the model_runs.yml file.
 
+    Parameters
+    ----------
+    defaults_path : str
+        The full path to the file containing default values for a CDDS request.
+    dataset : str
+        The model run to be extracted from MASS and processed.
+    mip_table_dir : str
+        The path to the MIP table directory to use from CDDS.
+    model_runs_yml_fp : str
+        The full path to the YAML file containing details of the model runs.
+    root_proc_dir : str
+        The directory for use when processing data with CDDS.
+    root_data_dir : str
+        The root directory for CDDS to store data.
+    variables_file : str
+        The full path to the file containing
+        the list of variables to be processed.
+    raw_data_dir_mode : str
+        Whether to save or reuse raw CDDS data files.
+
     Returns
     -------
     dict
         CDDS request configuration.
     """
-    defaults = load_request_defaults()
-
-    mip_table_dir = os.environ["MIP_TABLE_DIR"]
+    defaults = load_request_defaults(defaults_path)
 
     # Read the model run information from the model_runs.yml file
-    model_runs_yaml = Path(os.environ["DATASETS_LIST_DIR"]) / "model_runs.yml"
-    with open(model_runs_yaml, "r") as f:
-        dataset_dict = yaml.safe_load(f)[model_run]
+    with open(model_runs_yml_fp, "r") as file_handle:
+        dataset_dict = yaml.safe_load(file_handle)[dataset]
     logger.debug(
         "Dataset % config:\n%s",
-        model_run,
+        dataset,
         dataset_dict,
     )
 
@@ -124,21 +151,21 @@ def create_request(model_run):
     request["common"] = {
         **defaults["common"],
         "mip_table_dir": os.path.expanduser(mip_table_dir),
-        "root_proc_dir": os.environ["ROOT_PROC_DIR"],
-        "root_data_dir": os.environ["ROOT_DATA_DIR"],
+        "root_proc_dir": root_proc_dir,
+        "root_data_dir": root_data_dir,
         "workflow_basename": dataset_dict["suite_id"],
     }
     request["data"] = {
         **defaults["data"],
         "start_date": f"{dataset_dict['start_year']}-01-01T00:00:00",
         "end_date": f"{int(dataset_dict['end_year'])+1}-01-01T00:00:00",
         "model_workflow_id": dataset_dict["suite_id"],
-        "streams": list_streams(os.environ["VARIABLES_PATH"]),
-        "variable_list_file": os.environ["VARIABLES_PATH"],
+        "streams": list_streams(variables_file),
+        "variable_list_file": variables_file,
     }
     request["misc"] = dict(defaults["misc"])
     request["conversion"] = dict(defaults["conversion"])
-    if os.environ["RAW_DATA_DIR_MODE"] == "use_saved":
+    if raw_data_dir_mode == "use_saved":
         request["conversion"]["skip_extract"] = "True"
     request["netcdf_global_attributes"] = dict(
         defaults["netcdf_global_attributes"]
@@ -148,15 +175,15 @@ def create_request(model_run):
     return request
 
 
-def write_request(request, target_path):
-    """Write the request configuration to a file at ``target_path``.
+def write_request(request, output_filepath):
+    """Write the request configuration to a file at ``output_filepath``.
 
     Parameters
     ----------
     request : dict
         The request configuration.
 
-    target_path: Path
+    output_filepath: Path
         The full path to the file
         where the request configuration will be written.
     """
@@ -165,25 +192,57 @@ def write_request(request, target_path):
 
     logger.debug("Writing request config:\n%s", cfg)
 
-    with open(target_path, mode="w") as file_handle:
+    with open(output_filepath, mode="w") as file_handle:
         cfg.write(file_handle)
 
 
-def main():
+def create_request_file(
+    dataset,
+    output_filepath,
+    defaults_path,
+    mip_table_dir,
+    model_runs_yml_fp,
+    root_proc_dir,
+    root_data_dir,
+    variables_file,
+    raw_data_dir_mode,
+):
     """
-    Generate and write the request file for the current task environment.
+    Create a request file to standardise model data with CDDS.
 
-    The output file location is taken from the REQUEST_PATH environment
-    variable. All other required inputs are read from the environment
-    by ``create_request()``.
+    Parameters
+    ----------
+    dataset : str
+        The model run to be extracted from MASS and processed.
+    output_filepath : str
+        The full path to the file where the
+        request configuration will be written.
+    defaults_path : str
+        The full path to the file containing default values for a CDDS request.
+    mip_table_dir : str
+        The path to the MIP table directory to use from CDDS.
+    model_runs_yml_fp : str
+        The full path to the YAML file containing details of the model runs.
+    root_proc_dir : str
+        The directory for use when processing data with CDDS.
+    root_data_dir : str
+        The root directory for CDDS to store data.
+    variables_file : str
+        The full path to the file containing
+        the list of variables to be processed.
+    raw_data_dir_mode : str
+        Whether to save or reuse raw CDDS data files.
     """
-    dataset = os.environ["CYLC_TASK_PARAM_dataset"].strip()
     logger.info("Creating CDDS request for dataset %s", dataset)
 
-    request = create_request(dataset)
-    target_path = Path(os.environ["REQUEST_PATH"])
-    write_request(request, target_path)
-
-
-if __name__ == "__main__":
-    main()
+    request = create_request(
+        defaults_path,
+        dataset,
+        mip_table_dir,
+        model_runs_yml_fp,
+        root_proc_dir,
+        root_data_dir,
+        variables_file,
+        raw_data_dir_mode,
+    )
+    write_request(request, output_filepath)