Aggregate RMSE over multiple cases/variables and produce timeseries + Hinton plot [Experimental]

src/CSET/cset_workflow/meta/verification/rose-meta.conf

@@ -79,6 +79,61 @@ trigger=template variables=SCORES_SPATIAL_RMSE: False;
 compulsory=true
 sort-key=scoresspatial1
 
+# Hinton plots
+[Verification/ModelVsObsScoresHinton]
+ns=Verification/ModelVsObsScoresHinton
+sort-key=hinton
+title=Hinton Summary Scorecards
+
+# Categorical Scores metrics against observations.
+[template variables=HINTON_VAR1]
+ns=Verification/ModelVsObsScoresHinton
+description=Hinton variable 1/resample method/freq
+help=TODO
+type=character
+compulsory=true
+sort-key=hinton1
+
+[template variables=HINTON_VAR2]
+ns=Verification/ModelVsObsScoresHinton
+description=Hinton variable 2/resample method/freq
+help=TODO
+type=character
+compulsory=true
+sort-key=hinton1
+
+[template variables=HINTON_VAR3]
+ns=Verification/ModelVsObsScoresHinton
+description=Hinton variable 3/resample method/freq
+help=TODO
+type=character
+compulsory=true
+sort-key=hinton1
+
+[template variables=HINTON_VAR4]
+ns=Verification/ModelVsObsScoresHinton
+description=Hinton variable 4/resample method/freq
+help=TODO
+type=character
+compulsory=true
+sort-key=hinton1
+
+[template variables=HINTON_VAR5]
+ns=Verification/ModelVsObsScoresHinton
+description=Hinton variable 5/resample method/freq
+help=TODO
+type=character
+compulsory=true
+sort-key=hinton1
+
+[template variables=HINTON_RMSE]
+ns=Verification/ModelVsObsScoresHinton
+title=HINTON_RMSE
+description=Produce RMSE plot
+help=TODO
+type=python_boolean
+compulsory=true
+sort-key=hintona
 
 # Scores spatial plots
 [Verification/ScoresSpatial]
@@ -268,7 +323,6 @@ sort-key=scores1_model_vs_obs
 
 ##########################################################
 
-
 # Model Vs Obs Scores timeseries plots
 [Verification/ModelVsObsScoresTimeseries]
 ns=Verification/ModelVsObsScoresTimeseries
@@ -287,6 +341,19 @@ type=python_boolean
 compulsory=true
 sort-key=sc-ts-1
 
+[template variables=SCORES_TIMESERIES_RMSE_MODEL_VS_OBS_AGGREGATION]
+ns=Verification/ModelVsObsScoresTimeseries
+title=Root Mean Square Error Model Vs Obs Aggregated
+description=SCORES COUNTINOUS METRIC.
+            Experimental: Aggregation over all forecasts by taking the average.
+            Plots a timeseries plot of the RMSE between models and observations.
+            The computation is implemented by the scores package.
+help=This computes the metric using the scores package. Details of this can
+     be found at https://scores.readthedocs.io/en/stable/included.html#continuous
+type=python_boolean
+compulsory=true
+sort-key=sc-ts-1
+
 #################################################
 
 # Model Vs Obs Scores spatial plots

src/CSET/cset_workflow/rose-suite.conf.example

@@ -61,6 +61,13 @@ HISTOGRAM_SURFACE_FIELD=False
 HISTOGRAM_SURFACE_FIELD_AGGREGATION=False,False,False,False
 HISTOGRAM_SURFACE_FIELD_SEQUENCE=False
 HOUSEKEEPING_MODE=2
+HINTON_RMSE=False
+HINTON_VAR1=''
+HINTON_VAR2=''
+HINTON_VAR3=''
+HINTON_VAR4=''
+HINTON_VAR5=''
+SCORES_TIMESERIES_RMSE_MODEL_VS_OBS_AGGREGATION=False
 !!LATITUDE_POINT=0
 !!LATLON_IN_TYPE="realworld"
 LEVEL_BASIC_QQ_PLOT=False

src/CSET/cset_workflow/test_aggregation.yaml

@@ -0,0 +1,81 @@
+category: Hinton
+title: ('temperature_at_screen_level',) scores rmse Hinton for model vs 
+  observation points UK_domain
+description: TODO experimental
+steps:
+  - operator: read.read_cubes
+    file_paths:
+      - $DATADIR/CSET_testdata/UM_LFRic_OBS/*.nc
+      - $DATADIR/CSET_testdata/UM_LFRic_OBS/um/*
+      - $DATADIR/CSET_testdata/UM_LFRic_OBS/lf/*
+    model_names: &id002
+      - OBS
+      - UM
+      - LF
+    constraint:
+      operator: constraints.combine_constraints
+      varname_constraint:
+        operator: constraints.generate_var_constraint
+        varname:
+          - temperature_at_screen_level
+          - observed_temperature_at_screen_level
+      cell_methods_constraint:
+        operator: constraints.generate_cell_methods_constraint
+        cell_methods: []
+      pressure_level_constraint:
+        operator: constraints.generate_level_constraint
+        coordinate: pressure
+        levels: []
+    subarea_type: realworld
+    subarea_extent: &id003
+      - 50
+      - 55
+      - -6
+      - 2
+
+  - operator: misc.combine_cubes_into_cubelist
+    first:
+      operator: aggregate.combine_obs_across_forecasts
+      cubes:
+        operator: filters.filter_multiple_cubes
+        constraint:
+          operator: constraints.generate_var_constraint
+          varname: observed_temperature_at_screen_level
+    second:
+      operator: regrid.interpolate_to_point_cube
+      fld:     
+        operator: aggregate.ensure_aggregatable_across_cases
+        cubes:
+          operator: filters.filter_multiple_cubes
+          constraint:
+            operator: constraints.generate_var_constraint
+            varname: temperature_at_screen_level
+
+      point_cube:
+        operator: aggregate.combine_obs_across_forecasts
+        cubes:
+          operator: filters.filter_multiple_cubes
+          constraint:
+            operator: constraints.generate_var_constraint
+            varname: observed_temperature_at_screen_level
+
+  - operator: scoreswrappers.scores_rmse_model_obs
+    preserved_coordinates: [forecast_period]
+
+  - operator: collapse.collapse
+    coordinate: "forecast_reference_time"
+    method: MEAN
+
+    # Plot the data.
+  - operator: plot.hinton_plot
+    base: UM
+    other: LF
+    magnitudes: True
+
+
+  # - operator: write.write_cube_to_nc
+  #   overwrite: true
+
+
+
+#Run with cset -vv bake --recipe test_hinton.yaml --output-dir $DATADIR/2408_csettesting --plot-resolution=100 --DATADIR=$DATADIR

src/CSET/cset_workflow/test_hinton.yaml

@@ -0,0 +1,151 @@
+category: Hinton
+title: Full RMSE Hinton for something
+description: TODO experimental
+steps:
+  - operator: read.read_cubes
+    file_paths:
+      - $DATADIR/CSET_testdata/UM_LFRic_OBS/*.nc
+      - $DATADIR/CSET_testdata/UM_LFRic_OBS/um/*
+      - $DATADIR/CSET_testdata/UM_LFRic_OBS/lf/*
+    model_names:
+      - OBS
+      - UM
+      - LF
+    constraint:
+      operator: constraints.combine_constraints
+      varname_constraint:
+        operator: constraints.generate_var_constraint
+        varname:
+          - temperature_at_screen_level
+          - observed_temperature_at_screen_level
+          - air_pressure_at_mean_sea_level
+          - observed_air_pressure_at_mean_sea_level
+          - wind_speed_at_10m
+          - observed_wind_speed_at_10m
+          - visibility_in_air
+          - observed_visibility_in_air
+      cell_methods_constraint:
+        operator: constraints.generate_cell_methods_constraint
+        cell_methods: []
+      pressure_level_constraint:
+        operator: constraints.generate_level_constraint
+        coordinate: pressure
+        levels: []
+    subarea_type: realworld
+    subarea_extent:
+      - 50
+      - 55
+      - -6
+      - 2
+
+  - operator: misc.combine_cubes_into_cubelist
+    first:
+      operator: collapse.collapse
+      coordinate: forecast_reference_time
+      method: MEAN
+      cubes:
+        operator: scoreswrappers.scores_rmse_model_obs
+        preserved_coordinates: [forecast_period]
+        cubes:
+          operator: misc.combine_cubes_into_cubelist
+          first:
+            operator: aggregate.combine_obs_across_forecasts
+            cubes:
+              operator: filters.filter_multiple_cubes
+              constraint:
+                operator: constraints.generate_var_constraint
+                varname: observed_temperature_at_screen_level
+          second:
+            operator: regrid.interpolate_to_point_cube
+            fld:
+              operator: aggregate.ensure_aggregatable_across_cases
+              cubes:
+                operator: filters.filter_multiple_cubes
+                constraint:
+                  operator: constraints.generate_var_constraint
+                  varname: temperature_at_screen_level
+            point_cube:
+              operator: aggregate.combine_obs_across_forecasts
+              cubes:
+                operator: filters.filter_multiple_cubes
+                constraint:
+                  operator: constraints.generate_var_constraint
+                  varname: observed_temperature_at_screen_level
+
+    second:
+      operator: collapse.collapse
+      coordinate: forecast_reference_time
+      method: MEAN
+      cubes:
+        operator: scoreswrappers.scores_rmse_model_obs
+        preserved_coordinates: [forecast_period]
+        cubes:
+          operator: misc.combine_cubes_into_cubelist
+          first:
+            operator: aggregate.combine_obs_across_forecasts
+            cubes:
+              operator: filters.filter_multiple_cubes
+              constraint:
+                operator: constraints.generate_var_constraint
+                varname: observed_air_pressure_at_mean_sea_level
+          second:
+            operator: regrid.interpolate_to_point_cube
+            fld:
+              operator: aggregate.ensure_aggregatable_across_cases
+              cubes:
+                operator: filters.filter_multiple_cubes
+                constraint:
+                  operator: constraints.generate_var_constraint
+                  varname: air_pressure_at_mean_sea_level
+            point_cube:
+              operator: aggregate.combine_obs_across_forecasts
+              cubes:
+                operator: filters.filter_multiple_cubes
+                constraint:
+                  operator: constraints.generate_var_constraint
+                  varname: observed_air_pressure_at_mean_sea_level
+
+    third:
+      operator: collapse.collapse
+      coordinate: forecast_reference_time
+      method: MEAN
+      cubes:
+        operator: scoreswrappers.scores_rmse_model_obs
+        preserved_coordinates: [forecast_period]
+        cubes:
+          operator: misc.combine_cubes_into_cubelist
+          first:
+            operator: aggregate.combine_obs_across_forecasts
+            cubes:
+              operator: filters.filter_multiple_cubes
+              constraint:
+                operator: constraints.generate_var_constraint
+                varname: observed_visibility_in_air
+          second:
+            operator: regrid.interpolate_to_point_cube
+            fld:
+              operator: aggregate.ensure_aggregatable_across_cases
+              cubes:
+                operator: filters.filter_multiple_cubes
+                constraint:
+                  operator: constraints.generate_var_constraint
+                  varname: visibility_in_air
+            point_cube:
+              operator: aggregate.combine_obs_across_forecasts
+              cubes:
+                operator: filters.filter_multiple_cubes
+                constraint:
+                  operator: constraints.generate_var_constraint
+                  varname: observed_visibility_in_air
+
+
+    # Plot the data.
+  - operator: plot.hinton_plot
+    base: UM
+    other: LF
+    vars: ["temperature_at_screen_level","air_pressure_at_mean_sea_level","visibility_in_air"]
+    magnitude: True
+
+
+  # - operator: write.write_cube_to_nc
+  #   overwrite: true

src/CSET/loaders/verification.py

@@ -325,3 +325,55 @@ def load(conf: Config):
                 model_ids=["OBS"] + [model["id"] for model in models],
                 aggregation=False,
             )
+
+    if conf.HINTON_RMSE:
+        base_model = models[0]
+
+        hinton_opts = [
+            conf.HINTON_VAR1,
+            conf.HINTON_VAR2,
+            conf.HINTON_VAR3,
+            conf.HINTON_VAR4,
+            conf.HINTON_VAR5,
+        ]
+
+        for model in models[1:]:
+            variables = {
+                "MODEL_NAME": ["OBS"] + [model["id"] for model in models],
+                "SUBAREA_NAME": conf.SUBAREA_NAME if conf.SELECT_SUBAREA else "",
+                "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None,
+                "SUBAREA_EXTENT": (
+                    conf.SUBAREA_EXTENT if conf.SELECT_SUBAREA else None
+                ),
+            }
+
+            for i, item in enumerate(hinton_opts, start=1):
+                varname, method, timefreq = item.split("/")
+
+                variables[f"VARNAME{i}"] = varname
+                variables[f"METHOD{i}"] = method
+                variables[f"TIMEFREQ{i}"] = timefreq
+
+            yield RawRecipe(
+                recipe="hinton_rmse.yaml",
+                variables=variables,
+                model_ids=["OBS"] + [model["id"] for model in models],
+                aggregation=True,
+            )
+
+    if conf.SCORES_TIMESERIES_RMSE_MODEL_VS_OBS_AGGREGATION:
+        for field in conf.POINT_OBS_FIELDS:
+            yield RawRecipe(
+                recipe="timeseries_surface_scores_model_vs_obs_RMSE_aggregation.yaml",
+                variables={
+                    "VARNAME": field,
+                    "MODEL_NAME": ["OBS"] + [model["name"] for model in models],
+                    "SUBAREA_NAME": conf.SUBAREA_NAME if conf.SELECT_SUBAREA else "",
+                    "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None,
+                    "SUBAREA_EXTENT": conf.SUBAREA_EXTENT
+                    if conf.SELECT_SUBAREA
+                    else None,
+                },
+                model_ids=["OBS"] + [model["id"] for model in models],
+                aggregation=True,
+            )