Merge pull request #2424 from MetOffice/2403_freqbias

Scores [Categorical] Frequency Bias

Author: jwarner8

src/CSET/cset_workflow/meta/verification/rose-meta.conf

@@ -357,6 +357,13 @@ compulsory=true
 sort-key=scoresPOD3
 
 [Verification/Scores Categorical]
+[template variables=SCORES_CATEGORICAL_FB]
+ns=Verification/Scores Categorical
+description=Compute Scores Frequency Bias.
+type=python_boolean
+compulsory=true
+sort-key=scoresPOD4
+
 [template variables=SCORES_CATEGORICAL_PFD]
 ns=Verification/Scores Categorical
 description=Compute Scores Probability of False Detection.

src/CSET/cset_workflow/rose-suite.conf.example

@@ -129,6 +129,7 @@ POINT_OBS=False
 !!POINT_OBS_USE_WMO_STATION_NUMBERS=False
 !!POINT_OBS_WMO_BLOCK_STTN_NUMBERS=[]
 PRESSURE_LEVELS=[]
+SCORES_CATEGORICAL_FB=False
 SCORES_CATEGORICAL_ETS=False
 SCORES_CATEGORICAL_PFD=False
 SCORES_CATEGORICAL_POD=False

src/CSET/loaders/verification.py

@@ -71,6 +71,8 @@ def _get_scores_timeseries_categorical(conf):
         scores_timeseries_categorical.append("pod")
     if conf.SCORES_CATEGORICAL_ETS or conf.SCORES_ALL:
         scores_timeseries_categorical.append("ets")
+    if conf.SCORES_CATEGORICAL_FB or conf.SCORES_ALL:
+        scores_timeseries_categorical.append("fb")
     if conf.SCORES_CATEGORICAL_PFD or conf.SCORES_ALL:
         scores_timeseries_categorical.append("pfd")
     return scores_timeseries_categorical

src/CSET/recipes/verification/surface_categorical_model_obs_fb.yaml

@@ -0,0 +1,100 @@
+category: Scores Categorical
+title: Timeseries of FB for "$VARNAME over $SUBAREA_NAME, using threshold $OPERATOR $THRESHOLD"
+description: |
+
+    Extracts and plots the FB for $VARNAME for each model against observations as a timeseries.
+
+    The Frequency Bias (FB) in [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.categorical.BasicContingencyManager.frequency_bias) measures whether a forecasting system predicts an
+    event too frequently or too infrequently compared to observations. Unlike
+    metrics such as the Probability of Detection (POD) or Equitable Threat Score (ETS),
+    Frequency Bias does not assess the accuracy of forecast locations or timings,
+    only the overall frequency with which events are forecast.
+
+    For example, if the threshold is 290 K and op_func is gt (greater than), and
+    events exceeding this threshold are forecast twice as often as they are observed,
+    the frequency bias would be approximately 2. Conversely, if events are forecast
+    only half as often as they occur, the frequency bias would be approximately 0.5.
+
+    It is calculated as:
+
+    .. math::
+
+    Frequency\ Bias = \frac{hits + false\ alarms}
+                            {hits + misses}
+
+    where
+
+    hits
+        Number of occasions where an event was forecast and observed.
+
+    false alarms
+        Number of occasions where an event was forecast but not observed.
+
+    misses
+        Number of occasions where an event was observed but not forecast.
+
+    Frequency Bias ranges from 0 to infinity, where:
+
+    * 1 indicates the forecast predicts events at the correct frequency.
+    * Greater than 1 indicates overforecasting of events.
+    * Less than 1 indicates underforecasting of events.
+
+    A perfect frequency bias score is therefore 1, although a value of 1 does not
+    necessarily imply a skillful forecast, as hits and false alarms may compensate
+    for one another.
+
+    References
+    https://scores.readthedocs.io/en/stable/api.html#scores.categorical.BasicContingencyManager.frequency_bias
+
+steps:
+  - operator: read.read_cubes
+    file_paths: $INPUT_PATHS
+    model_names: $MODEL_NAME
+    constraint:
+      operator: constraints.combine_constraints
+      varname_constraint:
+        operator: constraints.generate_var_constraint
+        varname: ['observed_$VARNAME', '$VARNAME']
+      cell_methods_constraint:
+        operator: constraints.generate_cell_methods_constraint
+        cell_methods: []
+      pressure_level_constraint:
+        operator: constraints.generate_level_constraint
+        coordinate: "pressure"
+        levels: []
+    subarea_type: $SUBAREA_TYPE
+    subarea_extent: $SUBAREA_EXTENT
+
+  - operator: misc.extract_common_points
+    coordinate: time
+
+  - operator: misc.combine_cubes_into_cubelist
+    first:
+        operator: filters.filter_cubes
+        constraint:
+            operator: constraints.generate_var_constraint
+            varname: observed_$VARNAME
+    second:
+       operator: regrid.interpolate_to_point_cube
+       fld:
+           operator: filters.filter_multiple_cubes
+           constraint:
+               operator: constraints.combine_constraints
+               var_constraint:
+                   operator: constraints.generate_var_constraint
+                   varname: $VARNAME
+       point_cube:
+           operator: filters.filter_cubes
+           constraint:
+               operator: constraints.generate_var_constraint
+               varname: observed_$VARNAME
+
+  - operator: scoreswrappers.scores_frequency_bias
+    preserved_coordinates: "time"
+    threshold: $THRESHOLD
+    op_func: $OPERATOR
+
+  - operator: plot.plot_line_series
+
+  - operator: write.write_cube_to_nc
+    overwrite: True