Add CLI for ingesting tabular forecasts and observations into EMOS

doc/source/conf.py

@@ -398,6 +398,7 @@
     "https://scitools.org.uk/cf-units/docs/latest/": None,
     "https://docs.scipy.org/doc/numpy/": None,
     "https://docs.scipy.org/doc/scipy/reference/": None,
+    "https://pandas.pydata.org/pandas-docs/dev/": None,
 }
 
 # Get napoleon to document constructor methods.

doc/source/extended_documentation/calibration/calibration_data_ingestion.rst

@@ -0,0 +1,59 @@
+#######################################
+Ingestion of DataFrames into iris cubes
+#######################################
+
+DataFrames of the forecasts and truths (observations) can be provided
+for use with Ensemble Model Output Statistics (EMOS). The format
+expected for the forecast and truths DataFrames is described below.
+The forecasts are ensemble site forecasts in percentile format at
+a set of observation sites. The truths are observations from
+observation sites.
+
+****************************
+Forecast DataFrame
+****************************
+
+The forecast DataFrame is expected to contain the following compulsory
+columns: forecast, blend_time, forecast_period, forecast_reference_time,
+time, wmo_id, percentile, diagnostic, latitude, longitude, period, height,
+cf_name and units. Other columns will be ignored.
+
+A summary of the expected contents of a forecast table is shown below.
+
+.. csv-table::
+    :file: ./forecast_dataframe_metadata_info.csv
+    :widths: 25, 22, 53
+    :header-rows: 1
+
+An example forecast table for an instantaneous diagnostic is shown below.
+
+.. csv-table::
+    :file: ./forecast_dataframe_instantaneous_example.csv
+    :header-rows: 1
+
+An example forecast table for a period diagnostic is shown below.
+
+.. csv-table::
+    :file: ./forecast_dataframe_period_example.csv
+    :header-rows: 1
+
+****************************
+Truth DataFrame
+****************************
+
+The truth DataFrame is expected to contain the following compulsory
+columns: ob_value, time, wmo_id, diagnostic, latitude, longitude and
+altitude. Other columns will be ignored.
+
+A summary of the expected contents of a truth table is shown below.
+
+.. csv-table::
+    :file: ./truth_dataframe_metadata_info.csv
+    :widths: 30, 30, 40
+    :header-rows: 1
+
+An example truth table is shown below.
+
+.. csv-table::
+    :file: ./truth_dataframe_example.csv
+    :header-rows: 1

doc/source/extended_documentation/calibration/forecast_dataframe_instantaneous_example.csv

@@ -0,0 +1,6 @@
+index,forecast,altitude,blend_time,forecast_period,forecast_reference_time,latitude,longitude,time,wmo_id,cf_name,units,percentile,period,height,diagnostic
+0,282.69,15,2021-08-01 18:00:00+00:00,1 days,2021-08-01 18:00:00+00:00,60,-5,2021-08-02 18:00:00+00:00,03001,air_temperature,K,5,NaT,1.5,temperature_at_screen_level
+1,283.2,82,2021-08-01 18:00:00+00:00,1 days,2021-08-01 18:00:00+00:00,59,-4,2021-08-02 18:00:00+00:00,03002,air_temperature,K,5,NaT,1.5,temperature_at_screen_level
+2,282.62,30,2021-08-01 18:00:00+00:00,1 days,2021-08-01 18:00:00+00:00,58,-3,2021-08-02 18:00:00+00:00,03003,air_temperature,K,5,NaT,1.5,temperature_at_screen_level
+3,286.17,4,2021-08-01 18:00:00+00:00,1 days,2021-08-01 18:00:00+00:00,57,-2,2021-08-02 18:00:00+00:00,03004,air_temperature,K,5,NaT,1.5,temperature_at_screen_level
+4,284.43,15,2021-08-01 18:00:00+00:00,1 days,2021-08-01 18:00:00+00:00,56,-1,2021-08-02 18:00:00+00:00,03005,air_temperature,K,5,NaT,1.5,temperature_at_screen_level

doc/source/extended_documentation/calibration/forecast_dataframe_metadata_info.csv

@@ -0,0 +1,16 @@
+Column,Dtype,Notes
+forecast,float64,The value for a particular forecast.
+altitude,float32,The altitude in metres.
+blend_time,"datetime64[ns,UTC]",The time at which a blend of models was produced.
+forecast_period,timedelta64[ns],The difference between the blend time (and forecast reference time) and the validity time.
+forecast_reference_time,"datetime64[ns,UTC]",The time at which the forecast analysis was made for a forecast from a single source. Equal to the blend_time for a forecast created from blending multiple forecast sources.
+latitude,float32,The latitude in degrees.
+longitude,float32,The longitude in degrees.
+time,"datetime64[ns,UTC]",The validity time of the forecasts. Signifies the end of the forecast period for period diagnostics.
+wmo_id,object,The five digit WMO ID.
+cf_name,object,The CF name for the diagnostic. From DataFrames consisting of one diagnostic this is expected to be constant.
+units,object,The units of the forecast value. From DataFrames consisting of one diagnostic this is expected to be constant.
+percentile,float64,The percentile value.
+period,timedelta64[ns],The period the forecast valid is over. Set to missing data for instantaneous forecasts.
+height,float32,The height of the forecast value. From DataFrames consisting of one diagnostic this is expected to be constant.
+diagnostic,category,The name of the diagnostic. From DataFrames consisting of one diagnostic this is expected to be constant.

doc/source/extended_documentation/calibration/forecast_dataframe_period_example.csv

@@ -0,0 +1,6 @@
+index,forecast,altitude,blend_time,forecast_period,forecast_reference_time,latitude,longitude,time,wmo_id,cf_name,units,percentile,period,height,diagnostic
+0,282.69,15,2021-08-01 00:00:00+00:00,0 days 09:00:00,2021-08-01 00:00:00+00:00,60,-5,2021-08-01 21:00:00+00:00,03001,temperature_at_screen_level_daytime_max,K,5,0 days 12:00:00,1.5,temperature_at_screen_level_max-daytime
+1,283.2,82,2021-08-01 00:00:00+00:00,0 days 09:00:00,2021-08-01 00:00:00+00:00,59,-4,2021-08-01 21:00:00+00:00,03002,temperature_at_screen_level_daytime_max,K,5,0 days 12:00:00,1.5,temperature_at_screen_level_max-daytime
+2,282.62,30,2021-08-01 00:00:00+00:00,0 days 09:00:00,2021-08-01 00:00:00+00:00,58,-3,2021-08-01 21:00:00+00:00,03003,temperature_at_screen_level_daytime_max,K,5,0 days 12:00:00,1.5,temperature_at_screen_level_max-daytime
+3,286.17,4,2021-08-01 00:00:00+00:00,0 days 09:00:00,2021-08-01 00:00:00+00:00,57,-2,2021-08-01 21:00:00+00:00,03004,temperature_at_screen_level_daytime_max,K,5,0 days 12:00:00,1.5,temperature_at_screen_level_max-daytime
+4,284.43,15,2021-08-01 00:00:00+00:00,0 days 09:00:00,2021-08-01 00:00:00+00:00,56,-1,2021-08-01 21:00:00+00:00,03005,temperature_at_screen_level_daytime_max,K,5,0 days 12:00:00,1.5,temperature_at_screen_level_max-daytime

doc/source/extended_documentation/calibration/truth_dataframe_example.csv

@@ -0,0 +1,6 @@
+index,ob_value,altitude,latitude,longitude,time,wmo_id,diagnostic
+0,283.45,15,60,-5,2021-08-02 18:00:00+00:00,03001,temperature_at_screen_level
+1,283.91,82,59,-4,2021-08-02 18:00:00+00:00,03002,temperature_at_screen_level
+2,281.63,30,58,-3,2021-08-02 18:00:00+00:00,03003,temperature_at_screen_level
+3,286.55,4,57,-2,2021-08-02 18:00:00+00:00,03004,temperature_at_screen_level
+4,283.19,15,56,-1,2021-08-02 18:00:00+00:00,03005,temperature_at_screen_level

doc/source/extended_documentation/calibration/truth_dataframe_metadata_info.csv

@@ -0,0 +1,8 @@
+Column,Dtype,Notes
+time,"datetime64[ns,UTC]",The time of the observation.
+wmo_id,object,The five digit WMO ID.
+latitude,float32,The latitude in degrees.
+longitude,float32,The longtitude in degrees.
+altitude,float32,The altitude in metres.
+ob_value,float32,The value for a particular observation.
+diagnostic,category,The name of the diagnostic.

envs/environment_py37_iris30.yml

@@ -15,13 +15,15 @@ dependencies:
   - iris=3.0
   - netCDF4
   - numpy<1.21
+  - pandas
   - python-dateutil
   - python-stratify
   - pytz=2020.5
   - scipy=1.6
   - sigtools
   - sphinx
   # Optional
+  - fastparquet
   - numba
   - pysteps=1.4.1
   - statsmodels

envs/environment_py38_iris30.yml

@@ -15,6 +15,7 @@ dependencies:
   - iris=3.0
   - netCDF4
   - numpy<1.21
+  - pandas
   - python-dateutil
   - python-stratify
   - pytz=2020.5

improver/calibration/__init__.py

@@ -28,7 +28,9 @@
 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
-"""init for calibration"""
+"""init for calibration that contains functionality to split forecast, truth
+and coefficient inputs.
+"""
 
 from collections import OrderedDict
 from typing import Dict, List, Optional, Tuple