Merge pull request #3547 from divine7022/era5-cdsapi-tweaks

Enhance ERA5 Download Function: Direct NetCDF, Flexible Parameters, and Improved Docs

Author: dlebauer

CHANGELOG.md

@@ -55,6 +55,7 @@ section for the next release.
     * Modules `PEcAn.allometry`, `PEcAn.assim.batch`, `PEcAn.data.mining`, `PEcAn.emulator`, `PEcAn.MA`, `PEcAn.photosynthesis`, `PEcAn.priors`, and `PEcAn.RTM`.
 - Renamed master branch to main
 - `PEcAn.all::pecan_version()` now reports commit hashes as well as version numbers for each installed package.
+- `download.ERA5_cds` now uses the R package ecmwfr (replacing python dependency of cdsapi via reticulate), enabling direct NetCDF downloads; and made flexible for both reanalysis and ensemble data product.
 
 ### Removed
 

docker/depends/pecan_package_dependencies.csv

@@ -64,6 +64,7 @@
 "dplyr","*","modules/uncertainty","Imports",FALSE
 "dplyr",">= 0.8.1","modules/data.atmosphere","Imports",FALSE
 "dplyr",">= 1.1.2","base/db","Imports",FALSE
+"ecmwfr",">= 2.0.0","modules/data.atmosphere","Suggests",FALSE
 "ellipse","*","modules/assim.batch","Imports",FALSE
 "exactextractr","*","modules/assim.sequential","Suggests",FALSE
 "foreach","*","base/remote","Imports",FALSE

modules/data.atmosphere/DESCRIPTION

@@ -19,6 +19,7 @@ Description: The Predictive Ecosystem Carbon Analyzer (PEcAn) is a scientific
     package converts climate driver data into a standard format for models
     integrated into PEcAn. As a standalone package, it provides an interface to
     access diverse climate data sets.
+Depends: R (>= 4.1.0)
 Imports:
     abind (>= 1.4.5),
     amerifluxr,
@@ -61,6 +62,7 @@ Imports:
     zoo
 Suggests:
     doParallel,
+    ecmwfr (>= 2.0.0),
     doSNOW,
     furrr,
     future,

modules/data.atmosphere/NAMESPACE

@@ -113,7 +113,6 @@ export(wide2long)
 export(write_noaa_gefs_netcdf)
 importFrom(dplyr,"%>%")
 importFrom(foreach,"%dopar%")
-importFrom(purrr,"%>%")
 importFrom(rlang,.data)
 importFrom(rlang,.env)
 importFrom(sf,st_crs)

modules/data.atmosphere/NEWS.md

@@ -2,6 +2,7 @@
 
 * `ERA5_met_process()` can now process ensemble data efficiently in parallel using new option `n_cores`
 * Dependency `ggplot2` is now suggested rather than required. It is used in two vignettes and for optional diagnostic plots from `debias_met_regression`.
+* `download.ERA5_cds` now uses the R package ecmwfr (replacing python dependency of cdsapi via reticulate), enabling direct NetCDF downloads; and made flexible for both reanalysis and ensemble data product.
 * New function `sat_vapor_pressure()` added for computing saturation vapor pressure from temperature using various methods.
 
 # PEcAn.data.atmosphere 1.9.0

modules/data.atmosphere/R/ERA5_download.R

@@ -1,133 +1,149 @@
+#' Download ERA5 Climate Data from the Copernicus CDS API
+#'
 #' @description
-#' This function helps to download the yearly ERA5 data based on the prescribed features using the CDS API.
-#' @title ERA5_cds_annual_download
-#' 
-#' @param outfolder Character: physical path where the ERA5 data are stored.
+#' Download ERA5 climate data from the Copernicus Climate Data Store (CDS) API as NetCDF files, year by year, according to user-specified parameters.
+#' The function saves one NetCDF file per year in the specified output directory.
+#'
+#' @details
+#' This function requires a valid CDS API key and the \code{ecmwfr} package for accessing the Copernicus Climate Data Store.
+#' To get a Copernicus CDS API key, register at \url{https://cds.climate.copernicus.eu/profile}.
+#' You must provide both \code{user} (UID) and \code{key} parameters from your CDS profile.
+#'
+#' You can check the "CC-BY" license under the \href{https://cds.climate.copernicus.eu/profile?tab=licences}{'licences' tab of your profile page}.
+#' @param outfolder Character. Directory where downloaded NetCDF files will be saved.
 #' @param start_date character: the start date of the data to be downloaded. Format is YYYY-MM-DD (will only use the year part of the date)
 #' @param end_date character: the end date of the data to be downloaded. Format is YYYY-MM-DD (will only use the year part of the date)
-#' @param extent numeric: a vector of numbers contains the bounding box (formatted as xmin, xmax, ymin, ymax) to be downloaded.
+#' @param extent numeric: a vector of numbers contains the bounding box (formatted as xmin, xmax, ymin, ymax) (longitude and latitude in degrees).
 #' @param variables character: a vector contains variables to be downloaded (e.g., c("2m_temperature","surface_pressure")).
-#' @param auto.create.key Boolean: decide if we want to generate the CDS RC file if it doesn't exist, the default is TRUE.
+#' @param time Character vector or NULL. Hours of the day to download (e.g., c("00:00", "12:00")). Default to NULL to download all hours.
+#' @param dataset Character. Name of the CDS dataset to use (default: "reanalysis-era5-single-levels").
+#' @param product_type Character. Product type to request from CDS (default: "ensemble_members").
+#' @param user Character. CDS user ID (UID) from your CDS profile. Required for authentication.
+#' @param key Character. CDS API key from your CDS profile. Required for authentication.
 #' @param timeout numeric: the maximum time (in seconds) allowed to download the data. The default is 36000 seconds.
 #'
-#' @return A vector containing file paths to the downloaded files.
+#' @return
+#' A list where each element is a list containing:
+#'   \item{file}{File path to the downloaded NetCDF file.}
+#'   \item{host}{Host name where the file was downloaded.}
+#'   \item{startdate}{Start date and time of the data in the file.}
+#'   \item{enddate}{End date and time of the data in the file.}
+#'   \item{mimetype}{MIME type of the file ("application/x-netcdf").}
+#'   \item{formatname}{Format name ("ERA5_year.nc").}
+#'
+#' @examples
+#' \dontrun{
+#' # Download ERA5 reanalysis data for 2020
+#' output_dir <- withr::local_tempdir()
+#' era5_files <- download.ERA5_cds(
+#'   outfolder = output_dir,
+#'   start_date = "2020-01-01",
+#'   end_date = "2020-06-30",
+#'   extent = c(-72.2215, -72.1215, 42.4878, 42.5878),
+#'   variables = c("2m_temperature", "surface_pressure"),
+#'   user = "your_cds_user_id",
+#'   key = "your_cds_api_key",
+#'   product_type = "reanalysis"
+#' )
+#' 
+#' # Download ensemble data for specificed hours only
+#' era5_files <- download.ERA5_cds(
+#'   outfolder = output_dir,
+#'   start_date = "2020-01-01",
+#'   end_date = "2020-12-31",
+#'   extent = c(-83.05, -82.95, 42.95, 43.05),
+#'   variables = "surface_solar_radiation_downwards",
+#'   user = "your_cds_user_id",
+#'   key = "your_cds_api_key",
+#'   time = c("00:00", "12:00")
+#' )
+#' }
 #' @export
 #' 
-#' @importFrom purrr %>%
-#' @author Dongchen Zhang
-download.ERA5_cds <- function(outfolder, start_date, end_date, extent, variables, auto.create.key = T, timeout = 36000) {
-  # check shell environments.
-  if ("try-error" %in% class(try(system("grib_to_netcdf"), silent = T))) {
-    PEcAn.logger::logger.info("The grib_to_netcdf function is not detected in shell command.")
-    return(NA)
+#' @author Dongchen Zhang, Akash
+
+download.ERA5_cds <- function(outfolder, start_date, end_date,
+                              extent, variables, user, key, time = NULL,
+                              dataset = "reanalysis-era5-single-levels",
+                              product_type = "ensemble_members",
+                              timeout = 36000) {
+  
+  # check for required package
+  if (!requireNamespace("ecmwfr", quietly = TRUE)) {
+    PEcAn.logger::logger.severe(
+      "Package 'ecmwfr' is required for ERA5 downloads. ",
+      "Install with: install.packages('ecmwfr'). ",
+      "Get CDS credentials from: https://cds.climate.copernicus.eu/profile"
+    )
   }
+  
+  if (!dir.exists(outfolder)) dir.create(outfolder, recursive = TRUE)
+  
   # setup timeout for download.
   options(timeout=timeout)
   # convert arguments to CDS API specific arguments.
   years <- sort(unique(lubridate::year(seq(lubridate::date(start_date), lubridate::date(end_date), "1 year"))))
-  months <- sort(unique(lubridate::month(seq(lubridate::date(start_date), lubridate::date(end_date), "1 month")))) %>% 
+  months <- sort(unique(lubridate::month(seq(lubridate::date(start_date), lubridate::date(end_date), "1 month")))) |> 
     purrr::map(function(d)sprintf("%02d", d))
-  days <- sort(unique(lubridate::day(seq(lubridate::date(start_date), lubridate::date(end_date), "1 day")))) %>% 
+  days <- sort(unique(lubridate::day(seq(lubridate::date(start_date), lubridate::date(end_date), "1 day")))) |> 
     purrr::map(function(d)sprintf("%02d", d))
-  times  <- list('00:00','03:00','06:00',
-                 '09:00','12:00','15:00',
-                 '18:00','21:00')
-  area <- paste(c(extent[4], extent[1], extent[3], extent[2]), collapse = "/")
-  variables <- as.list(variables)
-  #load cdsapi from python environment.
-  tryCatch({
-    cdsapi <- reticulate::import("cdsapi")
-  }, error = function(e) {
-    PEcAn.logger::logger.severe(
-      "Failed to load `cdsapi` Python library. ",
-      "Please make sure it is installed to a location accessible to `reticulate`.",
-      "You should be able to install it with the following command: ",
-      "`pip install --user cdsapi`.",
-      "The following error was thrown by `reticulate::import(\"cdsapi\")`: ",
-      conditionMessage(e)
-    )
-  })
-  #define function for building credential file.
-  #maybe as a helper function.
-  getnetrc <- function (dl_dir) {
-    netrc <- file.path(dl_dir, ".cdsapirc")
-    if (file.exists(netrc) == FALSE ||
-        any(grepl("https://cds.climate.copernicus.eu/api/v2",
-                  readLines(netrc))) == FALSE) {
-      netrc_conn <- file(netrc)
-      writeLines(c(
-        sprintf(
-          "url: %s",
-          getPass::getPass(msg = "Enter URL from the following link \n (https://cds.climate.copernicus.eu/api-how-to#install-the-cds-api-key):")
-        ),
-        sprintf(
-          "key: %s",
-          getPass::getPass(msg = "Enter KEY from the following link \n (https://cds.climate.copernicus.eu/api-how-to#install-the-cds-api-key):")
-        )
-      ),
-      netrc_conn)
-      close(netrc_conn)
-      message(
-        "A netrc file with your CDS Login credentials was stored in the output directory "
-      )
-    }
-    return(netrc)
+  
+  # handle time argument: all hours if Null
+  if (is.null(time)) {
+    times <- sprintf("%02d:00", 0:23)
+  } else {
+    times <- time
   }
-  #check if the token exists for the cdsapi.
-  if (!file.exists(file.path(Sys.getenv("HOME"), ".cdsapirc")) & auto.create.key) {
-    if ("try-error" %in% class(try(find.package("getPass")))) {
-      PEcAn.logger::logger.info("The getPass pacakge is not installed for creating the API key.")
-      return(NA)
-    } else {
-      getnetrc(Sys.getenv("HOME"))
-    }
-  } else if (!file.exists(file.path(Sys.getenv("HOME"), ".cdsapirc")) & !auto.create.key) {
+  
+  # Format area for CDS API (North, West, South, East)
+  area <- round(c(extent[4], extent[1], extent[3], extent[2]), 2)
+  variables <- as.list(variables)
+  
+  # Set CDS credentials
+  if (is.null(user) || is.null(key)) {
     PEcAn.logger::logger.severe(
-      "Please create a `${HOME}/.cdsapirc` file as described here:",
-      "https://cds.climate.copernicus.eu/api-how-to#install-the-cds-api-key ."
+      "CDS 'user' and 'key' must be provided. ",
+      "Get them from: https://cds.climate.copernicus.eu/profile"
     )
   }
-  #grab the client object.
-  tryCatch({
-    c <- cdsapi$Client()
-  }, error = function(e) {
-    PEcAn.logger::logger.severe(
-      "The following error was thrown by `cdsapi$Client()`: ",
-      conditionMessage(e)
-    )
-  })
+  ecmwfr::wf_set_key(user = user, key = key)
+
   # loop over years.
   nc.paths <- c()
   for (y in years) {
-    fname <- file.path(outfolder, paste0("ERA5_", y, ".grib"))
-    # start retrieving data.
-    # you need to have an account for downloaing the files
-    # Read the documantion for how to setup your account and settings before trying this
-    # https://confluence.ecmwf.int/display/CKB/How+to+download+ERA5#HowtodownloadERA5-3-DownloadERA5datathroughtheCDSAPI
-    c$retrieve(
-      'reanalysis-era5-single-levels',
-      list(
-        'product_type' = 'ensemble_members',
-        'data_format' = 'grib',
-        "download_format" = "unarchived",
-        'day' = days,
-        'time' = times,
-        'month' = months,
-        'year' = as.character(y),
-        "area" = area,
-        'variable' = variables
-      ),
-      fname
+    fname <- file.path(outfolder, paste0("ERA5_", y, ".nc"))
+
+    request <- list(
+      dataset_short_name = dataset,
+      product_type = list(product_type),
+      data_format = 'netcdf',
+      download_format = "unarchived",
+      day = days,
+      time = times,
+      month = months,
+      year = list(as.character(y)),
+      area = area,
+      variable = variables,
+      target = basename(fname)
     )
-    # convert grib to nc file.
-    nc.path <- gsub(".grib", ".nc", fname, fixed = T)
-    cmd <- paste("grib_to_netcdf", fname, "-o", nc.path)
-    out <- system(cmd, intern = F, ignore.stdout = T, ignore.stderr = T)
-    # store the path.
-    nc.paths <- c(nc.paths, nc.path)
-    # remove previous grib file.
-    unlink(fname)
+    
+    # Submit request using ecmwfr
+    tryCatch({
+      ecmwfr::wf_request(
+        request = request,
+        user = user,
+        path = outfolder,
+        time_out = timeout
+      )
+      nc.paths <- c(nc.paths, fname)
+    }, error = function(e) {
+      PEcAn.logger::logger.error(
+        "Failed to download data for year ", y, ": ",
+        conditionMessage(e)
+      )
+    })
   }
+  
   # construct results to meet the requirements of pecan.met workflow.
   results <- vector("list", length = length(years))
   for (i in seq_along(results)) {