Skip to content

Commit e043879

Browse files
authored
Merge pull request #3592 from Sr31bu/Debias-Branch
Debias branch
2 parents d8f0ba6 + 239b708 commit e043879

17 files changed

Lines changed: 1468 additions & 12 deletions

docker/depends/pecan_package_dependencies.csv

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,6 @@
180180
"magrittr","*","base/db","Imports",FALSE
181181
"magrittr","*","base/utils","Imports",FALSE
182182
"magrittr","*","models/ed","Imports",FALSE
183-
"magrittr","*","modules/assim.sequential","Imports",FALSE
184183
"magrittr","*","modules/benchmark","Imports",FALSE
185184
"magrittr","*","modules/data.land","Imports",FALSE
186185
"magrittr","*","modules/data.remote","Imports",FALSE
@@ -477,6 +476,7 @@
477476
"reshape2","*","modules/benchmark","Imports",FALSE
478477
"reshape2","*","modules/data.atmosphere","Imports",FALSE
479478
"reshape2",">= 1.4.2","modules/assim.sequential","Suggests",FALSE
479+
"reticulate","*","modules/assim.sequential","Imports",FALSE
480480
"reticulate","*","modules/data.atmosphere","Suggests",FALSE
481481
"reticulate","*","modules/data.land","Suggests",FALSE
482482
"reticulate","*","modules/data.remote","Imports",FALSE

modules/assim.sequential/DESCRIPTION

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ Imports:
1919
future,
2020
ggplot2,
2121
lubridate (>= 1.6.0),
22-
magrittr,
2322
Matrix,
2423
ncdf4,
2524
nimble,
@@ -30,6 +29,7 @@ Imports:
3029
PEcAn.uncertainty,
3130
PEcAn.workflow,
3231
purrr,
32+
reticulate,
3333
rlang,
3434
stringr
3535
Suggests:

modules/assim.sequential/NAMESPACE

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,5 +71,4 @@ import(nimble)
7171
importFrom(dplyr,"%>%")
7272
importFrom(foreach,"%dopar%")
7373
importFrom(lubridate,"%m+%")
74-
importFrom(magrittr,"%>%")
7574
importFrom(rlang,.data)

modules/assim.sequential/R/Helper.functions.R

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
#' @return A list the same dimension as X, with each column of each dataframe
88
#' modified by replacing outlier points with the column median
99
#' @export
10-
#' @importFrom magrittr %>%
10+
#' @importFrom dplyr %>%
1111
#'
1212
outlier.detector.boxplot<-function(X) {
1313
X <- X %>%
@@ -86,7 +86,7 @@ SDA_control <-
8686
#' @description This function uses a set of scaling factors defined in the pecan XML to scale a given matrix
8787
#' @return rescaled Matrix
8888
#' @export
89-
#' @importFrom magrittr %>%
89+
#' @importFrom dplyr %>%
9090
rescaling_stateVars <- function(settings, X, multiply=TRUE) {
9191

9292
FUN <- ifelse(multiply, .Primitive('*'), .Primitive('/'))

modules/assim.sequential/R/Prep_OBS_SDA.R

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
#'
88
#' @return mean and covariance of observations
99
#'
10-
#' @importFrom magrittr %>%
10+
#' @importFrom dplyr %>%
1111
#' @export
1212
#'
1313
Prep_OBS_SDA <- function(settings, out_dir, AGB_dir, Search_Window=30){
Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
#' Generate site-year covariates from yearly GeoTIFF stacks (internal)
2+
#'
3+
#' Scans `cov_dir` for files like `<file_prefix><YYYY>.tiff`, extracts raster
4+
#' values at the provided site coordinates, and returns a long tibble.
5+
#'
6+
#' @param site_coords data.frame with columns: site (chr or coercible), lon (num), lat (num).
7+
#' @param cov_dir directory containing yearly multi-layer GeoTIFFs.
8+
#' @param crs CRS string for the input points (default "EPSG:4326").
9+
#' @param file_prefix character prefix before the 4-digit year (default "covariates_").
10+
#' Ignored if `file_regex` is provided.
11+
#' @param file_regex optional full regex to match files (must contain a 4-digit year).
12+
#'
13+
#' @return tibble with columns site, year, and per-layer covariates.
14+
#' @keywords internal
15+
#' @noRd
16+
generate_covariates_df <- function(site_coords,
17+
cov_dir,
18+
crs = "EPSG:4326",
19+
file_prefix = "covariates_",
20+
file_regex = NULL) {
21+
if (!dir.exists(cov_dir)) stop("`cov_dir` does not exist: ", cov_dir)
22+
23+
# validate site_coords
24+
if (!all(c("lon", "lat") %in% names(site_coords))) {
25+
stop("`site_coords` must have columns: lon, lat (and ideally site).")
26+
}
27+
if (!("site" %in% names(site_coords))) site_coords$site <- seq_len(nrow(site_coords))
28+
site_coords$site <- as.character(site_coords$site)
29+
30+
site_coords$lon <- suppressWarnings(as.numeric(site_coords$lon))
31+
site_coords$lat <- suppressWarnings(as.numeric(site_coords$lat))
32+
if (anyNA(site_coords$lon) || anyNA(site_coords$lat)) {
33+
bad <- site_coords$site[is.na(site_coords$lon) | is.na(site_coords$lat)]
34+
stop("Found non-numeric lon/lat for sites: ", paste(bad, collapse = ", "))
35+
}
36+
37+
# build points
38+
coords_mat <- as.matrix(site_coords[, c("lon", "lat")])
39+
pts <- terra::vect(coords_mat, type = "points", crs = crs)
40+
pts$site <- site_coords$site
41+
42+
# discover files/years
43+
pattern <- if (is.null(file_regex)) {
44+
# escape any regex chars in prefix, then expect YYYY.tiff
45+
paste0("^",
46+
stringr::str_replace_all(file_prefix, "([\\^$.|?*+()\\[\\]{}])", "\\\\\\1"),
47+
"\\d{4}\\.tiff$")
48+
} else {
49+
file_regex
50+
}
51+
tif_files <- list.files(cov_dir, pattern = pattern, full.names = TRUE)
52+
if (length(tif_files) == 0) {
53+
stop("No files matched pattern in: ", cov_dir, " (pattern: ", pattern, ")")
54+
}
55+
56+
years <- as.integer(stringr::str_extract(basename(tif_files), "\\d{4}"))
57+
if (any(is.na(years))) {
58+
stop("Could not parse years from filenames: ",
59+
paste(basename(tif_files)[is.na(years)], collapse = ", "))
60+
}
61+
ord <- order(years)
62+
tif_files <- tif_files[ord]
63+
years <- years[ord]
64+
65+
# per-year extractor
66+
extract_year <- function(tif_path, year) {
67+
r <- terra::rast(tif_path)
68+
vals <- terra::extract(r, pts)
69+
70+
if ("ID" %in% names(vals)) {
71+
vals <- vals[, setdiff(names(vals), "ID"), drop = FALSE]
72+
}
73+
74+
out <- dplyr::as_tibble(vals)
75+
if (nrow(out) != nrow(site_coords)) {
76+
stop("Row mismatch for year ", year, ": expected ", nrow(site_coords),
77+
" but got ", nrow(out), ". Check CRS/coordinates or raster extent.")
78+
}
79+
80+
dplyr::mutate(out, site = site_coords$site, year = as.integer(year)) |>
81+
dplyr::select("site", "year", dplyr::everything())
82+
}
83+
84+
purrr::map2_dfr(tif_files, years, extract_year)
85+
}
Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#' Load debias Python module (internal)
2+
#'
3+
#' Locate and import the Python residual-model module used by the SDA debias step.
4+
#' The search order prefers installed package paths and then falls back to a
5+
#' developer tree when running from source.
6+
#'
7+
#' **Search logic (first match wins):**
8+
#' 1. Installed package dirs: `system.file("python", pkg)`, `system.file("python_models", pkg)`
9+
#' 2. Dev fallbacks (from namespace path or `inst/`):
10+
#' - `<pkg>/python`, `<pkg>/python_models`
11+
#' - `inst/python`, `inst/python_models`
12+
#'
13+
#' **Module names recognized:**
14+
#' - Package directory: `pecan_debias/__init__.py` → imports `"pecan_debias"`
15+
#' - Single file: `debias.py` → imports `"debias"`
16+
#'
17+
#' The imported module is cached across calls. Pass `reload = TRUE` to force
18+
#' re-import (e.g., after editing the Python code in development).
19+
#'
20+
#' @param reload Logical; if `TRUE`, force re-import even if a cached module exists.
21+
#'
22+
#' @return A reticulate Python module object. Errors if no suitable path is found
23+
#' or if the import returns a null Python pointer.
24+
#'
25+
#' @keywords internal
26+
#' @noRd
27+
.get_debias_mod <- local({
28+
mod <- NULL
29+
function(reload = FALSE) {
30+
if (reload || is.null(mod) || reticulate::py_is_null_xptr(mod)) {
31+
32+
pkg <- "PEcAnAssimSequential"
33+
34+
# Installed package roots
35+
roots <- Filter(nzchar, c(
36+
system.file("python", package = pkg),
37+
system.file("python_models", package = pkg)
38+
))
39+
40+
# Dev fallbacks (if running from source)
41+
if (!length(roots)) {
42+
ns_path <- tryCatch(getNamespaceInfo(pkg, "path"), error = function(e) NA_character_)
43+
roots <- unique(stats::na.omit(c(
44+
file.path(ns_path, "python"),
45+
file.path(ns_path, "python_models"),
46+
normalizePath(file.path("inst", "python"), mustWork = FALSE),
47+
normalizePath(file.path("inst", "python_models"), mustWork = FALSE)
48+
)))
49+
roots <- roots[dir.exists(roots)]
50+
}
51+
52+
if (!length(roots)) {
53+
stop("Could not find a python dir (inst/python or inst/python_models) in ", pkg, ".")
54+
}
55+
56+
root <- roots[1]
57+
# Prefer package dir; else single file
58+
if (dir.exists(file.path(root, "pecan_debias")) &&
59+
file.exists(file.path(root, "pecan_debias", "__init__.py"))) {
60+
mod_name <- "pecan_debias"
61+
} else if (file.exists(file.path(root, "debias.py"))) {
62+
mod_name <- "debias"
63+
} else {
64+
stop("Expected either 'pecan_debias/__init__.py' or 'debias.py' under: ", root)
65+
}
66+
67+
mod <<- reticulate::import_from_path(mod_name, path = root, convert = TRUE)
68+
if (reticulate::py_is_null_xptr(mod)) {
69+
stop("Import returned a null Python object (py_is_null_xptr == TRUE).")
70+
}
71+
}
72+
mod
73+
}
74+
})
75+

0 commit comments

Comments
 (0)