confusion between metric name and metric type

[topepo]

For custom metrics, tune doesn't work well, mostly because it can't differentiate between the metric type and the name.

library(tidymodels)

set.seed(1)
rs <- vfold_cv(two_class_dat)

custom_cost <- function(
  data,
  truth,
  ...,
  na_rm = TRUE,
  event_level = "Class1",
  case_weights = NULL
) {
  custom_cost_vals <- tibble::tribble(
    ~truth   , ~estimate , ~cost ,
    "Class1" , "Class2"  ,    10 ,
    "Class2" , "Class1"  ,     2
  )

  classification_cost(
    data = data,
    truth = !!rlang::enquo(truth),
    ...,
    na_rm = na_rm,
    costs = custom_cost_vals
  )
}

custom_cost <- new_prob_metric(custom_cost, "minimize")

cls_mtr <- metric_set(custom_cost, accuracy, roc_auc)

cart_wflow <- workflow(
  Class ~ .,
  decision_tree(min_n = tune(), mode = "classification")
)

set.seed(2)
res <-
  cart_wflow |>
  tune_grid(resamples = rs, metrics = cls_mtr)

# Errors in finding the metric: 

select_best(res)
#> Warning in select_best(res): No value of `metric` was given; "custom_cost" will
#> be used.
#> Error in `.filter_perf_metrics()`:
#> ! No results are available. Please use `collect_metrics()` to see if
#>   there were any issues.
autoplot(res)

# This works though:
.get_tune_metric_names(res)
#> [1] "custom_cost" "accuracy"    "roc_auc"

# The .metric value is "classification_cost": 
res |> collect_metrics() |> count(.metric)
#> # A tibble: 3 × 2
#>   .metric                 n
#>   <chr>               <int>
#> 1 accuracy               10
#> 2 classification_cost    10
#> 3 roc_auc                10

# When subsetting, it looks for `metric = "custom_cost"` but the data has
# a `.metric` value of "classification_cost" 

set.seed(2)
res_bayz <-
  cart_wflow |>
  tune_bayes(resamples = rs, metrics = cls_mtr)
#> → A | error:   subscript out of bounds
#> There were issues with some computations   A: x1
#> Error in `check_gp_failure()`:
#> ! Gaussian process model was not fit.
#> ✖ Optimization stopped prematurely; returning current results.
#> There were issues with some computations   A: x1There were issues with some computations   A: x1

^{Created on 2026-01-12 with reprex v2.1.1}

[topepo]

It looks like it comes down to using the instructions in ?metric_set and using metric_tweak():

library(tidymodels)

tidymodels_prefer()
theme_set(theme_bw())
options(pillar.advice = FALSE, pillar.min_title_chars = Inf)

ccc_manual <- function(data, truth, estimate, na_rm = TRUE, ...) {
  ccc(
    data = data,
    truth = !!rlang::enquo(truth),
    estimate = !!rlang::enquo(estimate),
    # set bias = TRUE
    bias = TRUE,
    na_rm = na_rm,
    ...
  )
}

ccc_manual <- new_numeric_metric(ccc_manual, "maximize")
mtr_manual <- metric_set(rmse, rsq, ccc, ccc_manual)
mtr_manual
#> A metric set, consisting of:
#> - `rmse()`, a numeric metric       | direction: minimize
#> - `rsq()`, a numeric metric        | direction: maximize
#> - `ccc()`, a numeric metric        | direction: maximize
#> - `ccc_manual()`, a numeric metric | direction: maximize

ccc_tweak <- metric_tweak("ccc_tweak", ccc, bias = TRUE)
mtr_tweak <- metric_set(rmse, rsq, ccc, ccc_tweak)
mtr_tweak
#> A metric set, consisting of:
#> - `rmse()`, a numeric metric      | direction: minimize
#> - `rsq()`, a numeric metric       | direction: maximize
#> - `ccc()`, a numeric metric       | direction: maximize
#> - `ccc_tweak()`, a numeric metric | direction: maximize

set.seed(82)
dat <- sim_regression(200)
rs <- vfold_cv(dat)
ctrl <- control_resamples(save_workflow = TRUE)

res_manual<- 
  linear_reg() |> 
  fit_resamples(
    outcome ~ .,
    resamples = rs,
    metrics = mtr_manual,
    control = ctrl
  )

# Double the ccc
res_manual |> collect_metrics()
#> # A tibble: 3 × 6
#>   .metric .estimator   mean     n std_err .config        
#>   <chr>   <chr>       <dbl> <int>   <dbl> <chr>          
#> 1 ccc     standard    0.424    20  0.0289 pre0_mod0_post0
#> 2 rmse    standard   17.7      10  1.27   pre0_mod0_post0
#> 3 rsq     standard    0.254    10  0.0393 pre0_mod0_post0

# No can do
res_manual |> fit_best(metric = "ccc_manual")
#> Error in `.filter_perf_metrics()` at tune/R/select_best.R:95:3:
#> ! No results are available. Please use `collect_metrics()` to see if
#>   there were any issues.

res_tweak <- 
  linear_reg() |> 
  fit_resamples(
    outcome ~ .,
    resamples = rs,
    metrics = mtr_tweak,
    control = ctrl
  )

res_tweak |> collect_metrics()
#> # A tibble: 4 × 6
#>   .metric   .estimator   mean     n std_err .config        
#>   <chr>     <chr>       <dbl> <int>   <dbl> <chr>          
#> 1 ccc       standard    0.424    10  0.0420 pre0_mod0_post0
#> 2 ccc_tweak standard    0.424    10  0.0420 pre0_mod0_post0
#> 3 rmse      standard   17.7      10  1.27   pre0_mod0_post0
#> 4 rsq       standard    0.254    10  0.0393 pre0_mod0_post0

res_tweak |> fit_best(metric = "ccc_tweak")
#> ══ Workflow [trained] ══════════════════════════════════════════════════════════
#> Preprocessor: Formula
#> Model: linear_reg()
#> 
#> ── Preprocessor ────────────────────────────────────────────────────────────────
#> outcome ~ .
#> 
#> ── Model ───────────────────────────────────────────────────────────────────────
#> 
#> Call:
#> stats::lm(formula = ..y ~ ., data = data)
#> 
#> Coefficients:
#>  (Intercept)  predictor_01  predictor_02  predictor_03  predictor_04  
#>     13.60549       0.13797       0.15002      -0.51715       0.63060  
#> predictor_05  predictor_06  predictor_07  predictor_08  predictor_09  
#>     -0.10119      -0.29861      -0.54978      -0.27574      -0.41859  
#> predictor_10  predictor_11  predictor_12  predictor_13  predictor_14  
#>     -0.10499       0.60764       0.49683      -0.43613       2.43012  
#> predictor_15  predictor_16  predictor_17  predictor_18  predictor_19  
#>     -0.22484       0.82360      -0.30524      -2.44882       0.09636  
#> predictor_20  
#>      0.70345

^{Created on 2026-02-03 with reprex v2.1.1}

I think that we should remove the instructions and reference metric_tweak() for everything.