Adding pred_measure feature

.Rbuildignore

@@ -18,7 +18,7 @@ vignettes/loo2-non-factorized_cache/*
 .github/*
 .vscode/*
 ^\.github$
-^vignettes/online-only$
+^vignettes/articles-online-only$
 
 ^CRAN-SUBMISSION$
 ^release-prep\.R$
@@ -27,3 +27,11 @@ vignettes/loo2-non-factorized_cache/*
 ^touchstone$
 ^.git-blame-ignore-revs
 ^notes-release.md
+^notes$
+^\.agents$
+^agent$
+^\.claude$
+^\.refactor-review$
+^internal-notes$
+^scratch-files$
+^skills-lock\.json$

.github/workflows/pkgdown.yaml

@@ -39,7 +39,7 @@ jobs:
             local::.
             any::bayesplot
             any::BH
-            any::brms
+            paul-buerkner/brms
             any::ggplot2
             any::rmarkdown
             any::Rcpp
@@ -52,6 +52,7 @@ jobs:
             any::StanHeaders
             any::knitr
             any::withr
+            any::lme4
             stan-dev/pkgdown-config
 
       - name: Build site

.gitignore

@@ -19,11 +19,17 @@ vignettes/loo2-non-factorizable_cache/*
 vignettes/*.html
 vignettes/*.pdf
 inst/doc
+*.html
 
 revdep/*
 tests/testthat/Rplots.pdf
 
 cran-comments.md
 CRAN-RELEASE
 release-prep.R
+
+# personal maintainer scratch (not shared)
+internal-notes/
+notes/loo_se.pdf
+notes/loo-compare-se-diff.md
 CRAN-SUBMISSION

DESCRIPTION

@@ -36,6 +36,7 @@ Depends:
     R (>= 3.5)
 Imports:
     checkmate,
+    cli (>= 3.4.0),
     matrixStats (>= 0.52),
     parallel,
     posterior (>= 1.7.0),
@@ -46,6 +47,7 @@ Suggests:
     ggplot2,
     graphics,
     knitr,
+    lme4,
     rmarkdown,
     rstan,
     rstanarm (>= 2.19.0),

NAMESPACE

@@ -26,6 +26,7 @@ S3method(crps,numeric)
 S3method(dim,importance_sampling)
 S3method(dim,kfold)
 S3method(dim,loo)
+S3method(dim,pred_measure)
 S3method(dim,psis_loo)
 S3method(dim,waic)
 S3method(elpd,array)
@@ -47,18 +48,22 @@ S3method(loo_moment_match,default)
 S3method(loo_predictive_metric,matrix)
 S3method(loo_scrps,matrix)
 S3method(loo_subsample,"function")
+S3method(model_compare,default)
+S3method(model_compare,psis_loo_ss_list)
 S3method(nobs,psis_loo_ss)
 S3method(plot,loo)
 S3method(plot,psis)
 S3method(plot,psis_loo)
 S3method(pointwise,loo)
 S3method(print,compare.loo)
-S3method(print,compare.loo_ss)
 S3method(print,importance_sampling)
 S3method(print,importance_sampling_loo)
 S3method(print,kfold)
 S3method(print,loo)
+S3method(print,loo_pred_measure)
+S3method(print,measure)
 S3method(print,pareto_k_table)
+S3method(print,pred_measure)
 S3method(print,pseudobma_bb_weights)
 S3method(print,pseudobma_weights)
 S3method(print,psis)
@@ -102,12 +107,14 @@ export(.thin_draws)
 export(E_loo)
 export(compare)
 export(crps)
+export(custom_measure)
 export(elpd)
 export(example_loglik_array)
 export(example_loglik_matrix)
 export(extract_log_lik)
 export(find_model_names)
 export(gpdfit)
+export(insample_pred_measure)
 export(is.kfold)
 export(is.loo)
 export(is.psis)
@@ -116,6 +123,7 @@ export(is.sis)
 export(is.tis)
 export(is.waic)
 export(kfold)
+export(kfold_pred_measure)
 export(kfold_split_grouped)
 export(kfold_split_random)
 export(kfold_split_stratified)
@@ -134,27 +142,45 @@ export(loo_model_weights)
 export(loo_model_weights.default)
 export(loo_moment_match)
 export(loo_moment_match.default)
+export(loo_pred_measure)
 export(loo_predictive_metric)
 export(loo_scrps)
 export(loo_subsample)
 export(loo_subsample.function)
 export(mcse_loo)
+export(measure_acc)
+export(measure_bacc)
+export(measure_brier)
+export(measure_elpd)
+export(measure_ic)
+export(measure_mae)
+export(measure_mlpd)
+export(measure_mse)
+export(measure_r2)
+export(measure_rmse)
+export(measure_rps)
+export(measure_srps)
+export(model_compare)
 export(nlist)
 export(obs_idx)
 export(pareto_k_ids)
 export(pareto_k_influence_values)
 export(pareto_k_table)
 export(pareto_k_values)
 export(pointwise)
+export(pred_measure)
 export(print_dims)
 export(pseudobma_weights)
 export(psis)
 export(psis_n_eff_values)
 export(psislw)
+export(ptw_log_pred_density)
 export(relative_eff)
 export(scrps)
 export(sis)
 export(stacking_weights)
+export(supported_measures_list)
+export(test_pred_measure)
 export(tis)
 export(waic)
 export(waic.array)

NEWS.md

@@ -1,5 +1,25 @@
 # loo (development version)
 
+* Add a predictive performance API by @florence-bockting in #363:
+  - new functions `insample_pred_measure()`, `loo_pred_measure()`,
+    `kfold_pred_measure()`, `test_pred_measure()`, and `pred_measure()`
+  - new built-in measures `measure_*()`, listed in `supported_measures_list`
+* Deprecate `elpd()`, `crps()`, `scrps()`, `loo_crps()`, `loo_scrps()`, and
+  `loo_predictive_metric()`. Use `measure_*()` and `*_pred_measure()`. See
+  `vignette("migration-guide")`. By @florence-bockting in #363
+* Add `model_compare()` by @florence-bockting in #380:
+  - compares models on each measure of `*_pred_measure()` results
+  - compares each measure against its own best model
+  - shows loss measures on the utility scale
+  - ranks models by `elpd`, else by the first shared measure (alphabetical)
+  - new `print()` argument `measures`
+* Deprecate `loo_compare()`. Use `model_compare()`. Methods in other packages
+  (e.g. **brms**) still work. By @florence-bockting in #380
+* Add `custom_measure()` to set the name, loss flag, and SE of the difference
+  for a custom measure by @florence-bockting in #380
+* `loo_pred_measure()` warns when `loo_moment_match()` or `reloo()` changed the
+  `loo` object and the measure is not `elpd`, `mlpd`, or `ic`, by
+  @florence-bockting in #380
 * Improve numerical stability in `loo()`, `psis()`, model weighting, subsampling, 
 and moment matching in #395 
 * Fix `loo_compare()` when used with subsampling: compute model comparison by 

R/compare.R

@@ -1,7 +1,8 @@
 #' Model comparison (deprecated, old version)
 #'
-#' **This function is deprecated**. Please use the new [loo_compare()] function
-#' instead.
+#' **This function is deprecated**. Please use the new [model_compare()] function
+#' instead. See `vignette("migration-guide", package = "loo")` for details.
+#' `compare()` and `R/compare.R` are scheduled for removal in a future release.
 #'
 #' @export
 #' @param ... At least two objects returned by [loo()] (or [waic()]).
@@ -59,7 +60,7 @@
 #' }
 #'
 compare <- function(..., x = list()) {
-  .Deprecated("loo_compare")
+  .Deprecated("model_compare")
   dots <- list(...)
   if (length(dots)) {
     if (length(x)) {

R/crps.R

@@ -1,4 +1,15 @@
-#' Continuously ranked probability score
+#' Continuously ranked probability score (deprecated)
+#'
+#' As of loo 3.0.0, `crps()`, `scrps()`, `loo_crps()`, and `loo_scrps()` are
+#' **deprecated**. Please use [measure_rps()] and [measure_srps()] instead,
+#' or [loo_pred_measure()] for leave-one-out predictive performance.
+#' See `vignette("migration-guide", package = "loo")` for a full mapping table.
+#'
+#' @details
+#' The new functions use a single matrix of posterior predictive draws
+#' (`ypred`) and a PWM estimator. The deprecated functions require two
+#' independent draw matrices (`x` and `x2`) and use a permutation-based
+#' estimator. See [measure_rps()] for migration details.
 #'
 #' The `crps()` and `scrps()` functions and their `loo_*()` counterparts can be
 #' used to compute the continuously ranked probability score (CRPS) and scaled
@@ -34,16 +45,19 @@
 #'   The former reports estimator and standard error and latter the pointwise
 #'   values. Following Bolin & Wallin (2023), a larger value is better.
 #'
+#' @seealso [measure_rps()], [measure_srps()], [loo_pred_measure()]
+#'
 #' @examples
 #' \dontrun{
-#' # An example using rstanarm
+#' # Deprecated:
 #' library(rstanarm)
 #' data("kidiq")
 #' fit <- stan_glm(kid_score ~ mom_hs + mom_iq, data = kidiq)
 #' ypred1 <- posterior_predict(fit)
 #' ypred2 <- posterior_predict(fit)
 #' crps(ypred1, ypred2, y = fit$y)
-#' loo_crps(ypred1, ypred2, y = fit$y, log_lik = log_lik(fit))
+#' # ->
+#' measure_rps(y = fit$y, ypred = ypred1)
 #' }
 #'
 #' @references
@@ -80,6 +94,7 @@ loo_scrps <- function(x, ...) {
 #' @rdname crps
 #' @export
 crps.matrix <- function(x, x2, y, ..., permutations = 1) {
+  .Deprecated("measure_rps")
   validate_crps_input(x, x2, y)
   repeats <- replicate(permutations, EXX_compute(x, x2), simplify = F)
   EXX <- Reduce(`+`, repeats) / permutations
@@ -114,6 +129,7 @@ loo_crps.matrix <-
            permutations = 1,
            r_eff = 1,
            cores = getOption("mc.cores", 1)) {
+  .Deprecated("loo_pred_measure")
   validate_crps_input(x, x2, y, log_lik)
   repeats <- replicate(permutations,
                        EXX_loo_compute(x, x2, log_lik, r_eff = r_eff, ...),
@@ -128,6 +144,7 @@ loo_crps.matrix <-
 #' @rdname crps
 #' @export
 scrps.matrix <- function(x, x2, y, ..., permutations = 1) {
+  .Deprecated("measure_srps")
   validate_crps_input(x, x2, y)
   repeats <- replicate(permutations, EXX_compute(x, x2), simplify = F)
   EXX <- Reduce(`+`, repeats) / permutations
@@ -156,6 +173,7 @@ loo_scrps.matrix <-
     permutations = 1,
     r_eff = 1,
     cores = getOption("mc.cores", 1)) {
+  .Deprecated("loo_pred_measure")
   validate_crps_input(x, x2, y, log_lik)
   repeats <- replicate(permutations,
                        EXX_loo_compute(x, x2, log_lik, r_eff = r_eff, ...),

R/diagnostics.R

@@ -69,16 +69,15 @@
 #'   the MCMC draws from the posterior distribution to obtain more
 #'   reliable importance sampling estimates. This results in a smaller
 #'   shape parameter \eqn{k}.  See [loo_moment_match()] and the
-#'   vignette *Avoiding model refits in leave-one-out cross-validation
-#'   with moment matching* for an example of this.
+#'   vignette *Avoiding model refits with moment matching* for an
+#'   example of this.
 #'
 #' * Sampling from a leave-one-out mixture distribution (see the
-#'   vignette *Mixture IS leave-one-out cross-validation for
-#'   high-dimensional Bayesian models*), directly from \eqn{p(\theta^s
-#'   | y_{-i})} for the problematic observations \eqn{i}, or using
-#'   \eqn{K}-fold cross-validation (see the vignette *Holdout
-#'   validation and K-fold cross-validation of Stan programs with the
-#'   loo package*) will generally be more stable.
+#'   vignette *Mixture IS leave-one-out cross-validation*),
+#'   directly from \eqn{p(\theta^s | y_{-i})} for the problematic
+#'   observations \eqn{i}, or using \eqn{K}-fold cross-validation
+#'   (see the vignette *Holdout validation and K-fold cross-validation*)
+#'   will generally be more stable.
 #'
 #' * Using a model that is more robust to anomalous observations will
 #'   generally make approximate LOO-CV more stable.

R/elpd.R

@@ -1,26 +1,37 @@
-#' Generic (expected) log-predictive density
+#' Generic (expected) log-predictive density (deprecated)
+#'
+#' As of loo 3.0.0, `elpd()` is **deprecated**. Please use [measure_elpd()]
+#' instead. For full predictive performance workflows, see
+#' [insample_pred_measure()] and [loo_pred_measure()].
+#' See `vignette("migration-guide", package = "loo")` for a full mapping table.
+#'
+#' @details
+#' The return type differs: `elpd()` returns class `"elpd_generic"` with
+#' `elpd` and `ic` in `pointwise`; `measure_elpd()` returns class `"measure"`.
 #'
 #' The `elpd()` methods for arrays and matrices can compute the expected log
 #' pointwise predictive density for a new dataset or the log pointwise
 #' predictive density of the observed data (an overestimate of the elpd).
+#' The `elpd()` function is an S3 generic and methods are provided for
+#' 3-D pointwise log-likelihood arrays and matrices.
 #'
 #' @export
 #' @param x A log-likelihood array or matrix. The **Methods (by class)**
 #'   section, below, has detailed descriptions of how to specify the inputs for
 #'   each method.
 #' @param ... Currently ignored.
 #'
-#' @details The `elpd()` function is an S3 generic and methods are provided for
-#'   3-D pointwise log-likelihood arrays and matrices.
-#'
-#' @seealso The vignette *Holdout validation and K-fold cross-validation of Stan
-#'   programs with the loo package* for demonstrations of using the `elpd()`
-#'   methods.
+#' @seealso [measure_elpd()], [insample_pred_measure()], [loo_pred_measure()],
+#'   and the vignette *Holdout validation and K-fold cross-validation*.
 #'
 #' @examples
-#' # Calculate the lpd of the observed data
+#' \dontrun{
+#' # Deprecated:
 #' LLarr <- example_loglik_array()
 #' elpd(LLarr)
+#' # ->
+#' measure_elpd(LLarr)
+#' }
 #'
 elpd <- function(x, ...) {
   UseMethod("elpd")
@@ -31,22 +42,28 @@ elpd <- function(x, ...) {
 #' @template array
 #'
 elpd.array <- function(x, ...) {
+  .Deprecated("measure_elpd")
   ll <- llarray_to_matrix(x)
-  elpd.matrix(ll)
+  .elpd_matrix_impl(ll)
 }
 
 #' @export
 #' @templateVar fn elpd
 #' @template matrix
 #'
 elpd.matrix <- function(x, ...) {
-  pointwise <- pointwise_elpd_calcs(x)
-  elpd_object(pointwise, dim(x))
+  .Deprecated("measure_elpd")
+  .elpd_matrix_impl(x)
 }
 
 
-
 # internal ----------------------------------------------------------------
+# used to avoid duplicated deprecation warning messages
+.elpd_matrix_impl <- function(x) {
+  pointwise <- pointwise_elpd_calcs(x)
+  elpd_object(pointwise, dim(x))
+}
+
 pointwise_elpd_calcs <- function(ll){
   elpd <- colLogSumExps(ll) - log(nrow(ll))
   ic <- -2 * elpd