Merge pull request #289 from stan-dev/249-use-more-functions-from-the-posterior-package

Use testthat 3e and rely on posterior for ESS

Author: jgabry

DESCRIPTION

@@ -1,35 +1,35 @@
-Package: loo
 Type: Package
+Package: loo
 Title: Efficient Leave-One-Out Cross-Validation and WAIC for Bayesian Models
 Version: 2.8.0.9000
 Date: 2024-07-03
-Authors@R: c(person("Aki", "Vehtari", email = "Aki.Vehtari@aalto.fi", role = c("aut")),
-             person("Jonah", "Gabry", email = "jsg2201@columbia.edu", role = c("cre", "aut")),
-             person("Måns", "Magnusson", role = c("aut")),
-             person("Yuling", "Yao", role = c("aut")),
-             person("Paul-Christian", "Bürkner", role = c("aut")),
-             person("Topi", "Paananen", role = c("aut")),
-             person("Andrew", "Gelman", role = c("aut")),
-             person("Ben", "Goodrich", role = c("ctb")),
-             person("Juho", "Piironen", role = c("ctb")), 
-             person("Bruno", "Nicenboim", role = c("ctb")), 
-             person("Leevi", "Lindgren", role = c("ctb")))
+Authors@R: c(
+    person("Aki", "Vehtari", , "Aki.Vehtari@aalto.fi", role = "aut"),
+    person("Jonah", "Gabry", , "jsg2201@columbia.edu", role = c("cre", "aut")),
+    person("Måns", "Magnusson", role = "aut"),
+    person("Yuling", "Yao", role = "aut"),
+    person("Paul-Christian", "Bürkner", role = "aut"),
+    person("Topi", "Paananen", role = "aut"),
+    person("Andrew", "Gelman", role = "aut"),
+    person("Ben", "Goodrich", role = "ctb"),
+    person("Juho", "Piironen", role = "ctb"),
+    person("Bruno", "Nicenboim", role = "ctb"),
+    person("Leevi", "Lindgren", role = "ctb")
+  )
 Maintainer: Jonah Gabry <jsg2201@columbia.edu>
-URL: https://mc-stan.org/loo/, https://discourse.mc-stan.org
-BugReports: https://github.com/stan-dev/loo/issues
 Description: Efficient approximate leave-one-out cross-validation (LOO)
-    for Bayesian models fit using Markov chain Monte Carlo, as 
-    described in Vehtari, Gelman, and Gabry (2017) 
-    <doi:10.1007/s11222-016-9696-4>. 
-    The approximation uses Pareto smoothed importance sampling (PSIS), 
-    a new procedure for regularizing importance weights. 
-    As a byproduct of the calculations, we also obtain approximate 
-    standard errors for estimated predictive errors and for the comparison 
-    of predictive errors between models. The package also provides methods 
-    for using stacking and other model weighting techniques to average 
-    Bayesian predictive distributions.
+    for Bayesian models fit using Markov chain Monte Carlo, as described
+    in Vehtari, Gelman, and Gabry (2017) <doi:10.1007/s11222-016-9696-4>.
+    The approximation uses Pareto smoothed importance sampling (PSIS), a
+    new procedure for regularizing importance weights.  As a byproduct of
+    the calculations, we also obtain approximate standard errors for
+    estimated predictive errors and for the comparison of predictive
+    errors between models. The package also provides methods for using
+    stacking and other model weighting techniques to average Bayesian
+    predictive distributions.
 License: GPL (>=3)
-LazyData: TRUE
+URL: https://mc-stan.org/loo/, https://discourse.mc-stan.org
+BugReports: https://github.com/stan-dev/loo/issues
 Depends:
     R (>= 3.1.2)
 Imports:
@@ -50,8 +50,13 @@ Suggests:
     rstantools,
     spdep,
     testthat (>= 2.1.0)
-VignetteBuilder: knitr
+VignetteBuilder: 
+    knitr
+Config/testthat/edition: 3
+Config/testthat/parallel: true
+Config/testthat/start-first: loo_subsampling_cases, loo_subsampling
 Encoding: UTF-8
-SystemRequirements: pandoc (>= 1.12.3), pandoc-citeproc
-RoxygenNote: 7.3.2
+LazyData: TRUE
 Roxygen: list(markdown = TRUE)
+RoxygenNote: 7.3.2
+SystemRequirements: pandoc (>= 1.12.3), pandoc-citeproc

R/effective_sample_sizes.R

@@ -63,14 +63,14 @@ relative_eff.array <- function(x, ..., cores = getOption("mc.cores", 1)) {
   S <- prod(dim(x)[1:2]) # posterior sample size = iter * chains
 
   if (cores == 1) {
-    n_eff_vec <- apply(x, 3, ess_rfun)
+    n_eff_vec <- apply(x, 3, posterior::ess_mean)
   } else {
     if (!os_is_windows()) {
       n_eff_list <-
         parallel::mclapply(
           mc.cores = cores,
           X = seq_len(dim(x)[3]),
-          FUN = function(i) ess_rfun(x[, , i, drop = TRUE])
+          FUN = function(i) posterior::ess_mean(x[, , i, drop = TRUE])
         )
     } else {
       cl <- parallel::makePSOCKcluster(cores)
@@ -79,7 +79,7 @@ relative_eff.array <- function(x, ..., cores = getOption("mc.cores", 1)) {
         parallel::parLapply(
           cl = cl,
           X = seq_len(dim(x)[3]),
-          fun = function(i) ess_rfun(x[, , i, drop = TRUE])
+          fun = function(i) posterior::ess_mean(x[, , i, drop = TRUE])
         )
     }
     n_eff_vec <- unlist(n_eff_list, use.names = FALSE)
@@ -192,82 +192,3 @@ psis_n_eff.matrix <- function(w, r_eff = NULL, ...) {
   }
   1 / ss * r_eff
 }
-
-#' MCMC effective sample size calculation
-#'
-#' @noRd
-#' @param sims An iterations by chains matrix of draws for a single parameter.
-#'   In the case of the **loo** package, this will be the **exponentiated**
-#'   log-likelihood values for the ith observation.
-#' @return MCMC effective sample size based on RStan's calculation.
-#'
-ess_rfun <- function(sims) {
-  if (is.vector(sims)) dim(sims) <- c(length(sims), 1)
-  chains <- ncol(sims)
-  n_samples <- nrow(sims)
-  acov <- lapply(1:chains, FUN = function(i) posterior::autocovariance(sims[,i]))
-  acov <- do.call(cbind, acov)
-  chain_mean <- colMeans(sims)
-  mean_var <- mean(acov[1,]) * n_samples / (n_samples - 1)
-  var_plus <- mean_var * (n_samples - 1) / n_samples
-  if (chains > 1)
-    var_plus <- var_plus + var(chain_mean)
-  # Geyer's initial positive sequence
-  rho_hat_t <- rep.int(0, n_samples)
-  t <- 0
-  rho_hat_even <- 1
-  rho_hat_t[t + 1] <- rho_hat_even
-  rho_hat_odd <- 1 - (mean_var - mean(acov[t + 2, ])) / var_plus
-  rho_hat_t[t + 2] <- rho_hat_odd
-  while (t < nrow(acov) - 5 && !is.nan(rho_hat_even + rho_hat_odd) &&
-         (rho_hat_even + rho_hat_odd > 0)) {
-    t <- t + 2
-    rho_hat_even = 1 - (mean_var - mean(acov[t + 1, ])) / var_plus
-    rho_hat_odd = 1 - (mean_var - mean(acov[t + 2, ])) / var_plus
-    if ((rho_hat_even + rho_hat_odd) >= 0) {
-      rho_hat_t[t + 1] <- rho_hat_even
-      rho_hat_t[t + 2] <- rho_hat_odd
-    }
-  }
-  max_t <- t
-  # this is used in the improved estimate
-  if (rho_hat_even>0)
-      rho_hat_t[max_t + 1] <- rho_hat_even
-
-  # Geyer's initial monotone sequence
-  t <- 0
-  while (t <= max_t - 4) {
-    t <- t + 2
-    if (rho_hat_t[t + 1] + rho_hat_t[t + 2] >
-        rho_hat_t[t - 1] + rho_hat_t[t]) {
-      rho_hat_t[t + 1] = (rho_hat_t[t - 1] + rho_hat_t[t]) / 2;
-      rho_hat_t[t + 2] = rho_hat_t[t + 1];
-    }
-  }
-  ess <- chains * n_samples
-  # Geyer's truncated estimate
-  # tau_hat <- -1 + 2 * sum(rho_hat_t[1:max_t])
-  # Improved estimate reduces variance in antithetic case
-  tau_hat <- -1 + 2 * sum(rho_hat_t[1:max_t]) + rho_hat_t[max_t+1]
-  # Safety check for negative values and with max ess equal to ess*log10(ess)
-  tau_hat <- max(tau_hat, 1/log10(ess))
-  ess <- ess / tau_hat
-  ess
-}
-
-
-fft_next_good_size <- function(N) {
-  # Find the optimal next size for the FFT so that
-  # a minimum number of zeros are padded.
-  if (N <= 2)
-    return(2)
-  while (TRUE) {
-    m = N
-    while ((m %% 2) == 0) m = m / 2
-    while ((m %% 3) == 0) m = m / 3
-    while ((m %% 5) == 0) m = m / 5
-    if (m <= 1)
-      return(N)
-    N = N + 1
-  }
-}

tests/testthat/_snaps/E_loo.md

@@ -0,0 +1,110 @@
+# E_loo.default equal to snapshots
+
+    WAoAAAACAAQFAAACAwAAAAITAAAAAgAAAA4AAAABP5YkyJk2Uw4AAAAOAAAAAT/CCf5d2lYl
+    AAAEAgAAAAEABAAJAAAABW5hbWVzAAAAEAAAAAIABAAJAAAABXZhbHVlAAQACQAAAAhwYXJl
+    dG9fawAAAP4=
+
+---
+
+    WAoAAAACAAQFAAACAwAAAAITAAAAAgAAAA4AAAABP+99QfBwyEoAAAAOAAAAAT/CwzqDy8zd
+    AAAEAgAAAAEABAAJAAAABW5hbWVzAAAAEAAAAAIABAAJAAAABXZhbHVlAAQACQAAAAhwYXJl
+    dG9fawAAAP4=
+
+---
+
+    WAoAAAACAAQFAAACAwAAAAITAAAAAgAAAA4AAAABP+++XajbmJQAAAAOAAAAAT/CwzqDy8zd
+    AAAEAgAAAAEABAAJAAAABW5hbWVzAAAAEAAAAAIABAAJAAAABXZhbHVlAAQACQAAAAhwYXJl
+    dG9fawAAAP4=
+
+---
+
+    WAoAAAACAAQFAAACAwAAAAITAAAAAgAAAA4AAAABP5Q/UH4+kokAAAAOAAAAAT+pC/0t4hZY
+    AAAEAgAAAAEABAAJAAAABW5hbWVzAAAAEAAAAAIABAAJAAAABXZhbHVlAAQACQAAAAhwYXJl
+    dG9fawAAAP4=
+
+---
+
+    WAoAAAACAAQFAAACAwAAAAITAAAAAgAAAA4AAAADv/RdTp/OKVA/lD9Qfj6SiT/0B+XkzCLI
+    AAAADgAAAAE/qQv9LeIWWAAABAIAAAABAAQACQAAAAVuYW1lcwAAABAAAAACAAQACQAAAAV2
+    YWx1ZQAEAAkAAAAIcGFyZXRvX2sAAAD+
+
+# E_loo.matrix equal to snapshots
+
+    WAoAAAACAAQFAAACAwAAAAITAAAAAgAAAA4AAAAgP5YkyJk2Uw4/qWAe5+slHb+UGxtRYEj6
+    v3phLSMHz52/odiRUA18Oz+j5kMZY+OZv6UiBYpZA1c/gOCbIdkALD+aGlgT7KOMv6le704r
+    1Mk/lQ5ZQDf3uj+ewlXQr8iOv7Cx0vAv3x8/qBb/vlaNmD+RVknGML4AP5YmXOYjb6E/oOks
+    rHS+3b+SlEsEisU6v5de/GugPA6/poyaPupPOr+GCwyWu+Dpv3vFGSbfabK/mtljUf0Mlz+K
+    vNxsu3xMv6N5GBqP57a/jGhAZ3mD+L+mlNu35k+lP3ecrpcfLVC/dzGNuHKBtD+g/BS/OEnN
+    v4Igw4FPLoM/WQ0Eo+5f+gAAAA4AAAAgP8IJ/l3aViU/pca2RdOM3j+/8rXkBqCrv6pIWFgx
+    3Ro/zBDTTM3Ckz+e+pZvzROmP8SF8jCU81Y/2kxN82LpUD+ht5lbUInzP8PjBotEZAk/sBld
+    t19woT+1Fp9rH8ZgP7mOpdZ+VZ8/uQOl7kpZID/YnY9OInbSP9Gy/iCM9TY/4pVb5TMm8D/h
+    I59hPK5oP9i6DzMC02U/2d2gH1IGej/Fs7T0mf6XP80BWgAOvZg/wVDbzlrO2z/I8p50vNhw
+    P82+MASgNhQ/vbA2GxXFDz+ISvs8AG4SP9R81IdrKFQ/x2BUQaDJEz+7JHLaPhQmP8TX8Aiv
+    VNQ/wcnwypNNIwAABAIAAAABAAQACQAAAAVuYW1lcwAAABAAAAACAAQACQAAAAV2YWx1ZQAE
+    AAkAAAAIcGFyZXRvX2sAAAD+
+
+---
+
+    WAoAAAACAAQFAAACAwAAAAITAAAAAgAAAA4AAAAgP+99QfBwyEo/8GLAkD/xED/urslTY/r8
+    P+9KWn8Ems8/8CFJIrn2dz/vm+wdgqz5P/AqS8QTOzQ/8Cbqm5CTOT/xSPL9dvdTP+8H16ot
+    cmc/8Eke14BJfj/vYHX1dPLcP/BiQhlcUl4/78dCMDeGID/vSig0D8UoP/AOt7Yy0Nc/8QBL
+    caz6WD/v2Rg/ai74P+7+Midd/Bg/7b36n0mRlT/wXcau4TfXP+/m6IR7cp4/8DQ+EMd7Xj/u
+    46MOX9w+P/Gsf0/jcxw/7yZac/NVID/u8Trrn7y5P/AFJDL0n7s/72gByARvRT/wDbVLG9Ph
+    P+8uXczdPuI/7yVeIEvf5gAAAA4AAAAgP8LDOoPLzN0/vC9PGiC2mD/Dw665YwtSv0/zEf9g
+    gzs/wrKXeFX04D+6oCb1eqeGP9VkIEZUNbU/2kxN82LpUD+ht5lbUInzP8CPzHjviWW/oC1v
+    Ox8Noz+3SaHHVF8tv5miuj3t2Pk/yY0i+C8iSD/YnY9OInbSP9Gy/iCM9TY/3teTRr7GQT/l
+    5tcuLT5aP9i6DzMC02U/1PsEKxeyET+/YoKHfssaP80BWgAOvZg/uJULoaFQPT/I8p50vNhw
+    P9eBi3nBuZc/x4JCmLKJjj+5nmB9gzl/P9R81IdrKFQ/w+sDN0nZGz+1z55LjIrMP8TX8Aiv
+    VNQ/zWAJ7qX34gAABAIAAAABAAQACQAAAAVuYW1lcwAAABAAAAACAAQACQAAAAV2YWx1ZQAE
+    AAkAAAAIcGFyZXRvX2sAAAD+
+
+---
+
+    WAoAAAACAAQFAAACAwAAAAITAAAAAgAAAA4AAAAgP+++XajbmJQ/8DEU/8Lr5j/vVZ8WRY5+
+    P++kqukYCOA/8BCb8nKMzD/vzc6xkMxBP/AVF/p+31Y/8BNphxvXqz/woUx0rrHkP++C95Ag
+    m9k/8CRmBO/JwD/vr9aTD09nP/Aw1oOFlzI/7+OUeVFCtz/vpJF6/sziP/AHWiqornE/8H4z
+    /wmU2T/v7IYye0D5P+9+EUnmiwk/7tm0BTqf0j/wLp9qBaS7P+/zccuuIkY/8BoJ2GEYeT/v
+    cJAOK3HZP/DQ66mN/dI/75Jxr/yrxj/vd3o8ODlDP/ACkeSklNY/77OlzUqqMT/wBtkuWosN
+    P++WgQC9Cqo/75Hxz1uVVwAAAA4AAAAgP8LDOoPLzN0/vC9PGiC2mD/Dw665YwtSv0/zEf9g
+    gzs/wrKXeFX04D+6oCb1eqeGP9VkIEZUNbU/2kxN82LpUD+ht5lbUInzP8CPzHjviWW/oC1v
+    Ox8Noz+3SaHHVF8tv5miuj3t2Pk/yY0i+C8iSD/YnY9OInbSP9Gy/iCM9TY/3teTRr7GQT/l
+    5tcuLT5aP9i6DzMC02U/1PsEKxeyET+/YoKHfssaP80BWgAOvZg/uJULoaFQPT/I8p50vNhw
+    P9eBi3nBuZc/x4JCmLKJjj+5nmB9gzl/P9R81IdrKFQ/w+sDN0nZGz+1z55LjIrMP8TX8Aiv
+    VNQ/zWAJ7qX34gAABAIAAAABAAQACQAAAAVuYW1lcwAAABAAAAACAAQACQAAAAV2YWx1ZQAE
+    AAkAAAAIcGFyZXRvX2sAAAD+
+
+---
+
+    WAoAAAACAAQFAAACAwAAAAITAAAAAgAAAA4AAAAgP5Q/UH4+kok/rJhyoKzLQ7+mS88H3cZO
+    v4Diw77SBPK/nJmNhpoF3z+eJuXpnME8v6uKXkbg5Hc/iwobiZz0Yz+CcwHscaAbv7nU2Akt
+    GX+/kI6eu9QpOj+rntL08OAEv7OnDaPGnrk/qZJuk37o6j+khJ9TfTs9v3VOr8Mp0w4/Y4WW
+    2hIb6L+5piRwV8y9v51UE9Vh3d6/vDKUx9urWT+Vff+IHf9IP4PB4HUxxjK/rgu97ZVdnD91
+    l2lfiMfQv6ABPz28O8C/sFJbkh7yZL+o2Wq0N3S+P4Zn3piChxE/bj6AYsnnNj93X8jkk/2u
+    v5UhfPojTuu/pJCpz+t4lwAAAA4AAAAgP6kL/S3iFli/rllsqlAP3D+xkRFa2o0Dv6pIWFgx
+    3Rq/vbbHaFe7lL+3wSib9eF2P7e2HDVDyo8/2kxN82LpUD+ht5lbUInzv76dY0fV19G/oC1v
+    Ox8No7+xUxB4TL1Bv7MRS5NxFNQ/uQOl7kpZID/YnY9OInbSP9Gy/iCM9TY/3teTRr7GQT/V
+    Uo3UFZa8P9i6DzMC02U/1PsEKxeyET+mimNy3QAxP80BWgAOvZg/uEa4KLePjD/I8p50vNhw
+    v5z9lPVoNWw/vbA2GxXFDz+ISvs8AG4SP9R81IdrKFQ/w+sDN0nZGz+1z55LjIrMP8TX8Aiv
+    VNQ/taMpWVYn7wAABAIAAAABAAQACQAAAAVuYW1lcwAAABAAAAACAAQACQAAAAV2YWx1ZQAE
+    AAkAAAAIcGFyZXRvX2sAAAD+
+
+---
+
+    WAoAAAACAAQFAAACAwAAAAITAAAAAgAAAg4AAABAv/RdTp/OKVA/9Afl5MwiyL/z0FnOG0F+
+    P/W3tESq+hG/83dZE3d+GD/zW0+Mmka2v/XZilmZkLg/9Anod0Btdb/0f6W0vqw/P/OZiJ83
+    G76/9JCX02s4vD/z/c7djgcNv/TDQRow++g/86vcLJmh6L/0wjA4NlBjP/RMb7F0RzK/9HAc
+    YG8N8D/1mlKBbWwFv/SagAvlYpw/88LV9Rj1xb/0mH86EFjXP/QlJsXLmXy/9FUITMCdhT/0
+    YOU3NGbzv/XZiRsmTKE/8/w2ct9kJL/zwk+t/SPSP/Y2Nkb1kwi/9Jccz9A3RT/z1vUQMbyE
+    v/Szng+3zeY/9MciBArOKL/0f9xw2jLPP/Vga+lxCxC/8tVQDeo7Az/1p5eybgglv/SJCtxH
+    yEM/9AVPifNnSr/zjLE8SzISP/Ohe1nttQ6/9S6bPIHJgT/0hd4sDJmuv/RsPduiC1E/89vZ
+    GzKrar/1FZjeT/k2P/S7KA0QeGi/83Y/x5KlBz/ztNqmF3ywv/ZCUvZXPyI/9PI83KF96L/0
+    Ml3NWrX1P/Rnu9LXaiq/9WcGWiOq+D/zkg6tBDLov/T2vDTMsMs/9Og0XHuRtr/04y833CW8
+    P/NqXO3hEbG/9HEUw/KPQD/1y+CeGQebv/SL1cuvxX8/9SbAzpRiIb/zufzsmkU1P/S0K91V
+    63IAAAQCAAAAAQAEAAkAAAADZGltAAAADQAAAAIAAAACAAAAIAAAAP4AAAAOAAAAID+pC/0t
+    4hZYv65ZbKpQD9w/sZERWtqNA7+qSFhYMd0av722x2hXu5S/t8Eom/Xhdj+3thw1Q8qPP9pM
+    TfNi6VA/obeZW1CJ87++nWNH1dfRv6AtbzsfDaO/sVMQeEy9Qb+zEUuTcRTUP7kDpe5KWSA/
+    2J2PTiJ20j/Rsv4gjPU2P97Xk0a+xkE/1VKN1BWWvD/Yug8zAtNlP9T7BCsXshE/popjct0A
+    MT/NAVoADr2YP7hGuCi3j4w/yPKedLzYcL+c/ZT1aDVsP72wNhsVxQ8/iEr7PABuEj/UfNSH
+    ayhUP8PrAzdJ2Rs/tc+eS4yKzD/E1/AIr1TUP7WjKVlWJ+8AAAQCAAAAAQAEAAkAAAAFbmFt
+    ZXMAAAAQAAAAAgAEAAkAAAAFdmFsdWUABAAJAAAACHBhcmV0b19rAAAA/g==
+

tests/testthat/_snaps/compare.md

@@ -0,0 +1,56 @@
+# loo_compare returns expected results (2 models)
+
+    WAoAAAACAAQFAAACAwAAAAMOAAAAEAAAAAAAAAAAwBA6U1+cRe4AAAAAAAAAAD+2ake0LxMB
+    wFTh8N3JQljAVeWWE8MGuUARCD2zEXBfQBEalRIN2T9ACijAYdW5U0AmZ5XrANCKP/H9Zexy
+    814/8ZtgnG1nx0Bk4fDdyUJYQGXllhPDBrlAIQg9sxFwX0AhGpUSDdk/AAAEAgAAAAEABAAJ
+    AAAAA2RpbQAAAA0AAAACAAAAAgAAAAgAAAQCAAAAAQAEAAkAAAAIZGltbmFtZXMAAAATAAAA
+    AgAAABAAAAACAAQACQAAAAZtb2RlbDEABAAJAAAABm1vZGVsMgAAABAAAAAIAAQACQAAAAll
+    bHBkX2RpZmYABAAJAAAAB3NlX2RpZmYABAAJAAAACWVscGRfd2FpYwAEAAkAAAAMc2VfZWxw
+    ZF93YWljAAQACQAAAAZwX3dhaWMABAAJAAAACXNlX3Bfd2FpYwAEAAkAAAAEd2FpYwAEAAkA
+    AAAHc2Vfd2FpYwAABAIAAAABAAQACQAAAAVjbGFzcwAAABAAAAADAAQACQAAAAtjb21wYXJl
+    LmxvbwAEAAkAAAAGbWF0cml4AAQACQAAAAVhcnJheQAAAP4=
+
+# loo_compare returns expected result (3 models)
+
+    WAoAAAACAAQFAAACAwAAAAMOAAAAGAAAAAAAAAAAwBA6U1+cRe7AMA3KkbYEGAAAAAAAAAAA
+    P7ZqR7QvEwE/y6/t4TTtXsBU4fDdyUJYwFXllhPDBrnAWOVjgjbDYkARCD2zEXBfQBEalRIN
+    2T9AEPIF3GigE0AKKMBh1blTQCZnlesA0IpAQcjYUhrdCj/x/WXscvNeP/GbYJxtZ8c/8YDQ
+    kmfJX0Bk4fDdyUJYQGXllhPDBrlAaOVjgjbDYkAhCD2zEXBfQCEalRIN2T9AIPIF3GigEwAA
+    BAIAAAABAAQACQAAAANkaW0AAAANAAAAAgAAAAMAAAAIAAAEAgAAAAEABAAJAAAACGRpbW5h
+    bWVzAAAAEwAAAAIAAAAQAAAAAwAEAAkAAAAGbW9kZWwxAAQACQAAAAZtb2RlbDIABAAJAAAA
+    Bm1vZGVsMwAAABAAAAAIAAQACQAAAAllbHBkX2RpZmYABAAJAAAAB3NlX2RpZmYABAAJAAAA
+    CWVscGRfd2FpYwAEAAkAAAAMc2VfZWxwZF93YWljAAQACQAAAAZwX3dhaWMABAAJAAAACXNl
+    X3Bfd2FpYwAEAAkAAAAEd2FpYwAEAAkAAAAHc2Vfd2FpYwAABAIAAAABAAQACQAAAAVjbGFz
+    cwAAABAAAAADAAQACQAAAAtjb21wYXJlLmxvbwAEAAkAAAAGbWF0cml4AAQACQAAAAVhcnJh
+    eQAAAP4=
+
+# compare returns expected result (2 models)
+
+    Code
+      comp1
+    Output
+      elpd_diff        se 
+            0.0       0.0 
+
+---
+
+    Code
+      comp2
+    Output
+      elpd_diff        se 
+           -4.1       0.1 
+
+# compare returns expected result (3 models)
+
+    WAoAAAACAAQFAAACAwAAAAMOAAAAGAAAAAAAAAAAwBA6U1+cRe7AMA3KkbYEGAAAAAAAAAAA
+    P7ZqR7QvEwE/y6/t4TTtXsBU4fDdyUJYwFXllhPDBrnAWOVjgjbDYkARCD2zEXBfQBEalRIN
+    2T9AEPIF3GigE0AKKMBh1blTQCZnlesA0IpAQcjYUhrdCj/x/WXscvNeP/GbYJxtZ8c/8YDQ
+    kmfJX0Bk4fDdyUJYQGXllhPDBrlAaOVjgjbDYkAhCD2zEXBfQCEalRIN2T9AIPIF3GigEwAA
+    BAIAAAABAAQACQAAAANkaW0AAAANAAAAAgAAAAMAAAAIAAAEAgAAAAEABAAJAAAACGRpbW5h
+    bWVzAAAAEwAAAAIAAAAQAAAAAwAEAAkAAAACdzEABAAJAAAAAncyAAQACQAAAAJ3MwAAABAA
+    AAAIAAQACQAAAAllbHBkX2RpZmYABAAJAAAAB3NlX2RpZmYABAAJAAAACWVscGRfd2FpYwAE
+    AAkAAAAMc2VfZWxwZF93YWljAAQACQAAAAZwX3dhaWMABAAJAAAACXNlX3Bfd2FpYwAEAAkA
+    AAAEd2FpYwAEAAkAAAAHc2Vfd2FpYwAABAIAAAABAAQACQAAAAVjbGFzcwAAABAAAAAEAAQA
+    CQAAAAtjb21wYXJlLmxvbwAEAAkAAAAGbWF0cml4AAQACQAAAAVhcnJheQAEAAkAAAAPb2xk
+    X2NvbXBhcmUubG9vAAAA/g==
+

tests/testthat/_snaps/crps.md

@@ -0,0 +1,32 @@
+# crps matches snapshots
+
+    WAoAAAACAAQFAAACAwAAAAITAAAAAgAAAg4AAAACv+IeiUeluMc/vSY1w5IjJgAABAIAAAAB
+    AAQACQAAAAVuYW1lcwAAABAAAAACAAQACQAAAAhFc3RpbWF0ZQAEAAkAAAACU0UAAAD+AAAA
+    DgAAAAq/zdHHAHZD6L/K3ky9mEk4v/B2uF/xc76/38/N1JUUkL/Z4VW5uFHUv/D3XaD33ZO/
+    yPJtFZYXlL/RgM+YTul0v+9F9Rv30V+/6Q2h5ndyqgAABAIAAAH/AAAAEAAAAAIABAAJAAAA
+    CWVzdGltYXRlcwAEAAkAAAAJcG9pbnR3aXNlAAAA/g==
+
+---
+
+    WAoAAAACAAQFAAACAwAAAAITAAAAAgAAAg4AAAACv/ELnetrJtw/uJgUmYONYAAABAIAAAAB
+    AAQACQAAAAVuYW1lcwAAABAAAAACAAQACQAAAAhFc3RpbWF0ZQAEAAkAAAACU0UAAAD+AAAA
+    DgAAAAq/6MKGUxChmr/pWmzfnhawv/dXJCu4gh2/7/dH2oOKIL/tR104AbV4v/fX3UkYDWm/
+    5+ZWJz0Y3r/qbdDewNjFv/aQu+wUiEO/89yOK7F4DAAABAIAAAH/AAAAEAAAAAIABAAJAAAA
+    CWVzdGltYXRlcwAEAAkAAAAJcG9pbnR3aXNlAAAA/g==
+
+---
+
+    WAoAAAACAAQFAAACAwAAAAITAAAAAgAAAg4AAAACv+IVUtAszNU/vWbB6hSQFgAABAIAAAAB
+    AAQACQAAAAVuYW1lcwAAABAAAAACAAQACQAAAAhFc3RpbWF0ZQAEAAkAAAACU0UAAAD+AAAA
+    DgAAAAq/zKEONUJbrL/KuMhrCsFsv/BrsoFFUDu/31EWnT302L/ZnTwwjfZiv/E78Fc0R+6/
+    yM8apxh02L/RTacKeM0kv+6ksQycKl+/6bkMJjTk9gAABAIAAAH/AAAAEAAAAAIABAAJAAAA
+    CWVzdGltYXRlcwAEAAkAAAAJcG9pbnR3aXNlAAAA/g==
+
+---
+
+    WAoAAAACAAQFAAACAwAAAAITAAAAAgAAAg4AAAACv/EL1OkLv00/uNvlkZb58AAABAIAAAAB
+    AAQACQAAAAVuYW1lcwAAABAAAAACAAQACQAAAAhFc3RpbWF0ZQAEAAkAAAACU0UAAAD+AAAA
+    DgAAAAq/6JdmIec4s7/pNG4XiJ0yv/diwSdPiFi/77pjBRqbxr/tO+gNa/rMv/glDLlafB6/
+    5+SScRdhOb/qdN1KnaG+v/Y6AtSCgWa/9Ca44XM7bgAABAIAAAH/AAAAEAAAAAIABAAJAAAA
+    CWVzdGltYXRlcwAEAAkAAAAJcG9pbnR3aXNlAAAA/g==
+