diff --git a/AGENTS.md b/AGENTS.md index ecc90e84..cc9e9baf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -111,8 +111,9 @@ directly on the shared stack whenever it is picked up. History lives in `notes/`: the annotated commit log (`notes/2026-07-22-ds-refactor2-annotated-commit-log.md`) is the source of -truth for what was done and how it was verified; `notes/refactor-ideas.md` -holds all open threads and future refactor designs; the remaining dated +truth for what was done and how it was verified; `notes/ROADMAP.md` lists +open tasks and tech debt, `notes/refactor-ideas.md` holds the refactor +designs behind them; the remaining dated files are experiment records. CLAUDE.md describes only the current state. ## Refactoring practice @@ -131,6 +132,19 @@ whole grid a dependency of every branch); `tar_target`'s default `tidy_eval = FALSE` to defer); `rlang::syms` on the trainer column is load-bearing. +## Working incrementally + +Most work here is a chain of small decisions about data, edge cases and +forecasters, and full runs (explore sweeps, evaluation replays, exact +calibration sweeps) take hours, so a wrong early choice is expensive to find. +Build up from the cheapest check that can answer the current question: a +standalone script on one geo and a few dates, then one forecaster on a few +dates (`EVALUATION_N_DATES`, a filtered grid), then one forecaster on all +dates, and only last the full run. Look at each stage's output (plots, a +diff against a known value) before scaling up, and write each data or +edge-case decision into the relevant note when it is made, so it can be +revisited. + ## Conventions - Testing focuses on utility functions; forecaster quality is assessed by inspecting results/reports, not unit tests. diff --git a/Justfile b/Justfile new file mode 100644 index 00000000..9d8e22b0 --- /dev/null +++ b/Justfile @@ -0,0 +1,6 @@ +# R isn't installed on the host; run recipes inside the distrobox: `distrobox enter rocker -- just `. +rscript := env_var_or_default("RSCRIPT", "Rscript") + +# Render calibration experiment notebooks (all, or the named ones, e.g. e18_ref_op_bridge) and the index. +calibration-experiments *notebooks: + {{rscript}} scripts/calibration/render_experiments.R {{notebooks}} diff --git a/R/calibration/calibrate.R b/R/calibration/calibrate.R index 54435487..39764639 100644 --- a/R/calibration/calibrate.R +++ b/R/calibration/calibrate.R @@ -64,7 +64,18 @@ hub_series_matrix <- function(series, round_date, n_levels) { #' Online-calibrate a hub model's submitted forecasts, per (location, horizon). #' #' @param forecasts [hub_read_forecasts()] output. -#' @param truth tibble with columns `target_end_date`, `location` (FIPS), `truth` (raw count). +#' @param truth tibble with columns `target_end_date`, `location` (FIPS), `truth` +#' (raw count). Always used for scoring (the `truth` column of the output); +#' also used for learning when `learn_truth` is `NULL`. +#' @param learn_truth what the tracker learns from. `NULL` learns from `truth` +#' (finalized values, not a realistic backtest). A [nhsn_read_vintages()] +#' table (has a `version` column) learns each outcome from its value as of the +#' round it is revealed at, where round `t` sees the data published by +#' `round_date[t] - 3` (the Wednesday submission); as-of dates before the +#' first version use the first version. The warm start then uses the vintage +#' as of the first live round. A table without `version` is used as-is for +#' learning, which is how [calibrate_hub_forecasts_exact()] passes one +#' snapshot per round. See E00 in `notes/calibration-ledger.md`. #' @param settle_days how long after `target_end_date` the outcome is trusted. #' 14 gives exactly `horizon + 2` rounds of delay on hub coordinates. #' @param burn_in_seasons season labels (as produced by [hub_label_seasons()]) @@ -85,6 +96,12 @@ hub_series_matrix <- function(series, round_date, n_levels) { #' still step when revealed inside the off window. The hidden offsets carry #' unchanged into the next season. `"04-01"` switches calibration off for the #' spring tail, where the peak-tuned offsets are out of regime. +#' @param late_decay `NULL`, or `list(from = "MM-DD", factor = )`. At each round +#' whose reference date falls on or after `from` (and before July), the stored +#' fast offset is multiplied by `factor` in `[0, 1]`, so the shrink persists: +#' later rounds, and the next season under `carry`, start from the shrunk +#' value. Learning continues as usual. The slow term is not decayed. Mutually +#' exclusive with `off_after`. #' @param lr_seasonal `NULL`, or `list(half_width_weeks = 5)`. Pools into the #' eta residual window the rounds of every earlier season that fall within #' `half_width_weeks` of the same point in the season (same calendar date @@ -138,6 +155,7 @@ hub_series_matrix <- function(series, round_date, n_levels) { calibrate_hub_forecasts <- function( forecasts, truth, + learn_truth = NULL, settle_days = 14L, burn_in_seasons = character(0), season_policy = c("carry", "reset", "shrink"), @@ -148,6 +166,7 @@ calibrate_hub_forecasts <- function( projection = c("isotonic", "sort", "none"), nonneg = TRUE, off_after = NULL, + late_decay = NULL, lr_seasonal = NULL, transform = c("identity", "log1p", "sqrt", "quartic_root"), scales = NULL, @@ -178,6 +197,15 @@ calibrate_hub_forecasts <- function( if (!is.null(off_after) && fast_decay > 0) { cli::cli_abort("{.arg off_after} and {.arg fast_decay} cannot be combined; choose one way of retiring stale offsets.") } + if (!is.null(late_decay)) { + if (!is.null(off_after)) { + cli::cli_abort("{.arg off_after} and {.arg late_decay} cannot be combined.") + } + f <- late_decay$factor + if (!is.character(late_decay$from) || !(is.numeric(f) && length(f) == 1L && f >= 0 && f <= 1)) { + cli::cli_abort("{.arg late_decay} must be {.code list(from = \"MM-DD\", factor = <0..1>)}.") + } + } rounds <- hub_label_seasons(unique(forecasts$reference_date)) round_date <- rounds$round_date @@ -197,6 +225,12 @@ calibrate_hub_forecasts <- function( shrink = shrink_factor ) + if (!is.null(late_decay)) { + mmdd <- format(round_date, "%m-%d") + late <- mmdd >= late_decay$from & mmdd < "07-01" + hidden_scale[late] <- hidden_scale[late] * late_decay$factor + } + # The slow term may train through burn-in; the fast term never does. slow_update_from <- if (burn_in_learns_slow) rep(TRUE, n) else update_from apply <- rep(TRUE, n) @@ -224,9 +258,17 @@ calibrate_hub_forecasts <- function( season gap policy {.val {season_policy}}." ) - truth_lookup <- stats::setNames( - truth$truth, paste(truth$location, truth$target_end_date) - ) + truth_lookup <- hub_truth_lookup(truth) + vintage_mode <- !is.null(learn_truth) && "version" %in% names(learn_truth) + learn_lookup <- if (is.null(learn_truth)) { + truth_lookup + } else if (!vintage_mode) { + hub_truth_lookup(learn_truth) + } + if (vintage_mode) { + vintages_by_loc <- split(learn_truth, learn_truth$location) + first_version <- min(learn_truth$version) + } grid <- forecasts %>% distinct(.data$location, .data$horizon) %>% @@ -244,8 +286,14 @@ calibrate_hub_forecasts <- function( if (!any(rounds$season %in% burn_in_seasons)) { cli::cli_abort("{.arg slow_init} needs at least one burn-in season.") } + warm_lookup <- if (vintage_mode) { + first_live <- round_date[which(update_from)[1L]] + hub_truth_lookup(hub_vintage_snapshot(learn_truth, hub_round_asof(first_live))) + } else { + learn_lookup + } init_slow_by_h <- hub_burn_in_offsets( - chunks, keys, round_date, m, levels, truth_lookup, + chunks, keys, round_date, m, levels, warm_lookup, burn = rounds$season %in% burn_in_seasons, scales = scales, fwd = fwd ) } @@ -266,13 +314,19 @@ calibrate_hub_forecasts <- function( # truth yet (the live tail of a season), or no base forecast at that round # (nothing was played, so there is no coverage). A round pruned here is not # "revealed late", it is never learned from at all. - learnable <- !is.na(Y) & !apply(is.na(Yhat), 2, any) delay <- qt_delay_from_dates(round_date, target_end_date, settle_days) + Y_learn <- if (vintage_mode) { + hub_vintage_at_reveal(vintages_by_loc[[loc]], target_end_date, round_date, delay, first_version) + } else { + unname(learn_lookup[paste(loc, target_end_date)]) + } + learnable <- !is.na(Y_learn) & !apply(is.na(Yhat), 2, any) delay <- lapply(delay, function(idx) idx[learnable[idx]]) + revealed <- seq_len(n) %in% unlist(delay) s_t <- hub_round_scales(scales, loc, round_date) res <- qt_track( - Y = fwd(Y / s_t), Yhat = fwd(sweep(Yhat, 2L, s_t, "/")), levels = levels, delay = delay, + Y = fwd(Y_learn / s_t), Yhat = fwd(sweep(Yhat, 2L, s_t, "/")), levels = levels, delay = delay, lr = lr, lr_window = lr_window, lr_args = lr_args, projection = projection, nonneg = FALSE, update_from = update_from, hidden_scale = hidden_scale, @@ -298,7 +352,8 @@ calibrate_hub_forecasts <- function( slow = as.vector(res$slow), fast = as.vector(res$fast), lr_level = as.vector(res$lr_levels), - truth = rep(Y, each = m) + truth = rep(Y, each = m), + truth_learned = rep(ifelse(revealed, Y_learn, NA_real_), each = m) ) diagnostics[[i]] <- tibble( location = loc, @@ -326,7 +381,7 @@ calibrate_hub_forecasts <- function( "location", "horizon", "reference_date", "target_end_date", "season", "season_round", "round_index", "level_index", "level", "value_base", "value_cal", "offset", "hidden", "slow", "fast", "lr_level", - "truth", "is_burn_in" + "truth", "truth_learned", "is_burn_in" ) list( @@ -420,61 +475,116 @@ hub_quantile_loss <- function(cal, drop_burn_in = TRUE, by = character(0)) { } -#' Rolling calibration error against rolling quantile loss, the trade-off curve. +#' Calibrate as prod does: re-run the tracker at every round on that round's +#' vintage and keep only the forecast it plays at that round. #' -#' Both quantities are rolling averages over the last `window` rounds of one -#' series (or of a geo-pooled set of series): -#' calibration error = mean over levels of |rolling coverage at a - a| -#' quantile loss = mean over levels and rounds of the pinball loss -#' Plotting one against the other shows whether calibration is being bought at -#' the cost of sharpness, which is the thing a tracker can quietly get wrong. +#' Prod re-runs [calibrate_hub_forecasts()] from scratch each week on the +#' latest data, so at round `t` every revealed outcome carries its value as of +#' `t`. Passing a vintage table as `learn_truth` to [calibrate_hub_forecasts()] +#' instead fixes each outcome at its value as of the round it was revealed at; +#' this function is the reference that approximation is checked against. It +#' costs one full run per round. #' -#' @param cal [calibrate_hub_forecasts()] output. -#' @param window number of rounds in the rolling window. -#' @param by grouping columns; `character(0)` pools all locations, which is the -#' default because per-series rolling coverage over 20 rounds is mostly noise. +#' @param forecasts,truth as in [calibrate_hub_forecasts()]; `truth` is used for +#' scoring only. +#' @param vintages [nhsn_read_vintages()] output. +#' @param ... passed to [calibrate_hub_forecasts()]. +#' @param workers number of forked processes (rounds are independent runs). +#' @return list with `forecasts` (one round from each run) and `rounds` (from +#' the last run), which the metric functions accept. #' @export -hub_rolling_tradeoff <- function(cal, window = 20L, by = character(0), drop_burn_in = TRUE) { - fc <- if (is.list(cal) && !is.data.frame(cal)) cal$forecasts else cal - # Rounds a location was not forecast at are all-NA in value_base/value_cal. - fc <- fc %>% filter(!is.na(.data$truth), !is.na(.data$value_base)) - if (drop_burn_in) fc <- fc %>% filter(!.data$is_burn_in) - pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) - - # Per (round, level) within each group: coverage indicator and pinball loss, - # pooled over whatever `by` does not name (locations, by default). - per_round <- fc %>% - group_by(across(all_of(c(by, "horizon", "round_index", "reference_date", "level")))) %>% - summarize( - cov_base = mean(.data$truth <= .data$value_base), - cov_cal = mean(.data$truth <= .data$value_cal), - loss_base = mean(pinball(.data$truth, .data$value_base, .data$level)), - loss_cal = mean(pinball(.data$truth, .data$value_cal, .data$level)), - .groups = "drop" +calibrate_hub_forecasts_exact <- function(forecasts, truth, vintages, ..., workers = 1L) { + rounds <- hub_label_seasons(unique(forecasts$reference_date)) + # A run needs at least one live round, and burn-in rounds are not scored, so + # burn-in rounds are taken from the first live round's run. + burn <- rounds$season %in% (list(...)$burn_in_seasons %||% character(0)) + live <- which(!burn) + run_at <- function(t) { + snapshot <- hub_vintage_snapshot(vintages, hub_round_asof(rounds$round_date[t])) + # Later rounds cannot affect what is played at t, so they are dropped to + # save time; the round axis up to t is unchanged. + cal <- suppressMessages(calibrate_hub_forecasts( + forecasts %>% filter(.data$reference_date <= rounds$round_date[t]), + truth, + learn_truth = snapshot, progress = FALSE, ... + )) + keep <- if (t == live[1L]) c(which(burn), t) else t + list( + forecasts = cal$forecasts %>% filter(.data$round_index %in% keep), + rounds = cal$rounds ) + } + # Later rounds are the slowest runs, so they are started first. + order <- rev(seq_along(live)) + runs <- parallel::mclapply(live[order], run_at, mc.cores = workers, mc.preschedule = FALSE) + failed <- purrr::map_lgl(runs, inherits, "try-error") + if (any(failed)) { + cli::cli_abort("Exact calibration failed at round{?s} {.val {live[order][failed]}}: {runs[[which(failed)[1]]]}") + } + runs <- runs[order(order)] + list( + forecasts = bind_rows(purrr::map(runs, "forecasts")), + rounds = runs[[length(runs)]]$rounds + ) +} - roll <- function(x) slider::slide_dbl(x, mean, .before = window - 1L, .complete = TRUE) - per_round %>% - group_by(across(all_of(c(by, "horizon", "level")))) %>% - arrange(.data$round_index, .by_group = TRUE) %>% - mutate( - roll_cov_base = roll(.data$cov_base), - roll_cov_cal = roll(.data$cov_cal), - roll_loss_base = roll(.data$loss_base), - roll_loss_cal = roll(.data$loss_cal) - ) %>% + +#' The as-of date of a hub round: the Wednesday submission before the Saturday +#' reference date. +#' @keywords internal +hub_round_asof <- function(round_date) round_date - 3L + + +#' Named lookup of `truth` by `"location target_end_date"`. +#' @keywords internal +hub_truth_lookup <- function(truth) { + stats::setNames(truth$truth, paste(truth$location, truth$target_end_date)) +} + + +#' Values as of one date from a [nhsn_read_vintages()] table; dates before the +#' first version use the first version. +#' @keywords internal +hub_vintage_snapshot <- function(vintages, as_of) { + as_of <- max(as_of, min(vintages$version)) + vintages %>% + filter(.data$version <= as_of) %>% + group_by(.data$location, .data$target_end_date) %>% + slice_max(.data$version, n = 1L, with_ties = FALSE) %>% ungroup() %>% - filter(!is.na(.data$roll_cov_base)) %>% - # Now collapse the level axis: calibration error is the mean over levels of - # |rolling coverage - nominal|. - group_by(across(all_of(c(by, "horizon", "round_index", "reference_date")))) %>% - summarize( - cal_error_base = mean(abs(.data$roll_cov_base - .data$level)), - cal_error_cal = mean(abs(.data$roll_cov_cal - .data$level)), - qloss_base = mean(.data$roll_loss_base), - qloss_cal = mean(.data$roll_loss_cal), - .groups = "drop" + filter(!is.na(.data$truth)) %>% + select("location", "target_end_date", "truth") +} + + +#' Each round's outcome as of the round that reveals it, for one location. +#' +#' @param vintages one location's rows of a [nhsn_read_vintages()] table. +#' @param delay from [qt_delay_from_dates()], before pruning. +#' @return numeric of length `length(round_date)`; `NA` for rounds that are +#' never revealed or whose value was not yet published at the reveal round. +#' @keywords internal +hub_vintage_at_reveal <- function(vintages, target_date, round_date, delay, first_version) { + n <- length(round_date) + out <- rep(NA_real_, n) + if (is.null(vintages)) { + return(out) + } + reveal_at <- rep(NA_integer_, n) + for (t in seq_len(n)) reveal_at[delay[[t]]] <- t + i <- which(!is.na(reveal_at)) + queries <- tibble( + i = i, + target_end_date = target_date[i], + as_of = pmax(hub_round_asof(round_date[reveal_at[i]]), first_version) + ) + hits <- queries %>% + left_join( + vintages %>% select("target_end_date", "version", "truth"), + by = dplyr::join_by("target_end_date", closest("as_of" >= "version")) ) + out[hits$i] <- hits$truth + out } diff --git a/R/calibration/hub_data.R b/R/calibration/hub_data.R index 1d7fa7cf..632c2746 100644 --- a/R/calibration/hub_data.R +++ b/R/calibration/hub_data.R @@ -29,6 +29,18 @@ HUB_QUANTILE_LEVELS <- c( HUB_FLU_TARGET <- "wk inc flu hosp" HUB_COVID_TARGET <- "wk inc covid hosp" +# Submitted forecasts broken by a pipeline bug rather than by the model; see +# notes/spoiled-submissions.md. `NA` location or horizon matches all of them. +HUB_SPOILED_SUBMISSIONS <- tibble::tribble( + ~model, ~target, ~reference_date, ~location, ~horizon, + "CMU-TimeSeries", HUB_FLU_TARGET, "2025-11-22", NA, -1L, + "CMU-TimeSeries", HUB_FLU_TARGET, "2024-12-14", "US", NA, + "CMU-TimeSeries", HUB_FLU_TARGET, "2024-12-21", "US", NA, + "CMU-TimeSeries", HUB_FLU_TARGET, "2025-01-04", "US", NA, + "CMU-TimeSeries", HUB_COVID_TARGET, "2024-11-23", NA, NA, + "CMU-TimeSeries", HUB_COVID_TARGET, "2026-04-25", NA, -1L +) %>% dplyr::mutate(reference_date = as.Date(.data$reference_date)) + # Parent directory containing the flu hub checkout. Override with DELPHI_HUB_PARENT_DIR # or HUB_FLU_DIR to point at a different layout. @@ -67,13 +79,14 @@ hub_level_index <- function(x, levels = HUB_QUANTILE_LEVELS, tol = 1e-8) { #' Read one model's quantile forecasts for one target out of a hub checkout. #' #' @param hub_dir local hub checkout. For flu, use `HUB_FLU_DIR` -#' (cdcepi/FluSight-forecast-hub); for covid, use `HUB_COVID_DIR` -#' (cdcepi/covid19-forecast-hub). Only the model-output directory is needed. +#' (cdcepi/FluSight-forecast-hub); for covid, pass a checkout of +#' cdcepi/covid19-forecast-hub. Only the model-output directory is needed. #' @param model model-output subdirectory. #' @param target hub target string. 2023-24 files also carry #' `wk flu hosp rate change` pmf rows, so both `target` and #' `output_type == "quantile"` are filtered. #' @param locations optionally restrict to these hub location codes. +#' @param drop_spoiled drop the rows listed in `HUB_SPOILED_SUBMISSIONS`. #' @return long tibble: `reference_date`, `horizon`, `target_end_date`, #' `location`, `level_index`, `level`, `value`. #' @export @@ -81,7 +94,8 @@ hub_read_forecasts <- function( hub_dir = HUB_FLU_DIR, model = "CMU-TimeSeries", target = HUB_FLU_TARGET, - locations = NULL + locations = NULL, + drop_spoiled = TRUE ) { model_dir <- file.path(path.expand(hub_dir), "model-output", model) files <- list.files(model_dir, pattern = "\\.csv$", full.names = TRUE) @@ -122,6 +136,17 @@ hub_read_forecasts <- function( if (!is.null(locations)) { out <- out %>% filter(.data$location %in% locations) } + if (drop_spoiled) { + spoiled <- HUB_SPOILED_SUBMISSIONS %>% filter(.data$model == !!model, .data$target == !!target) + for (i in seq_len(nrow(spoiled))) { + sp <- spoiled[i, ] + out <- out %>% filter(!( + .data$reference_date == sp$reference_date & + (is.na(sp$location) | .data$location == sp$location) & + (is.na(sp$horizon) | .data$horizon == sp$horizon) + )) + } + } # The hub's own invariant: target_end_date is determined by the other two. bad <- out %>% filter(.data$target_end_date != .data$reference_date + 7L * .data$horizon) @@ -136,6 +161,54 @@ hub_read_forecasts <- function( } +#' Latest-vintage NHSN admissions as calibration truth, on hub coordinates. +#' +#' NHSN `time_value` is the Saturday week end, the same grid as the hub's +#' `target_end_date`. +#' @param archive an NHSN `epi_archive`, by default [get_nhsn_data_archive()]. +#' @return tibble with `target_end_date`, `location` (FIPS), `truth`. +#' @export +nhsn_read_truth <- function(disease = "flu", archive = get_nhsn_data_archive(disease)) { + archive %>% + epix_as_of(archive$versions_end) %>% + as_tibble() %>% + nhsn_to_hub_locations() %>% + select(target_end_date = "time_value", "location", truth = "value") %>% + filter(!is.na(.data$truth)) %>% + arrange(.data$location, .data$target_end_date) +} + + +#' Every NHSN vintage, on hub coordinates, for vintage-aware calibration. +#' +#' One row per (location, week, version) at which the value changed (the +#' archive is compactified), so the value as of a date is the row with the +#' latest `version` at or before it. `NA` rows mean the value was absent in +#' that version. +#' @inheritParams nhsn_read_truth +#' @return tibble with `target_end_date`, `location` (FIPS), `version`, `truth`. +#' @export +nhsn_read_vintages <- function(disease = "flu", archive = get_nhsn_data_archive(disease)) { + archive$DT %>% + as_tibble() %>% + nhsn_to_hub_locations() %>% + select(target_end_date = "time_value", "location", "version", truth = "value") %>% + arrange(.data$location, .data$target_end_date, .data$version) +} + + +#' Map NHSN `geo_value` (state abbreviations, `usa`) to hub FIPS `location`. +#' @keywords internal +nhsn_to_hub_locations <- function(df) { + df %>% + mutate(geo_value = ifelse(.data$geo_value == "usa", "us", .data$geo_value)) %>% + inner_join( + get_population_data() %>% distinct(.data$state_id, .keep_all = TRUE) %>% select("state_id", location = "state_code"), + by = c("geo_value" = "state_id") + ) +} + + #' Convert an internal-format forecast tibble to hub forecast format. #' #' The internal format uses geo_value (state abbreviation), forecast_date diff --git a/R/calibration/views.R b/R/calibration/views.R new file mode 100644 index 00000000..4e01dfa1 --- /dev/null +++ b/R/calibration/views.R @@ -0,0 +1,619 @@ +# Named reference configs and standard views for the calibration experiment +# notebooks. The experiments, their reference configs and the view ids (V-head, +# V-month, ...) are indexed in notes/calibration-ledger.md. + + +#' Arguments to [calibrate_hub_forecasts()] for a named reference config. +#' +#' `"paper"` is REF-paper (count space, single term); `"op"` is REF-op, the +#' operating point in prod. Experiments vary one axis from one of these via +#' `...` in [cal_run()]. +#' @export +cal_ref_args <- function(ref = c("paper", "op")) { + ref <- rlang::arg_match(ref) + paper <- list( + burn_in_seasons = "2023-2024", settle_days = 14L, transform = "identity", + lr = "adaptive+", lr_window = 20, lr_args = list(mult = 0.03), + season_policy = "carry" + ) + switch(ref, + paper = paper, + op = utils::modifyList(paper, list( + transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3), + slow_init = "burn_in_quantile", lr_slow = list(mult = 0.003), fast_decay = 0.1 + )) + ) +} + + +#' Run a reference config with some arguments overridden. +#' +#' @param learn `"final"`, `"vintage"` (value as of the reveal round) or +#' `"exact"` (re-run per round on that round's vintage, as prod does). The +#' last two need `vintages`; `workers` parallelizes `"exact"`. +#' @param ... overrides of [cal_ref_args()]. `lr_args` replaces the reference's +#' `lr_args` whole rather than merging into it. +#' @export +cal_run <- function(forecasts, truth, ref = "paper", ..., learn = c("final", "vintage", "exact"), vintages = NULL, workers = 1L) { + learn <- rlang::arg_match(learn) + args <- cal_ref_args(ref) + overrides <- list(...) + args[names(overrides)] <- overrides + if (learn != "final" && is.null(vintages)) { + cli::cli_abort("{.arg learn} = {.val {learn}} needs {.arg vintages}.") + } + suppressMessages(switch(learn, + final = do.call(calibrate_hub_forecasts, c(list(forecasts, truth, progress = FALSE), args)), + vintage = do.call(calibrate_hub_forecasts, c(list(forecasts, truth, learn_truth = vintages, progress = FALSE), args)), + exact = do.call(calibrate_hub_forecasts_exact, c(list(forecasts, truth, vintages), args, workers = workers)) + )) +} + + +#' [cal_run()], cached on disk under a hash of its inputs. +#' +#' Keeps only the forecast columns the views use, plus the tracker's per-row +#' state (`offset`, `fast`, `slow`, `lr_level`) when `internals` is set; a +#' cached run without them is re-run and overwritten. Clear `cache_dir` after +#' changing code in `R/calibration/`: the key covers the arguments and data, +#' not the code. +#' @export +cal_run_cached <- function(forecasts, truth, ref = "paper", ..., learn = "exact", vintages = NULL, workers = 1L, + internals = FALSE, cache_dir = here::here("cache/calibration/experiments/runs")) { + key <- rlang::hash(list(forecasts, truth, ref, list(...), learn, if (learn != "final") vintages)) + path <- file.path(cache_dir, paste0(key, ".rds")) + internal_cols <- c("offset", "fast", "slow", "lr_level") + if (file.exists(path)) { + cal <- readRDS(path) + if (!internals || all(internal_cols %in% names(cal$forecasts))) { + return(cal) + } + } + cal <- cal_run(forecasts, truth, ref, ..., learn = learn, vintages = vintages, workers = workers) + cal <- list(forecasts = cal$forecasts %>% select( + "location", "horizon", "reference_date", "target_end_date", "season", "level", + "value_base", "value_cal", "truth", "is_burn_in", any_of(if (internals) internal_cols else character(0)) + )) + dir.create(cache_dir, showWarnings = FALSE, recursive = TRUE) + saveRDS(cal, path) + cal +} + + +#' The fixed V-state locations, so state panels are comparable across +#' experiments: national, then states from large to very small. +#' @export +CAL_STATES <- c(US = "US", CA = "06", GA = "13", MN = "27", VT = "50") + +CAL_FAN_COLS <- c(base = "#0072B2", calibrated = "#D55E00") + + +#' Each round's NHSN snapshot for one location, as the forecaster saw it +#' (published by `reference_date - 3`), limited to `weeks` before that. +#' @keywords internal +cal_round_snapshots <- function(vintages, loc, ref_dates, weeks = 12L) { + v <- vintages %>% filter(.data$location == loc) + purrr::map(ref_dates, function(rd) { + hub_vintage_snapshot(v, hub_round_asof(rd)) %>% + filter(.data$target_end_date >= rd - 7L * weeks) %>% + mutate(reference_date = rd) + }) %>% bind_rows() +} + + +#' V-state: one location over one season. Every other round's fan (h−1–h3, +#' base 80% band and median, calibrated 10/50/90%), finalized truth in grey and, +#' in black, the last two reported weeks of the snapshot each plotted round saw, +#' with a dot on the latest. That is normally the fan's h−1 week, so each black +#' tail leads into its own fan; after a late release it ends a week earlier. +#' @export +cal_state_panel <- function(cal, loc, season, truth, vintages, title = NULL) { + refs <- cal$forecasts %>% + filter(.data$season == !!season, .data$location == loc, !is.na(.data$value_base)) %>% + distinct(.data$reference_date) %>% + arrange(.data$reference_date) %>% + pull() + refs <- refs[seq(1L, length(refs), by = 2L)] + fan <- cal$forecasts %>% + filter(.data$location == loc, .data$reference_date %in% refs, .data$horizon %in% -1:3, .data$level %in% c(0.1, 0.5, 0.9)) %>% + mutate(lvl = c(`0.1` = "lo", `0.5` = "med", `0.9` = "hi")[as.character(.data$level)]) %>% + select("reference_date", "target_end_date", "lvl", base = "value_base", calibrated = "value_cal") %>% + tidyr::pivot_longer(c("base", "calibrated"), names_to = "which") %>% + tidyr::pivot_wider(names_from = "lvl", values_from = "value") + fin <- truth %>% + filter(.data$location == loc, .data$target_end_date >= min(refs) - 21L, .data$target_end_date <= max(refs) + 28L) + snap <- cal_round_snapshots(vintages, loc, refs, weeks = 4L) %>% + group_by(.data$reference_date) %>% + slice_max(.data$target_end_date, n = 2L) %>% + ungroup() + snap_end <- snap %>% group_by(.data$reference_date) %>% slice_max(.data$target_end_date, n = 1L) %>% ungroup() + ggplot2::ggplot() + + ggplot2::geom_line(data = fin, ggplot2::aes(.data$target_end_date, .data$truth), colour = "grey65") + + ggplot2::geom_ribbon( + data = filter(fan, .data$which == "base"), + ggplot2::aes(.data$target_end_date, ymin = .data$lo, ymax = .data$hi, fill = .data$which, group = .data$reference_date), + alpha = 0.2 + ) + + ggplot2::geom_line( + data = filter(fan, .data$which == "calibrated") %>% tidyr::pivot_longer(c("lo", "hi"), names_to = "edge", values_to = "q"), + ggplot2::aes(.data$target_end_date, .data$q, colour = .data$which, group = interaction(.data$reference_date, .data$edge)), + linewidth = 0.3 + ) + + ggplot2::geom_line( + data = fan, + ggplot2::aes(.data$target_end_date, .data$med, colour = .data$which, group = interaction(.data$reference_date, .data$which)) + ) + + ggplot2::geom_line(data = snap, ggplot2::aes(.data$target_end_date, .data$truth, group = .data$reference_date), colour = "black") + + ggplot2::geom_point(data = snap_end, ggplot2::aes(.data$target_end_date, .data$truth), colour = "black", size = 1) + + ggplot2::scale_fill_manual(values = CAL_FAN_COLS, aesthetics = c("fill", "colour")) + + ggplot2::labs(x = NULL, y = "admissions", colour = NULL, fill = NULL, subtitle = title %||% paste(names(CAL_STATES)[CAL_STATES == loc], season)) +} + + +#' V-gallery selection: a few forecasts, the best and the worst per season +#' phase and location size, each from a different location. +#' +#' Ranking by a single metric fills a gallery with small states late in the +#' season, where a stale offset on a tiny base inflates relative error, and +#' ranking by WIS per 100k favours small places too (their rates are noisier). +#' Here each forecast is scored by its WIS reduction as a share of its own +#' location-season's base WIS, so a late-season forecast on a tiny base only +#' ranks high if it moved a real part of that location's season error. Small +#' places still vary more, so slots are split by size (population above or +#' below the median) as well as by phase (before, around, after the location's +#' peak in finalized truth). +#' +#' @param population tibble of `location`, `population`. +#' @param peak_weeks half-width of the "peak" phase. +#' @return one row per picked forecast: `location`, `reference_date`, `phase`, +#' `size`, `kind` (gain/loss), `share` (% of the location-season's base WIS), +#' `wis_pct`. +#' @export +cal_gallery_pick <- function(cal, population, peak_weeks = 3L) { + pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) + fc <- cal$forecasts %>% filter(!.data$is_burn_in, !is.na(.data$truth), !is.na(.data$value_base)) + median_pop <- stats::median(population$population[population$location %in% fc$location]) + peaks <- fc %>% + distinct(.data$location, .data$season, .data$target_end_date, .data$truth) %>% + group_by(.data$location, .data$season) %>% + slice_max(.data$truth, n = 1L, with_ties = FALSE) %>% + ungroup() %>% + select("location", "season", peak = "target_end_date") + scored <- fc %>% + group_by(.data$location, .data$season, .data$reference_date) %>% + summarize( + wis_base = sum(pinball(.data$truth, .data$value_base, .data$level)), + wis_cal = sum(pinball(.data$truth, .data$value_cal, .data$level)), + .groups = "drop" + ) %>% + group_by(.data$location, .data$season) %>% + mutate(share = 100 * (.data$wis_base - .data$wis_cal) / sum(.data$wis_base)) %>% + ungroup() %>% + inner_join(peaks, by = c("location", "season")) %>% + inner_join(population, by = "location") %>% + mutate( + size = if_else(.data$population >= median_pop, "large", "small"), + weeks_from_peak = as.numeric(.data$reference_date - .data$peak) / 7, + phase = factor( + dplyr::case_when( + .data$weeks_from_peak < -peak_weeks ~ "before peak", + .data$weeks_from_peak > peak_weeks ~ "after peak", + TRUE ~ "around peak" + ), + levels = c("before peak", "around peak", "after peak") + ), + wis_pct = 100 * (.data$wis_base - .data$wis_cal) / .data$wis_base + ) + # Greedy, one slot at a time, skipping locations already picked. + slots <- tidyr::expand_grid(phase = levels(scored$phase), size = c("large", "small"), kind = c("gain", "loss")) + used <- character(0) + picks <- purrr::map(seq_len(nrow(slots)), function(i) { + cand <- scored %>% filter(.data$phase == slots$phase[i], .data$size == slots$size[i], !(.data$location %in% used)) + cand <- if (slots$kind[i] == "gain") arrange(cand, desc(.data$share)) else arrange(cand, .data$share) + pick <- cand[1, ] %>% mutate(kind = slots$kind[i]) + used <<- c(used, pick$location) + pick + }) + bind_rows(picks) %>% + select("location", "reference_date", "phase", "size", "kind", "share", "wis_pct") +} + + +#' V-gallery panel: one forecast (all horizons), base and calibrated 50% and +#' 90% bands, the snapshot the forecaster saw (black) and finalized truth (grey). +#' @export +cal_gallery_panel <- function(cal, loc, ref_date, truth, vintages, subtitle = NULL) { + levels_map <- c("0.05" = "lo90", "0.25" = "lo50", "0.5" = "med", "0.75" = "hi50", "0.95" = "hi90") + fin <- truth %>% filter(.data$location == loc, .data$target_end_date >= ref_date - 84L, .data$target_end_date <= ref_date + 28L) + snap <- cal_round_snapshots(vintages, loc, ref_date) + fan <- cal$forecasts %>% + filter(.data$location == loc, .data$reference_date == ref_date, .data$level %in% as.numeric(names(levels_map))) %>% + mutate(lvl = levels_map[as.character(.data$level)]) %>% + select("target_end_date", "lvl", base = "value_base", calibrated = "value_cal") %>% + tidyr::pivot_longer(c("base", "calibrated"), names_to = "which") %>% + tidyr::pivot_wider(names_from = "lvl", values_from = "value") + ggplot2::ggplot() + + ggplot2::geom_line(data = fin, ggplot2::aes(.data$target_end_date, .data$truth), colour = "grey65") + + ggplot2::geom_point(data = fin, ggplot2::aes(.data$target_end_date, .data$truth), colour = "grey65", size = 0.7) + + ggplot2::geom_ribbon(data = fan, ggplot2::aes(.data$target_end_date, ymin = .data$lo90, ymax = .data$hi90, fill = .data$which), alpha = 0.16) + + ggplot2::geom_ribbon(data = fan, ggplot2::aes(.data$target_end_date, ymin = .data$lo50, ymax = .data$hi50, fill = .data$which), alpha = 0.3) + + ggplot2::geom_line(data = fan, ggplot2::aes(.data$target_end_date, .data$med, colour = .data$which)) + + ggplot2::geom_line(data = snap, ggplot2::aes(.data$target_end_date, .data$truth), colour = "black") + + ggplot2::geom_point(data = snap, ggplot2::aes(.data$target_end_date, .data$truth), colour = "black", size = 0.7) + + ggplot2::geom_vline(xintercept = hub_round_asof(ref_date), linetype = "dashed", colour = "grey40") + + ggplot2::scale_fill_manual(values = CAL_FAN_COLS, aesthetics = c("fill", "colour")) + + ggplot2::labs(x = NULL, y = "admissions", colour = NULL, fill = NULL, subtitle = subtitle) +} + + +#' V-month3: the month view. Per config × season × calendar month × horizon: +#' WIS reduction %, L1 coverage bias within the month (base and calibrated), +#' and the absolute error of the median, reduction % vs base. +#' +#' Seasons are August–July years of the reference date, so hub runs and the +#' clean replay are cut the same way. Burn-in rounds are dropped. +#' +#' @param cals named list of runs (each with `$forecasts`) or of forecast +#' frames with `location`, `horizon`, `reference_date`, `level`, `truth`, +#' `value_base`, `value_cal` and optionally `is_burn_in`. +#' @param horizons horizons to keep; `NULL` keeps all. +#' @param states_only drop the US national series. +#' @return list of `month` (one row per config, season, month, horizon) and +#' `cov` (season-pooled and month-average L1 coverage bias per config, +#' season and horizon; season `"both"` pools the seasons). +#' @export +cal_month_view <- function(cals, seasons = c("2024-2025", "2025-2026"), horizons = 0:3, states_only = TRUE) { + pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) + per_level <- purrr::imap(cals, function(x, nm) { + fc <- if (is.data.frame(x)) x else x$forecasts + if (!"is_burn_in" %in% names(fc)) fc$is_burn_in <- FALSE + if (!is.null(horizons)) fc <- fc %>% filter(.data$horizon %in% horizons) + if (states_only) fc <- fc %>% filter(.data$location != "US") + yr <- as.integer(format(fc$reference_date, "%Y")) - (as.integer(format(fc$reference_date, "%m")) < 8L) + fc %>% + mutate(season = paste0(yr, "-", yr + 1L)) %>% + filter(!.data$is_burn_in, !is.na(.data$truth), !is.na(.data$value_base), .data$season %in% seasons) %>% + mutate(month = factor(format(.data$reference_date, "%b"), levels = month.abb[c(8:12, 1:7)])) %>% + group_by(.data$season, .data$month, .data$horizon, .data$level) %>% + summarize( + wis_base = sum(pinball(.data$truth, .data$value_base, .data$level)), + wis_cal = sum(pinball(.data$truth, .data$value_cal, .data$level)), + ae_base = sum(abs(.data$truth - .data$value_base)), ae_cal = sum(abs(.data$truth - .data$value_cal)), + below_base = sum(.data$truth <= .data$value_base), below_cal = sum(.data$truth <= .data$value_cal), + n = n(), rounds = n_distinct(.data$reference_date), .groups = "drop" + ) %>% + mutate(config = nm, .before = 1) + }) %>% + bind_rows() %>% + mutate(config = factor(.data$config, levels = names(cals))) + month <- per_level %>% + group_by(.data$config, .data$season, .data$month, .data$horizon) %>% + summarize( + wis_base = sum(.data$wis_base), wis_cal = sum(.data$wis_cal), + cov_base = mean(abs(.data$below_base / .data$n - .data$level)), + cov_cal = mean(abs(.data$below_cal / .data$n - .data$level)), + ae_base = .data$ae_base[.data$level == 0.5], ae_cal = .data$ae_cal[.data$level == 0.5], + n = first(.data$n), rounds = first(.data$rounds), .groups = "drop" + ) %>% + group_by(.data$config, .data$season, .data$horizon) %>% + mutate( + wis_pct = 100 * (.data$wis_base - .data$wis_cal) / .data$wis_base, + ae_pct = 100 * (.data$ae_base - .data$ae_cal) / .data$ae_base, + share = 100 * .data$wis_base / sum(.data$wis_base) + ) %>% + ungroup() + both <- function(d) bind_rows(d, d %>% mutate(season = "both")) + pooled <- both(per_level) %>% + group_by(.data$config, .data$season, .data$horizon, .data$level) %>% + summarize(b = sum(.data$below_base) / sum(.data$n), c = sum(.data$below_cal) / sum(.data$n), .groups = "drop") %>% + group_by(.data$config, .data$season, .data$horizon) %>% + summarize(pooled_base = mean(abs(.data$b - .data$level)), pooled_cal = mean(abs(.data$c - .data$level)), .groups = "drop") + mavg <- both(month) %>% + group_by(.data$config, .data$season, .data$horizon) %>% + summarize(mavg_base = mean(.data$cov_base), mavg_cal = mean(.data$cov_cal), months = n(), .groups = "drop") + cov <- left_join(pooled, mavg, by = c("config", "season", "horizon")) %>% + mutate(season = factor(.data$season, levels = c(seasons, "both"))) + list(month = month, cov = cov) +} + + +#' Caption for [cal_month_cov_table()]. +#' @export +CAL_MONTH_COV_CAPTION <- paste( + "L1 coverage bias (lower is better), states only, as season-pooled / month-avg.", + "Season-pooled scores all of a season's rounds together. Month-avg is the mean over calendar months", + "of the bias computed within each month, every month weighted equally ('both': every season-month weighted equally)." +) + + +#' Season-pooled / month-average L1 coverage bias as a wide table: configs as +#' rows (base first; one base row per distinct base), `season hX` as columns. +#' @export +cal_month_cov_table <- function(mv, seasons = NULL) { + cov <- mv$cov + if (!is.null(seasons)) cov <- cov %>% filter(.data$season %in% seasons) + cell <- function(p, m) sprintf("%.3f / %.3f", p, m) + wide <- function(d) { + d %>% + arrange(.data$season, .data$horizon) %>% + transmute(.data$config, col = paste0(.data$season, " h", .data$horizon), cell = .data$cell) %>% + tidyr::pivot_wider(names_from = "col", values_from = "cell") + } + base <- wide(cov %>% mutate(cell = cell(.data$pooled_base, .data$mavg_base))) + base <- base[!duplicated(base[-1]), ] + base$config <- if (nrow(base) == 1L) "base" else paste0("base (", base$config, ")") + cal <- wide(cov %>% mutate(cell = cell(.data$pooled_cal, .data$mavg_cal))) %>% mutate(config = as.character(.data$config)) + bind_rows(base, cal) +} + + +#' WIS reduction % per season and horizon as a wide table: configs as rows, +#' `season hX` as columns, `both` pooling the seasons. Same rows as the month +#' view (states only, burn-in dropped). +#' @export +cal_month_wis_table <- function(mv, seasons = NULL) { + m <- mv$month + if (!is.null(seasons)) m <- m %>% filter(.data$season %in% seasons) + bind_rows(m, m %>% mutate(season = "both")) %>% + mutate(season = factor(.data$season, levels = c(sort(unique(m$season)), "both"))) %>% + group_by(.data$config, .data$season, .data$horizon) %>% + summarize(pct = 100 * (sum(.data$wis_base) - sum(.data$wis_cal)) / sum(.data$wis_base), .groups = "drop") %>% + arrange(.data$season, .data$horizon) %>% + transmute(.data$config, col = paste0(.data$season, " h", .data$horizon), cell = sprintf("%+.1f", .data$pct)) %>% + tidyr::pivot_wider(names_from = "col", values_from = "cell") +} + + +#' The month view plot for one season: rows are WIS reduction %, L1 coverage +#' bias and median absolute-error reduction %; columns are horizons; one line +#' per config. The base's coverage bias is grey (dashed unless `linetype` is +#' given), and grey bars are each month's share of the season's base WIS. +#' +#' @param month_tbl the `month` element of [cal_month_view()]. +#' @param colours named colours, keyed by the `colour_by` values; defaults to +#' a fixed palette. +#' @param colour_by column of `month_tbl` mapped to colour; each config is +#' still its own line. +#' @param linetype optional column of `month` mapped to linetype (e.g. the +#' forecaster); the base line is then drawn once per linetype group. +#' @param clip WIS and AE reductions are squished into this range; clipped +#' points are drawn open. +#' @export +cal_month_plot <- function(month_tbl, season, colours = NULL, linetype = NULL, colour_by = "config", clip = c(-40, 40)) { + m_wis <- "WIS reduction %\n(positive = calibrated\nWIS lower than base)" + m_cov <- "L1 coverage bias\nreduction %\n(positive = calibrated\nbias lower than base)" + m_ae <- "median AE reduction %\n(positive = calibrated\nAE lower than base)" + lv <- c(m_wis, m_cov, m_ae) + d <- month_tbl %>% + filter(.data$season == !!season) %>% + mutate(month = droplevels(.data$month), h = factor(paste0("h", .data$horizon), levels = paste0("h", sort(unique(month_tbl$horizon))))) + d$grp <- if (is.null(linetype)) "base" else as.character(d[[linetype]]) + d$col <- d[[colour_by]] + configs <- levels(droplevels(d$config)) + if (is.null(colours)) { + keys <- if (is.factor(d$col)) levels(droplevels(d$col)) else unique(d$col) + pal <- c("black", "#0072B2", "#D55E00", "#009E73", "#CC79A7", "#E69F00", "#56B4E9", "#999933") + colours <- setNames(rep_len(pal, length(keys)), keys) + } + sq <- function(x) pmin(pmax(x, clip[1]), clip[2]) + long <- bind_rows( + d %>% transmute(.data$config, .data$col, .data$grp, .data$month, .data$h, metric = m_wis, value = sq(.data$wis_pct), clipped = .data$wis_pct != value), + d %>% mutate(cov_pct = 100 * (.data$cov_base - .data$cov_cal) / .data$cov_base) %>% + transmute(.data$config, .data$col, .data$grp, .data$month, .data$h, metric = m_cov, value = sq(.data$cov_pct), clipped = .data$cov_pct != value), + d %>% transmute(.data$config, .data$col, .data$grp, .data$month, .data$h, metric = m_ae, value = sq(.data$ae_pct), clipped = .data$ae_pct != value) + ) %>% mutate(metric = factor(.data$metric, levels = lv)) + share <- d %>% + filter(.data$config == configs[1]) %>% + transmute(.data$month, .data$h, metric = factor(m_wis, levels = lv), value = sq(.data$share)) + zero <- tibble(metric = factor(lv, levels = lv), y = 0) + p <- ggplot2::ggplot(long, ggplot2::aes(.data$month, .data$value)) + + ggplot2::geom_col(data = share, fill = "grey85", width = 0.7) + + ggplot2::geom_hline(data = zero, ggplot2::aes(yintercept = .data$y), colour = "grey40", linewidth = 0.3) + p <- if (is.null(linetype)) { + p + ggplot2::geom_line(ggplot2::aes(colour = .data$col, group = .data$config)) + } else { + p + ggplot2::geom_line(ggplot2::aes(colour = .data$col, group = .data$config, linetype = .data$grp)) + + ggplot2::scale_linetype_manual(values = c("solid", "22", "13")[seq_along(unique(d$grp))], name = NULL) + } + p + + ggplot2::geom_point(ggplot2::aes(colour = .data$col, shape = .data$clipped), size = 1.2) + + ggplot2::scale_shape_manual(values = c(`FALSE` = 16, `TRUE` = 1), guide = "none") + + ggplot2::scale_colour_manual(values = colours, name = NULL) + + ggplot2::facet_grid(metric ~ h, scales = "free_y", switch = "y") + + ggplot2::labs( + x = "month of reference date", y = NULL, title = season, + caption = sprintf( + "States only. All rows: positive = better than the uncalibrated base. Grey bars: month's share of the season's base WIS (%%). Reductions clipped to [%d, %d]; open points are clipped.", + clip[1], clip[2] + ) + ) + + ggplot2::theme( + legend.position = "bottom", legend.box = "vertical", strip.placement = "outside", strip.text.y.left = ggplot2::element_text(angle = 0), + axis.text.x = ggplot2::element_text(angle = 45, hjust = 1) + ) +} + + +CAL_NOTEBOOK_DIR <- "reports/writeups/calibration_experiments" + +#' Markdown heading marks for a heading written by `cat()`: level `n`, pushed +#' down when knit inside `cal_month_section()`. +#' @export +cal_h <- function(n) strrep("#", n + getOption("cal.hshift", 0L)) + +# Add `shift` levels to every markdown heading outside code chunks. +cal_shift_headings <- function(lines, shift) { + in_code <- FALSE + for (i in seq_along(lines)) { + if (grepl("^```", lines[[i]])) { + in_code <- !in_code + } else if (!in_code && grepl("^#+ ", lines[[i]])) { + lines[[i]] <- paste0(strrep("#", shift), lines[[i]]) + } + } + lines +} + +#' Per-state WIS (summed pinball) of the base and each run, per (season, +#' location, horizon), live seasons, states only. Seasons run August to July. +#' @export +cal_state_wis <- function(cals, seasons = c("2024-2025", "2025-2026")) { + pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) + purrr::imap(cals, function(cal, nm) { + cal$forecasts %>% + mutate(season = cal_season_of(.data$reference_date)) %>% + filter(.data$location != "US", .data$season %in% seasons, !is.na(.data$truth), !is.na(.data$value_base)) %>% + group_by(across(all_of(c("season", "location", "horizon")))) %>% + summarize( + wis_base = sum(pinball(.data$truth, .data$value_base, .data$level)), + wis_cal = sum(pinball(.data$truth, .data$value_cal, .data$level)), .groups = "drop" + ) %>% + mutate(config = nm) + }) %>% bind_rows() +} + +#' Round bootstrap for WIS reduction %, states only, live seasons, h0–h3. +#' +#' Rounds (reference dates) are resampled with replacement within each +#' season; `"both"` pools the two seasons' resampled rounds. Every config is +#' scored on the same draws, so differences between configs taken draw by +#' draw are paired. Each draw's WIS reduction % is `100 × (WIS_base − +#' WIS_cal) / WIS_base` over the resampled rounds. The interval is the +#' central `level` percentile band of the draws; see +#' notes/calibration-ledger.md, "How to read the numbers". +#' +#' @param cals named list of runs (each with `$forecasts`) or forecast frames. +#' @return list of `ci` (one row per config, season, h, with `wis_pct`, `lo`, +#' `hi`) and `draws` (one row per config, season, h, draw, with `wis_pct`). +#' @export +cal_wis_boot <- function(cals, seasons = c("2024-2025", "2025-2026"), horizons = 0:3, n_draws = 1000, level = 0.9, seed = 1, states_only = TRUE) { + pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) + per_round <- purrr::imap(cals, function(x, nm) { + fc <- if (is.data.frame(x)) x else x$forecasts + if (!"is_burn_in" %in% names(fc)) fc$is_burn_in <- FALSE + if (states_only) fc <- fc %>% filter(.data$location != "US") + fc %>% + mutate(season = cal_season_of(.data$reference_date)) %>% + filter(!.data$is_burn_in, .data$horizon %in% horizons, .data$season %in% seasons, !is.na(.data$truth), !is.na(.data$value_base)) %>% + group_by(.data$season, .data$reference_date, .data$horizon) %>% + summarize( + wis_base = sum(pinball(.data$truth, .data$value_base, .data$level)), + wis_cal = sum(pinball(.data$truth, .data$value_cal, .data$level)), .groups = "drop" + ) %>% + mutate(config = nm, .before = 1) + }) %>% bind_rows() + # One weight matrix per season (draw × round), shared by every config. + rounds <- per_round %>% distinct(.data$season, .data$reference_date) %>% arrange(.data$season, .data$reference_date) + set.seed(seed) + weights <- purrr::map(split(rounds$reference_date, rounds$season), function(r) { + n <- length(r) + w <- t(replicate(n_draws, tabulate(sample.int(n, n, replace = TRUE), nbins = n))) + colnames(w) <- as.character(r) + w + }) + # A config or horizon missing a round (e.g. h3 at the season's end) just + # contributes nothing for it. + score <- function(d, s) { + w <- weights[[s]][, as.character(d$reference_date), drop = FALSE] + list(base = as.vector(w %*% d$wis_base), cal = as.vector(w %*% d$wis_cal)) + } + draws <- per_round %>% + group_by(.data$config, .data$horizon) %>% + group_modify(function(d, key) { + by_season <- purrr::map(split(d, d$season), function(ds) score(ds, ds$season[1])) + by_season$both <- list( + base = Reduce(`+`, purrr::map(by_season, "base")), + cal = Reduce(`+`, purrr::map(by_season, "cal")) + ) + purrr::imap(by_season, function(x, s) { + tibble::tibble(season = s, draw = seq_len(n_draws), wis_pct = 100 * (x$base - x$cal) / x$base) + }) %>% bind_rows() + }) %>% + ungroup() %>% + mutate( + config = factor(.data$config, levels = names(cals)), + season = factor(.data$season, levels = c(seasons, "both")), + h = factor(paste0("h", .data$horizon), levels = paste0("h", sort(horizons))) + ) %>% + select("config", "season", "h", "draw", "wis_pct") + point <- per_round %>% + bind_rows(per_round %>% mutate(season = "both")) %>% + group_by(.data$config, .data$season, .data$horizon) %>% + summarize(wis_pct = 100 * (sum(.data$wis_base) - sum(.data$wis_cal)) / sum(.data$wis_base), .groups = "drop") %>% + mutate( + config = factor(.data$config, levels = names(cals)), + season = factor(.data$season, levels = c(seasons, "both")), + h = factor(paste0("h", .data$horizon), levels = paste0("h", sort(horizons))) + ) %>% + select("config", "season", "h", "wis_pct") + a <- (1 - level) / 2 + ci <- draws %>% + group_by(.data$config, .data$season, .data$h) %>% + summarize(lo = quantile(.data$wis_pct, a), hi = quantile(.data$wis_pct, 1 - a), .groups = "drop") %>% + inner_join(point, by = c("config", "season", "h")) %>% + select("config", "season", "h", "wis_pct", "lo", "hi") + list(ci = ci, draws = draws, level = level, n_draws = n_draws) +} + +#' Paired bootstrap interval for the WIS reduction % gap between two configs +#' of a [cal_wis_boot()] result, `a − b`, draw by draw. +#' @return one row per season and h with `diff`, `lo`, `hi`. +#' @export +cal_wis_boot_diff <- function(boot, a, b) { + al <- (1 - boot$level) / 2 + d <- boot$draws %>% + filter(.data$config %in% c(a, b)) %>% + mutate(config = ifelse(.data$config == a, "a", "b")) %>% + tidyr::pivot_wider(names_from = "config", values_from = "wis_pct") %>% + group_by(.data$season, .data$h) %>% + summarize(lo = quantile(.data$a - .data$b, al), hi = quantile(.data$a - .data$b, 1 - al), .groups = "drop") + p <- boot$ci %>% + filter(.data$config %in% c(a, b)) %>% + mutate(config = ifelse(.data$config == a, "a", "b")) %>% + select("config", "season", "h", "wis_pct") %>% + tidyr::pivot_wider(names_from = "config", values_from = "wis_pct") %>% + transmute(.data$season, .data$h, diff = .data$a - .data$b) + inner_join(p, d, by = c("season", "h")) +} + +#' Wide table of a [cal_wis_boot()] result: configs as rows, `season hX` as +#' columns, cells `point [lo, hi]`. +#' @export +cal_wis_boot_table <- function(boot) { + boot$ci %>% + transmute(.data$config, col = paste(.data$season, .data$h), cell = sprintf("%+.1f [%+.1f, %+.1f]", .data$wis_pct, .data$lo, .data$hi)) %>% + tidyr::pivot_wider(names_from = "col", values_from = "cell") +} + +#' Caption for [cal_wis_boot_table()]. +#' @export +cal_wis_boot_caption <- function(boot) { + sprintf( + "WIS reduction %% (positive = calibrated WIS lower than base), states only, with a %d%% interval from %d round bootstrap draws (rounds resampled within each season; `both` pools the seasons' draws).", + round(100 * boot$level), boot$n_draws + ) +} + +# August-July season label, as `season_of()` in the replay scripts. +cal_season_of <- function(d) { + y <- as.integer(format(d, "%Y")) - (as.integer(format(d, "%m")) < 8) + paste0(y, "-", y + 1) +} + +#' Knit the month view (`_month_view.Rmd`) in the calling environment under +#' its own heading, `level` levels below the notebook's top. For notebooks that +#' show more than one month view; uses `mv` and the optional `mv_*` settings +#' from `envir`. Call from a chunk with `results = "asis"`. +#' @export +cal_month_section <- function(title, level = 2L, envir = parent.frame()) { + lines <- readLines(here::here(CAL_NOTEBOOK_DIR, "_month_view.Rmd")) + lines[grep("^# Month view", lines)] <- paste("#", title) + shift <- getOption("cal.hshift", 0L) + level - 1L + old <- options(cal.hshift = shift) + on.exit(options(old)) + slug <- gsub("[^a-z0-9]+", "-", tolower(title)) + cat(knitr::knit_child( + text = cal_shift_headings(lines, shift), envir = envir, quiet = TRUE, + options = list(fig.path = paste0(knitr::opts_chunk$get("fig.path"), slug, "-")) + )) +} diff --git a/notes/CALIBRATION.md b/notes/CALIBRATION.md deleted file mode 100644 index 1cf9af8a..00000000 --- a/notes/CALIBRATION.md +++ /dev/null @@ -1,587 +0,0 @@ -# Online quantile calibration (MultiQT) - -Post-hoc online calibration of our submitted forecasts, following MultiQT -(Ding, Gibbs & Tibshirani 2025): per (location, horizon) series, an additive -offset vector over the 23 hub quantile levels is updated by online gradient -descent on pinball loss, with an adaptive learning rate and isotonic -projection. Full repro commands live in `notes/calibration-runbook.md`. - -# State - -## Code - -- `R/calibration/qt.R` — the tracker (`qt_track`, `qt_learning_rate`, - `qt_project`, `qt_delay_from_dates`). Bit-exact against the authors' Python - on the `delphi-fixes` branch of `~/repos/delphi/multiQT` (their published - code had several bugs we fixed and reported; see that branch's commit - message). Oracle fixtures in `tests/testthat/test-qt.R`, plus a real-series - cross-check (`scripts/calibration_export_series.R`). -- `R/calibration/hub_data.R` — FluSight hub adapters: `hub_read_forecasts()` - (CMU-TimeSeries submissions from the sparse hub checkout), - `hub_read_truth()` (hub finalized truth CSV), `hub_label_seasons()`. -- `R/calibration/calibrate.R` — `calibrate_hub_forecasts()` driver (burn-in, - season policy, ~5 s for all 265 series) and metrics: `hub_coverage()`, - `hub_coverage_summary()`, `hub_quantile_loss()`, `hub_rolling_tradeoff()`. - Method options beyond the paper: `transform` (power/log space), - `scales` (per-location, per-era divisors so rounds from different sources - share one tracker), `lr_slow` + `fast_decay` + `burn_in_learns_slow` (the - two-term offset, below), `off_after`, `lr_seasonal`. (`lr_geo_pool`, the - geo-pooled eta in the sweep table below, was removed 2026-09-17: it never - beat the baseline.) -- `scripts/calibration_ws_backfill.R` — replays flu prod's `windowed_seasonal` - over the NHSN seasons (hub schema, cached to - `cache/calibration/ws_pseudo_hub_forecasts.parquet`); `scripts/calibration_ws_experiments.R` - runs the one-forecaster-across-eras comparison (below). -- `scripts/calibration_ili_backfill.R` — runs flu prod's `windowed_seasonal` - forecaster on the ILI+ state history (2010–2024, Wednesday labels, snapshot - truncated one week to mimic reporting lag) and writes a pseudo-hub - `forecasts`/`truth` pair in hub schema to `cache/calibration/ili_pseudo_hub_*.parquet` - (~30 min). Multi-season burn-in material for the slow term. -- `scripts/calibration_harness.R` — targets-based harness for calibrating our - own forecasters outside the hub path (covid - `windowed_seasonal_extra_sources`, aheads 0:4, `covid_hosp_evaluation` - store). WIP. - -## Data - -FluSight CMU-TimeSeries submissions: 83 rounds (2023-10-14 … 2026-05-30), 53 -locations, horizons −1…3, 23 quantile levels. Season 2023-24 is a burn-in -(incomplete horizons/locations): its residuals warm the learning rate but no -gradient steps are taken. Waiting 14 days for NHSN to settle makes the reveal -delay exactly `horizon + 2` rounds, so the offset at h3 is always 5 rounds -stale. - -As-of NHSN vintages for the gallery come from -`cache/calibration/nhsn_archive_flu.parquet`, copied from the oracle capture -`cache/oracle/flu_hosp_prod/today-main-prmpwlvz/nhsn_archive_data.parquet` -(versions 2024-11-19 … 2026-07-22, so both live seasons are covered; no -vintages exist before 2024-11-19 anywhere). Note: the S3 -`nhsn_data_archive.parquet` object stalled at version 2026-01-30 — the -polling job appears to have stopped; refresh the copy from a newer oracle -capture if one exists. - -## Findings (factorial sweep, `scripts/reports/calibration_qt_flu.Rmd`) - -Sweep: `season_policy` {carry, reset} × `lr_window` {8, 20, 50, Inf} × -`lr_mult` {0.3, 0.1, 0.03, 0.01}; results in `cache/calibration/sweep.rds`. - -- **The multiplier is the control knob; the window barely matters.** Per - horizon, WIS-improvement spread across multipliers is 8–63 points; across - windows 1.5–5. Windows 20/50/Inf are nearly indistinguishable at this data - size. The expanding-window learning-rate ratchet is real but immaterial. -- **The paper's `mult = 0.1` overpays for calibration**: calibration error - improves 70–85% but WIS degrades −3.6% (h0) to −22.9% (h3). `mult = 0.03` - gets equal-or-better calibration error at h ≥ 1 at a fraction of the WIS - cost (−2 to −6%). `mult = 0.01` is near WIS-parity but converges too slowly - (bad first-live-season calibration error, 0.07–0.10). The real tradeoff is - convergence speed vs steady-state sharpness cost. -- **Horizon −1 calibration improves WIS outright (+4%)** — calibration is - free there. -- **Carry vs reset across the off-season is minor** (carry slightly better at - h3, otherwise a wash). -- The WIS cost grows with horizon, consistent with `h + 2` staleness: a stale - additive offset does the most damage where the seasonal ramp is steep. -- Residual scale swings 1–2 orders of magnitude within a season, so an - additive offset learned near the peak is badly sized for the trough. - -**Original operating point (count space): `lr_mult = 0.03`, `lr_window = 20`, -`season_policy = "carry"`** (20 over the untested 15 because it is a -validated grid point and the window is immaterial). At this multiplier the -tracker needs most of a season to warm up, which makes burn-in/state-carrying -the central integration design issue. - -**Current operating point (2026-09-16): `transform = "sqrt"`, `lr_mult = 0.03`, -`floor = 1e-3`, `lr_window = 20`, carry, `slow_init = "burn_in_quantile"`, -`lr_slow = list(mult = 0.003)`, `fast_decay = 0.1`.** WIS-positive at every -horizon in both live seasons (+9.0/+5.3/+1.9/+0.8/+0.5 at h−1…3) with -calibration error 0.044–0.072 against 0.095–0.118 uncalibrated. See "Warm -start" below for the table. - -## Notebooks - -- `scripts/reports/calibration_qt_flu.Rmd` — the parameter-sweep EDA. Frozen as - the sweep record; rendered HTML alongside. -- `scripts/reports/calibration_qt_seasons_flu.Rmd` — whole-season views of four - states × two live seasons, one panel per (state, season): every-other-round - fans (80% band) for h 0–3, the NHSN vintage each round saw painted over its - fortnight, finalized truth, and an eta strip; plus collapsible internals - tables at h2 (eta, pre-PAVA hidden offset, post-PAVA offset, base, - calibrated at 5 levels). The states are picked from the baseline run: best - and worst by WIS change and best and worst by median absolute-error change, - pooled over horizons and both live seasons, deduplicated (as of 2026-09-16: - AK best on both, so HI is the best-AE panel; NY worst WIS; CT worst AE). - Repeated for six method variants (baseline, off after March 1 / February 15, - per-level eta, geo-pooled eta, seasonal window) with a headline - WIS/calibration-error comparison at the top, plus a by-month breakdown of the - baseline (WIS change, share of base WIS, median shift) that shows where in - the season the method gains and loses. -- `scripts/reports/calibration_qt_gallery_flu.Rmd` — fixed operating point. Slim - headline table (WIS + calibration error per horizon per season), then a - ranked per-forecast gallery: top-N (location, round) panels ordered by mean - absolute quantile displacement relative to the base median. Each panel: - as-of NHSN snapshot (data the forecaster saw), finalized truth, base and - calibrated fans (50% + 90% ribbons, horizons −1…3), captioned with MAD, WIS - delta, and whether truth escaped the base 90% band. Seasons 2024-25 and - 2025-26 only, flu only. A dynamic/paginated app version was considered and - rejected for now in favor of static top-N HTML. - -## Method variants (`calibrate_hub_forecasts()` options; 2026-08-27) - -All at the operating point above; WIS change vs base, both live seasons, by -horizon −1/0/1/2/3: - -| variant | option | h−1 | h0 | h1 | h2 | h3 | -|---|---|---|---|---|---|---| -| baseline | — | +4.3 | +1.8 | −1.9 | −5.3 | −6.7 | -| off after April 1 | `off_after = "04-01"` | +4.9 | +1.4 | −1.3 | −2.8 | −3.9 | -| off after March 1 | `off_after = "03-01"` | +4.5 | +2.0 | +0.4 | 0.0 | +0.2 | -| off after February 15 | `off_after = "02-15"` | +3.0 | +2.5 | +1.3 | +0.8 | +0.9 | -| per-level eta | `lr_args$per_level = TRUE` | +4.2 | +1.8 | −1.3 | −3.4 | −5.4 | -| geo-pooled eta (option since removed) | `lr_geo_pool = ` | +4.4 | +2.0 | −2.2 | −5.5 | −7.7 | -| seasonal eta window (carry) | `lr_window = 10, lr_seasonal = list(half_width_weeks = 5)` | +2.9 | +0.1 | −3.0 | −5.5 | −7.5 | -| seasonal eta window (reset) | same + `season_policy = "reset"` | +3.2 | +1.1 | −2.1 | −4.7 | −7.0 | -| log-space tracker | `transform = "log1p", lr_args = list(mult = 0.03, floor = 1e-3)` | +10.8 | +1.8 | −5.7 | −9.0 | −8.8 | -| log-space, mult 0.1 | `transform = "log1p", lr_args = list(mult = 0.1, floor = 1e-3)` | +5.3 | −5.8 | −21.8 | −30.9 | −37.8 | -| sqrt-space | `transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3)` | +8.0 | +4.1 | −0.4 | −3.1 | −3.4 | -| fourth-root space | `transform = "quartic_root", lr_args = list(mult = 0.03, floor = 1e-3)` | +9.6 | +4.1 | −2.2 | −4.9 | −5.0 | -| fourth-root, mult 0.1 | same, `mult = 0.1` | +12.3 | +0.9 | −9.8 | −17.8 | −17.6 | - -- Eta was never pooled across horizons: each (location, horizon) series has its - own tracker and its own eta (pooled over the 23 levels and the window). -- **Switching off after April 1** (play base, no updates, offsets carry to next - season) roughly halves the WIS cost at h2/h3 — the spring tail, where - peak-tuned additive offsets are out of regime, is where most of the damage - was. -- **Switching off after March 1 removes the WIS cost entirely** (h1–h3 within - ±0.4 of base, h−1/h0 still positive) — the out-of-regime tail starts around - March, not April. First variant that is WIS-neutral-or-better at every - horizon. -- **February 15 is better still at h0–h3** (+2.5/+1.3/+0.8/+0.9, - WIS-*positive* everywhere) at the cost of ~1.5 points at h−1 vs March 1 — - the post-peak descent is already out of regime, not just the spring tail. - The notebook now runs the March-1 and Feb-15 cutoffs (April 1 dropped). -- **Per-level eta** is a modest gain at h ≥ 1: the pooled 0.9 quantile of - |residual| is dominated by the outer levels and over-steps the median. -- **Geo-pooled eta** gives visibly smoother eta trajectories but no WIS gain. -- **Seasonal window** is worse everywhere on the headline. In the panels its - eta at season start is *smaller* than baseline's (last year's same weeks were - a slow ramp), so the carried-over spring offsets take longer to unwind. - Reset instead of carry recovers some of that (h2 −4.7) but still trails - baseline; 2024-25 is fine (+5.7/+2.8/+0.1 at h−1/0/1), 2025-26 is uniformly - worse. The notebook's seasonal section now runs the reset variant. -- **Log-space tracking** (offsets relative instead of in counts; 2026-09-16) - is worse at every horizon except h−1 and degrades fast with the multiplier - (mult 0.3: −50 to −270%). Calibration error does fall with a larger - multiplier (0.011–0.030 at mult 0.1 vs 0.022–0.035 baseline), so it buys - coverage, but relative residuals at trough counts (base 1–5) are large and - noisy and the staleness problem is untouched. Not in the notebook. If - scale-awareness is retried, normalize by a smooth scale proxy (trailing - 4-week truth, floored) rather than a log. -- **Power transforms** (2026-09-16; `transform = "sqrt"` / `"quartic_root"`, - the latter being the forecasters' whitening map `(x + 0.01)^0.25`). The - bookkeeping: Y and all 23 base quantiles are mapped, residuals/eta/offsets/ - PAVA all live in the mapped space, the played quantiles are mapped back and - clamped, and scoring is on counts. Coverage indicators are invariant under - a monotone map, so the gradient is identical to count space; only the - step geometry changes. Multipliers are not comparable across spaces, so - compare at matched calibration error. `sqrt` at mult 0.03 matches the - baseline's calibration error (0.029–0.043 vs 0.022–0.035) with better WIS - at every horizon, and beats the baseline in the Nov–Feb core at every - horizon (+7.8/+4.5/+0.5/−1.5/−1.2 vs +5.3/+1.9/−1.4/−3.2/−3.1); the spring - tail is still lost at h2/h3 (−19/−33% vs −26/−55%). Fourth root at 0.03 - under-steps (calibration error 0.037–0.049) and at 0.1 over-pays like every - other aggressive setting. Staleness is untouched by any transform; the - transform only bounds how badly a stale offset is sized. Window 20 vs 50 - is again immaterial. Worth a small sweep over power × mult. - -## Where the WIS moves (2026-09-16, baseline, both live seasons pooled) - -By calendar month of the reference date; scratch numbers, reproduced live in -the seasons notebook's by-month tables. - -- **The mass is in December–February.** At h3, Dec + Jan carry ~75% of the - season's base WIS and Mar–May ~7%. Calibration is −2% in Dec/Jan at h2/h3 - and −33/−120/−84% in Mar/Apr/May. Of the −6.7% headline at h3, roughly 56% - comes from the spring on tiny mass and 44% from small losses on the huge - Dec–Feb mass. The gains are Feb at h−1/h0 (+7.6/+4.1%) and Mar at h−1 - (+11.5%). -- **The base median is biased low almost everywhere**, by 10–60% of truth - (Dec: 37–59% at h0–h3, truth above the base median in 90% of December - forecasts). The direction of the needed correction is stable; only its - magnitude swings, by two orders of magnitude within a season. -- **The tracker gets the sign wrong at every turn**, because the offset is a - 5-round-stale count. NY h3, 2025-26: Oct 18 opens with hidden = +159 carried - from spring; the five May outcomes revealed that round (where the played - forecast had over-covered) knock it to +15, then −9 by Nov 22. Then - Nov 29 – Dec 20 reveal *nothing* (the 5-week Oct/Nov gap plus the 5-round - delay), so the offset sits at −9 while base under-predicts by 500–1900. - January pushes it to +148 chasing December, when truth is already falling; - February over-corrects back to −13; spring climbs to +281 on a base of - 26–47 with eta stuck at 85 (the 20-round window still holds January - residuals in the hundreds). The calibrated h3 median is *below* base in 90% - of December forecasts across all states. -- **Eta is not the lever.** The gradient is a coverage indicator whose sign is - set by outcomes 5 rounds old; eta only scales how fast the offset chases - it. Any eta large enough to unwind a stale offset quickly is large enough - to have built it. That is why the seasonal-eta variants lose (small eta at - season start slows the unwind of last spring's offsets) and why the - cutoff variants win (they stop the chase where it is wrong-signed and, - with carry, hand the next season a mid-February offset instead of a May - one). The multiplier sweep in `calibration_qt_flu.Rmd` said the same thing - from the other side: mult trades convergence speed against overshoot with - no setting that wins both. - -### Two-term offset: slow + fast (2026-09-16) - -`qt_track()` now carries `hidden = slow + fast`. Both take the same coverage -gradient; `fast` is the paper's iterate (short eta window, obeys the season -policy, optional per-round leak `fast_decay`), `slow` has its own long window -and small multiplier, never decays, ignores the season policy, and may train -through burn-in (`burn_in_learns_slow`). Without `lr_slow` and with -`fast_decay = 0` the output is bit-identical to before (tests pin this). - -All on `transform = "sqrt"`, mult 0.03, floor 1e-3, window 20, carry, hub -2023-24 burn-in. WIS change vs base by horizon −1…3, then calibration error -range, then Mar–May WIS change at h2/h3: - -| variant | h−1 | h0 | h1 | h2 | h3 | cal err | spring h2/h3 | -|---|---|---|---|---|---|---|---| -| sqrt, single term | +8.0 | +4.1 | −0.4 | −3.1 | −3.4 | 0.029–0.043 | −19 / −33 | -| + fast_decay 0.1 | +8.2 | +4.3 | +1.3 | +0.1 | −0.8 | 0.074–0.089 | −1 / −9 | -| + fast_decay 0.2 | +6.5 | +3.2 | +1.1 | +0.3 | −0.1 | 0.084–0.101 | 0 / −5 | -| + slow 0.01 (no decay) | +8.9 | +4.2 | −1.3 | −3.5 | −4.8 | 0.024–0.034 | −20 / −44 | -| + slow 0.01, decay 0.1 | +9.9 | +5.1 | +0.8 | −0.9 | −2.2 | 0.056–0.066 | −7 / −21 | -| + slow 0.01, decay 0.1, slow trains in burn-in | +10.1 | +5.7 | +1.1 | −0.8 | −1.6 | 0.052–0.066 | −8 / −24 | -| + slow 0.03, decay 0.2, slow trains in burn-in | +11.6 | +5.7 | −0.1 | −2.7 | −3.0 | 0.034–0.041 | −20 / −48 | - -- **The leak is what fixes the spring and h2/h3.** Decay 0.1–0.2 alone makes - the tracker WIS-neutral-or-better at every horizon and removes most of the - spring loss, but it gives back most of the coverage gain (calibration error - doubles): a correction that must be continually refreshed cannot hold - coverage through a 5-round reveal lag. -- **The slow term alone does nothing visible** at mult 0.01 over one burn-in - season plus one live season: it has not had time to move. At mult 0.03 it - is no longer slow and behaves like a second fast term (spring loss is back). -- **Slow + leaky fast is the balance point**: WIS-positive at h ≤ 1, - −1 to −2 at h2/h3, calibration error about half-way between the single-term - tracker and uncalibrated. Letting the slow term train through burn-in - helps a little with one season; the whole point of the ILI+ backfill is to - give it thirteen. - -### Is a learned seasonal offset viable? - -Checked on the only pair of live seasons with the same base forecaster -(2024-25 → 2025-26, 53 locations × 27 season-rounds, aligned by -`season_round`), base-median residual relative to the base median: - -| horizon | cor(rel residual, same season_round, YoY) | Spearman | -|---|---|---| -| −1 | −0.01 | 0.02 | -| 0 | 0.05 | 0.13 | -| 1 | 0.13 | 0.21 | -| 2 | 0.32 | 0.25 | -| 3 | 0.30 | 0.26 | - -Raw count residuals correlate at 0.4–0.7, but that is geography (big states -have big residuals every year), not seasonality. Smoothing last year's -residual over ±2…8 season-rounds does not raise the correlation (it drifts -down slightly), and aligning seasons on each location's peak round instead of -the calendar changes nothing, because the two seasons peaked within 3 rounds -of each other (median season_round 11 vs 8) so the two axes coincide. -Pooling the prior nationally helps a little (0.45 at h2). In fourth-root -residuals the correlations are higher (0.14–0.19 at h0, 0.36–0.44 at -h2/h3), since that is a variance-stabilized quantity. So: weak-to-moderate -year-over-year signal at long horizons, none at short, from one season pair -whose phases happened to line up; the test of whether phase-alignment matters -needs a season that peaks late. The learned hidden offsets -themselves correlate YoY at 0.11 (h2 median). Pooled nationally by 4-round -bin, the relative residual is positive in nearly every bin both years but -its size differs by year (bin 5: +56% vs +20%; bin 17: +23% vs −29%), and -the 2025-26 axis is shifted by the 5-week Oct/Nov gap. Conclusion: a -per-(location, horizon, season-week) offset learned from one prior season is -noise at h ≤ 1 and weak at h2/h3; the stable, learnable component is a -*constant* low bias, not a seasonal profile. Revisit after more seasons of a -stable base forecaster, and align on peak-relative rather than calendar -weeks if so. - -### Warm start from the burn-in season (2026-09-16) - -`slow_init = "burn_in_quantile"` sets the slow term per (horizon, level) to the -conformal offset implied by the burn-in rounds pooled over locations: the -`level`-quantile of `Y − Yhat[level]` in working units. The tracker then runs -on top of it. Same setup as the two-term table (sqrt, mult 0.03, window 20, -carry, hub 2023-24 burn-in); uncalibrated calibration error is 0.095–0.118. - -| variant | h−1 | h0 | h1 | h2 | h3 | cal err | 2024-25 h1/h2/h3 | 2025-26 h1/h2/h3 | -|---|---|---|---|---|---|---|---|---| -| sqrt, single term | +8.0 | +4.1 | −0.4 | −3.1 | −3.4 | 0.029–0.043 | +0.5/−1.8/−3.6 | −2.3/−5.4/−3.1 | -| warm start only, no tracking | +0.5 | +1.2 | +0.9 | +1.1 | +2.0 | 0.062–0.104 | +1.2/+1.7/+3.0 | +0.3/+0.1/+0.1 | -| warm start + fast, no decay | +8.1 | +4.7 | +0.1 | −2.2 | −1.9 | 0.016–0.038 | +1.4/−0.3/−0.9 | −2.5/−5.6/−3.6 | -| warm start + fast, decay 0.1 | +8.4 | +5.0 | +1.9 | +0.9 | +0.8 | 0.048–0.079 | +2.1/+1.0/+1.0 | +1.6/+0.9/+0.5 | -| warm start + slow 0.003 + fast decay 0.1 | +9.0 | +5.3 | +1.9 | +0.8 | +0.5 | 0.044–0.072 | +2.0/+0.7/+0.6 | +1.5/+0.8/+0.3 | -| warm start + slow 0.01 + fast decay 0.1 | +10.1 | +5.7 | +1.3 | −0.1 | −0.6 | 0.036–0.058 | +1.9/+0.1/−0.4 | +0.1/−0.5/−0.8 | -| warm start + slow 0.01 + fast decay 0.2 | +9.1 | +5.4 | +1.5 | +0.5 | +0.4 | 0.040–0.064 | +2.2/+0.8/+0.7 | 0.0/+0.1/0.0 | - -- **One season of batch conformal offsets is already WIS-positive at every - horizon** (+0.5 to +2.0) while removing a third of the coverage error: the - persistent low bias is real and cheap to correct. -- **Warm start + leaky fast tracker is the first configuration that is - WIS-positive at every horizon in *both* seasons** and it halves the - coverage error. The leak is still what keeps the spring in check - (h3 spring: −19% with decay 0.1 vs −45% without); the warm start is what - lets the leak be affordable, since coverage no longer depends on the fast - term holding a level. -- Adding a slow gradient term on top of the warm start is a small dial - between WIS (mult 0.003) and coverage (mult 0.01); the chosen point is - 0.003 with decay 0.1. -- The h3 warm-start offsets at the outer levels are large (0.95 level: +2.9 - scaled-sqrt units for CA) because the 2023-24 ensemble was badly - under-dispersed at long horizons; it is the correct conformal answer for - that season but a reminder that a one-season warm start inherits that - season's base forecaster. In prod the warm start would come from the - previous live season(s) of the *current* forecaster. - -### ILI+ burn-in (2026-09-16): negative result - -The prod flu forecasters already train on ILI+ (percent positive × percent -ILI, state level, 2010–2024) as faux-versioned augmentation rows folded into -`nhsn_prod_archive`. For calibration the analogous move is to *forecast* the -ILI+ seasons with the same `windowed_seasonal` forecaster and use those -residuals as burn-in: 13 usable seasons (2020/21 and 2021/22 dropped as -non-seasons; 2023/24 dropped as overlapping the hub's own burn-in) instead -of one. The pseudo-hub tables are on the ILI+ percent scale, so the tracker -must run in scaled units: `scales` divides each location's rounds by an -era-specific divisor (90th percentile of in-season truth: ILI+ seasons for the -ILI+ era, the hub 2023-24 season for the NHSN era) before the sqrt transform, -so offsets and eta learned on ILI+ carry over as fractions of a typical -season-peak level. Caveats: (1) the pseudo-hub base is one component -(`windowed_seasonal`), the hub base is the submitted ensemble; (2) ILI+ has -no revisions, so the pseudo-hub residuals lack the as-of under-reporting -that drives part of the live bias; (3) the h−1 pseudo-forecast is a model -forecast of an already-observed week, as in prod. - -Result: caveat (1) is fatal for this use. In-season, in scaled sqrt units: - -| era | base | truth > median | median scaled residual (h0 … h3) | -|---|---|---|---| -| ILI+ 2010–20 | windowed_seasonal on ILI+ | 0.52–0.54 | 0.014 … 0.031 | -| hub 2023-24 | submitted ensemble | 0.61–0.67 | 0.030 … 0.112 | -| hub 2024-26 | submitted ensemble | 0.68–0.75 | 0.058 … 0.209 | - -The pseudo-base is essentially unbiased in the median, and its by-season bias -swings from −213% (2015-16) to +66% with no stable component, whereas the -submitted ensemble runs low in every hub season. So ILI+ residuals carry no -information about the live base's bias. Worse, thirteen seasons of near-zero -drift (slightly more under- than over-coverage, and a residual scale 2–3× -NHSN's inflating eta) still summed to a slow term of ~+0.1 scaled-sqrt units -at the first live round, which is +25% at peak and ~3× at trough: WIS −130 to -−500%. Eta warm-up from ILI+ alone (no slow term) changed only h−1, where the -hub burn-in has almost no rounds, and made it worse. Kept: -`scripts/calibration_ili_backfill.R` and the `scales` machinery, which would -be the right tools if the live base were the `windowed_seasonal` component -itself (calibrating per forecaster rather than post-ensemble, roadmap item 3), -since then the pseudo-base and the live base are the same model. - -The useful implication is the opposite one: the hub's own 2023-24 burn-in -*does* share the live bias, so the slow term should be warm-started from it -directly (`slow_init = "burn_in_quantile"`: the conformal offset per horizon -and level, pooled over locations) rather than accumulated by gradient steps. - -### Correctness fixes (2026-09-17) - -A review of the calibration code found two defects that affect every table -above dated 2026-09-16; the fixes are on `ds/calibrate` as separate commits. - -1. **`sqrt` inverse was `x^2`, not `pmax(x, 0)^2`.** The isotonic projection - runs on the sqrt scale, so a played vector with negative low quantiles is - monotone there but squaring folds the negative part back up and the - count-space quantiles cross. On the hub run this hit 578 of 20,393 forecast - sets in the single-term baseline and 1,708 at the operating point, almost - all at levels 0.01–0.05 in small geos at the trough. `quartic_root` already - clamped; `sqrt` now does the same, and `tests/testthat/test-calibrate.R` - exercises the negative regime. -2. **Gating was by reveal round, not issue round.** `update_from[t]` gated the - step *taken at* round `t`, but that step applies gradients of the rounds in - `delay[[t]]`, which at the first live round are the last `h + 2` burn-in - rounds (revealed in a burst after the off-season gap). So the live season - opened with several stale spring gradients, and `off_after` rounds played at - base were learned from the following October. Both masks are now indexed by - the round a forecast was issued at; burn-in and switched-off outcomes still - feed the eta pool but never step. - -Re-measured after both fixes (sqrt, mult 0.03, floor 1e-3, window 20, carry, -hub 2023-24 burn-in), WIS change vs base by horizon −1…3 and calibration error -range. The issue-round gating is what removes most of the h2/h3 loss of the -single-term tracker: the stale burst at season start was a large part of it. - -| variant | h−1 | h0 | h1 | h2 | h3 | cal err | -|---|---|---|---|---|---|---| -| sqrt, single term (was +8.0/+4.1/−0.4/−3.1/−3.4) | +8.0 | +4.5 | +0.8 | −0.6 | −1.4 | 0.022–0.043 | -| operating point: warm start + slow 0.003 + fast decay 0.1 (was +9.0/+5.3/+1.9/+0.8/+0.5) | +9.0 | +5.6 | +2.7 | +1.9 | +1.9 | 0.040–0.072 | - -Also in the same set of commits: `lr_geo_pool` removed (never beat baseline); -`lr_slow` aborts with a constant `lr` or per-level eta (it silently ignored its -own `mult` before); `off_after` and `fast_decay` abort when combined (the leak -would run through the off window, so "frozen offset" would be false); the -hub reader aborts on duplicate (round, level) rows; the accidentally committed -`covid_hosp_prod/workspaces/` binaries were dropped from history and the -pattern is now ignored. The notebooks have not been re-rendered since. - -### One forecaster across eras (2026-09-17) - -Question: are the hub-season gains an artifact of three seasons of one -ensemble, and does the tracker behave the same on a decade of the *same* -forecaster? `scripts/calibration_ws_backfill.R` replays flu prod's -`windowed_seasonal` (prod archive, seed, substitutions) over the NHSN seasons -in hub schema: 2024-25 and 2025-26 as honest as-of replays, 2023-24 with the -`"cheating"` policy (NHSN vintages start 2024-11-19). With the ILI+ pseudo-hub -(`scripts/calibration_ili_backfill.R`) this is one forecaster from 2011 to -2026. `scripts/calibration_ws_experiments.R` runs three setups, all sqrt, -mult 0.03, window 20, carry, on the 50 locations common to all sources and -on the hub's submission rounds only: - -- **A** submitted ensemble, burn-in 2023-24 (the reference). -- **B** `windowed_seasonal` on NHSN, burn-in 2023-24. -- **C** `windowed_seasonal` on the ILI+ decade (2011-12 burn-in, 2012-2020 and - 2022-23 tracked live) then NHSN 2023-26, whitened per location and era by - the 90th percentile of in-season truth (`scales`), offsets carried across - the source switch. - -Variants: `single` (paper's tracker), `leaky` (`fast_decay = 0.1`), -`operating` (warm start + slow 0.003 + leaky fast). WIS change % vs base: - -| setup / variant | 2024-25 h−1…h3 | 2025-26 h−1…h3 | -|---|---|---| -| A single | +11.7 / +2.6 / +1.1 / −0.1 / −1.3 | −2.6 / +2.0 / −2.1 / −3.4 / −2.8 | -| A operating | +11.2 / +3.8 / +2.6 / +2.5 / +3.8 | +0.4 / +3.9 / +0.2 / −0.5 / −0.7 | -| B single | +0.7 / +2.0 / +0.3 / −0.7 / −1.5 | −1.4 / +2.3 / −0.5 / −0.7 / −0.3 | -| B leaky | +1.4 / +2.1 / +0.8 / +0.3 / −0.3 | −0.3 / +1.7 / +0.4 / +0.8 / +1.3 | -| B operating | +2.1 / +2.7 / +1.1 / +0.7 / +0.8 | −0.1 / +2.3 / +0.4 / +0.5 / +0.6 | -| C single | −6.0 / −0.3 / −0.3 / +0.4 / +1.3 | −11.4 / −3.8 / −4.0 / −3.4 / −2.9 | -| C leaky | +1.5 / +1.9 / +0.4 / −0.5 / −1.2 | −0.3 / +1.7 / +0.4 / +0.8 / +1.3 | -| C operating | −0.7 / +5.1 / +2.9 / +3.3 / +4.1 | −10.1 / −1.1 / −2.0 / −1.3 / −0.6 | - -C on the first NHSN season (2023-24, live there): single −42 / −41 / −27 / -−22 / −19, operating −24 / −23 / −15 / −10 / −6, leaky +1.4 / +0.5 / +0.4 / -+0.7 / +0.6. Calibration error (mean over levels; base first) for B: -2024-25 base 0.074–0.168, single 0.053–0.088, operating 0.048–0.093; -2025-26 base 0.044–0.088, single 0.007–0.016, operating 0.016–0.042. - -Fraction of truth above the base median (the sign of the bias): - -| base | 2023-24 h−1…h3 | 2024-25 | 2025-26 | -|---|---|---|---| -| ensemble | 0.56 / 0.55 / 0.55 / 0.53 / 0.52 | 0.83 / 0.74 / 0.73 / 0.74 / 0.76 | 0.58 / 0.63 / 0.59 / 0.60 / 0.60 | -| windowed_seasonal on NHSN | 0.55 / 0.54 / 0.58 / 0.62 / 0.68 | 0.59 / 0.68 / 0.68 / 0.72 / 0.76 | 0.58 / 0.66 / 0.65 / 0.65 / 0.65 | -| windowed_seasonal on ILI+ (2014-2019 seasons) | 0.25–0.45 | | | - -`windowed_seasonal` calibrated on ILI+ alone (2011-12 burn-in, ten seasons): -`single` improves WIS in 8 of 10 seasons (+5 to +26%, losses in 2012-13 and -2022-23) and cuts calibration error at h ≥ 0 from 0.019–0.036 to 0.013–0.017; -`operating` is negative in 6 of 10 seasons and triples calibration error -(0.046–0.053). Same numbers appear as the ILI+ rows of setup C. - -Reading: - -- **The low bias is not the ensemble's alone.** `windowed_seasonal` on NHSN - under-predicts in all three seasons (0.55–0.76), growing with horizon, so - there is something to correct; the gains are smaller than the ensemble's - (+0.4 to +2.7% at the operating point vs up to +11%) because the component - is less miscalibrated to begin with (base cal err 0.07–0.17 vs the - ensemble's 0.13–0.19 in 2024-25) and because the ensemble's h−1 is a - different, worse model (climate_linear). -- **The same forecaster's bias flips sign with the data source.** On ILI+ - it over-predicts (truth above median 25–45%); on NHSN it under-predicts. - Carrying ILI-era offsets into the NHSN era (C) therefore costs 20–40% WIS - in the first NHSN season, and the warm start learned on the ILI decade - keeps hurting through 2025-26. The ILI+ history is not a proxy for the - NHSN regime even with the same model, so the earlier negative result on - ILI+ burn-in was not a plumbing bug (date and geo alignment were checked - and hold) but a regime difference. -- **What transfers across seasons is the plain tracker, not the warm start.** - Across ten ILI+ seasons the single-term tracker is reliably positive; the - warm start + slow term, chosen on two hub seasons, is reliably negative - there. On NHSN with one forecaster (B) the two are close, with `operating` - slightly ahead on WIS and `single` far ahead on coverage. The leaky - variant is the only one indifferent to history (identical in B and C): - small, consistent WIS gains, weakest coverage gains. -- **Overfitting verdict.** The tracker itself is not overfit to the three - hub seasons; the operating point's extra machinery is fit to the ensemble's - persistent low bias and should not be assumed to carry over to a new base - or a new data source. A prod design should warm-start from the *current* - forecaster's most recent NHSN season only, or use the leaky tracker with no - warm start when that season is unavailable. - -### Directions this points to, ranked - -1. **Split the offset into a slow level term and a fast tracker.** Done - (two-term tracker + conformal warm start, above); this is the current - operating point. Open: warm-starting from more than one season, and a - relative (log-space) slow term so the warm start transfers across the - scale swing more evenly than a sqrt-space constant does. -2. **Attack the staleness directly, not eta.** h−1 is free because its lag is - one round; h3 loses because its lag is five. Short-horizon coverage - indicators for the same target weeks are known ~4 rounds before h3's - own, and the base forecaster's bias direction is shared across horizons - (all horizons under-predict in the ramp). Feeding the fresh h−1/h0 - gradient into h2/h3's update (a cross-horizon proxy signal) is the only - idea on the list that changes *when* the offset turns rather than how fast. -3. **Phase-gate updates on data, not the calendar.** The Feb-15 cutoff works - because both live seasons peaked around the turn of the year; a - late-peaking season would break it. Gating on the base median's trailing - trend (post-peak ⇒ stop updating, or shrink) expresses the same rule - without a hard-coded date. -4. **Scale-aware offsets via a smooth proxy**, given the log-space failure. -5. **Eta variants** (seasonal, per-level, pooled): parked. None can fix a - wrong-signed step. - -## Production integration (2026-09-21) - -Calibrated forecasts are now threaded into both flu and covid production pipelines as a secondary submission, `CMU-TimeSeries-Calibrated`, leaving the primary `CMU-TimeSeries` ensemble unchanged. - -**Architecture:** three new targets added to each prod script, executed after the ensemble: - -- `calibrated_ensemble_nhsn` (`cue = tar_cue("always")`) — reads all past `CMU-TimeSeries` submissions from the hub checkout via `hub_read_forecasts()`, appends the current round's `ensemble_mix` output converted to hub schema via `internal_to_hub_forecasts()` if not already present, reads finalized truth, and calls `calibrate_hub_forecasts()`. - Skips (returns `NULL`) when `g_submission_directory == "cache"` (no hub checkout set). -- `make_calibrated_submission_csv` — writes the calibrated quantiles to `model-output/CMU-TimeSeries-Calibrated/-CMU-TimeSeries-Calibrated.csv` in the hub checkout. -- `local_calibrated_scores_nhsn` — scores the calibrated forecasts against `nhsn_latest_data` via `score_forecasts()`; fed into the ongoing score report alongside the base forecasters. - -**Operating point (flu):** `transform = "sqrt"`, `lr_args = list(mult = 0.03, floor = 1e-3)`, `lr_window = 20`, `season_policy = "carry"`, `slow_init = "burn_in_quantile"`, `lr_slow = list(mult = 0.003)`, `fast_decay = 0.1`, `burn_in_seasons = "2023-2024"`. -Flu gets the 2023-24 burn-in for free: the hub checkout already contains all past CMU-TimeSeries flu submissions. - -**Operating point (covid):** same except `burn_in_seasons = character(0)`, `slow_init = NULL` (no hub burn-in available; covid submissions begin 2024-11-23). - -**Notebooks:** `CMU-TimeSeries-Calibrated` is added to `ongoing_score_report.Rmd` (`our_forecasters`, score table styling, line width). -The per-date `forecast_report.Rmd` notebook receives calibrated forecasts for the current round's reference date, converted back to internal schema (state abbr, Wednesday `forecast_date`, count-space quantile values) via the `notebook` target in `prod_shared.R`. - -**Model metadata:** `CMU-TimeSeries-Calibrated.yml` created in both `../FluSight-forecast-hub/model-metadata/` and `../covid19-forecast-hub/model-metadata/` (`designated_model: false`). - -**Bug fixed en route:** `nssp_archive` in both `flu_data_targets.R` and `covid_data_targets.R` had a duplicate-key error (pre-existing since August 2026): `compactify = TRUE` can retain multiple rows per `(geo_value, time_value)` when the value changed across issues, and the subsequent `mutate(version = time_value + 7)` flattened all to the same version, producing duplicate keys in `as_epi_archive()`. -Fix: `group_by(geo_value, time_value) %>% slice_max(version, n = 1L, with_ties = FALSE) %>% ungroup()` before the version clobber. - -# Roadmap - -Possible next steps, roughly ordered: - -1. **Review the ranked gallery** — does the WIS cost concentrate at turning - points (the `h + 2` staleness prediction)? Are the biggest offsets fixing - real miscalibration or chasing data-revision artifacts? -2. **Send the collaborator email** (sweep findings; h −1 free win; carry vs - reset a wash). -3. **`windowed_seasonal_extra_sources` retrospective** via - `scripts/calibration_harness.R` (covid, evaluation store) — does - calibration help our best component forecaster, not just the submitted - ensemble? -4. **Integration design** for prod: where the tracker lives (per-forecaster - vs post-ensemble), how state persists week to week, and how to handle the - warm-up (burn-in on a past season? carry state across seasons? a - burn-in-free learning-rate schedule?). -5. **Method extensions**, in the order argued under "Directions this points - to": slow relative level term + fast tracker; cross-horizon proxy - gradients for staleness; data-driven phase gating; scale proxy. Pooling - offsets across locations and per-level learning rates are lower priority. -6. **Agentic triage** of gallery panels against a fixed rubric (data revision - visible? base missed the turn? calibration helped/hurt?) if manual - skimming of the ranked gallery proves insufficient. -7. **Dynamic gallery app** (Shiny/Posit or a served page with dynamic data - fetch) if the static top-N format becomes limiting. diff --git a/notes/ROADMAP.md b/notes/ROADMAP.md new file mode 100644 index 00000000..561bb7b3 --- /dev/null +++ b/notes/ROADMAP.md @@ -0,0 +1,146 @@ +# Roadmap + +Repo-wide tasks, TODOs and tech debt, roughly in priority order within each +section. Calibration-specific threads live under "Open threads" in +`notes/calibration-ledger.md`; refactor designs (E2 ensemble sweep, snapshot +validator, simplification inventory) live in `notes/refactor-ideas.md`. + +## Forecast evaluation + +1. **Calibration in the evaluation pipeline.** The operating point is a + provisional disease-specific choice (covid REF-op cold, flu sqrt 0.018 + warm + leak), not yet implemented. The plan, the pipeline's current + state and the open decisions are in `notes/calibration-ledger.md`, + "TODO: calibration in the evaluation pipeline". The clean replay + (`EVALUATION_FORECASTERS`, `EVALUATION_SUBSTITUTIONS=false`, AGENTS.md) + and the harness-side 2023-24 burn-in are done; see the ledger's "Code, + data and prod". The fixed-weight ensemble and hand-edits-on-vs-off + questions (old 1c, 1d) are in Parked; ILINet as a held-out target (old + 1e) is in the ledger's "Ideas". + +2. **NSSP-target backtesting is second-class and needs dedicated + attention.** + - Explore forecasts NHSN only: every family sets `outcome = "hhs"`. NSSP + as a target exists only for the prod components, in the prod and + evaluation pipelines and the `*_nssp_backtesting_*` reports, so no + NSSP-target sweep has ever been run. + - Flu NSSP scoring looks broken. In `flu_nssp_backtesting_2024_2025_on_2026-05-06` + and `2026-07-01_flu_nssp_scoring_*`, every model, the hub ensemble + included, has 90% coverage of 0 and WIS about equal to absolute error. + Most likely the truth and the forecasts are on different scales + (proportion vs percent). Fix this before reading any flu NSSP result. + - Covid NSSP scoring looks sane (May 2026 backtest, 2024-25): our best + component is `windowed_seasonal_latest` (mean WIS 0.11), + `CMU-TimeSeries` 0.14, the CovidHub ensemble 0.06. + +2b. **US is handled several different ways; make it one declared policy** + (audited 2026-10-01). + - `score_forecasts()` drops location `US` on purpose + (`R/targets/score_targets.R:84`), so every local, external and + calibrated NHSN score omits US. Behind that, its location → geo join + goes through `get_population_data()`, which has both `us` and an + `usa` alias for code `US`, so dropping only the filter double-counts + US. The calibrated-forecast joins in `prod_shared.R` and both prod + pipelines have the same latent duplicate. A patch (one-row-per-location + `hub_location_crosswalk()`, used in all those joins) is in + `_local/patches/us_scoring.patch`, checked on 2 dates per disease. It + roughly doubles cross-geo mean WIS (US WIS ≈ sum of states), so ship it + with a `geo_level` column and state-only means in the reports. + - Submissions are not affected: US is present in all covid files and + all flu files since 2024 (US ÷ sum of states median ratio 1.00 flu, + 1.03 covid). But flu 2025-11-22 to 12-06 had US at 0.58–0.71 of the + state sum at h2–h3, and nothing checks that. + - `climate_geo_agged` pools US into the state climatology unscaled, so + CMU-climate_baseline's US is about state-sized relative to the pool + (submitted US 0.5–0.9 of the state sum). + - Flu explore: 4 `scaled_pop_data_augmented` forecasters + (`filter_agg_level = ""`) forecast and score US; the other 129 drop it. + The comparison notebook ranks them by mean WIS, so these 4 look worst. + Covid explore drops US everywhere. + - Proposed design: one canonical `"us"` from archive build on, with one + crosswalk for every geo ↔ location join (then delete the `usa` alias + and the redundant downstream renames). A spec column `us_method` + (`direct` default, `sum_of_states`, `none`) in + `FORECASTER_SPEC_DEFAULTS` and the ensemble specs, applied once in + `run_forecaster()`, replacing the data-dependent `filter_agg_level` + drop. Contracts: `validate_forecast_output()` requires `us` unless + `us_method = "none"`; prod health warns when the US median is outside + about 0.8–1.25 × the sum of state medians; `score_forecasts()` never + drops a location silently and has unique keys. + +## Revision-aware forecasting and h−1 + +3. **Revision-method backtests.** `revision_aware` against the + `revision_ratio` baseline at h−1 (explore h−1 families plus + `nowcast_notebook`, both diseases), then choose prod's h−1 component. + Calibrating h−1 waits on this (`notes/calibration-ledger.md`). +4. **Flu explore has no current scores for the revision families.** In the + flu explore store on S3 (`joined_scores_2024_2026`, 2026-09-19), none of + `revision_aware`, `revision_aware_augmented`, `revision_aware_nssp`, + `revision_aware_no_season`, `revision_aware_beds_no_season`, + `revision_aware_beds_seasonal` or `revision_ratio` has scores under its + current id. The 32 orphaned ids in that store score 20–60× flatline in + 2024-25, consistent with the pre-fix US population-scaling bug + (`notes/revision_aware_forecaster_devlog.md`). The two `beds` grids are + 24 forecasters each and have never been scored for flu. +5. **Re-run covid explore families that take NSSP as an input.** Covid + explore NSSP now has real vintages (`up_to_date_nssp_state_archive`); the + covid store on S3 (2026-09-28) predates that. `scaled_pop_exogenous` + scores 0.55 relative WIS against flatline there, better than the + CovidHub ensemble (0.62), which suggests leakage from finalized NSSP. + `revision_aware_nssp` needs the same re-run. + +## Explore grid + +6. **Trim `climate_linear`.** It is 73 of 182 flu forecasters and 49 of 97 + covid forecasters. In the S3 stores (WIS relative to flatline, h0–h4, + mean of 2024-25 and 2025-26): + - `drop_non_seasons` gives identical scores either way, for both + diseases. Either it is a no-op on these dates or it is not wired + through; check before dropping it. + - `nonlin_method` moves the best score by less than 0.001. + - `model_used` is the axis that matters, and the winner differs by + disease: `climate` for flu (best 0.865; `climate_linear` 0.963, + `climatological_forecaster` 1.04), `climate_linear` for covid (best + 0.854; `climate` 1.99, `climatological_forecaster` 5.3). + - `quantiles_by_geo = TRUE` helps flu, hurts covid. + Fixing `drop_non_seasons` and `nonlin_method` alone cuts the family to + about a quarter (flu ~19, covid ~13). + +## Parked (2026-10-01, not on the calibration path) + +- Covid `windowed_seasonal_extra_sources` calibration (calibration-ledger open + threads). It is ~86% of covid `ensemble_mix` but has no 2023-24 NSSP, so + no burn-in. +- CMU-climate_baseline US forecast is too low (item 2b). +- `us_method` spec column and removing the `usa` alias (item 2b). +- Ensemble-base calibration headline tables (E00, E03, E09, E10 part 2) pool + raw counts over all locations, so US is about half of their "WIS change + %". The replay experiments report states only (except E12). +- *The ensemble with fixed weights* (was item 1c, parked 2026-10-07). The + geo-exclusions CSVs carry the ensemble weights as well as exclusions, so "no hand edits" for + `ensemble_mix` means one fixed weight block for every date. Decide + which block (the default one at the top of the file, or the current + one), check the resolved weights on one date, then replay. Not needed + to choose the operating point: E10 already scores the finalists on + the submitted ensemble, and E07 shows calibration acts the same way + on the ensemble and the single-component replay. Revisit if the calibration pipeline plan (ledger TODO) needs + a clean ensemble history. +- *Hand edits on vs off as its own question* (was item 1d, parked + 2026-10-07). A separate evaluation project (like the `_regr` ones) replaying with the edits, to measure + whether the weekly substitutions and weight changes help. The only + past check found is the 2025 revision report + (`revision_summary_report_2025.Rmd`), which scored substitutions by + whether they moved values toward the final value. Calibration needs + only the clean replay. + +## Tech debt + +7. renv warns on every run (library renv 1.1.6, lockfile 1.2.3; some + lockfile packages not installed). Harmless so far. +8. Low priority: `windowed_seasonal`'s h−1 forecast ignores the latest + reported week, so a data substitution to that week never reaches h−1. + Most substitutions are to that week (flu 20 of 25 rows, covid 10 of + 10). Submitted h−1 comes from other components, so nothing shipped is + affected; revisit with the revision-aware h−1 work (item 3). +9. Refactor and cleanup threads: `notes/refactor-ideas.md`. diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md new file mode 100644 index 00000000..3a4be6b8 --- /dev/null +++ b/notes/calibration-ledger.md @@ -0,0 +1,515 @@ +# Calibration ledger + +Post-hoc online calibration of our hub forecasts with MultiQT (Ding, Gibbs & +Tibshirani 2025). Each (location, horizon) series gets an additive offset over +the 23 quantile levels, updated by online gradient descent on pinball loss +and projected to be monotone. Dropped hub rounds are in +`notes/spoiled-submissions.md`; how to run the tests and notebooks is under +"Running things". + +Every number below was re-read from the rendered notebook (or, for E12/E13, +the score CSV) on 2026-10-02; E19's come from its CSVs on 2026-10-06. +Older notes, pre-fix tables and the summary-of-reviews layers were +removed; they are in VCS history at change `uxposooz` +(`notes/CALIBRATION.md`, `notes/calibration-review.md`, +`notes/calibration-summary-2026-10-01.md`). + +## How to read the numbers + +- **WIS change %** is against the uncalibrated base; positive is better. + **Coverage bias** is mean |coverage − nominal| over levels; lower is + better. +- **Uncertainty is a round bootstrap (E10, E20; `cal_wis_boot()`).** Two + live seasons, 27 and 28 rounds; rounds are resampled within a season, + 1000 draws, 90% intervals, the same draws for every config so gaps + between configs are paired. A single cell's interval is wide: about ±1 to + ±4 points per season on the adaptive anchor, ±0.5 to ±2 on the constant + one, and a 2025-26 cell is wider than a 2024-25 one. A paired gap between + two similar configs is much tighter, about ±0.5 to ±1 pooled. The old + convention, a gap under 1 WIS point is "no clear difference", holds for + pooled paired gaps and is too optimistic for single cells. The round is the + resampling unit, which ignores dependence between neighbouring rounds, so + the intervals are if anything too narrow. +- **Coverage bias is the selection criterion; WIS is the tie-breaker + (2026-10-07).** Pick the config on the Pareto front where both improve, + and never trade month-avg coverage for WIS: WIS rewards a one-directional + shift when the base runs low (see "The correction is a small upward + shift") and is the less trusted metric. A config whose month-avg coverage + is worse than base in any season at h1–h3 is not a finalist, whatever its + WIS. +- **Use month-avg coverage, not season-pooled.** Pooling over a season (or + two) lets ramp under-prediction cancel post-peak over-prediction, and lets + one season's error cancel the other's. Season-pooled gains of 2–5× mostly + vanish month by month. Month-level noise is about 0.01. +- **Report each season.** Most rankings flip between 2024-25 and 2025-26. + Pooled numbers hide that. +- **Cold starts dilute 2024-25.** Burn-in rounds warm the learning rate but + never step the offset, so every config without a warm start equals the + base through Nov–Dec 2024 (35–40% of that season's WIS). Warm-vs-cold in + 2024-25 is "some offset vs none". +- **Tuning is in-sample.** The constant rates (0.018, 0.032) and the + `off_after` dates were picked on the same two seasons they are scored on. +- **Two base forecasters.** E00, E03–E05 and E09–E11 calibrate the + submitted ensemble (h−1…h3, all locations in headline tables). E01, E02, + E07 and E12–E19 calibrate the clean `windowed_seasonal` replay (h0–h3, + states only, except E12). + +## What the numbers support + +Large and consistent (several points, both seasons): + +- **Too-large steps hurt.** Adaptive multiplier 0.1 on sqrt costs 7–17 WIS + points at h1–h3, 0.3 costs 17–81 (E01). A constant per-100k rate above + about 0.1 loses steeply: −3 to −4 at h3 by 0.18 (E14), −12 to −23 at 1.0 + (E09). +- **sqrt beats count.** 2–5 points at h1–h3 on the clean replay (E01), + 1.4–3.9 pooled on the ensemble, mostly from 2025-26 (E03). +- **h−1 is where calibration pays.** Covid +10 to +11% WIS and about +20% + median absolute error, both seasons, either config (E10); flu ensemble +3 + to +6 (E00, E03). +- **Stateful learning needs a delay.** A tracker that learns once from the + first report loses all of REF-op's h−1 gain (0.0 vs +6.1) and makes h−1 + coverage worse than base; one extra week fixes it (E00, E09). Prod re-runs + from scratch each week, so this does not apply to prod. +- **Some covid configs are harmful.** REF-paper loses 10–19 points at h1–h3 + in 2025-26 (E10); constant 0.1 with a warm start loses 8–27 (E13). + +Small, or dependent on season (0–3 points): + +- **Among reasonable trackers nothing clearly wins.** REF-op (warm or cold), + sqrt constant 0.018, rate constant 0.018–0.032 and sqrt adaptive 0.01 are + within about 1 point of each other pooled, and the order flips by season. + Configs with a warm start lead in 2024-25; the others lead in 2025-26 + (E07, E16, E18). +- **Warm start: first live season only.** +1 to +3 points at h2–h3 in + 2024-25, −0.7 to +0.4 in 2025-26 (E05, E13, E18). Warm configs under-cover + the 50% interval (0.39–0.44, E13). +- **Leak (`fast_decay` 0.1): season-dependent.** On the ensemble it avoids + a 3–5 point loss at h1–h3 in 2025-26 and costs 0.4–1.3 in 2024-25 (E04, + E05). On the clean replay it adds 0.5–1.3 at h1–h3 in both seasons and + costs 0.4 at h0 (E18). +- **Why the leak and a small constant rate land close (E19).** Both limit + the adaptive step's late-season overreaction, in different ways. The + adaptive eta comes from base residuals, so the leak does not change it; + relative to interval width it rises from about 0.015 in January to 0.10 + at h3 in May 2025, 3–4× the constant rate's. Without the leak the offset + keeps growing into March–April, when the base over-predicts (h3 median + offset 0.24 of width in April 2025 vs REF-op 0.11). REF-op tracks the + ramp as fast (0.045 in February) and the leak empties it by May. Constant + 0.018 barely corrects the ramp (0.013 in February) and has little to + overshoot with. In April–May 2026 the base under-predicted and the leak + dropped an offset that was still right (0.01 vs 0.14 at h3), which fits + sqrt 0.018 leading in 2025-26. The link to WIS is inferred, not scored. +- **Switching off from 1 Mar or 15 Feb.** On the clean replay: +2 to +4 at + h1–h3 over sqrt adaptive 0.03 (E02), +0.4 to +0.7 over sqrt constant 0.018 + (E17). The date was chosen after seeing the March losses. About half of + the 2025-26 gain at h2 is offsets frozen in February 2025 carried into + December, not a spring fix. On the ensemble it is the weakest candidate in + 2024-25 (E07). +- **Per-level eta:** about +1 point at h1–h3 (E02). +- **March costs every tracker.** Up to −28% WIS within March at h2, about 1 + point of the season (E16). The base already over-predicts after the peak + and the offsets are still positive. +- **The correction is a small upward shift.** Offsets are a few percent of + the 90% interval width; the width ratio is 1.00–1.04. REF-op's gain comes + from raising the lower half; the upper half loses beyond h0 (E15). + +Coverage: + +- **Month-avg, no flu config moves coverage much.** Every config is within + about 0.025 of base, most within 0.01. REF-op and the warm configs on the + ensemble are 0.012–0.025 better than base at h1–h3 (E05, E09). Constant + rates are no better than base, or slightly worse at h3 (E09, E14). +- **Season-pooled coverage gains are cancellation.** For example, E11's + "4–5× lower bias" is about 2× per season, and nothing month by month. + +Finalists (E10 part 1, E18), states only, h1/h2/h3 pooled over both seasons: + +- **Flu: sqrt 0.018 warm + leak leads on both bases (E10, E20).** + Ensemble +2.8/+3.0/+4.8 vs sqrt 0.018 warm +2.5/+2.7/+4.3 and REF-op warm + +1.9/+1.5/+2.3; replay +1.8/+1.9/+2.9 vs +1.6/+1.5/+2.7 and +1.1/+0.8/+1.2. + The leak repairs the warm constant's 2025-26 on the ensemble: +0.7/−0.0/ + −0.8 vs −0.2/−0.9/−1.7 (REF-op warm +0.1/−0.7/−0.9). Month-avg coverage on + the ensemble, pooled: 0.108/0.126/0.140 at h1–h3 vs REF-op warm + 0.110/0.124/0.138 and base 0.123/0.137/0.151; in 2025-26 it is 0.004–0.007 + below REF-op warm at h1–h2. Ranks first pooled on both bases, with REF-op + warm first on ensemble coverage by 0.001. **Bootstrap (2026-10-07):** the + paired gap to REF-op warm, pooled h1/h2/h3, is +0.9 [+0.3, +1.3] / +1.6 + [+0.9, +2.1] / +2.4 [+1.4, +3.2] on the ensemble and +0.6 [−0.1, +1.7] / + +0.9 [+0.1, +2.1] / +1.9 [+1.2, +2.8] on the replay, so the interval + excludes zero at h2–h3 on both bases; per season it excludes zero at + h2–h3 in 2024-25 and only at h3 on the replay in 2025-26. The leak's own + gap over sqrt 0.018 warm is +0.3 [−0.4, +1.1] / +0.4 [−0.3, +1.2] / +0.5 + [−0.1, +1.2] on the ensemble and +0.2 / +0.3 / +0.4 with the same spread + on the replay: for WIS the leak is a coin flip, and it is in the finalist + for its 2025-26 coverage (E20), not its WIS. Against the warm adaptive + cell at the same leak, the constant anchor's gap is +0.6 [+0.0, +1.5] / + +0.8 [+0.2, +1.7] / +1.6 [+1.0, +2.3] pooled (E20). +- **Covid: no grid cell helps at h1–h3, and the warm start hurts (E20).** + See the E20 row; REF-op cold stays the least bad config overall. The + bootstrap puts constant cold leak above REF-op cold on the replay by + +0.5 [+0.1, +0.9] / +0.9 [+0.3, +1.4] / +1.1 [+0.2, +1.9] pooled at + h1–h3, all from 2025-26, and both are below the base there (−1.2 to −1.7 + vs −2.4 to −3.2). That cell has not been run on the covid ensemble, where + the constant anchor without a leak loses to REF-op cold by 2.6–3.4 (next + bullet), so it does not change the operating point. +- **Covid: sqrt 0.018 is worse than REF-op cold on the ensemble.** + −2.7/−3.7/−4.2 vs −0.1/−0.3/−0.8, from 2025-26 (−6.9 to −8.3 at h1–h3; + REF-op has the lower WIS in 44–45 of 52 states). At h−1 sqrt 0.018 gains + the most of any config, +14.9 vs REF-op +10.2 (E10 part 2). No config + helps covid at h1–h3 on either base. +- **The season flip is broad, but the pair is unmatched.** REF-op warm + beats sqrt 0.018 cold in 41–45 of 52 flu states in 2024-25 and 14–18 in + 2025-26 (E18). Cold configs play the base through Nov–Dec 2024, and on the + same replay sqrt 0.018 warm beats REF-op warm at every horizon in both + seasons, so the per-state flip is the start, not the tracker: matched, + REF-op wins 23 of 52 states in 2024-25 and 15 in 2025-26 warm vs warm, + 17 and 15 cold vs cold (E18). + +- **On the replay, matched on the start, the constant rate leads (E20).** + Constant 0.018 beats adaptive 0.03 in 29–47 of 52 states in every cell + and season. Warm plus leak is the best cell for either anchor; for the + constant rate +3.3/+1.8/+1.9/+2.9 pooled vs REF-op warm +2.8/+1.1/+0.9/ + +1.1, with month-avg coverage 0.012–0.017 above base at h2–h3 in + 2025-26. +- **The constant rate's 2025-26 lead is mostly inherited (E19).** Resetting + its offset at the season boundary takes 2025-26 from +3.0/+1.0/+1.1/+1.5 + to +1.0/+0.1/+0.2/+0.3. Cold configs enter 2024-25 with nothing, so the + two seasons are not the same test of a tracker. + +Operating point (preliminary, chosen 2026-10-07 for that week's +submission; the open threads below can still move it): + +- **Flu: sqrt 0.018 cold** (in `pipelines/flu_hosp_prod.R`). Chosen on + coverage, not WIS: it is the only flu config whose month-avg coverage is + better than base in both seasons on both bases (by 0.005–0.014 at h1–h3) + and whose WIS is never below base. It gives up the warm start's 2024-25 + WIS (sqrt 0.018 warm + leak is +2.8/+3.0/+4.8 pooled on the ensemble vs + +0.6/+0.1/−0.0), because the warm configs are worse than base on + month-avg coverage in 2025-26 on the replay (+0.008/+0.014 at h2–h3 with + the leak, +0.022/+0.027 without). The rate was picked in-sample (E16). +- **Covid: REF-op cold** (prod, unchanged). No config improves covid + coverage at h1–h3 on the ensemble (every one is 0.000–0.008 worse than + base) and every one loses WIS there; REF-op cold loses least (−2.0 to + −2.9 in 2025-26 vs −6.9 to −8.3 for sqrt 0.018 cold, which is only + competitive on the replay). Its value is h0 (+0.8) and h−1 (+10 to +15, + E10 part 2). + +## Reference configs + +- **REF-paper**: count scale, adaptive eta (mult 0.03, window 20), carry, + 2023-24 burn-in, `settle_days` 14 (one extra week of revision delay), + single offset term. +- **REF-op** (prod): REF-paper with sqrt, eta floor 1e-3, warm start + `slow_init = "burn_in_quantile"`, slow term mult 0.003, leak + `fast_decay = 0.1`. "REF-op cold" drops the warm start and burn-in. + +Both are `cal_ref_args()` in `R/calibration/views.R`; notebooks run them via +`cal_run()`, which also picks the learning truth (`"exact"`: re-run from +scratch each round on that round's data, as prod does). + +**Notebooks.** One notebook per experiment, `eNN_*.Rmd` in +`reports/writeups/calibration_experiments/`, rendered to +`reports/calibration_experiments/` with `just calibration-experiments [name]`. +The index groups them by topic (finalists, flu and covid, learning rate, +tracker structure, late season, learning truth). + +Each comparison notebook leads with the month view (`_month_view.Rmd`): +WIS, coverage bias and median error by month of reference date, then +by-horizon tables of WIS reduction % and season-pooled / month-avg coverage +bias. All scores are states only. Sweep notebooks (E01, E09, E16) lead +with their coverage-vs-WIS curves and follow with a month view of a few rates +around the elbow. Per-state fan panels and the gallery of best and worst +forecasts are only in `prod_views.Rmd`, for the prod configs. Conclusions +live in this ledger, not in the notebooks. + +## Experiments + +All flu unless stated; all post-fix (2026-09-17 sqrt clamp and issue-round +gating), exact learning truth, spoiled submissions excluded. + +| id | notebook | base | question | what the numbers show | caveats | +|---|---|---|---|---|---| +| E00 | `e00_vintage_backtest` | ensemble | learn from final, vintage or exact data? | exact ≈ final within 0.6 points except REF-op h−1 (2.2, 2024-25 only). Vintage at settle 7 loses the h−1 gain. Settle 7 vs 14: REF-paper +1 to +3 at every horizon; REF-op tie (flips by season) | | +| E01 | `e01_eta_settings` | replay, cold | adaptive eta settings | mult ≥ 0.1 costs 7–81 at h1–h3. 0.01 beats 0.03 by 0.5–2.4 at h1–h3 in both seasons; 0.03 better at h0 in 2024-25 (0.8). Window and carry vs reset: ≤ 2.4, mostly < 1. Count is 2–5 worse than sqrt | month-avg coverage of 0.01 vs 0.03 within noise | +| E02 | `e02_eta_variants` | replay, cold | `off_after`, per-level eta, seasonal window | off 1 Mar / 15 Feb +2 to +4 at h1–h3 over the 0.03 reference, both seasons, no month-avg coverage cost. Per-level +1. Seasonal window, burn-in, off 1 Apr: < 1 | dates in-sample; half the 2025-26 gain is December carry-over; "seasonal window" also changes `lr_window` 20 → 10 | +| E03 | `e03_scale` | ensemble | count vs sqrt vs rate | sqrt +1.4 to +3.9 over count pooled, mostly 2025-26. Coverage: neither better (sqrt worse per season pooled, better month-avg only in 2024-25) | rate identical to count in every cell (adaptive rate is scale invariant) | +| E04 | retired 2026-10-07 (superseded by E20) | ensemble | leak, slow term | leak +2.5 to +5.6 at h1–h3 in 2025-26, −0.4 to −1.3 in 2024-25; month-avg coverage flips the same way (±0.015). Slow term without warm start: ≤ 0.5 | | +| E05 | retired 2026-10-07 (superseded by E20) | ensemble | warm start from burn-in | warm +1.7 to +3.2 at h2–h3 in 2024-25, ±0.4 in 2025-26, for every tracker. REF-op ≈ warm + leak (≤ 0.4). Month-avg coverage: all within 0.01 of each other | | +| E07 | `e07_across_eras`; the candidate ranking (formerly E07b) is in `e10_covid` | both, same rows | do results transfer between ensemble and replay? | Same direction on both; ensemble gains about 1 point more with REF-op warm. Replay base 5–7% worse pooled, 10–33% worse at h0–h1 in 2025-26. Candidate order flips by season on both; rank correlation of coverage in 2025-26 is 0.31 | | +| E09 | `e09_lr_delay` | ensemble | constant rate × revision delay | knee 0.056: +3 to +5.5 over REF-paper at h1–h3; REF-op +1.2 to +2.4 over knee. Extra delay costs about 1 h−1 point per week. Above 0.1 loses steeply | month-avg: no constant rate beats base at h1–h3 | +| E10 | `e10_covid` | flu + covid, ensemble + replay | the finalists (now including sqrt 0.018 warm + leak, E20's best flu cell) on both diseases and bases, with round-bootstrap intervals and paired gaps (part 1); REF-paper, REF-op and sqrt 0.018 on the covid ensemble at h−1–h3 (part 2) | see "Finalists" above, including the bootstrap gaps. Part 2: h−1 +10 to +15 (sqrt 0.018 largest). REF-op h1–h3: +1.9 to +2.8 in 2024-25, −3.0 to −4.3 in 2025-26. REF-paper −10 to −19 and sqrt 0.018 −7.7 to −9.5 at h1–h3 in 2025-26 | part 1 is h0–h3 on rows both bases have; part 2 tables run to 2026-09-19, including the 2026 summer wave | +| E11 | retired 2026-10-07 (superseded by E20) | ensemble | constant 0.1 vs adaptive | REF-op best pooled WIS at every horizon (up to 2 over warm + sqrt). Constant 0.1 gives up 1.5–2.3 at h−1/h0 vs adaptive sqrt | constant 0.1 is effectively cold; its coverage edge is season-pooled only | +| E12 | none (`calibration_ws_replay.R`) | replay, cold, flu + covid | first look at the clean replay | flu: REF-op cold leads constant 0.1 by 0–2.5. Covid: REF-op cold h0 +1.3 states, losses at h1–h3 in 2025-26 | **all locations** (US is 47% of WIS); covid h0 gains are mostly US | +| E13 | none (`calibration_ws_replay.R burn_in`) | replay, flu + covid | 2023-24 HHS burn-in | flu warm +1 to +2.6 in 2024-25, ≤ 0 in 2025-26. Covid: warm worse at h0–h1, better at h2–h3 (+1.3, +4.4); constant 0.1 warm −8 to −27 | HHS used as-is (3–9% below NHSN); coverage is season-pooled only | +| E14 | folded into `e16_sqrt_constant_lr` | replay, cold | constant rate grid by season | every rate 0.0032–0.056 is ≥ 0 in both seasons and within 1.2 of each other; 0.018 vs 0.032 ≤ 0.3. Above 0.1 loses | month-avg: no rate ≤ 0.032 beats base beyond 0.01 | +| E15 | `e15_wis_sources` | replay, cold | where the WIS gain comes from | offsets a few % of interval width; REF-op gain from the lower half and the median shift | width-only variant's coverage not measured; split is not additive | +| E16 | `e16_sqrt_constant_lr` | replay, cold | constant rate on sqrt scale | sqrt 0.018 about +1 over rate constants at h0; at h1–h3 within 0.2 pooled, up to +0.7 in 2025-26. vs REF-op cold: behind in 2024-25 h0–h1, ahead elsewhere; month-avg coverage tie | 0.018 picked in-sample | +| E17 | `e17_late_decay` | replay, cold | shrink or stop offsets from 1 Mar | `late_decay` fixes March but costs about 0.5 at h0, otherwise ±0.2. Off 1 Mar +0.4 to +0.7 at h1–h3 for sqrt 0.018 (mostly 2025-26), ≤ 0 for rate 0.018 | `late_decay` shrinks the stored offset, so it carries into the next season | +| E18 | `e18_ref_op_bridge` | replay | which piece of REF-op matters | REF-op vs sqrt 0.018 within 0.6 pooled; REF-op ahead in 2024-25 by 1.3–1.5, behind in 2025-26 by 0.8–1.3. Warm start +0.9 to +2.0 in 2024-25, −0.2 to −0.7 in 2025-26. No step moves month-avg coverage beyond 0.008. Branch sqrt 0.018 warm: +3.4/+1.6/+1.6/+2.5 pooled, best of all, but month-avg coverage 0.011–0.018 worse than base at h2–h3. States where REF-op wins, 2024-25 / 2025-26: chain ends (warm vs cold) 45 / 13 of 52; warm vs warm 23 / 15; cold vs cold 17 / 15 | steps 0 and 4 are the same cached runs as E13; steps 4 and 5 identical (scale invariance). The chain-ends pair is unmatched on the start | +| E19 | `e19_tracker_internals` | replay | why do the leak and constant 0.018 score alike? Step size, offset, base bias and WIS by month on the E20 grid; carry vs reset on both anchors | Step / width at h3 rises from 0.015–0.02 (Dec–Feb) to 0.06–0.09 (Apr–May) on the adaptive rate, 0.005–0.007 to 0.013–0.023 on the constant, in both seasons. Offset / width at h3, Mar/Apr/May 2025: adaptive cold no leak 0.105/0.218/0.186, with leak 0.064/0.078/−0.012, constant cold 0.026/0.055/0.072; Apr/May 2026: 0.160/0.150, 0.057/−0.011, 0.097/0.143. Base bias (share above the base median, h3): Mar 0.43 and 0.27, Apr–May 0.54–0.81 and 0.62–0.71. March WIS at h3: adaptive no leak −48 and −81 (2025, 2026), with leak −27 and −22, constant cold −9 and −35, with leak −5 and −8; Apr–May 2026 constant no leak +4.8/+11.0. **Carry vs reset:** resetting at the season boundary takes the constant tracker's 2025-26 from +3.0/+1.0/+1.1/+1.5 to +1.0/+0.1/+0.2/+0.3, the adaptive one's from +2.2/−0.6/−0.7/−0.2 to +0.7/−0.9/−1.3/−1.6; the carried offset is 0.01–0.025 of width. Carry's gain is Oct–Feb (h3 Nov–Dec 2025 +5.7/+3.8 vs +0.6/+0.4); reset is better after the peak | offsets from the exact runs, step sizes from one pass on final truth per config. `lr_level` for the two-term REF-op configs reports the slow term's rate (about a tenth of the fast one), so their step-size rows are not comparable; REF-op's fast step equals the adaptive anchor's by construction | +| E20 | `e20_structure_grid` | replay | warm start and leak on both anchors: {adaptive 0.03, constant 0.018} × {cold, warm} × {no leak, leak 0.1}, plus REF-op warm and cold | Warm start: +1.0 to +2.6 (adaptive) and +1.5 to +3.8 (constant) in 2024-25, −0.5 to +0.6 in 2025-26, on both anchors at either leak. Leak: on the adaptive rate +0.7 to +2.0 at h1–h3 in both seasons; on the constant rate −0.2 to +0.7 warm and −0.6 to −1.7 in 2025-26 cold. Best cell constant warm leak: +3.3/+1.8/+1.9/+2.9 pooled vs REF-op warm +2.8/+1.1/+0.9/+1.1, but month-avg coverage 0.012–0.017 above base at h2–h3 in 2025-26 (the warm constant without leak is 0.024–0.037 above). Constant beats adaptive in 29–47 of 52 states in every cell and season. **Bootstrap (90%, rounds):** flu single cells are ±1.5 to ±4 per season on the adaptive anchor and ±0.5 to ±2 on the constant; paired gaps: best cell − REF-op warm +0.5/+0.7/+1.0/+1.8 pooled at h0–h3 with the interval excluding zero at h2–h3; best cell − warm constant without leak +0.2/+0.3/+0.4 at h1–h3, intervals include zero (2024-25 alone excludes it, +0.4 to +0.7); warm − cold on the constant with leak +1.8 to +3.8 in 2024-25 excluding zero, +0.4 to +1.6 in 2025-26 including it. **Covid:** the warm start costs 2–7 points at h0–h2 in 2024-25 (constant) and 1–3 (adaptive), and 0.01–0.04 of month-avg coverage; the leak adds 1–3 on the adaptive anchor and on the cold constant in 2024-25, costs 1–3 on the warm constant. No cell beats base at h1–h3 pooled; least bad is constant cold leak −0.5/−0.8/−0.5 vs REF-op cold −1.1/−1.7/−1.6, adaptive warm leak +3.4 at h3 but −1.9 at h1 | replaces E04, E05 and E11 on the replay (retired). Not a scale comparison: every cell runs on sqrt, so E03 stays. The slow term is left out (E18 step 2). All constant-rate configs tuned in-sample (E16) | +| E21 | `e21_data_by_season` | replay data | how the two live seasons differ as data: shape, revisions, learning-truth gap at lags 1–4, HHS-stitched burn-in vs NHSN | 2024-25: 27 rounds from 2024-11-23, peak 2025-02-08 at 55.6k (states summed), total 559k; 2025-26: 28 rounds from 2025-10-18, peak 2026-01-03 at 42.5k, total 336k. State peaks within ±1 week of the national one: 37 vs 47 of 52. Revisions: 26% vs 40% of (state, week) keys never revised; relative spread > 10% for 48% vs 39%. Value at lag 2 (what the tracker learns from) vs final: median gap −1.1% vs 0.0%, \|gap\| > 10% for 19% vs 14% of keys; at lag 1, 44% vs 41%. HHS vs NHSN final over 2023-24: 0.957–0.978 in Nov–Apr (states summed), per-state median 1.004 | describes the data only; no tracker runs. Revision keys start at the first archive version (2024-11-19) | + +E06 (ILI+ burn-in) and E08 (old covid) were run on code or data later found +broken and were not rebuilt. + +## Known issues + +- **Run cache ignores code.** `cal_run_cached()` keys on inputs and + arguments, not code; the E00 and E09 caches are keyed by chunk code or + name. Clear `cache/calibration/experiments/` after changing + `R/calibration/`. + +Checked on 2026-10-02: the whole cache was cleared and every experiment +re-run from scratch (E12, E13 and every notebook). Every table matched +the cached results cell for cell, so no result above came from a stale +run. + +Checked and resolved on 2026-10-02: + +- **`scales` is applied.** `rate` equals `count` to every digit with the + adaptive rate (E03, E09, E18 steps 4–5) because the adaptive rate is + scale invariant: all 53 locations match a population scale, adaptive + runs agree to 1e-11 with and without it, and with a constant rate the + calibrated quantiles differ by up to 300 admissions. +- **E10 pools through the 2026 summer on purpose**, to include covid's + summer wave; its headline and month tables run to 2026-09-19. +- **Month-view base column** in E07 (one base shown for configs on two + forecasters) and the **`late_decay` docstring** (now says it shrinks the + stored offset) were fixed. + +## Review status + +| notebook | status | +|---|---| +| E00, E01 | verified | +| E02, E07, E09 | commented; changes made, not re-verified | +| E10 part 1, E18 | changes made (finalists runs, matched pairs), rendered 2026-10-07, not reviewed | +| E20 (flu and covid), E10, E15 | rendered 2026-10-07 with the covid grid, the new finalist and (E10, E20) the bootstrap tables, not reviewed | +| E19 | rendered 2026-10-07, not reviewed | +| E21 | rendered 2026-10-06, not reviewed | +| E03, E10, E14, E15, E16, E17 | not reviewed | +| E04, E05, E11 | retired 2026-10-07, never reviewed | + +On 2026-10-02 every notebook's introduction and Findings were rewritten to +state setup and measured sizes only; the rewritten text has not been +reviewed. + +## Code, data and prod + +- `R/calibration/qt.R`: the tracker (`qt_track`, `qt_learning_rate`, + `qt_project`). A port of the authors' Python, tested against it (see + "Running things"). +- `R/calibration/calibrate.R`: `calibrate_hub_forecasts()` and + `calibrate_hub_forecasts_exact()`, plus metrics. Options beyond the + paper: `transform`, `scales`, `lr_slow` / `fast_decay` / + `burn_in_learns_slow` / `slow_init` (two-term offset and warm start), + `off_after`, `late_decay`, `lr_seasonal`. +- `R/calibration/hub_data.R`: hub readers (`hub_read_forecasts()` drops + spoiled submissions), `nhsn_read_truth()`. +- `R/calibration/views.R`: reference configs, cached runs, standard views, + and `cal_wis_boot()` (round bootstrap of WIS reduction %, shared draws + across configs) with `cal_wis_boot_diff()` for paired gaps. +- `scripts/calibration/calibration_ws_replay.R`: E12/E13. Scores in + `cache/calibration/ws_replay_scores_*.csv`. +- `scripts/calibration/calibration_ws_replay.R` also holds the notebook + helpers: `ws_grid_configs()` (the E19/E20 structure grid), + `ws_prod_specs()`, `ws_run_spec()`, `ws_wis_by_season()` and + `ws_internals()` (per-round eta and offsets from an exact run). +- `scripts/calibration/calibration_ili_backfill.R`: replays + `windowed_seasonal` over the ILI+ state history (2010–2024) into + hub-schema parquets in `cache/calibration/`, for an ILI+ burn-in (thread closed 2026-10-02, unused). +- The pre-experiment notebooks (`reports/writeups/calibration/`) and the old + E07 scripts were deleted on 2026-10-02; they are in VCS history. Their + only analyses not in the current suite are the staleness lagged + correlation, the ILI+ base-bias table, per-round tracker internals (eta, + offsets before and after projection) and a per-level reliability plot. +- **Data.** Flu hub submissions 2023-10-14 … 2026-05-30, 53 locations, + h−1…h3. 2023-24 is burn-in. NHSN vintages from + `get_nhsn_data_archive()` start 2024-11-19. With `settle_days` 14 the h3 + offset is always 5 rounds stale. +- **Prod (2026-09-21).** Flu and covid prod write a secondary submission, + `CMU-TimeSeries-Calibrated`, via the targets `calibrated_ensemble_nhsn`, + `make_calibrated_submission_csv` and `local_calibrated_scores_nhsn`. Flu + runs REF-op with the 2023-24 burn-in; covid runs REF-op without burn-in + or warm start. The covid submission is mostly + `windowed_seasonal_extra_sources` at h1–h3 and `revision_aware` at h−1 + (weights in `pipelines/covid_geo_exclusions.csv`, edited weekly). + +## TODO: calibration in the evaluation pipeline + +The evidence for the operating point comes from the harness +(`scripts/calibration/`), which reads the evaluation store's forecasts, +learns from per-round NHSN snapshots and scores states only. The pipeline's +calibration targets (`pipelines/{flu,covid}_hosp_prod.R`) run in evaluation +mode but unchanged from prod: they calibrate the hub checkout's submitted +history plus only the current replayed round, learn from +`hhs_evaluation_data` (final truth), hard-code REF-op, and are skipped when +`EVALUATION_FORECASTERS` drops an ensemble component. The goal is a +calibrated row in the store beside the other forecasters, so the score +notebooks pick it up. + +Three steps, each a run and a row in "Experiments": + +1. **Covid, REF-op cold, on the store's ensemble.** In evaluation mode the + target takes `ensemble_mix` over all replayed dates from the store, + calls `calibrate_hub_forecasts_exact()` with truth and vintages from + `nhsn_archive_data` (as `ch_hub_truth()` does), and takes the config + from a per-disease list. No burn-in. Score it like any other forecaster; + states-only is a scoring filter, so keep the US row. Check against E10's + covid ensemble REF-op cold row, the same computation in the harness. +2. **Flu, sqrt 0.018 warm + leak, on the store's ensemble.** Same target, + plus the hub checkout's 2023-24 rounds as the burn-in season (the prod + target already reads them). The burn-in learns from the NHSN archive's + first version, which carries 2023-24 as final, as E10's ensemble warm + runs do (`fin_inputs()`); no HHS stitch on this base. Check against + E10's ensemble sqrt 0.018 warm + leak row. +3. **Prod's config beside the finalist.** REF-op warm (flu) and REF-op cold + (covid) through the same target as a second row, so the evaluation + reports show both. Then decide whether prod switches. + +Not in these steps: a calibrated row on the clean `windowed_seasonal` base. +The store has nothing before 2024-11-20; the harness backfills 2023-24 by +rerunning the forecaster on the HHS-stitched archive +(`ws_burn_in_forecasts()`). Doing that in the pipeline needs two decisions +already made but not implemented: ILI+/flusurv extras are folded into +`nhsn_prod_archive` with `version = time_value`, so a 2023-24 snapshot +would train on finalized ILI+ (version each extras row at its season's end +and replace the archive-build `stopifnot` with a version check; dates from +2024-11-21 on are unaffected); and NSSP has no vintages before 2024-04-18, +so every NSSP-reading component needs an explicit skip for 2023-24 (a +snapshot before the first vintage silently returns 0 rows). Parked unless +the evaluation reports need the replay base. + +## Running things + +Commands run from the repo root (inside the rocker container, +`distrobox enter rocker -- `, if R isn't on the host). The hub +checkouts are siblings of this repo: `../FluSight-forecast-hub` (sparse: +CMU model-output only) and `../covid19-forecast-hub`. + +**Port tests.** `R/calibration/qt.R` ports the authors' Python +(`projectedQT`). The oracle is `../multiQT` on branch `delphi-fixes`, which +fixes several defects in the published code (see that branch's commit +message); the R port implements only the fixed behavior. + +```sh +# tracker tests: bit-exact oracle fixtures plus properties +Rscript -e 'testthat::test_file("tests/testthat/test-qt.R")' +# whole suite +Rscript -e 'testthat::test_dir("tests/testthat")' + +# regenerate the fixtures (made with lr_window = 50, which test-qt.R spells out) +(cd ../multiQT && uv run --with numpy --with scikit-learn --with matplotlib python make_r_fixtures.py) +cp ../multiQT/r_fixtures/*.csv tests/testthat/fixtures/qt/ + +# cross-check on real hub series; expect ALL MATCH within 1e-8 +Rscript scripts/calibration/calibration_export_series.R +(cd ../multiQT && uv run --with numpy --with scikit-learn --with matplotlib python check_r_port_real_series.py) +``` + +One porting trap: numpy broadcasts `Y - Yhat` along the last axis, while R +recycles a vector down columns. Getting it wrong silently corrupts every +learning rate; `test-qt.R` pins `eta` against a hand-built residual matrix. + +**Notebooks and scripts.** + +```sh +just calibration-experiments # all notebooks and the index +just calibration-experiments finalists # one notebook and the index +Rscript scripts/calibration/calibration_ws_replay.R 12 # E12 (12 workers) +Rscript scripts/calibration/calibration_ws_replay.R 12 burn_in # E13 +``` + +Tracker runs are cached under `cache/calibration/` by `cal_run_cached()`, +keyed on inputs and arguments but not code: clear the cache after changing +`R/calibration/`. E00 and E09 keep their own caches, cleared by hand. + +## Open threads + +Checked against the code on 2026-10-02. Repo-wide items are in +`notes/ROADMAP.md`. Add a thread only if its answer could change the +operating point. + +| item | status | +|---|---| +| **Reorganization (2026-10-06).** The constant-rate line was never given the structure analysis (E03–E05, E11 ran the adaptive rate on the ensemble; E16 inherited "cold" from E11's framing and E13's constant 0.1 warm covid loss). Plan: (1) E18 matched-pair scatters, warm vs warm and cold vs cold; (2) E20, the {anchor} × {start} × {leak} grid on the replay; (3) E19 as a notebook on that grid with WIS by month and carry vs reset; (4) E21, the seasons as data. Then re-read E03–E05 and E11 against E20 and retire or keep them | all rendered 2026-10-07 (E21 on 2026-10-06) and read into this file; E20 covers covid too, and sqrt 0.018 warm + leak is a finalist in E10 and E15. E03–E05 and E11 re-read against E20 on 2026-10-07: E20 reproduces E04, E05 and E11's directions on the replay (warm start first-season only, leak season-dependent on the adaptive rate, slow term nothing, constant beats adaptive), so those three were deleted; their rows stay in "Experiments". E03 stays: E20 runs every cell on sqrt and never compares scales | Render with `just calibration-experiments e18_ref_op_bridge e19_tracker_internals e20_structure_grid` inside the rocker container | +| Uncertainty for WIS differences (bootstrap over rounds) | closed 2026-10-07: `cal_wis_boot()` in every E10 and E20 season table, with paired gaps. The flu finalist's lead over REF-op warm survives at h2–h3 on both bases; the leak's WIS gain does not (its case is coverage); the constant anchor's lead over the adaptive one survives at h2–h3; the warm start's gain is 2024-25 only. See "How to read the numbers" and "Finalists" | +| Shrink only the played offset after the peak, keep the stored state | open; `late_decay` shrinks the stored state | +| Disease-specific operating point | preliminary choice made 2026-10-07 (see "Operating point"): flu sqrt 0.018 cold on coverage grounds, covid REF-op cold. Earlier provisional reading, WIS-led: covid REF-op cold (no grid cell helps at h1–h3, E20); flu sqrt 0.018 warm + leak (E10, E20), about 1/1.5/2.5 points over REF-op warm at h1–h3 on the ensemble, coverage within 0.002. Rate and leak picked in-sample; two seasons. The bootstrap backs the flu lead at h2–h3 on both bases but not the leak's WIS share of it. Not implemented; see "TODO: calibration in the evaluation pipeline" | +| HHS burn-in level | closed 2026-10-07: E21 measures it at 2–4% below the NHSN final in Nov–Apr 2023-24 (states summed), per-state median ratio 1.004, within the warm start's own noise. Used as-is. The pipeline's flu step (the TODO section) calibrates the ensemble base, whose 2023-24 rounds are hub submissions and whose burn-in truth is the NHSN archive's first version, so the stitch is only needed for a replay-base burn-in | +| `windowed_seasonal_extra_sources` retrospective | parked (ROADMAP) | +| Re-rank the finalists coverage-first (2026-10-07) | acted on the same day: flu moved to sqrt 0.018 cold (see "Operating point"). Still open as a question, since the replay 2025-26 excess is near the noise floor. Under the coverage rule the flu finalist (sqrt 0.018 warm + leak) fails on the replay in 2025-26: month-avg coverage +0.008/+0.014 above base at h2–h3 (warm without leak +0.022/+0.027), while on the ensemble it is within 0.004 of base there and 0.024–0.028 better in 2024-25. The cold configs (sqrt 0.018, REF-op cold) improve coverage in both seasons on both bases by 0.005–0.019 but give up 2–4 WIS points at h2–h3 in 2024-25, partly because cold = base through Nov–Dec 2024. Month-level noise is about 0.01, so the replay 2025-26 excess is at the edge. To settle: a coverage bootstrap like `cal_wis_boot()`, and the E20 grid read by coverage first | + +**Priorities (2026-10-07).** The configs tested are within about a point +of each other and differ only in how they handle the rise and the fall, so +further config-vs-config distinctions are deprioritized. Order of work: + +1. **Mechanistic view of one tracker on one season.** The per-round view + (offset relative to width, base bias, truth vs base median, WIS by + round) for the operating config, with the real-time offsets against + what the finalized data later showed. The 2025-26 zigzag reads cleanly + this way: a one-directional offset against a base whose error changes + sign at the peak, the plateau and the final decline. Build this as a + standing notebook (E19 is the start) and run it on every new season + before any config comparison. +2. **Coverage as the criterion.** A coverage bootstrap beside + `cal_wis_boot()`, so the replay 2025-26 excess (0.008–0.014, noise about + 0.01) can be called. Then re-read E20 by coverage. +3. **Covid at h1–h3.** Nothing helps; decide whether to stop calibrating + those horizons and keep h0/h−1. +4. Everything else in this table and in "Ideas" waits. Do not add a config + unless the mechanistic view predicts where it would help; do not tune + for a single season's anomaly. + +**TODO: rewrite this ledger and the notebooks around the mechanistic view +(noted 2026-10-07, not started).** The experiment summaries above are +detailed, expensive to read, and mostly record that the scores are +indifferent to the config; every session starts by triaging them. Slim the +ledger to the data facts, the operating point and its reasoning, the +coverage rule, and the per-season mechanistic reads; drop the E-by-E result +prose (it stays recoverable from the commit log and the rendered +notebooks). Decide what to keep in a dedicated session, not in passing. + +Closed 2026-10-02 without further work: ramp vs post-peak split (the month +view shows it), h−1 calibration (left to the revision-aware methods, +ROADMAP 3), ILI+ burn-in (the HHS burn-in covers it), gallery review (review +uses the month view), speeding up exact runs. + +## Ideas + +Method ideas with no code and no notebook that would test them. None of +E18–E21 points at any of them; they are kept so they are not re-invented. + +- Cross-horizon gradients as a staleness signal: the h0 offset is fresher + than the h3 one by three rounds, so their gap could steer the h3 step. +- Data-driven phase gating: switch the offset off (or shrink what is + played) when the season's phase changes, instead of a fixed date + (E02, E17) or a leak. +- A scale proxy: a per-location scale from the base's own interval width + rather than population, so the sqrt-per-100k transform is not needed. + +Housekeeping, low priority: + +- Rename the tracker's `slow` / `fast` terms to `init_offset` / + `tracked_offset` (`lr_slow`, `fast_decay`, `slow_init`, `init_slow`, + `burn_in_learns_slow`). The slow term's online learning never registered + (E04, E18); what it does is hold the warm start where the leak cannot + drain it, so the real split is a fixed burn-in offset plus a learned, + leaky one. "Constant" / "adaptive" are taken by the learning rates. + +Held-out data, no code yet: + +- **ILINet as a target**, to check the trackers on a dataset they were not + tuned on. `fluview` has real issue history back to about 2010 (CA wILI + for week 2023-50 is revised weekly through May 2024). Source checked + 2026-10-02 (`notes/ilinet-sources.md`): v5 `fluview_ilinet` matches + epidatr `fluview` at every issue outside NY, version date = CDC release + Friday; state version history starts 2017w40 in both, so a vintage + backtest has 2017-18 on; use state `ili` (v5 has no state `wili`). + Would follow the `nssp_target_archive` + `primary_source` pattern, drop + the ILI+ training rows in this mode, and face the percentage-scale + question NSSP-target scoring has (ROADMAP 2). Prototype on one forecaster + and a few seasons; `calibration_ili_backfill.R` (ILI+, no vintages) is a + reference, not the base to extend. diff --git a/notes/calibration-runbook.md b/notes/calibration-runbook.md deleted file mode 100644 index 0568cf14..00000000 --- a/notes/calibration-runbook.md +++ /dev/null @@ -1,233 +0,0 @@ -# Calibration runbook - -Every command needed to reproduce the online-calibration work from scratch. -Three repos are involved: - -| path | role | -|---|---| -| `~/allHail/delphi/exploration-tooling` | the R implementation, tests, driver, notebook | -| `~/allHail/delphi/multiQT` | the authors' code plus our upstreamed bugfixes (branch `delphi-fixes`) — the oracle for the R tests | -| `~/allHail/delphi/FluSight-forecast-hub` | flu submitted forecasts (sparse checkout: CMU model-output only) | -| `~/allHail/delphi/covid19-forecast-hub` | covid submitted forecasts + truth CSV | - -The multiQT clone must be on `delphi-fixes`. The as-published code had a -handful of defects (a learning-rate window that sliced the level axis instead -of time, an unpopulated first played column, a default `lr='adaptive++'` with -no implementation, a numpy read-only-view crash), all reported to the authors -and fixed in that branch's single commit — see its message for the details. -The R port implements the corrected semantics only; there are no -compatibility knobs for the as-published behavior. - -R is not on `$PATH`; it lives in the rocker container. Every R command below is -shown with the `distrobox` prefix. Drop it if you are already inside the -container. - ---- - -## 0. One-time setup - -```sh -# Python deps are supplied per-invocation by uv; nothing to install. -# The multiQT oracle must be on the bugfix branch: -cd ~/allHail/delphi/multiQT && git checkout delphi-fixes - -# Refresh the hub submissions (83 CMU-TimeSeries rounds, 2023-10-14 .. 2026-05-30): -cd ~/allHail/delphi/FluSight-forecast-hub && git pull --ff-only - -# The hub's truth file is downloaded and cached on first use by hub_read_truth(); -# to force a refresh: -cd ~/allHail/delphi/exploration-tooling -rm -f cache/calibration/target-hospital-admissions.csv -``` - -The hub checkout is sparse (`hub-config`, `model-output/CMU-TimeSeries`, -`model-output/CMU-climate_baseline`). Leave it that way — `target-data/time-series.csv` -alone is 84 MB, and we fetch the 570 KB truth file over HTTP instead. - ---- - -## 1. R implementation and its oracle tests - -```sh -cd ~/allHail/delphi/exploration-tooling - -# just the tracker's tests (bit-exact oracle fixtures + properties) -distrobox enter rocker -- Rscript -e \ - 'testthat::set_max_fails(Inf); testthat::test_file("tests/testthat/test-qt.R")' - -# the whole suite, to confirm nothing else moved (expect 183 pass / 0 fail / 9 skip) -distrobox enter rocker -- Rscript -e 'testthat::test_dir("tests/testthat")' -``` - -### Regenerating the oracle fixtures - -`tests/testthat/fixtures/qt/` is the output of `projectedQT` on the -`delphi-fixes` branch, generated with `lr_window = 50` — the same default the -R side uses, so `test-qt.R` calls `qt_track()` with its defaults (plus -`lr_window = 50` spelled out). If you change the fixture set, regenerate and -copy: - -```sh -cd ~/allHail/delphi/multiQT # on delphi-fixes -uv run --with numpy --with scikit-learn --with matplotlib \ - python make_r_fixtures.py -cp r_fixtures/*.csv ~/allHail/delphi/exploration-tooling/tests/testthat/fixtures/qt/ -``` - ---- - -## 2. Cross-check the R port against Python on **real** hub series - -The fixtures are synthetic. This runs the same comparison on real submitted -forecasts, real hub truth, and the real `horizon + 2` delay. - -```sh -cd ~/allHail/delphi/exploration-tooling -distrobox enter rocker -- Rscript scripts/calibration_export_series.R -# optional: Rscript scripts/calibration_export_series.R - -cd ~/allHail/delphi/multiQT -uv run --with numpy --with scikit-learn --with matplotlib \ - python check_r_port_real_series.py -``` - -Expect `ALL MATCH`, max abs diff ~1e-12 (CSV round-trip noise) against a -tolerance of 1e-8. Only series complete across all 83 rounds are exported (52 of -265), because the reference Python has no notion of an inactive round. - ---- - -## 3. Run the calibration - -```sh -cd ~/allHail/delphi/exploration-tooling -distrobox enter rocker -- Rscript -e ' -suppressPackageStartupMessages(source("R/load_all.R")) -fc <- hub_read_forecasts() -tr <- hub_read_truth() -cal <- calibrate_hub_forecasts(fc, tr, - burn_in_seasons = "2023-2024", - season_policy = "carry", # or "reset" / "shrink" - lr_window = 50, # Inf is an expanding window - lr_args = list(mult = 0.1)) -print(as.data.frame(hub_coverage_summary(cal))) -print(as.data.frame(hub_quantile_loss(cal))) -saveRDS(cal, "cache/calibration/cal.rds") -' -``` - -Takes ~5 s for all 265 (location, horizon) series. Other entry points: - -- `hub_coverage(cal, by = "location")` — per-location coverage, for the fan plot -- `hub_rolling_tradeoff(cal, window = 20)` — rolling calibration error vs quantile loss -- `cal$series` — per-series learning-rate trajectory and reveal counts -- `cal$rounds` — the global round axis with season labels, burn-in flags, `hidden_scale` - -## 3b. COVID calibration - -The COVID hub checkout at `~/allHail/delphi/covid19-forecast-hub` carries all -model-output for CMU-TimeSeries (89 rounds, 2024-11-23 … present) and a local -truth CSV — no HTTP download required. - -```sh -cd ~/allHail/delphi/exploration-tooling -distrobox enter rocker -- Rscript -e ' -suppressPackageStartupMessages(source("R/load_all.R")) -fc <- hub_read_forecasts(hub_dir = HUB_COVID_DIR, target = HUB_COVID_TARGET) -tr <- hub_read_covid_truth() -cal <- calibrate_hub_forecasts(fc, tr, - burn_in_seasons = character(0), # no off-season in covid data yet - season_policy = "carry", - transform = "sqrt", - lr_window = 20, - lr_args = list(mult = 0.03, floor = 1e-3)) -print(as.data.frame(hub_coverage_summary(cal))) -print(as.data.frame(hub_quantile_loss(cal))) -' -``` - -Takes ~5 s for all 265 (location, horizon) series. -Refresh the submissions before running: `cd ~/allHail/delphi/covid19-forecast-hub && git pull --ff-only`. - ---- - -## 4. The EDA notebook - -```sh -cd ~/allHail/delphi/exploration-tooling -distrobox enter rocker -- Rscript -e \ - 'rmarkdown::render("scripts/reports/calibration_qt_flu.Rmd", output_file = "calibration_qt_flu.html")' -``` - -Output: `scripts/reports/calibration_qt_flu.html`. Parameters (`burn_in_seasons`, -`season_policy`, `lr_window`, `lr_mult`, `roll_window`) are in the YAML header; -override per-render with: - -```sh -distrobox enter rocker -- Rscript -e ' -rmarkdown::render("scripts/reports/calibration_qt_flu.Rmd", - output_file = "calibration_qt_mult003.html", - params = list(lr_mult = 0.03, season_policy = "reset"))' -``` - -It renders the settings sweep itself, so it takes ~5 min (32 full calibration runs). - -## 5. The gallery notebook - -Fixed operating point (`lr_mult = 0.03`, `lr_window = 20`, carry) plus a -ranked per-forecast panel gallery; see `notes/CALIBRATION.md` for the design. - -```sh -cd ~/allHail/delphi/exploration-tooling -distrobox enter rocker -- Rscript -e \ - 'rmarkdown::render("scripts/reports/calibration_qt_gallery_flu.Rmd", output_file = "calibration_qt_gallery.html")' -``` - -As-of vintages come from `cache/calibration/nhsn_archive_flu.parquet` (a copy -of an oracle-capture NHSN archive; provenance and staleness caveats in -`notes/CALIBRATION.md`). `n_panels` is a YAML param (default 150). - -## 6. The seasons notebook - -Four states (best/worst by WIS and by median AE change, chosen from the -baseline run) × two live seasons × six method variants, plus a by-month -breakdown of where the baseline gains and loses. Chunks `load` and `variants` -are knitr-cached; clear `cache/calibration/knitr_cache_seasons/` after -changing anything in `R/calibration/`. - -```sh -cd ~/allHail/delphi/exploration-tooling -distrobox enter rocker -- Rscript -e \ - 'rmarkdown::render("scripts/reports/calibration_qt_seasons_flu.Rmd", output_dir = here::here("reports"))' -``` - -Output: `reports/calibration_qt_seasons.html`. - -## 7. ILI+ pseudo-hub backfill - -Runs flu prod's `windowed_seasonal` forecaster over the ILI+ state history and -caches hub-schema forecasts and truth for multi-season calibration burn-in -(design and caveats in `notes/CALIBRATION.md`, "ILI+ burn-in"). - -```sh -cd ~/allHail/delphi/exploration-tooling -distrobox enter rocker -- Rscript scripts/calibration_ili_backfill.R # ~30 min -distrobox enter rocker -- Rscript scripts/calibration_ili_backfill.R --refresh # rebuild -``` - -Output: `cache/calibration/ili_pseudo_hub_forecasts.parquet` and -`ili_pseudo_hub_truth.parquet`; read them with `ili_read_pseudo_hub()` after -sourcing the script. To combine with the hub tables, bind the rows, pass every -pre-2024-25 season as `burn_in_seasons`, and hand `calibrate_hub_forecasts()` -a `scales` table with one ILI+-era and one NHSN-era divisor per location. - ---- - -## Porting gotcha worth knowing about - -The one bug hit in the port itself, invisible by construction: numpy -broadcasts `Y - Yhat` along the **last** axis, R recycles a vector -**column-major down** the matrix. Getting it wrong transposes the outcome onto -the level axis and silently corrupts every learning rate while leaving the -algorithm's shape intact. `test-qt.R` pins `eta` against a hand-rolled residual -matrix to catch it — see "the adaptive lr is the 90th percentile of |Y - Yhat|". diff --git a/notes/ilinet-sources.md b/notes/ilinet-sources.md new file mode 100644 index 00000000..d7010e22 --- /dev/null +++ b/notes/ilinet-sources.md @@ -0,0 +1,158 @@ +# ILINet sources: epidata v4 `fluview` vs v5 `fluview_ilinet` + +Checks whether v5 can replace the v4 `fluview` endpoint as the issue-versioned ILINet archive for flu +backtests. Script: `scripts/one_offs/compare_ilinet_sources.py` (`fetch`, then `analyze`; the full table +dump is in `cache/ilinet/analysis_output.md`). Data pulled 2026-10-02. Latest issue in both: 202637 (report 2026-09-25). + +## The two sources + +**v4** `https://api.delphi.cmu.edu/epidata/fluview/` (`epidatr::pub_fluview`). It stores one row per +`(region, epiweek, issue)`, so every issue repeats the whole window of weeks that CDC re-published. Columns: +`wili, ili, num_ili, num_patients, num_providers, num_age_0..5, release_date, lag`. Regions: `nat`, `hhs1-10`, +`cen1-9`, states, `jfk` (NYC), `ny` (all of NY, which v4 builds itself). One call with `regions=ca`, +`epiweeks=199701-203053`, `issues=199701-203053` returns everything for a region (no row cap was hit at 93k rows). + +**v5** `https://delphi.cmu.edu/epidata/v5`, source `fluview_ilinet`. +- Signals: `ili, wili, num_ili, num_patients, num_providers`. Geo types: `nation` (`us`), `state`, `hhs`, `census_division`. +- Extra key `age_group` (`all`, plus `0-4, 5-24, 25-49, 25-64, 50-64, 65+` on `num_ili`). Filter on `age_group = all`. +- `fill_method` is `source`, except v5 `ny` (all of NY), which has fill_method `nyc_plus_ny_minus_nyc` because it is derived. +- `/archive/` stores changes only: you get a row when a `(signal, geo, reference_time, age_group)` value changes. + To rebuild an issue, carry the last change forward (as-of join). `/snapshot/?snapshot_date=D` does that on the server. +- `reference_time` is the Saturday that ends the MMWR week. `report_time` is a UTC timestamp, midnight except for the newest row. +- The deployed `/archive/` takes only `source, signal, geo_type, fill_method, extra_keys, limit, use_pagination, + report_time_query, columns, format, header`. The `geo_value` and `reference_times` filters in `../cast-api` are + not deployed yet, so pull a whole `(signal, geo_type)` at once. That's 10 requests and 36 MB of CSV for state plus nation. +- Auth: send `token: $DELPHI_EPIDATA_KEY` as a header (same key as v4). Anonymous archive calls are limited to 5/min and 60/hour. + +``` +GET /epidata/v5/archive/?source=fluview_ilinet&signal=ili&geo_type=state (header token: …) +GET /epidata/v5/metadata/?source=fluview_ilinet +``` + +### Geo mapping + +| v4 | v5 | note | +|---|---|---| +| `nat` | `us` (geo_type `nation`) | | +| `jfk` | `nyc` | NYC; v5 stops at week 2025-09-27 | +| `ny` | `ny` with fill_method `nyc_plus_ny_minus_nyc` | all of NY; v5 stops at week 2025-09-27 | +| none | `ny_minus_nyc` | upstate only; v5 continues past 2025-09 | +| other states, `dc pr vi` | same codes | | + +v4 `ny` and v5 `ny` are both sums of NYC and upstate, but they disagree slightly. Only 32% of `ili` and `num_patients` +values match exactly, with a median relative difference of 2e-5. The NYC component matches exactly, so v4's upstate +component (or the revision it summed) is a few patients off. Both sources stop publishing NYC and the NY total +after week 2025-39. Only v5 `ny_minus_nyc` continues, and I did not check whether it switches to the whole state. + +## Coverage + +| | v4 nat | v5 nat | v4 states | v5 states | +|---|---|---|---|---| +| geos | 1 | 1 | 54 | 54 + `ny_minus_nyc` | +| weeks | 1997-10-04 – 2026-09-19 | same | 2010-10-09 – 2026-09-19 | same | +| issues / reports | 200347 – 202637 | 2003-11-28 – 2026-09-25 | 201740 – 202637 | 2017-10-20 – 2026-09-25 | +| rows (ili) | 56,898 | 20,172 (changes) | 1,966,481 | 93,498 (changes) | +| state `wili` | n/a | n/a | yes, always equal to `ili` (100%) | absent | + +- Both sources have state issues only from 2017w40. Before that, state values exist only as the 2017w40 backfill. + "CA history back to 2010" therefore means weeks back to 2010, not issues. +- Both skip the same releases: 202539–202544 (October 2025, when CDC published nothing) and 202350 for every state except `pr`. + In 202350 (released 2023-12-22), both sources have only nat and `pr`. +- Value changes since 2019w40 are almost equal (state ili: 55,928 in v4 vs 56,915 in v5). v5 has slightly more + because of the extra `ny_minus_nyc` geo and a few off-cycle reports. +- In the latest version, these weeks exist in v4 only: + - `fl`: 574 weeks from 2010-10-09 to 2021-10-02 (v5 Florida starts at week 2021-10-09). + - `la`, `vi`, `pr`: 313, 261 and 156 old weeks (2010 to 2013–2016) that appear only in v4's 2017w40 backfill. + - `dc`: one week, 2022-02-26. +- `ny_minus_nyc` is in v5 only (833 weeks). + +## Version alignment + +**The v5 `report_time` date is the v4 `release_date`: the Friday CDC publishes FluView, normally the issue +week's Saturday plus 6 days.** 1,062 of 1,071 distinct v5 report dates equal a v4 national release date. In the +remaining 9, v4 stamped a release a day or more late for some geos (August 2025, 2017-10-24, 2014). Holiday-delayed +releases (9–10 days after week end, 34 issues) fall on the same date in both sources. + +Shift test (state + nat `ili`, issues ≥ 2019w40): agreement between v4 issue W and v5 read as of `release_date + k`: + +| k (days) | -7 | -1 | 0 | +1 | +6 | +7 | +|---|---|---|---|---|---|---| +| % within 5e-6 | 93.68 | 93.87 | **97.74** | 97.74 | 97.54 | 95.27 | + +(The ceiling of 97.7 comes from the NY sum difference. Without NY it is 99.99.) There is no off-by-one-week +offset. To reproduce v4 issue W from v5, take the last change with `report_time::date <= ew_end(W) + 6`, or +better, v4's own `release_date`, which covers delayed releases. In a backtest with a forecast date D, the v5 as-of-D +snapshot is the v4 data available on D, so no translation to issues is needed. + +## Value agreement + +Each v4 `(geo, week, issue)` row is compared with the v5 value as of that issue's release date. + +| geo | signal | n | % v5 missing | % exact | % within 5e-6 | max abs diff | +|---|---|---|---|---|---|---| +| nat | ili | 56,898 | 0 | 99.88 | 99.97 | 0.029 | +| nat | wili | 56,898 | 0 | 99.79 | 99.97 | 0.103 | +| nat | counts (each) | 56,898 | 0 | 99.97–99.98 | 99.97–99.98 | — | +| state | ili | 1,966,481 | 1.11 | 97.49 | 97.68 | 8.18 | +| state | num_ili | 1,966,481 | 1.11 | 98.86 | 98.86 | 1430 | +| state | num_patients | 1,966,481 | 1.11 | 97.67 | 97.67 | 104,269 | + +Excluding NY, the 2017w40–41 backfill issues and rows that v5 doesn't have: + +| geo | signal | n | % exact | max abs diff | +|---|---|---|---|---| +| nat | all 5 | 36,306 | 100 | 0 | +| state | ili | 1,888,666 | 99.99 | 8.18 (VT, below) | +| state | num_ili / num_patients / num_providers | 1,888,762 | 99.99 | 92 / 6886 / 14 | + +The two sources use the same units: ILI is a percent (e.g. 6.47), not a proportion. Neither rounds differently. +The national "exact vs within 5e-6" gap in 2009–2012 issues is float noise (≤1.3e-8). + +The residual mismatches, by cause: +1. **NY sum** (about 23.6k ili rows, median diff 7e-5): described in the geo mapping section. +2. **2017w40/41 backfill** (about 90 rows in 37 states and 14 nat weeks, weeks 2017-05 to 2017-10): v4's first state + issue (release_date 2017-10-24, filled in later) and nat 201741 differ by 0.002–1.2 ILI points from v5. +3. **VT, issues 202502–202513**: v4 has `num_patients = 10/11` for weeks 202440–202501, which pushes ili up to about + 8 points high. v5 never records those values and keeps 6375 until the zero in 202514, which both sources have. + v4 looks like it captured a corrupt CDC publication that v5 missed or rejected. +4. **DC week 2022-02-26**: v4 has `num_patients = 10` from 202208 to 202339, while v5 has 0 throughout. +5. **v5 missing** (1.1% of state rows): FL before 2021-10 (19.5k rows), and LA/PR/VI old weeks in v4's 201740 backfill. + A few hundred more rows come from 201740 issues released before v5's first report. + +## Revision spot checks + +The as-of trajectories of CA, TX and nat (wili) for weeks 201952, 202250 and 202350 match v4 exactly at every +issue: the same values, revised on the same Fridays. Examples: +- CA 201952: 3.803 → 4.924 → … → 4.699, with the last change in the 202026 release. +- CA 202350: first value in 202351 in both sources (202350 missing in both), last revision on 2024-10-04. +- nat wili 202350: 24 revisions through 202531, all identical. + +NY has the same revision timing with a constant offset of about 2e-4. The full trajectories are in `cache/ilinet/analysis_output.md`. + +## Finalized values (latest version, all weeks) + +| geo | signal | geo-weeks | v4 only | v5 only | % exact | max abs diff | +|---|---|---|---|---|---|---| +| nat | ili / wili / counts | 1417 | 0 | 0 | 99.7–100 (100 within 1e-8) | 1e-8 | +| state | ili | 45,356 | 1305 | 833 | 98.94 (99.3 within 5e-6) | 0.00096 (NY) | +| state | num_ili / num_patients / num_providers | 45,356 | 1304 | 833 | 99.99 / 99.3 / 100 | 2 / 56 / 0 | + +All finalized state differences are in NY, plus DC 2022-02-26. + +## Recommendation + +v5 can replace v4 for state and national ILINet backtests. Exact agreement is 99.99% outside NY, and version +dates and revision trajectories match one for one. Read v5 as of the forecast date, or as of the v4 `release_date` +to reproduce an issue. Caveats: +- **State wILI**: v5 has no state `wili`. In v4 it is a copy of `ili`, so use `ili` for states. +- **NY**: use v5 `ny` with fill_method `nyc_plus_ny_minus_nyc`. Small differences from v4 `ny` (≈2e-5 relative) + shift historical scores slightly. After 2025-09 there is no NYC or NY total in either source, so define what "NY" means from 2025-26 on. +- **Florida before 2021-10**: v4 has 574 weeks that v5 lacks. This only matters if a training window reaches before + 2021-22 for FL (it has no issue history before 2017w40 anyway). +- **State issue history starts 2017w40 in both sources.** For a state as-of before October 2017, neither source helps. +- **VT 2025 and DC 2022-02-26**: v4 contains bad values (num_patients = 10) that v5 doesn't have. v5 is the + cleaner archive here, but a backtest replayed on v5 won't reproduce whatever a 2025 forecast saw from v4 for VT + in January–March 2025. +- **API**: until `geo_value` and `reference_times` filters are deployed, pull whole `(signal, geo_type)` archives with + the token header (cheap: 36 MB) and do the as-of join locally (DuckDB `ASOF JOIN`). Don't use `/snapshot/` per + date (3/min anonymous). Not compared: HHS and census regions, and age-group counts (v5 has them only for national `num_ili`). diff --git a/notes/refactor-ideas.md b/notes/refactor-ideas.md index 8893aff7..cd2adbfa 100644 --- a/notes/refactor-ideas.md +++ b/notes/refactor-ideas.md @@ -1,6 +1,7 @@ # Open threads and future refactor ideas -The single home for everything not yet implemented. Consolidated 2026-07-22 +Refactor designs and cleanup inventories. The task list itself is +`notes/ROADMAP.md`. Consolidated 2026-07-22 from CLAUDE.md's "Open threads" and the three 2026-07-19 design/cleanup scouts (`e2-explore-ensemble-design`, `validate-snapshot-design`, `clarity-simplification-scout` — originals in git history); items the diff --git a/notes/spoiled-submissions.md b/notes/spoiled-submissions.md new file mode 100644 index 00000000..f76ea563 --- /dev/null +++ b/notes/spoiled-submissions.md @@ -0,0 +1,125 @@ +# Spoiled hub submissions + +Forecasts we submitted to the hubs that were broken by a pipeline bug, not by +the model. They stay in the hub forever, so anything that reads our hub +submissions (the calibration experiments, scoring) should drop them. +`hub_read_forecasts()` does this by default from `HUB_SPOILED_SUBMISSIONS` in +`R/calibration/hub_data.R`; add new cases there and here. + +Covid submissions were checked for the two flu patterns (all-zero quantiles, +halved US) and are clean. Of the cases the sweep found, the two covid ones +are excluded; the flu 2026-04-04 one is kept on purpose (see its section). + +## Flu, 2025-11-22 round: h−1 is zero everywhere + +Every quantile at horizon −1 is exactly 0, in all 53 locations. h0–h3 are +normal. + +Cause: the first forecast after the autumn 2025 data gap (forecast date +2025-11-19). That week the AR models were kept out of negative aheads, so h−1 +came only from the `climate_linear` ensemble. The weights block added that day +in `pipelines/flu_geo_exclusions.csv` (commit `492bade`) dated the `linear`, +`climate_base` and other climate rows `2024-11-19` instead of `2025-11-19`. +The weights parser matches dates exactly, so those forecasters had no weight, +and the `climate_linear` ensemble came out as 0. The AR models were let into +h−1 from 2025-11-26 (`24f366d`), and the typo was fixed on 2026-09-30. +Before that fix, replays of 2025-11-19 in the prod targets store reproduced +the zero `ensemble_clim_lin`. + +## Flu, US, rounds 2024-12-14, 2024-12-21 and 2025-01-04: halved + +The US forecast is about half the sum of the state forecasts at every horizon +(ratio 0.45–0.54). In every other live round the US median is within a few +percent of the state sum at h−1 and h0. The cause is not confirmed. Commits +from early 2025 fixed several `us` / `usa` / `US` mixups, and averaging the +US forecast with a zero duplicate would halve it exactly. + +## Covid, 2024-11-23 round: about 2.4x too high everywhere (excluded, all horizons) + +At h−1 and h0 the median is about 2.2–2.6x both the value known at the time +and the finalized value, in 41 of 49 locations; h1–h3 are off by a similar +factor. + +Cause: NHSN's 2024-11-20 release inflated week 2024-11-16 in many states +(covid: PA 6124, MI 3732, US 17593; corrected on 2024-11-21 to 1005, 600 and +7437). Flu had the same bad release (PA 729 → 111). The first run for this +round (2024-11-20) used that data; the AR models are pooled across states, +so a jump in a few large states raised every state's forecast. On 2024-11-21 +we re-ran on the corrected data and re-submitted flu (FluSight hub commit +`c160da00`, "updated 11-23 forecast for new data"), but the covid file was +never re-submitted, so the hub holds the 2024-11-20 version. A prod report +rendered on 2024-11-21 (`reports/2024-11-21_covid_prod_on_2024-11-21.html`) +shows the forecasts we would have sent (GA h−1 median 64 against the +submitted 339). A May 2025 replay of the store also gives forecasts about +half the submitted ones. + +## Covid, 2026-04-25 round: h−1 near zero in many states (excluded, h−1) + +The h−1 median is far below both the value known at the time and the +finalized value in about half the locations (WI 2 vs 27 known, VA 3 vs 28, +MA 9 vs 44, IN 9 vs 38), and far below the same round's own h0 (WI 10, VA 25). +h0–h3 look normal. The data were fine: the 2026-04-22 release already had +these values for week 2026-04-18, and later revisions barely changed them. + +Cause, not confirmed: at the time, covid h−1 came only from the +`climate_linear` ensemble (the AR models were filtered out of negative +aheads), and at short aheads that ensemble is mostly the linear baseline, +which extrapolates the last few reported weeks. The near-zeros are what a +linear trend through 2026-04-04 → 2026-04-11 gives (WI 23 → 14, VA 51 → 26) +if week 2026-04-18 is missing, so the run most likely did not have that +week. That day the cast API moved from v2 to v5; the fix (`6d91648`, "keep +up with cast-api updates") was committed at 16:11 PDT, nine minutes after +the submission (covid hub `a6b94bb`, 16:02), so the submission probably ran +on a stale archive fetch. There is no covid prod store for that date to +check. + +## Flu, 2026-04-04 round: h−1 about 1.8x too high (kept) + +The h−1 median is about 1.6–1.8x both the value known then and the finalized +value in 22 of 35 locations (OH 204 vs 60, CO 90 vs 34, MN 92 vs 36). h0 is +fine. The prod store replay reproduces the submission exactly, so this is +what the pipeline does, not a one-off failure. + +Cause: week 2026-03-28 fell abruptly (OH 257 → 60, US about 5900 → 3300; the +finalized data agree). At h−1 the AR models are run through +`filter_minus_one_ahead()`, which drops the week being nowcast, so they +extrapolate from 2026-03-21 and never see the drop, even though it was +already reported. The same mechanism puts 2025-26 h−1 above the reported +value by 10–30% in other weeks of the decline. This is a design choice in +how h−1 is built, not a bug in one round, so it is kept: it is a systematic +h−1 bias, which is what the calibrator should learn to correct. The fix +belongs in how prod builds h−1 (let the AR models see the reported week). + +## How to find more + +`scripts/one_offs/sweep_submission_deviations.R [flu|covid]` joins every +submitted forecast with the NHSN value known when it was made and the +finalized value, writes `cache/submission_sweep_{disease}.csv`, and prints +two flags at h−1 and h0: + +- **suspect**: the median is off by more than 2x from both the value known + then and the finalized value. This is the useful one for pipeline bugs. +- **missed**: the finalized value is outside the 95% interval and the median + is off by more than 2x. This also catches data problems (bad first + reports, holiday delays, the autumn 2025 gap). + +A round where many locations are flagged together usually means a pipeline +problem; scattered single locations are usually model error or bad data. The +halved-US check (US median vs the sum of state medians) is not in the script; +it is a one-line duckdb query over the hub CSVs. + +## Open threads + +- Halved US (flu 2024-12-14, 2024-12-21, 2025-01-04): cause unconfirmed; a + `us`/`usa` mixup is the best guess. +- Covid 2026-04-25: a stale fetch during the cast API v2→v5 change is likely + but unconfirmed; there is no store to replay. +- Covid 2024-11-23: the May 2025 replay doesn't reproduce the submission, for + reasons not yet explained. Don't read replays as "what was submitted". + +## To do + +- Run the same sweep on `windowed_seasonal` (the main ensemble component, + used as a proxy for the ensemble in the calibration harness) to check it + does not hit similar pathologies or data errors. Needs a variant of the + script that reads the harness forecasts instead of the hub files. diff --git a/pipelines/covid_hosp_prod.R b/pipelines/covid_hosp_prod.R index 955737fc..d5f0a865 100644 --- a/pipelines/covid_hosp_prod.R +++ b/pipelines/covid_hosp_prod.R @@ -587,7 +587,7 @@ joined_targets <- list2( combined_targets <- build_combined_targets(external_forecast_targets) # ============================== CALIBRATION TARGETS ============================ -# Same design as flu; see flu_hosp_prod.R and notes/CALIBRATION.md. No 2023-24 +# Same design as flu; see flu_hosp_prod.R and notes/calibration-ledger.md. No 2023-24 # burn-in for covid (hub started 2024-11-23), so slow_init is NULL. calibration_targets <- list( tar_target( @@ -614,8 +614,10 @@ calibration_targets <- list( by = c("geo_value" = "state_id") ) %>% select("target_end_date", "location", truth = "true_value") + # REF-op cold: see "Operating point" in notes/calibration-ledger.md. calibrate_hub_forecasts( historical_fc, truth, + settle_days = 14L, burn_in_seasons = character(0), transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3), diff --git a/pipelines/flu_hosp_prod.R b/pipelines/flu_hosp_prod.R index 965ea097..611101ef 100644 --- a/pipelines/flu_hosp_prod.R +++ b/pipelines/flu_hosp_prod.R @@ -587,7 +587,7 @@ combined_targets <- build_combined_targets(external_forecast_targets) # burn-in for free), appends the current week's ensemble_mix if not yet # submitted, and runs calibrate_hub_forecasts() over the full history. The # current round's value_cal is then written as the CMU-TimeSeries-Calibrated -# submission. Skipped in cache/evaluation mode. See notes/CALIBRATION.md for +# submission. Skipped in cache/evaluation mode. See notes/calibration-ledger.md for # the parameter choices. calibration_targets <- list( tar_target( @@ -614,16 +614,19 @@ calibration_targets <- list( by = c("geo_value" = "state_id") ) %>% select("target_end_date", "location", truth = "true_value") + # sqrt constant 0.018 per 100k, cold: see "Operating point" in + # notes/calibration-ledger.md (chosen 2026-10-07 on month-avg coverage). + rate_scales <- get_population_data() %>% + distinct(.data$state_code, .keep_all = TRUE) %>% + transmute(location = .data$state_code, from = as.Date("2000-01-01"), scale = .data$population / 1e5) calibrate_hub_forecasts( historical_fc, truth, - burn_in_seasons = "2023-2024", + settle_days = 14L, + burn_in_seasons = character(0), transform = "sqrt", - lr_args = list(mult = 0.03, floor = 1e-3), - lr_window = 20, - season_policy = "carry", - slow_init = "burn_in_quantile", - lr_slow = list(mult = 0.003), - fast_decay = 0.1 + scales = rate_scales, + lr = 0.018, + season_policy = "carry" ) }, cue = tar_cue("always") diff --git a/reports/writeups/calibration/calibration_qt_covid.Rmd b/reports/writeups/calibration/calibration_qt_covid.Rmd deleted file mode 100644 index 82397d1c..00000000 --- a/reports/writeups/calibration/calibration_qt_covid.Rmd +++ /dev/null @@ -1,492 +0,0 @@ ---- -title: Online quantile calibration (MultiQT) on submitted COVID hub forecasts -date: "`r format(Sys.time(), '%d %B %Y')`" -output: - html_document: - code_folding: hide - df_print: paged - toc: true - toc_float: true -params: - burn_in_seasons: "" - season_policy: "carry" - lr_window: 50 - lr_mult: 0.1 - roll_window: 20 -editor_options: - chunk_output_type: console ---- - -```{r setup, echo=FALSE} -# cache = TRUE chunks are keyed on chunk code + cache.extra, NOT on functions -# in R/ -- after changing calibration code, clear cache/calibration/knitr_cache_covid/. -knitr::opts_chunk$set( - fig.align = "center", fig.width = 10, fig.height = 6, - message = FALSE, warning = FALSE, cache = FALSE, - cache.path = here::here("cache/calibration/knitr_cache_covid/") -) -knitr::opts_knit$set(root.dir = here::here()) -ggplot2::theme_set(ggplot2::theme_bw()) -``` - -```{r load, cache=TRUE, cache.extra=list(params)} -suppressPackageStartupMessages(source(here::here("R/load_all.R"))) - -forecasts <- hub_read_forecasts(hub_dir = HUB_COVID_DIR, target = HUB_COVID_TARGET) -truth <- hub_read_covid_truth() -rounds <- hub_label_seasons(unique(forecasts$reference_date)) - -burn_in <- if (nzchar(params$burn_in_seasons)) { - strsplit(params$burn_in_seasons, ",")[[1]] -} else { - character(0) -} - -cal <- calibrate_hub_forecasts( - forecasts, truth, - burn_in_seasons = burn_in, - season_policy = params$season_policy, - lr_window = params$lr_window, - lr_args = list(mult = params$lr_mult), - progress = FALSE -) -``` - -We calibrate the forecasts CMU-TimeSeries submitted to the COVID hub. https://github.com/cdcepi/covid19-forecast-hub -The hub ships one consistent truth series (`target-data/covid-hospital-admissions.csv`). -The method is MultiQT (Ding, Gibbs & Tibshirani 2025); our port is `R/calibration/qt.R`, verified against the authors' Python in `tests/testthat/test-qt.R`. - -COVID submissions run year-round with no meaningful off-season gap, so there is no burn-in season. - -Data facts: - -```{r data-shape} -forecasts %>% - left_join(select(rounds, reference_date = round_date, season), by = "reference_date") %>% - group_by(season) %>% - summarize( - rounds = n_distinct(reference_date), - locations = n_distinct(location), - horizons = paste(sort(unique(horizon)), collapse = ","), - cell_completeness = sprintf( - "%.0f%%", 100 * (n() / 23) / (n_distinct(reference_date) * 265) - ), - .groups = "drop" - ) -``` - -Notes: - -1. Waiting for revisions costs us `horizon + 2` rounds. `reference_date` and - `target_end_date` are both Saturdays with - `target_end_date = reference_date + 7 * horizon`, so waiting 14 days for NHSN - to settle puts the delay at exactly `h + 2` rounds. The offset applied at - horizon 3 is therefore always 5 rounds stale. -2. Unlike flu, COVID has no off-season: all rounds are treated as live. - Season policy (`carry`/`reset`/`shrink`) therefore has no effect on these results. -3. Per-series coverage cannot be measured at the extreme levels: the 1% and 99% - levels see ~0.3 expected exceedances per season per series. Every coverage - number below is pooled over the 53 locations. - -# Score results - -Coverage improves at some cost in sharpness. Below we show that the sharpness -cost can be reduced by changing the learning rate. - -```{r headline} -headline <- left_join( - hub_coverage_summary(cal), - select(hub_quantile_loss(cal), horizon, loss_base, loss_cal, loss_pct = pct_improvement), - by = "horizon" -) -headline %>% - transmute( - horizon, - n_obs, - `cal err (base)` = round(cal_error_base, 4), - `cal err (calibrated)` = round(cal_error_cal, 4), - `cal err improvement %` = round(pct_improvement, 1), - `WIS (base)` = round(2 * loss_base, 1), - `WIS (calibrated)` = round(2 * loss_cal, 1), - `WIS improvement %` = round(loss_pct, 1) - ) -``` - -Calibration error here is the mean over the 23 quantile levels of -`|geo-pooled empirical coverage at a - a|`. WIS is reported with the standard -factor-of-two convention (2 x the level-averaged pinball loss on this level -set), so it is directly comparable to hub scores; improvement percentages are -identical either way. - -# 1. What feeds the learning rate - -`eta_t = max(floor, mult * quantile_0.9(|Y - Yhat|))` over a window of recent -observed rounds, pooled across all 23 quantile levels -- one step size per -(series, round). The residual distribution below therefore determines the -learning rate, up to the multiplier. - -```{r resid-series, fig.height=7} -resid <- cal$forecasts %>% - filter(!is.na(truth), !is.na(value_base)) %>% - mutate(abs_resid = abs(truth - value_base)) %>% - group_by(location, horizon, reference_date, season, is_burn_in) %>% - summarize(q90 = quantile(abs_resid, 0.9), .groups = "drop") - -show_geos <- c("06", "36", "48", "50", "US") -resid %>% - filter(location %in% show_geos) %>% - mutate(location = recode(location, - "06" = "CA", "36" = "NY", "48" = "TX", "50" = "VT", "US" = "US" - )) %>% - ggplot(aes(reference_date, q90, colour = factor(horizon))) + - geom_line() + - facet_wrap(~location, scales = "free_y", ncol = 1) + - scale_y_log10() + - labs( - title = "90th percentile of |truth - base forecast|, per round", - subtitle = "The quantity the adaptive learning rate is a fraction of. Log scale.", - x = NULL, y = "q90 |residual| (log)", colour = "horizon" - ) -``` - -## Does the window length matter? - -`lr_window` sets how many of the most recent observed rounds feed the adaptive -learning rate; `Inf` is an expanding window over the full history. - -```{r lr-window, cache=TRUE, dependson="load"} -lr_by_window <- purrr::map(c(8, 20, 50, Inf), function(w) { - purrr::map(show_geos, function(loc) { - series <- forecasts %>% filter(location == loc, horizon == 2) - if (nrow(series) == 0) { - return(NULL) - } - Yhat <- hub_series_matrix(series, rounds$round_date, 23) - ted <- rounds$round_date + 14L - Y <- truth$truth[match(paste(loc, ted), paste(truth$location, truth$target_end_date))] - learnable <- !is.na(Y) & !apply(is.na(Yhat), 2, any) - delay <- qt_delay_from_dates(rounds$round_date, ted, 14L) - delay <- lapply(delay, function(i) i[learnable[i]]) - res <- qt_track(Y, Yhat, HUB_QUANTILE_LEVELS, - delay = delay, lr_window = w, nonneg = TRUE - ) - tibble( - location = loc, round_date = rounds$round_date, - lr = res$lr, window = as.character(w) - ) - }) %>% bind_rows() -}) %>% bind_rows() - -lr_by_window %>% - filter(lr > 0) %>% - mutate( - location = recode(location, - "06" = "CA", "36" = "NY", "48" = "TX", "50" = "VT", "US" = "US" - ), - window = factor(window, levels = c("8", "20", "50", "Inf")) - ) %>% - ggplot(aes(round_date, lr, colour = window)) + - geom_line() + - facet_wrap(~location, scales = "free_y", ncol = 1) + - scale_y_log10() + - labs( - title = "Learning-rate trajectory by window length (horizon 2)", - subtitle = "A short window tracks the season; an expanding one (Inf) ratchets up and stays.", - x = NULL, y = "eta (log)", colour = "lr_window" - ) -``` - -# 2. Actual vs nominal coverage, per location - -```{r coverage-plot, fig.height=6} -cov_by_geo <- hub_coverage(cal, by = "location") - -cov_long <- cov_by_geo %>% - select(location, horizon, level, coverage_base, coverage_cal) %>% - tidyr::pivot_longer( - c(coverage_base, coverage_cal), - names_to = "which", values_to = "coverage" - ) %>% - mutate(which = factor( - if_else(which == "coverage_base", "pre-calibration", "post-calibration"), - levels = c("pre-calibration", "post-calibration") - )) - -cov_long %>% - filter(horizon %in% c(0, 2)) %>% - ggplot(aes(level, coverage, group = location)) + - geom_abline(slope = 1, intercept = 0, linetype = "dashed", colour = "grey30") + - geom_line(alpha = 0.28, colour = "steelblue4") + - facet_grid(horizon ~ which, labeller = label_both) + - coord_equal(xlim = c(0, 1), ylim = c(0, 1)) + - labs( - title = "Empirical vs nominal coverage, one line per location", - subtitle = "Dashed line is perfect calibration. Each line is one of 53 locations, over all rounds.", - x = "nominal level", y = "empirical coverage" - ) -``` - -```{r coverage-pooled} -hub_coverage(cal) %>% - select(horizon, level, coverage_base, coverage_cal) %>% - tidyr::pivot_longer(c(coverage_base, coverage_cal), - names_to = "which", values_to = "coverage" - ) %>% - mutate(which = if_else(which == "coverage_base", "pre", "post")) %>% - ggplot(aes(level, coverage, colour = which)) + - geom_abline(slope = 1, intercept = 0, linetype = "dashed", colour = "grey30") + - geom_line() + - geom_point(size = 1) + - facet_wrap(~horizon, labeller = label_both, nrow = 1) + - coord_equal() + - labs( - title = "Geo-pooled coverage, pre and post calibration", - x = "nominal level", y = "empirical coverage", colour = NULL - ) -``` - -# 3. Calibration error against quantile loss - -Both axes are rolling averages over the last `r params$roll_window` rounds, pooled -over locations. Moving down is better calibrated; moving left is sharper. - -```{r tradeoff, fig.height=6} -tradeoff <- hub_rolling_tradeoff(cal, window = params$roll_window) - -tradeoff %>% - select(horizon, reference_date, cal_error_base, cal_error_cal, qloss_base, qloss_cal) %>% - tidyr::pivot_longer( - -c(horizon, reference_date), - names_to = c(".value", "which"), names_pattern = "(.*)_(base|cal)$" - ) %>% - mutate(which = if_else(which == "base", "pre-calibration", "post-calibration")) %>% - ggplot(aes(qloss, cal_error, colour = which)) + - geom_path(alpha = 0.7) + - geom_point(aes(size = as.numeric(reference_date)), alpha = 0.5, show.legend = FALSE) + - scale_size_continuous(range = c(0.4, 2.2)) + - facet_wrap(~horizon, scales = "free_x", labeller = label_both) + - labs( - title = "Rolling calibration error vs rolling quantile loss", - subtitle = "Point size increases with time. Down = better calibrated, left = sharper.", - x = "rolling mean pinball loss (proportional to WIS)", y = "rolling calibration error", colour = NULL - ) -``` - -```{r tradeoff-time, fig.height=5} -tradeoff %>% - select(horizon, reference_date, cal_error_base, cal_error_cal) %>% - tidyr::pivot_longer(-c(horizon, reference_date), - names_to = "which", values_to = "cal_error" - ) %>% - mutate(which = if_else(which == "cal_error_base", "pre", "post")) %>% - ggplot(aes(reference_date, cal_error, colour = which)) + - geom_line() + - facet_wrap(~horizon, labeller = label_both, nrow = 1) + - labs( - title = "Rolling calibration error over time", - subtitle = "How fast the tracker converges and what it looks like without an off-season gap.", - x = NULL, y = "rolling calibration error", colour = NULL - ) -``` - -# 4. Interval widths - -The WIS cost has to show up somewhere in the quantiles; central interval widths -are the direct measure. - -```{r interval-widths} -interval_defs <- tibble::tribble( - ~interval, ~lo, ~hi, - "20% (0.4-0.6)", 0.4, 0.6, - "60% (0.2-0.8)", 0.2, 0.8, - "80% (0.1-0.9)", 0.1, 0.9 -) - -width_rows <- cal$forecasts %>% - filter(!is_burn_in, !is.na(value_base), !is.na(value_cal)) %>% - select(reference_date, season, location, horizon, level, value_base, value_cal) - -widths <- purrr::pmap(interval_defs, function(interval, lo, hi) { - inner_join( - width_rows %>% filter(level == lo) %>% - select(-level) %>% rename(base_lo = value_base, cal_lo = value_cal), - width_rows %>% filter(level == hi) %>% - select(-level) %>% rename(base_hi = value_base, cal_hi = value_cal), - by = c("reference_date", "season", "location", "horizon") - ) %>% - transmute( - reference_date, season, location, horizon, - interval, - width_base = base_hi - base_lo, - width_cal = cal_hi - cal_lo - ) -}) %>% - bind_rows() %>% - mutate(interval = factor(interval, levels = interval_defs$interval)) -``` - -```{r width-time, fig.height=6} -widths %>% - group_by(interval, horizon, reference_date) %>% - summarize( - base = mean(width_base), calibrated = mean(width_cal), - .groups = "drop" - ) %>% - tidyr::pivot_longer(c(base, calibrated), names_to = "which", values_to = "width") %>% - ggplot(aes(reference_date, width, colour = which)) + - geom_line() + - facet_grid(interval ~ horizon, scales = "free_y", labeller = label_both) + - labs( - title = "Geo-pooled mean central interval width, base vs calibrated", - x = NULL, y = "mean interval width (admissions)", colour = NULL - ) -``` - -```{r width-change, fig.height=4} -widths %>% - group_by(interval, horizon) %>% - summarize( - pct_change = 100 * (mean(width_cal) / mean(width_base) - 1), - .groups = "drop" - ) %>% - ggplot(aes(horizon, pct_change, colour = interval)) + - geom_hline(yintercept = 0, linetype = "dashed", colour = "grey40") + - geom_line() + - geom_point(size = 2) + - labs( - title = "Change in mean interval width from calibration", - subtitle = "Positive means the calibrated intervals are wider than the base intervals.", - x = "horizon", y = "width change (%)", colour = "central interval" - ) -``` - -# 5. Sensitivity: window or multiplier? - -```{r sweep, cache=TRUE, dependson="load"} -sweep_grid <- tidyr::expand_grid( - season_policy = c("carry", "reset"), - lr_window = c(8, 20, 50, Inf), - lr_mult = c(0.3, 0.1, 0.03, 0.01) -) -sweep <- purrr::pmap(sweep_grid, function(season_policy, lr_window, lr_mult) { - s <- calibrate_hub_forecasts(forecasts, truth, - burn_in_seasons = burn_in, season_policy = season_policy, - lr_window = lr_window, lr_args = list(mult = lr_mult), progress = FALSE - ) - overall <- left_join( - hub_coverage_summary(s), - select(hub_quantile_loss(s), horizon, loss_pct = pct_improvement), - by = "horizon" - ) %>% - mutate(season = "all") - by_season <- left_join( - hub_coverage_summary(s, by = "season"), - select(hub_quantile_loss(s, by = "season"), - season, horizon, - loss_pct = pct_improvement - ), - by = c("season", "horizon") - ) - bind_rows(overall, by_season) %>% - mutate(season_policy = season_policy, lr_window = lr_window, lr_mult = lr_mult) -}) %>% bind_rows() - -dir.create(here::here("cache/calibration"), showWarnings = FALSE, recursive = TRUE) -saveRDS(sweep, here::here("cache/calibration/sweep_covid.rds")) -``` - -```{r sweep-cal-error, fig.height=5} -sweep %>% - filter(season == "all") %>% - ggplot(aes(lr_mult, cal_error_cal, colour = factor(lr_window))) + - geom_line() + - geom_point(size = 1.6) + - scale_x_log10() + - facet_grid(season_policy ~ horizon, labeller = label_both) + - labs( - title = "Calibration error by multiplier, one line per window", - x = "lr mult (log)", y = "calibration error (calibrated)", - colour = "lr_window" - ) -``` - -```{r sweep-wis, fig.height=5} -sweep %>% - filter(season == "all") %>% - ggplot(aes(lr_mult, loss_pct, colour = factor(lr_window))) + - geom_hline(yintercept = 0, linetype = "dashed", colour = "grey40") + - geom_line() + - geom_point(size = 1.6) + - scale_x_log10() + - facet_grid(season_policy ~ horizon, labeller = label_both) + - labs( - title = "WIS improvement by multiplier, one line per window", - subtitle = "Dashed line is WIS parity with the raw submission.", - x = "lr mult (log)", y = "WIS improvement (%)", - colour = "lr_window" - ) -``` - -```{r sweep-corner, fig.height=5} -sweep %>% - filter(season == "all", season_policy == "carry") %>% - ggplot(aes(cal_error_cal, loss_pct, - colour = factor(lr_mult), shape = factor(lr_window) - )) + - geom_hline(yintercept = 0, linetype = "dashed", colour = "grey40") + - geom_point(size = 3, alpha = 0.85) + - facet_wrap(~horizon, scales = "free_x", labeller = label_both, nrow = 1) + - labs( - title = "Calibration error vs WIS change (season_policy = carry)", - subtitle = "Upper-left is the target region: well calibrated and no worse on WIS.", - x = "calibration error (calibrated)", y = "WIS change (%)", - colour = "lr mult", shape = "lr_window" - ) -``` - -```{r sweep-table} -sweep %>% - filter(season == "all") %>% - transmute( - season_policy, lr_window, lr_mult, horizon, - cal_error_cal = round(cal_error_cal, 4), - wis_pct = round(loss_pct, 1) - ) %>% - arrange(horizon, season_policy, lr_window, lr_mult) -``` - -# 6. Offset trajectories - -```{r offsets, fig.height=6} -cal$forecasts %>% - filter(location %in% show_geos, horizon == 2, level %in% c(0.025, 0.5, 0.975)) %>% - mutate(location = recode(location, - "06" = "CA", "36" = "NY", "48" = "TX", "50" = "VT", "US" = "US" - )) %>% - ggplot(aes(reference_date, offset, colour = factor(level))) + - geom_hline(yintercept = 0, colour = "grey50") + - geom_line() + - facet_wrap(~location, scales = "free_y", ncol = 1) + - labs( - title = "Realised offset (played - base), horizon 2", - x = NULL, y = "offset (admissions)", colour = "level" - ) -``` - -```{r offset-scale} -cal$forecasts %>% - filter(!is_burn_in, !is.na(value_base), value_base > 0) %>% - group_by(horizon, level) %>% - summarize( - median_abs_offset = median(abs(offset)), - median_rel = median(abs(offset) / pmax(value_base, 1)), - .groups = "drop" - ) %>% - ggplot(aes(level, median_rel, colour = factor(horizon))) + - geom_line() + - scale_y_continuous(labels = scales::percent) + - labs( - title = "Median |offset| as a fraction of the base forecast", - x = "quantile level", y = "median |offset| / base", colour = "horizon" - ) -``` diff --git a/reports/writeups/calibration/calibration_qt_flu.Rmd b/reports/writeups/calibration/calibration_qt_flu.Rmd deleted file mode 100644 index ce1e8946..00000000 --- a/reports/writeups/calibration/calibration_qt_flu.Rmd +++ /dev/null @@ -1,582 +0,0 @@ ---- -title: Online quantile calibration (MultiQT) on submitted FluSight forecasts -date: "`r format(Sys.time(), '%d %B %Y')`" -output: - html_document: - code_folding: hide - df_print: paged - toc: true - toc_float: true -params: - burn_in_seasons: "2023-2024" - season_policy: "carry" - lr_window: 50 - lr_mult: 0.1 - roll_window: 20 -editor_options: - chunk_output_type: console ---- - -```{r setup, echo=FALSE} -# cache = TRUE chunks are keyed on chunk code + cache.extra, NOT on functions -# in R/ -- after changing calibration code, clear cache/calibration/knitr_cache_flu/. -knitr::opts_chunk$set( - fig.align = "center", fig.width = 10, fig.height = 6, - message = FALSE, warning = FALSE, cache = FALSE, - cache.path = here::here("cache/calibration/knitr_cache_flu/") -) -knitr::opts_knit$set(root.dir = here::here()) -ggplot2::theme_set(ggplot2::theme_bw()) -``` - -```{r load, cache=TRUE, cache.extra=list(params)} -suppressPackageStartupMessages(source(here::here("R/load_all.R"))) - -forecasts <- hub_read_forecasts() -truth <- hub_read_truth() -rounds <- hub_label_seasons(unique(forecasts$reference_date)) - -burn_in <- if (nzchar(params$burn_in_seasons)) { - strsplit(params$burn_in_seasons, ",")[[1]] -} else { - character(0) -} - -cal <- calibrate_hub_forecasts( - forecasts, truth, - burn_in_seasons = burn_in, - season_policy = params$season_policy, - lr_window = params$lr_window, - lr_args = list(mult = params$lr_mult), - progress = FALSE -) -``` - -We calibrate the forecasts CMU-TimeSeries submitted to FluSight. https://github.com/cdcepi/FluSight-forecast-hub The hub -ships one consistent truth series (`target-data/target-hospital-admissions.csv`) -covering every season. The method is MultiQT (Ding, Gibbs & Tibshirani 2025); our -port is `R/calibration/qt.R`, verified against the authors' Python in -`tests/testthat/test-qt.R` (same input, same output). - -Data facts: - -```{r data-shape} -forecasts %>% - left_join(select(rounds, reference_date = round_date, season), by = "reference_date") %>% - group_by(season) %>% - summarize( - rounds = n_distinct(reference_date), - locations = n_distinct(location), - horizons = paste(sort(unique(horizon)), collapse = ","), - cell_completeness = sprintf( - "%.0f%%", 100 * (n() / 23) / (n_distinct(reference_date) * 265) - ), - .groups = "drop" - ) -``` - -Notes: - -1. Waiting for revisions costs us `horizon + 2` rounds. `reference_date` and - `target_end_date` are both Saturdays with - `target_end_date = reference_date + 7 * horizon`, so waiting 14 days for NHSN - to settle puts the delay at exactly `h + 2` rounds. The offset applied at - horizon 3 is therefore always 5 rounds stale. -2. 2023-24 is incomplete: horizon -1 exists in only 5 of 28 rounds and - locations drop in and out, so it is used as a burn-in season. Its residuals - warm up the learning rate, but no gradient step is taken and the offsets - stay at zero. -3. Per-series coverage cannot be measured at the extreme levels: ~27 rounds - per season minus delay means the 1% and 99% levels see ~0.3 expected - exceedances per series. Every coverage number below is pooled over the 53 - locations. - -# Score results - -Coverage improves at some cost in sharpness. Below we show that the sharpness -cost can be reduced by changing the learning rate. - -```{r headline} -headline <- left_join( - hub_coverage_summary(cal), - select(hub_quantile_loss(cal), horizon, loss_base, loss_cal, loss_pct = pct_improvement), - by = "horizon" -) -headline %>% - transmute( - horizon, - n_obs, - `cal err (base)` = round(cal_error_base, 4), - `cal err (calibrated)` = round(cal_error_cal, 4), - `cal err improvement %` = round(pct_improvement, 1), - `WIS (base)` = round(2 * loss_base, 1), - `WIS (calibrated)` = round(2 * loss_cal, 1), - `WIS improvement %` = round(loss_pct, 1) - ) -``` - -Calibration error here is the mean over the 23 quantile levels of -`|geo-pooled empirical coverage at a - a|`. WIS is reported with the standard -factor-of-two convention (2 x the level-averaged pinball loss on this level -set), so it is directly comparable to hub scores; improvement percentages are -identical either way. The two metrics move in opposite directions, and the WIS -cost grows with the horizon. This is consistent with the `h + 2` staleness: a -stale additive offset in count space does the most damage where the seasonal -ramp is steep relative to the offset. - -## By season - -```{r headline-by-season} -left_join( - hub_coverage_summary(cal, by = "season"), - select(hub_quantile_loss(cal, by = "season"), - season, horizon, loss_base, loss_cal, loss_pct = pct_improvement - ), - by = c("season", "horizon") -) %>% - transmute( - season, horizon, - `cal err (base)` = round(cal_error_base, 4), - `cal err (calibrated)` = round(cal_error_cal, 4), - `cal err improvement %` = round(pct_improvement, 1), - `WIS (base)` = round(2 * loss_base, 1), - `WIS (calibrated)` = round(2 * loss_cal, 1), - `WIS improvement %` = round(loss_pct, 1) - ) %>% - arrange(season, horizon) -``` - -The burn-in season (2023-24) is excluded throughout. In the per-season view, -season 2 is the tracker's first live season; season 3 shows the effect of a -second season of accumulated state (under `carry` across the off-season gap). - -# 1. What feeds the learning rate - -`eta_t = max(floor, mult * quantile_0.9(|Y - Yhat|))` over a window of recent -observed rounds, pooled across all 23 quantile levels -- one step size per -(series, round). The residual distribution below therefore determines the -learning rate, up to the multiplier. - -```{r resid-series, fig.height=7} -resid <- cal$forecasts %>% - filter(!is.na(truth), !is.na(value_base)) %>% - mutate(abs_resid = abs(truth - value_base)) %>% - group_by(location, horizon, reference_date, season, is_burn_in) %>% - summarize(q90 = quantile(abs_resid, 0.9), .groups = "drop") - -show_geos <- c("06", "36", "48", "50", "US") -resid %>% - filter(location %in% show_geos) %>% - mutate(location = recode(location, - "06" = "CA", "36" = "NY", "48" = "TX", "50" = "VT", "US" = "US" - )) %>% - ggplot(aes(reference_date, q90, colour = factor(horizon))) + - geom_line() + - facet_wrap(~location, scales = "free_y", ncol = 1) + - scale_y_log10() + - labs( - title = "90th percentile of |truth - base forecast|, per round", - subtitle = "The quantity the adaptive learning rate is a fraction of. Log scale.", - x = NULL, y = "q90 |residual| (log)", colour = "horizon" - ) -``` - -The residual scale swings by 1-2 orders of magnitude within a season, so an -additive offset learned near the peak is badly sized for the trough, and vice -versa. - -## Does the window length matter? - -`lr_window` sets how many of the most recent observed rounds feed the adaptive -learning rate; `Inf` is an expanding window over the full history. - -```{r lr-window, cache=TRUE, dependson="load"} -lr_by_window <- purrr::map(c(8, 20, 50, Inf), function(w) { - purrr::map(show_geos, function(loc) { - series <- forecasts %>% filter(location == loc, horizon == 2) - if (nrow(series) == 0) { - return(NULL) - } - Yhat <- hub_series_matrix(series, rounds$round_date, 23) - ted <- rounds$round_date + 14L - Y <- truth$truth[match(paste(loc, ted), paste(truth$location, truth$target_end_date))] - learnable <- !is.na(Y) & !apply(is.na(Yhat), 2, any) - delay <- qt_delay_from_dates(rounds$round_date, ted, 14L) - delay <- lapply(delay, function(i) i[learnable[i]]) - res <- qt_track(Y, Yhat, HUB_QUANTILE_LEVELS, - delay = delay, lr_window = w, nonneg = TRUE - ) - tibble( - location = loc, round_date = rounds$round_date, - lr = res$lr, window = as.character(w) - ) - }) %>% bind_rows() -}) %>% bind_rows() - -lr_by_window %>% - filter(lr > 0) %>% - mutate( - location = recode(location, - "06" = "CA", "36" = "NY", "48" = "TX", "50" = "VT", "US" = "US" - ), - window = factor(window, levels = c("8", "20", "50", "Inf")) - ) %>% - ggplot(aes(round_date, lr, colour = window)) + - geom_line() + - facet_wrap(~location, scales = "free_y", ncol = 1) + - scale_y_log10() + - labs( - title = "Learning-rate trajectory by window length (horizon 2)", - subtitle = "A short window tracks the season; an expanding one (Inf) ratchets up and stays.", - x = NULL, y = "eta (log)", colour = "lr_window" - ) -``` - -The expanding window is monotone by construction: once a peak-season residual -enters the pool it never leaves, so `eta` rises to peak scale and stays there -through the following low season. A short window decays back. On 83 rounds the -difference in outcome is small (below), but it would likely grow over more -seasons. - -# 2. Actual vs nominal coverage, per location - -```{r coverage-plot, fig.height=6} -cov_by_geo <- hub_coverage(cal, by = "location") - -cov_long <- cov_by_geo %>% - select(location, horizon, level, coverage_base, coverage_cal) %>% - tidyr::pivot_longer( - c(coverage_base, coverage_cal), - names_to = "which", values_to = "coverage" - ) %>% - mutate(which = factor( - if_else(which == "coverage_base", "pre-calibration", "post-calibration"), - levels = c("pre-calibration", "post-calibration") - )) - -cov_long %>% - filter(horizon %in% c(0, 2)) %>% - ggplot(aes(level, coverage, group = location)) + - geom_abline(slope = 1, intercept = 0, linetype = "dashed", colour = "grey30") + - geom_line(alpha = 0.28, colour = "steelblue4") + - facet_grid(horizon ~ which, labeller = label_both) + - coord_equal(xlim = c(0, 1), ylim = c(0, 1)) + - labs( - title = "Empirical vs nominal coverage, one line per location", - subtitle = "Dashed line is perfect calibration. Each line is one of 53 locations, over all non-burn-in rounds.", - x = "nominal level", y = "empirical coverage" - ) -``` - -```{r coverage-pooled} -hub_coverage(cal) %>% - select(horizon, level, coverage_base, coverage_cal) %>% - tidyr::pivot_longer(c(coverage_base, coverage_cal), - names_to = "which", values_to = "coverage" - ) %>% - mutate(which = if_else(which == "coverage_base", "pre", "post")) %>% - ggplot(aes(level, coverage, colour = which)) + - geom_abline(slope = 1, intercept = 0, linetype = "dashed", colour = "grey30") + - geom_line() + - geom_point(size = 1) + - facet_wrap(~horizon, labeller = label_both, nrow = 1) + - coord_equal() + - labs( - title = "Geo-pooled coverage, pre and post calibration", - x = "nominal level", y = "empirical coverage", colour = NULL - ) -``` - -The per-location spread is not visible in the pooled number: it shows whether -the pooled improvement is a per-series effect or an averaging artefact, and it -bears on whether the offsets themselves should be pooled across geos. - -# 3. Calibration error against quantile loss - -Both axes are rolling averages over the last `r params$roll_window` rounds, pooled -over locations. Moving down is better calibrated; moving left is sharper. A -trajectory that goes down and to the right means calibration is being bought -with sharpness. - -```{r tradeoff, fig.height=6} -tradeoff <- hub_rolling_tradeoff(cal, window = params$roll_window) - -tradeoff %>% - select(horizon, reference_date, cal_error_base, cal_error_cal, qloss_base, qloss_cal) %>% - tidyr::pivot_longer( - -c(horizon, reference_date), - names_to = c(".value", "which"), names_pattern = "(.*)_(base|cal)$" - ) %>% - mutate(which = if_else(which == "base", "pre-calibration", "post-calibration")) %>% - ggplot(aes(qloss, cal_error, colour = which)) + - geom_path(alpha = 0.7) + - geom_point(aes(size = as.numeric(reference_date)), alpha = 0.5, show.legend = FALSE) + - scale_size_continuous(range = c(0.4, 2.2)) + - facet_wrap(~horizon, scales = "free_x", labeller = label_both) + - labs( - title = "Rolling calibration error vs rolling quantile loss", - subtitle = "Point size increases with time. Down = better calibrated, left = sharper.", - x = "rolling mean pinball loss (proportional to WIS)", y = "rolling calibration error", colour = NULL - ) -``` - -```{r tradeoff-time, fig.height=5} -tradeoff %>% - select(horizon, reference_date, cal_error_base, cal_error_cal) %>% - tidyr::pivot_longer(-c(horizon, reference_date), - names_to = "which", values_to = "cal_error" - ) %>% - mutate(which = if_else(which == "cal_error_base", "pre", "post")) %>% - ggplot(aes(reference_date, cal_error, colour = which)) + - geom_line() + - facet_wrap(~horizon, labeller = label_both, nrow = 1) + - labs( - title = "Rolling calibration error over time", - subtitle = "How fast the tracker converges after the burn-in season, and what the off-season gap does to it.", - x = NULL, y = "rolling calibration error", colour = NULL - ) -``` - -# 4. Interval widths - -The WIS cost has to show up somewhere in the quantiles; central interval widths -are the direct measure. For three central intervals -- 20% (0.4 to 0.6), 60% -(0.2 to 0.8), 80% (0.1 to 0.9) -- we compare the calibrated width to the base -width, per (round, location, horizon). Widths are geo-pooled by mean, matching -the implicit weighting of the WIS aggregate (large-count series dominate both). - -```{r interval-widths} -interval_defs <- tibble::tribble( - ~interval, ~lo, ~hi, - "20% (0.4-0.6)", 0.4, 0.6, - "60% (0.2-0.8)", 0.2, 0.8, - "80% (0.1-0.9)", 0.1, 0.9 -) - -width_rows <- cal$forecasts %>% - filter(!is_burn_in, !is.na(value_base), !is.na(value_cal)) %>% - select(reference_date, season, location, horizon, level, value_base, value_cal) - -widths <- purrr::pmap(interval_defs, function(interval, lo, hi) { - inner_join( - width_rows %>% filter(level == lo) %>% - select(-level) %>% rename(base_lo = value_base, cal_lo = value_cal), - width_rows %>% filter(level == hi) %>% - select(-level) %>% rename(base_hi = value_base, cal_hi = value_cal), - by = c("reference_date", "season", "location", "horizon") - ) %>% - transmute( - reference_date, season, location, horizon, - interval, - width_base = base_hi - base_lo, - width_cal = cal_hi - cal_lo - ) -}) %>% - bind_rows() %>% - mutate(interval = factor(interval, levels = interval_defs$interval)) -``` - -```{r width-time, fig.height=6} -widths %>% - group_by(interval, horizon, reference_date) %>% - summarize( - base = mean(width_base), calibrated = mean(width_cal), - .groups = "drop" - ) %>% - tidyr::pivot_longer(c(base, calibrated), names_to = "which", values_to = "width") %>% - ggplot(aes(reference_date, width, colour = which)) + - geom_line() + - facet_grid(interval ~ horizon, scales = "free_y", labeller = label_both) + - labs( - title = "Geo-pooled mean central interval width, base vs calibrated", - subtitle = "Mean over the 53 locations, per round. The gap between the lines is the sharpness cost.", - x = NULL, y = "mean interval width (admissions)", colour = NULL - ) -``` - -```{r width-change, fig.height=4} -widths %>% - group_by(interval, horizon) %>% - summarize( - pct_change = 100 * (mean(width_cal) / mean(width_base) - 1), - .groups = "drop" - ) %>% - ggplot(aes(horizon, pct_change, colour = interval)) + - geom_hline(yintercept = 0, linetype = "dashed", colour = "grey40") + - geom_line() + - geom_point(size = 2) + - labs( - title = "Change in mean interval width from calibration", - subtitle = "Positive means the calibrated intervals are wider than the base intervals.", - x = "horizon", y = "width change (%)", colour = "central interval" - ) -``` - -Wider intervals where the base under-covers are the intended correction; the -question is whether the widening is concentrated in the intervals and horizons -where the coverage plots (section 2) show a deficit, or spread indiscriminately. - -# 5. Sensitivity: window or multiplier? - -The learning rate has two knobs: how far back the residual pool reaches -(`lr_window`) and what fraction of its 90th percentile becomes the step size -(`lr_mult`). The factorial sweep below shows which one matters: if the lines -for different windows sit on top of each other while the multiplier moves -everything, the multiplier is the tuning parameter and the window is a detail -(and vice versa). - -```{r sweep, cache=TRUE, dependson="load"} -sweep_grid <- tidyr::expand_grid( - season_policy = c("carry", "reset"), - lr_window = c(8, 20, 50, Inf), - lr_mult = c(0.3, 0.1, 0.03, 0.01) -) -sweep <- purrr::pmap(sweep_grid, function(season_policy, lr_window, lr_mult) { - s <- calibrate_hub_forecasts(forecasts, truth, - burn_in_seasons = burn_in, season_policy = season_policy, - lr_window = lr_window, lr_args = list(mult = lr_mult), progress = FALSE - ) - overall <- left_join( - hub_coverage_summary(s), - select(hub_quantile_loss(s), horizon, loss_pct = pct_improvement), - by = "horizon" - ) %>% - mutate(season = "all") - by_season <- left_join( - hub_coverage_summary(s, by = "season"), - select(hub_quantile_loss(s, by = "season"), - season, horizon, - loss_pct = pct_improvement - ), - by = c("season", "horizon") - ) - bind_rows(overall, by_season) %>% - mutate(season_policy = season_policy, lr_window = lr_window, lr_mult = lr_mult) -}) %>% bind_rows() - -# Keep the sweep around for analysis outside the notebook. -dir.create(here::here("cache/calibration"), showWarnings = FALSE, recursive = TRUE) -saveRDS(sweep, here::here("cache/calibration/sweep.rds")) -``` - -```{r sweep-cal-error, fig.height=5} -sweep %>% - filter(season == "all") %>% - ggplot(aes(lr_mult, cal_error_cal, colour = factor(lr_window))) + - geom_line() + - geom_point(size = 1.6) + - scale_x_log10() + - facet_grid(season_policy ~ horizon, labeller = label_both) + - labs( - title = "Calibration error by multiplier, one line per window", - subtitle = "Separated lines would indicate a window effect; sloped lines indicate a multiplier effect.", - x = "lr mult (log)", y = "calibration error (calibrated)", - colour = "lr_window" - ) -``` - -```{r sweep-wis, fig.height=5} -sweep %>% - filter(season == "all") %>% - ggplot(aes(lr_mult, loss_pct, colour = factor(lr_window))) + - geom_hline(yintercept = 0, linetype = "dashed", colour = "grey40") + - geom_line() + - geom_point(size = 1.6) + - scale_x_log10() + - facet_grid(season_policy ~ horizon, labeller = label_both) + - labs( - title = "WIS improvement by multiplier, one line per window", - subtitle = "Dashed line is WIS parity with the raw submission; above it calibration has no WIS cost.", - x = "lr mult (log)", y = "WIS improvement (%)", - colour = "lr_window" - ) -``` - -The target region is the combination of low calibration error and WIS at or -above parity. - -```{r sweep-corner, fig.height=5} -sweep %>% - filter(season == "all", season_policy == "carry") %>% - ggplot(aes(cal_error_cal, loss_pct, - colour = factor(lr_mult), shape = factor(lr_window) - )) + - geom_hline(yintercept = 0, linetype = "dashed", colour = "grey40") + - geom_point(size = 3, alpha = 0.85) + - facet_wrap(~horizon, scales = "free_x", labeller = label_both, nrow = 1) + - labs( - title = "Calibration error vs WIS change (season_policy = carry)", - subtitle = "Upper-left is the target region: well calibrated and no worse on WIS.", - x = "calibration error (calibrated)", y = "WIS change (%)", - colour = "lr mult", shape = "lr_window" - ) -``` - -If the expanding window's monotone behavior matters, its effect should grow -with history: season 2 starts with at most one season in the pool, season 3 -with two. The per-season view checks that. - -```{r sweep-by-season, fig.height=5} -sweep %>% - filter(season != "all", season_policy == "carry", horizon == 2) %>% - ggplot(aes(lr_mult, cal_error_cal, colour = factor(lr_window))) + - geom_line() + - geom_point(size = 1.6) + - scale_x_log10() + - facet_wrap(~season, labeller = label_both) + - labs( - title = "Calibration error by season (horizon 2, season_policy = carry)", - subtitle = "Does the window matter more once multiple seasons of residuals are available?", - x = "lr mult (log)", y = "calibration error (calibrated)", - colour = "lr_window" - ) -``` - -```{r sweep-table} -sweep %>% - filter(season == "all") %>% - transmute( - season_policy, lr_window, lr_mult, horizon, - cal_error_cal = round(cal_error_cal, 4), - wis_pct = round(loss_pct, 1) - ) %>% - arrange(horizon, season_policy, lr_window, lr_mult) -``` - -# 6. Offset trajectories - -```{r offsets, fig.height=6} -cal$forecasts %>% - filter(location %in% show_geos, horizon == 2, level %in% c(0.025, 0.5, 0.975)) %>% - mutate(location = recode(location, - "06" = "CA", "36" = "NY", "48" = "TX", "50" = "VT", "US" = "US" - )) %>% - ggplot(aes(reference_date, offset, colour = factor(level))) + - geom_hline(yintercept = 0, colour = "grey50") + - geom_line() + - facet_wrap(~location, scales = "free_y", ncol = 1) + - labs( - title = "Realised offset (played - base), horizon 2", - subtitle = "Zero through the burn-in season by construction. With season_policy='carry' the offset is held across the off-season gap.", - x = NULL, y = "offset (admissions)", colour = "level" - ) -``` - -```{r offset-scale} -cal$forecasts %>% - filter(!is_burn_in, !is.na(value_base), value_base > 0) %>% - group_by(horizon, level) %>% - summarize( - median_abs_offset = median(abs(offset)), - median_rel = median(abs(offset) / pmax(value_base, 1)), - .groups = "drop" - ) %>% - ggplot(aes(level, median_rel, colour = factor(horizon))) + - geom_line() + - scale_y_continuous(labels = scales::percent) + - labs( - title = "Median |offset| as a fraction of the base forecast", - subtitle = "How large a correction the tracker is applying, and how that scales with horizon.", - x = "quantile level", y = "median |offset| / base", colour = "horizon" - ) -``` diff --git a/reports/writeups/calibration/calibration_qt_gallery_covid.Rmd b/reports/writeups/calibration/calibration_qt_gallery_covid.Rmd deleted file mode 100644 index cbd9b641..00000000 --- a/reports/writeups/calibration/calibration_qt_gallery_covid.Rmd +++ /dev/null @@ -1,242 +0,0 @@ ---- -title: MultiQT calibration gallery — per-forecast panels, COVID hub -date: "`r format(Sys.time(), '%d %B %Y')`" -output: - html_document: - code_folding: hide - df_print: paged - toc: true - toc_float: true - toc_depth: 2 -params: - burn_in_seasons: "" - season_policy: "carry" - lr_window: 20 - lr_mult: 0.03 - n_panels: 150 -editor_options: - chunk_output_type: console ---- - -```{r setup, echo=FALSE} -# cache = TRUE chunks are keyed on chunk code + cache.extra, NOT on functions -# in R/ -- after changing calibration code, clear cache/calibration/knitr_cache_gallery_covid/. -knitr::opts_chunk$set( - fig.align = "center", fig.width = 10, fig.height = 6, - message = FALSE, warning = FALSE, cache = FALSE, - cache.path = here::here("cache/calibration/knitr_cache_gallery_covid/") -) -knitr::opts_knit$set(root.dir = here::here()) -ggplot2::theme_set(ggplot2::theme_bw()) -``` - -```{r load, cache=TRUE, cache.extra=list(params)} -suppressPackageStartupMessages(source(here::here("R/load_all.R"))) - -forecasts <- hub_read_forecasts(hub_dir = HUB_COVID_DIR, target = HUB_COVID_TARGET) -truth <- hub_read_covid_truth() -rounds <- hub_label_seasons(unique(forecasts$reference_date)) - -burn_in <- if (nzchar(params$burn_in_seasons)) { - strsplit(params$burn_in_seasons, ",")[[1]] -} else { - character(0) -} - -cal <- calibrate_hub_forecasts( - forecasts, truth, - burn_in_seasons = burn_in, - season_policy = params$season_policy, - lr_window = params$lr_window, - lr_args = list(mult = params$lr_mult), - transform = "sqrt", - progress = FALSE -) -``` - -Per-forecast panel gallery for COVID hub submissions at the chosen operating point -(`calibration_qt_covid.Rmd` has the parameter sweeps; see `notes/CALIBRATION.md` -for project state). - -Parameters: -- `lr_mult = `r params$lr_mult`` -- `lr_window = `r params$lr_window`` -- `season_policy = "`r params$season_policy`"` -- `transform = "sqrt"` - -# Summary scores - -```{r headline} -bind_rows( - left_join( - hub_coverage_summary(cal), - select(hub_quantile_loss(cal), horizon, loss_base, loss_cal, loss_pct = pct_improvement), - by = "horizon" - ) %>% mutate(season = "all"), - left_join( - hub_coverage_summary(cal, by = "season"), - select(hub_quantile_loss(cal, by = "season"), - season, horizon, loss_base, loss_cal, loss_pct = pct_improvement - ), - by = c("season", "horizon") - ) -) %>% - transmute( - season, horizon, - `cal err (base)` = round(cal_error_base, 4), - `cal err (calibrated)` = round(cal_error_cal, 4), - `cal err improvement %` = round(pct_improvement, 1), - `WIS (base)` = round(2 * loss_base, 1), - `WIS (calibrated)` = round(2 * loss_cal, 1), - `WIS improvement %` = round(loss_pct, 1) - ) %>% - arrange(season, horizon) -``` - -# Data vintages for the panels - -Each panel shows the NHSN data as of the Wednesday before the reference -date (`reference_date - 3`) alongside the finalized truth. - -```{r snapshots, cache=TRUE, dependson="load"} -archive <- nanoparquet::read_parquet( - here::here("cache/calibration/nhsn_archive_covid.parquet") -) %>% - as_epi_archive(compactify = TRUE) - -geo_map <- get_population_data() %>% - distinct(state_id, .keep_all = TRUE) %>% - transmute(state_id, abbrev = toupper(state_id), location = state_code) - -live_rounds <- rounds %>% filter(!season %in% burn_in) -min_version <- min(archive$DT$version) -max_version <- archive$versions_end - -snapshots <- purrr::map(live_rounds$round_date, function(rd) { - as_of <- rd - 3L - if (as_of < min_version || as_of > max_version) { - return(NULL) - } - epix_as_of(archive, as_of) %>% - as_tibble() %>% - inner_join(geo_map, by = c("geo_value" = "state_id")) %>% - transmute(reference_date = rd, location, time_value, value) -}) %>% bind_rows() -``` - -# Ranking - -Panels are ordered by mean absolute displacement between calibrated and base -quantiles, divided by the round's mean base median (floored at 20 admissions). - -```{r ranking} -pinball <- function(truth, value, level) { - ifelse(truth >= value, level * (truth - value), (1 - level) * (value - truth)) -} - -live <- cal$forecasts %>% - filter(!is_burn_in, !is.na(value_base), !is.na(value_cal)) - -escapes <- live %>% - filter(level %in% c(0.05, 0.95), !is.na(truth)) %>% - select(location, reference_date, horizon, level, value_base, truth) %>% - tidyr::pivot_wider(names_from = level, values_from = value_base, names_prefix = "q") %>% - group_by(location, reference_date) %>% - summarize(escape90 = any(truth < q0.05 | truth > q0.95), .groups = "drop") - -panel_stats <- live %>% - group_by(season, location, reference_date) %>% - summarize( - mad = mean(abs(value_cal - value_base)), - base_med = mean(value_base[level == 0.5]), - wis_base = 2 * mean(pinball(truth, value_base, level)[!is.na(truth)]), - wis_cal = 2 * mean(pinball(truth, value_cal, level)[!is.na(truth)]), - .groups = "drop" - ) %>% - mutate( - mad_rel = mad / pmax(base_med, 20), - wis_pct = if_else(wis_base > 0, 100 * (1 - wis_cal / wis_base), NA_real_) - ) %>% - left_join(escapes, by = c("location", "reference_date")) %>% - left_join(select(geo_map, location, abbrev), by = "location") %>% - arrange(desc(mad_rel)) - -panel_stats %>% - transmute( - season, abbrev, reference_date, - mad = round(mad, 1), mad_rel = round(mad_rel, 3), - wis_pct = round(wis_pct, 1), escape90 - ) -``` - -# Gallery: top `r params$n_panels` panels - -Black: NHSN data as of the forecast Wednesday (dashed vertical line). -Grey: finalized truth. Blue: submitted (base) forecast. Orange: calibrated. -Ribbons are the 50% and 90% central intervals; line is the median, across -horizons −1 to 3. - -```{r panel-fn} -fan_cols <- c(base = "#0072B2", calibrated = "#D55E00") -ribbon_levels <- c( - "0.05" = "lo90", "0.25" = "lo50", "0.5" = "med", - "0.75" = "hi50", "0.95" = "hi90" -) - -panel_plot <- function(loc, ref_date, caption) { - win_start <- ref_date - 84L - win_end <- ref_date + 28L - fin <- truth %>% - filter(location == loc, target_end_date >= win_start, target_end_date <= win_end) - snap <- snapshots %>% - filter(reference_date == ref_date, location == loc, time_value >= win_start) - fan <- cal$forecasts %>% - filter( - location == loc, reference_date == ref_date, - level %in% c(0.05, 0.25, 0.5, 0.75, 0.95) - ) %>% - mutate( - target_end_date = reference_date + 7L * horizon, - level = ribbon_levels[as.character(level)] - ) %>% - select(target_end_date, level, base = value_base, calibrated = value_cal) %>% - tidyr::pivot_longer(c(base, calibrated), names_to = "which") %>% - tidyr::pivot_wider(names_from = level, values_from = value) - - ggplot() + - geom_line(data = fin, aes(target_end_date, truth), colour = "grey65") + - geom_point(data = fin, aes(target_end_date, truth), colour = "grey65", size = 0.7) + - geom_ribbon( - data = fan, aes(target_end_date, ymin = lo90, ymax = hi90, fill = which), - alpha = 0.16 - ) + - geom_ribbon( - data = fan, aes(target_end_date, ymin = lo50, ymax = hi50, fill = which), - alpha = 0.3 - ) + - geom_line(data = fan, aes(target_end_date, med, colour = which)) + - geom_line(data = snap, aes(time_value, value), colour = "black") + - geom_point(data = snap, aes(time_value, value), colour = "black", size = 0.7) + - geom_vline(xintercept = ref_date - 3L, linetype = "dashed", colour = "grey40") + - scale_fill_manual(values = fan_cols, aesthetics = c("fill", "colour")) + - labs(x = NULL, y = "admissions", colour = NULL, fill = NULL, subtitle = caption) -} -``` - -```{r gallery, results='asis', fig.width=9, fig.height=4} -top <- head(panel_stats, params$n_panels) -for (i in seq_len(nrow(top))) { - p <- top[i, ] - cat(sprintf( - "\n\n### %d. %s — %s (%s)\n\n", - i, p$abbrev, format(p$reference_date), p$season - )) - caption <- sprintf( - "MAD %.1f admissions (base median %.1f) | WIS %s | truth escaped base 90%%: %s", - p$mad, p$base_med, - if (is.na(p$wis_pct)) "n/a" else sprintf("%+.1f%%", p$wis_pct), - if (isTRUE(p$escape90)) "yes" else "no" - ) - print(panel_plot(p$location, p$reference_date, caption)) -} -``` diff --git a/reports/writeups/calibration/calibration_qt_gallery_flu.Rmd b/reports/writeups/calibration/calibration_qt_gallery_flu.Rmd deleted file mode 100644 index 00b9cbc6..00000000 --- a/reports/writeups/calibration/calibration_qt_gallery_flu.Rmd +++ /dev/null @@ -1,248 +0,0 @@ ---- -title: MultiQT calibration gallery — per-forecast panels at the chosen operating point -date: "`r format(Sys.time(), '%d %B %Y')`" -output: - html_document: - code_folding: hide - df_print: paged - toc: true - toc_float: true - toc_depth: 2 -params: - burn_in_seasons: "2023-2024" - season_policy: "carry" - lr_window: 20 - lr_mult: 0.03 - n_panels: 150 -editor_options: - chunk_output_type: console ---- - -```{r setup, echo=FALSE} -# cache = TRUE chunks are keyed on chunk code + cache.extra, NOT on functions -# in R/ -- after changing calibration code, clear cache/calibration/knitr_cache_gallery_flu/. -knitr::opts_chunk$set( - fig.align = "center", fig.width = 10, fig.height = 6, - message = FALSE, warning = FALSE, cache = FALSE, - cache.path = here::here("cache/calibration/knitr_cache_gallery_flu/") -) -knitr::opts_knit$set(root.dir = here::here()) -ggplot2::theme_set(ggplot2::theme_bw()) -``` - -```{r load, cache=TRUE, cache.extra=list(params)} -suppressPackageStartupMessages(source(here::here("R/load_all.R"))) - -forecasts <- hub_read_forecasts() -truth <- hub_read_truth() -rounds <- hub_label_seasons(unique(forecasts$reference_date)) - -burn_in <- if (nzchar(params$burn_in_seasons)) { - strsplit(params$burn_in_seasons, ",")[[1]] -} else { - character(0) -} - -cal <- calibrate_hub_forecasts( - forecasts, truth, - burn_in_seasons = burn_in, - season_policy = params$season_policy, - lr_window = params$lr_window, - lr_args = list(mult = params$lr_mult), - progress = FALSE -) -``` - -This document contains a gallery of forecasts that were adjusted the most (in mean absolute deviation, see below) (`calibration_qt_flu.Rmd` has the parameter-sweeps, this notebook gets more granular about forecast adjustments; see -`notes/CALIBRATION.md` for the project state). - -The parameters for the calibration method are fixed: -- `lr_mult = `r params$lr_mult`` (the learning rate multiplier, which was found to be too slow at 0.01 and too fast at 0.1), -- `lr_window = `r params$lr_window`` (a window of 8 weeks paid the cost across aheads, while 20, 50, and Inf were essentially indistinguishable and had consistent calibration improvements across aheads) -- `season_policy = "`r params$season_policy`"` (this parameter was either "carry" for carrying the learning rate over from the previous season or "reset" to reset it; we found that this did not influence things very much) - -# Summary scores at the parameter sweep peak settings - -Calibration error is the mean over the 23 quantile levels of -`|geo-pooled empirical coverage at a - a|`; WIS uses the standard -factor-of-two convention. The burn-in season (2023-24) is excluded, so testing is across (2024-25 and 2025-26). - -```{r headline} -bind_rows( - left_join( - hub_coverage_summary(cal), - select(hub_quantile_loss(cal), horizon, loss_base, loss_cal, loss_pct = pct_improvement), - by = "horizon" - ) %>% mutate(season = "all"), - left_join( - hub_coverage_summary(cal, by = "season"), - select(hub_quantile_loss(cal, by = "season"), - season, horizon, loss_base, loss_cal, loss_pct = pct_improvement - ), - by = c("season", "horizon") - ) -) %>% - transmute( - season, horizon, - `cal err (base)` = round(cal_error_base, 4), - `cal err (calibrated)` = round(cal_error_cal, 4), - `cal err improvement %` = round(pct_improvement, 1), - `WIS (base)` = round(2 * loss_base, 1), - `WIS (calibrated)` = round(2 * loss_cal, 1), - `WIS improvement %` = round(loss_pct, 1) - ) %>% - arrange(season, horizon) -``` - -# Data vintages for the panels - -Each panel shows the NHSN data as of the Wednesday before the reference -date (`reference_date - 3`, which should be what the forecaster saw at submission -time, unless there were delays) alongside the finalized truth. - -```{r snapshots, cache=TRUE, dependson="load"} -archive <- nanoparquet::read_parquet( - here::here("cache/calibration/nhsn_archive_flu.parquet") -) %>% - as_epi_archive(compactify = TRUE) - -geo_map <- get_population_data() %>% - distinct(state_id, .keep_all = TRUE) %>% - transmute(state_id, abbrev = toupper(state_id), location = state_code) - -live_rounds <- rounds %>% filter(!season %in% burn_in) -min_version <- min(archive$DT$version) - -snapshots <- purrr::map(live_rounds$round_date, function(rd) { - as_of <- rd - 3L - if (as_of < min_version) { - return(NULL) - } - epix_as_of(archive, as_of) %>% - as_tibble() %>% - inner_join(geo_map, by = c("geo_value" = "state_id")) %>% - transmute(reference_date = rd, location, time_value, value) -}) %>% bind_rows() -``` - -# Ranking - -Panels are ordered by the size of the correction: the mean absolute -displacement between calibrated and base quantiles (over the 5 horizons x 23 -levels of a round), divided by the round's mean base median (this way we rank small states -on relative shift). We floored the denominator at 20 admissions (with out this the entire top of the list is -end-of-season rounds where the base median is ~1). - -Even with that correction, season edges (May, October) dominate: the mid-season offsets are carried into the very low signal values at the end. - -```{r ranking} -pinball <- function(truth, value, level) { - ifelse(truth >= value, level * (truth - value), (1 - level) * (value - truth)) -} - -live <- cal$forecasts %>% - filter(!is_burn_in, !is.na(value_base), !is.na(value_cal)) - -escapes <- live %>% - filter(level %in% c(0.05, 0.95), !is.na(truth)) %>% - select(location, reference_date, horizon, level, value_base, truth) %>% - tidyr::pivot_wider(names_from = level, values_from = value_base, names_prefix = "q") %>% - group_by(location, reference_date) %>% - summarize(escape90 = any(truth < q0.05 | truth > q0.95), .groups = "drop") - -panel_stats <- live %>% - group_by(season, location, reference_date) %>% - summarize( - mad = mean(abs(value_cal - value_base)), - base_med = mean(value_base[level == 0.5]), - wis_base = 2 * mean(pinball(truth, value_base, level)[!is.na(truth)]), - wis_cal = 2 * mean(pinball(truth, value_cal, level)[!is.na(truth)]), - .groups = "drop" - ) %>% - mutate( - mad_rel = mad / pmax(base_med, 20), - wis_pct = if_else(wis_base > 0, 100 * (1 - wis_cal / wis_base), NA_real_) - ) %>% - left_join(escapes, by = c("location", "reference_date")) %>% - left_join(select(geo_map, location, abbrev), by = "location") %>% - arrange(desc(mad_rel)) - -panel_stats %>% - transmute( - season, abbrev, reference_date, - mad = round(mad, 1), mad_rel = round(mad_rel, 3), - wis_pct = round(wis_pct, 1), escape90 - ) -``` - -# Gallery: top `r params$n_panels` panels - -In the plots below, black is the NHSN data as of the forecast Wednesday -(dashed vertical line), while gray is the finalized series. Blue is the submitted -(base) forecast, orange the calibrated one; ribbons are the 50% and 90% -central intervals, the line is the median, across horizons -1 to 3. - -```{r panel-fn} -fan_cols <- c(base = "#0072B2", calibrated = "#D55E00") -ribbon_levels <- c( - "0.05" = "lo90", "0.25" = "lo50", "0.5" = "med", - "0.75" = "hi50", "0.95" = "hi90" -) - -panel_plot <- function(loc, ref_date, caption) { - win_start <- ref_date - 84L - win_end <- ref_date + 28L - fin <- truth %>% - filter(location == loc, target_end_date >= win_start, target_end_date <= win_end) - snap <- snapshots %>% - filter(reference_date == ref_date, location == loc, time_value >= win_start) - fan <- cal$forecasts %>% - filter( - location == loc, reference_date == ref_date, - level %in% c(0.05, 0.25, 0.5, 0.75, 0.95) - ) %>% - mutate( - target_end_date = reference_date + 7L * horizon, - level = ribbon_levels[as.character(level)] - ) %>% - select(target_end_date, level, base = value_base, calibrated = value_cal) %>% - tidyr::pivot_longer(c(base, calibrated), names_to = "which") %>% - tidyr::pivot_wider(names_from = level, values_from = value) - - ggplot() + - geom_line(data = fin, aes(target_end_date, truth), colour = "grey65") + - geom_point(data = fin, aes(target_end_date, truth), colour = "grey65", size = 0.7) + - geom_ribbon( - data = fan, aes(target_end_date, ymin = lo90, ymax = hi90, fill = which), - alpha = 0.16 - ) + - geom_ribbon( - data = fan, aes(target_end_date, ymin = lo50, ymax = hi50, fill = which), - alpha = 0.3 - ) + - geom_line(data = fan, aes(target_end_date, med, colour = which)) + - geom_line(data = snap, aes(time_value, value), colour = "black") + - geom_point(data = snap, aes(time_value, value), colour = "black", size = 0.7) + - geom_vline(xintercept = ref_date - 3L, linetype = "dashed", colour = "grey40") + - scale_fill_manual(values = fan_cols, aesthetics = c("fill", "colour")) + - labs(x = NULL, y = "admissions", colour = NULL, fill = NULL, subtitle = caption) -} -``` - -```{r gallery, results='asis', fig.width=9, fig.height=4} -top <- head(panel_stats, params$n_panels) -for (i in seq_len(nrow(top))) { - p <- top[i, ] - cat(sprintf( - "\n\n### %d. %s — %s (%s)\n\n", - i, p$abbrev, format(p$reference_date), p$season - )) - caption <- sprintf( - "MAD %.1f admissions (base median %.1f) | WIS %s | truth escaped base 90%%: %s", - p$mad, p$base_med, - if (is.na(p$wis_pct)) "n/a" else sprintf("%+.1f%%", p$wis_pct), - if (isTRUE(p$escape90)) "yes" else "no" - ) - print(panel_plot(p$location, p$reference_date, caption)) -} -``` diff --git a/reports/writeups/calibration/calibration_qt_seasons_covid.Rmd b/reports/writeups/calibration/calibration_qt_seasons_covid.Rmd deleted file mode 100644 index 2c941d8c..00000000 --- a/reports/writeups/calibration/calibration_qt_seasons_covid.Rmd +++ /dev/null @@ -1,351 +0,0 @@ ---- -title: MultiQT calibration — whole-season views of four states, COVID hub -date: "`r format(Sys.time(), '%d %B %Y')`" -output: - html_document: - code_folding: hide - df_print: paged - toc: true - toc_float: true - toc_depth: 2 -params: - burn_in_seasons: "" - season_policy: "carry" - lr_window: 20 - lr_mult: 0.03 - lr_floor: 0.001 -editor_options: - chunk_output_type: console ---- - -```{r setup, echo=FALSE} -# cache = TRUE chunks are keyed on chunk code + cache.extra, NOT on functions -# in R/ -- after changing calibration code, clear cache/calibration/knitr_cache_seasons_covid/. -knitr::opts_chunk$set( - fig.align = "center", fig.width = 10, fig.height = 6, - message = FALSE, warning = FALSE, cache = FALSE, - cache.path = here::here("cache/calibration/knitr_cache_seasons_covid/") -) -knitr::opts_knit$set(root.dir = here::here()) -ggplot2::theme_set(ggplot2::theme_bw()) -``` - -This notebook takes the sqrt-space operating point (`transform = "sqrt"`, -`lr_mult = 0.03`, `lr_window = 20`, carry, no burn-in) and looks at four -states across the available season of COVID hub submissions. -COVID runs year-round with no off-season gap, so season policy has no practical -effect; burn-in-dependent variants (conformal warm start) are not included. -The states are chosen by how much the baseline calibration changed them, pooled -over horizons −1…3. - -```{r load, cache=TRUE} -suppressPackageStartupMessages(source(here::here("R/load_all.R"))) - -forecasts <- hub_read_forecasts(hub_dir = HUB_COVID_DIR, target = HUB_COVID_TARGET) -truth <- hub_read_covid_truth() -rounds <- hub_label_seasons(unique(forecasts$reference_date)) -burn_in <- if (nzchar(params$burn_in_seasons)) { - strsplit(params$burn_in_seasons, ",")[[1]] -} else { - character(0) -} - -geo_map <- get_population_data() %>% - distinct(state_id, .keep_all = TRUE) %>% - transmute(state_id, abbrev = toupper(state_id), location = state_code) - -archive <- nanoparquet::read_parquet( - here::here("cache/calibration/nhsn_archive_covid.parquet") -) %>% - as_epi_archive(compactify = TRUE) - -live_rounds <- rounds %>% filter(!season %in% burn_in) -min_version <- min(archive$DT$version) -max_version <- archive$versions_end -snapshots <- purrr::map(live_rounds$round_date, function(rd) { - as_of <- rd - 3L - if (as_of < min_version || as_of > max_version) { - return(NULL) - } - epix_as_of(archive, as_of) %>% - as_tibble() %>% - inner_join(geo_map, by = c("geo_value" = "state_id")) %>% - transmute(reference_date = rd, location, time_value, value) -}) %>% bind_rows() - -seasons <- unique(live_rounds$season) -``` - -```{r variants, cache=TRUE, dependson="load"} -base_args <- list( - burn_in_seasons = burn_in, season_policy = params$season_policy, - lr_window = params$lr_window, transform = "sqrt", - lr_args = list(mult = params$lr_mult, floor = params$lr_floor), - progress = FALSE -) -# Burn-in-dependent variants (warm_static, warm_fast, warm_slow_fast) are not -# included: COVID has no burn-in season so slow_init = "burn_in_quantile" has -# no data to warm from. off_feb is also excluded -- COVID has no spring tail. -variant_args <- list( - baseline = list(), - count_space = list(transform = "identity", lr_args = list(mult = params$lr_mult)), - slow_term = list(lr_slow = list(mult = 0.003)), - slow_fast = list(lr_slow = list(mult = 0.003), fast_decay = 0.1) -) -variant_titles <- c( - baseline = "Sqrt-space tracker (baseline)", - count_space = "Count-space tracker (the paper's method)", - slow_term = "Sqrt tracker + persistent slow term", - slow_fast = "Sqrt tracker + slow term + leaky fast term" -) - -cals <- purrr::map(variant_args, function(v) { - do.call(calibrate_hub_forecasts, c(list(forecasts, truth), modifyList(base_args, v))) -}) -``` - -```{r select-states, dependson="variants"} -pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) - -by_state <- cals$baseline$forecasts %>% - filter(!is_burn_in, !is.na(truth)) %>% - group_by(location) %>% - summarize( - wis_pct = 100 * (1 - sum(pinball(truth, value_cal, level)) / sum(pinball(truth, value_base, level))), - ae_pct = 100 * (1 - sum(abs(truth - value_cal)[level == 0.5]) / sum(abs(truth - value_base)[level == 0.5])), - .groups = "drop" - ) %>% - inner_join(select(geo_map, location, abbrev), by = "location") - -pick <- function(ranked, taken) ranked$location[!ranked$location %in% taken][1] -roles <- list( - `best WIS` = arrange(by_state, desc(wis_pct)), - `worst WIS` = arrange(by_state, wis_pct), - `best AE` = arrange(by_state, desc(ae_pct)), - `worst AE` = arrange(by_state, ae_pct) -) -taken <- character(0) -for (r in names(roles)) taken <- c(taken, pick(roles[[r]], taken)) -states <- tibble(location = taken, role = names(roles)) %>% - left_join(by_state, by = "location") %>% - mutate(label = sprintf("%s (%s: WIS %+.1f%%, AE %+.1f%%)", abbrev, role, wis_pct, ae_pct)) -``` - -# Which states, and why - -Per-state change from baseline calibration over all rounds, all horizons pooled. - -```{r state-table} -knitr::kable(states %>% transmute(state = abbrev, role, `WIS change %` = round(wis_pct, 1), `AE change %` = round(ae_pct, 1))) -``` - -# Summary across variants - -WIS % improvement over the base forecast (all rounds), per horizon. - -```{r variant-summary} -purrr::imap(cals, function(cal, nm) { - hub_quantile_loss(cal) %>% transmute(variant = variant_titles[[nm]], horizon, pct = round(pct_improvement, 1)) -}) %>% - bind_rows() %>% - tidyr::pivot_wider(names_from = horizon, values_from = pct, names_prefix = "h") -``` - -## Where in the year the baseline gains and loses - -Baseline variant, pooled over all locations, by calendar month of the reference -date. - -```{r by-month} -by_month <- cals$baseline$forecasts %>% - filter(!is_burn_in, !is.na(truth)) %>% - mutate(month = format(reference_date, "%b"), month_n = as.integer(format(reference_date, "%m"))) %>% - group_by(horizon, month, month_n) %>% - summarize( - base = sum(pinball(truth, value_base, level)), - cal = sum(pinball(truth, value_cal, level)), - shift = 100 * sum((value_cal - value_base)[level == 0.5]) / sum(truth[level == 0.5]), - .groups = "drop" - ) %>% - group_by(horizon) %>% - mutate(pct = 100 * (base - cal) / base, share = 100 * base / sum(base)) %>% - ungroup() %>% - mutate(month = factor(month, levels = unique(month[order(month_n)]))) - -show_wide <- function(col, digits = 1) { - by_month %>% - select(horizon, month, value = all_of(col)) %>% - mutate(value = round(value, digits)) %>% - tidyr::pivot_wider(names_from = horizon, values_from = value, names_prefix = "h") %>% - arrange(month) -} -knitr::kable(show_wide("pct"), caption = "WIS change % (calibrated vs base)") -knitr::kable(show_wide("share"), caption = "Share of base WIS %") -knitr::kable(show_wide("shift"), caption = "Median shift % of truth") -``` - -```{r helpers} -headline_table <- function(cal) { - bind_rows( - left_join( - hub_coverage_summary(cal), - select(hub_quantile_loss(cal), horizon, loss_base, loss_cal, loss_pct = pct_improvement), - by = "horizon" - ) %>% mutate(season = "all"), - left_join( - hub_coverage_summary(cal, by = "season"), - select(hub_quantile_loss(cal, by = "season"), - season, horizon, loss_base, loss_cal, loss_pct = pct_improvement - ), - by = c("season", "horizon") - ) - ) %>% - transmute( - season, horizon, - `cal err (base)` = round(cal_error_base, 4), - `cal err (calibrated)` = round(cal_error_cal, 4), - `cal err improvement %` = round(pct_improvement, 1), - `WIS (base)` = round(2 * loss_base, 1), - `WIS (calibrated)` = round(2 * loss_cal, 1), - `WIS improvement %` = round(loss_pct, 1) - ) %>% - arrange(season, horizon) -} - -fan_cols <- c(base = "#0072B2", calibrated = "#D55E00") - -season_plot <- function(cal, loc, season) { - rr <- cal$rounds %>% filter(season == !!season, season_round %% 2 == 1) - ref_dates <- sort(rr$round_date) - - fin <- truth %>% - filter(location == loc, target_end_date >= min(ref_dates) - 21L, target_end_date <= max(ref_dates) + 28L) - - fan <- cal$forecasts %>% - filter(location == loc, reference_date %in% ref_dates, horizon %in% 0:3, level %in% c(0.1, 0.5, 0.9)) %>% - mutate( - target_end_date = reference_date + 7L * horizon, - lvl = c(`0.1` = "lo", `0.5` = "med", `0.9` = "hi")[as.character(level)] - ) %>% - select(reference_date, target_end_date, lvl, base = value_base, calibrated = value_cal) %>% - tidyr::pivot_longer(c(base, calibrated), names_to = "which") %>% - tidyr::pivot_wider(names_from = lvl, values_from = value) - - vint_windows <- tibble(ref = ref_dates) %>% - mutate( - win_start = dplyr::lag(ref, default = ref[1] - 14L) - 3L, - win_end = ref - 3L - ) - vint <- purrr::pmap(vint_windows, function(ref, win_start, win_end) { - snapshots %>% - filter(reference_date == ref, location == loc, time_value >= win_start, time_value <= win_end) - }) %>% bind_rows() - - main <- ggplot() + - geom_line(data = fin, aes(target_end_date, truth), colour = "grey65") + - geom_point(data = fin, aes(target_end_date, truth), colour = "grey65", size = 0.7) + - geom_ribbon( - data = filter(fan, which == "base"), - aes(target_end_date, ymin = lo, ymax = hi, fill = which, group = reference_date), - alpha = 0.2 - ) + - geom_line( - data = filter(fan, which == "calibrated"), - aes(target_end_date, lo, colour = which, group = reference_date), linewidth = 0.3 - ) + - geom_line( - data = filter(fan, which == "calibrated"), - aes(target_end_date, hi, colour = which, group = reference_date), linewidth = 0.3 - ) + - geom_line( - data = fan, - aes(target_end_date, med, colour = which, group = interaction(reference_date, which)) - ) + - geom_line(data = vint, aes(time_value, value), colour = "black") + - geom_point(data = vint, aes(time_value, value), colour = "black", size = 0.6) + - geom_vline(xintercept = ref_dates - 3L, linetype = "dashed", colour = "grey40", linewidth = 0.3) + - scale_fill_manual(values = fan_cols, aesthetics = c("fill", "colour")) + - labs(x = NULL, y = "admissions", colour = NULL, fill = NULL) + - theme(legend.position = "top") - - eta <- cal$series %>% - filter(location == loc, season == !!season, horizon %in% 0:3) %>% - mutate(lr = if_else(lr > 0, lr, NA_real_)) %>% - ggplot(aes(round_date, lr, colour = factor(horizon))) + - geom_line(na.rm = TRUE) + - geom_point(size = 0.6, na.rm = TRUE) + - labs(x = NULL, y = "eta", colour = "horizon") + - theme(legend.position = "top") - - title <- paste0(geo_map$abbrev[match(loc, geo_map$location)], " — ", season) - cowplot::plot_grid( - cowplot::ggdraw() + cowplot::draw_label(title, fontface = "bold"), - cowplot::plot_grid(main, eta, ncol = 1, rel_heights = c(3, 1), align = "v", axis = "lr"), - ncol = 1, rel_heights = c(0.08, 1) - ) -} - -oh_table <- function(cal, loc, h, season) { - cal$forecasts %>% - filter(location == loc, horizon == h, season == !!season, level %in% c(0.05, 0.25, 0.5, 0.75, 0.95)) %>% - mutate(level = paste0("q", level)) %>% - select(reference_date, level, lr_level, truth, hidden, offset, value_base, value_cal) %>% - tidyr::pivot_wider( - names_from = level, - values_from = c(hidden, offset, value_base, value_cal, lr_level), - names_glue = "{level}_{.value}" - ) %>% - mutate( - eta = round(q0.5_lr_level, 3), - across(c(truth, matches("hidden|offset|value_base|value_cal")), ~ round(.x)) - ) %>% - select( - reference_date, eta, truth, - matches("^q.*_hidden$"), matches("^q.*_offset$"), - matches("^q.*_value_base$"), matches("^q.*_value_cal$") - ) %>% - arrange(reference_date) -} - -internals_block <- function(cal, season) { - for (i in seq_len(nrow(states))) { - loc <- states$location[i] - cat(sprintf("\n\n
%s internals (h=2)\n\n", states$label[i])) - print(knitr::kable(oh_table(cal, loc, 2L, season))) - cat("\n\n
\n\n") - } -} -``` - -```{r sections, results='asis'} -for (nm in names(cals)) { - cal <- cals[[nm]] - cat(sprintf("\n\n# %s\n\n", variant_titles[[nm]])) - - desc <- switch(nm, - baseline = "The single-term tracker run on sqrt(counts). lr_mult = 0.03, lr_window = 20, carry, isotonic projection.", - count_space = "The paper's method as originally run: additive offsets in count space, same multiplier and window.", - slow_term = "Sqrt tracker plus a slow persistent offset term (mult 0.003, expanding window). No fast decay.", - slow_fast = "Sqrt tracker plus slow persistent term (mult 0.003) and a 10% per-round leak on the fast term." - ) - cat(desc, "\n\n") - - cat("## Headline\n\n") - print(knitr::kable(headline_table(cal))) - - for (season in seasons) { - cat(sprintf("\n\n## %s\n\n", season)) - for (i in seq_len(nrow(states))) { - loc <- states$location[i] - cat(sprintf("\n\n### %s — %s\n\n", states$label[i], season)) - print(season_plot(cal, loc, season)) - } - } - - cat("\n\n## Internals\n\n") - for (season in seasons) { - cat(sprintf("\n\n### %s\n\n", season)) - internals_block(cal, season) - } -} -``` diff --git a/reports/writeups/calibration/calibration_qt_seasons_flu.Rmd b/reports/writeups/calibration/calibration_qt_seasons_flu.Rmd deleted file mode 100644 index 86b30f5e..00000000 --- a/reports/writeups/calibration/calibration_qt_seasons_flu.Rmd +++ /dev/null @@ -1,396 +0,0 @@ ---- -title: MultiQT calibration — whole-season views of four states, across method variants -date: "`r format(Sys.time(), '%d %B %Y')`" -output: - html_document: - code_folding: hide - df_print: paged - toc: true - toc_float: true - toc_depth: 2 -params: - burn_in_seasons: "2023-2024" - season_policy: "carry" - lr_window: 20 - lr_mult: 0.03 - lr_floor: 0.001 -editor_options: - chunk_output_type: console ---- - -```{r setup, echo=FALSE} -# cache = TRUE chunks are keyed on chunk code + cache.extra, NOT on functions -# in R/ -- after changing calibration code, clear cache/calibration/knitr_cache_seasons_flu/. -knitr::opts_chunk$set( - fig.align = "center", fig.width = 10, fig.height = 6, - message = FALSE, warning = FALSE, cache = FALSE, - cache.path = here::here("cache/calibration/knitr_cache_seasons_flu/") -) -knitr::opts_knit$set(root.dir = here::here()) -ggplot2::theme_set(ggplot2::theme_bw()) -``` - -This notebook takes the sqrt-space operating point from `notes/CALIBRATION.md` -(`transform = "sqrt"`, `lr_mult = 0.03`, `lr_window = 20`, carry, burn-in -2023-24) and looks at four -states across whole seasons rather than one panel at a time -(`calibration_qt_gallery_flu.Rmd` has the per-forecast gallery view). The states -are chosen by how much the baseline calibration changed them over the two live -seasons, pooled over horizons −1…3: the best and worst by WIS change, and the -best and worst by absolute error of the median. If one state is best (or -worst) on both, the next-ranked distinct state is taken so four panels are -shown. Method variants compared: the sqrt single-term tracker (baseline), the -paper's count-space tracker, a static conformal warm start from the burn-in -season with no tracking, the warm start plus a leaky fast tracker, the warm -start plus slow and leaky fast terms (the current operating point), and -switching off after February 15 as the earlier reference. - -```{r load, cache=TRUE} -suppressPackageStartupMessages(source(here::here("R/load_all.R"))) - -forecasts <- hub_read_forecasts() -truth <- hub_read_truth() -rounds <- hub_label_seasons(unique(forecasts$reference_date)) -burn_in <- strsplit(params$burn_in_seasons, ",")[[1]] - -geo_map <- get_population_data() %>% - distinct(state_id, .keep_all = TRUE) %>% - transmute(state_id, abbrev = toupper(state_id), location = state_code) - -archive <- nanoparquet::read_parquet( - here::here("cache/calibration/nhsn_archive_flu.parquet") -) %>% - as_epi_archive(compactify = TRUE) - -live_rounds <- rounds %>% filter(!season %in% burn_in) -min_version <- min(archive$DT$version) -max_version <- archive$versions_end -snapshots <- purrr::map(live_rounds$round_date, function(rd) { - as_of <- rd - 3L - if (as_of < min_version || as_of > max_version) { - return(NULL) - } - epix_as_of(archive, as_of) %>% - as_tibble() %>% - inner_join(geo_map, by = c("geo_value" = "state_id")) %>% - transmute(reference_date = rd, location, time_value, value) -}) %>% bind_rows() - -seasons <- c("2024-2025", "2025-2026") -``` - -```{r variants, cache=TRUE, dependson="load"} -base_args <- list( - burn_in_seasons = burn_in, season_policy = params$season_policy, - lr_window = params$lr_window, transform = "sqrt", - lr_args = list(mult = params$lr_mult, floor = params$lr_floor), - progress = FALSE -) -variant_args <- list( - baseline = list(), - count_space = list(transform = "identity", lr_args = list(mult = params$lr_mult)), - warm_static = list(slow_init = "burn_in_quantile", lr_args = list(mult = 1e-6, floor = 0)), - warm_fast = list(slow_init = "burn_in_quantile", fast_decay = 0.1), - warm_slow_fast = list(slow_init = "burn_in_quantile", lr_slow = list(mult = 0.003), fast_decay = 0.1), - off_feb = list(off_after = "02-15") -) -variant_titles <- c( - baseline = "Sqrt-space tracker (baseline)", - count_space = "Count-space tracker (the paper's method)", - warm_static = "Conformal warm start only, no tracking", - warm_fast = "Warm start + leaky fast tracker", - warm_slow_fast = "Warm start + slow term + leaky fast tracker (operating point)", - off_feb = "Calibration off after February 15" -) - -cals <- purrr::map(variant_args, function(v) { - do.call(calibrate_hub_forecasts, c(list(forecasts, truth), modifyList(base_args, v))) -}) -``` - -```{r select-states, dependson="variants"} -pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) - -by_state <- cals$baseline$forecasts %>% - filter(!is_burn_in, !is.na(truth)) %>% - group_by(location) %>% - summarize( - wis_pct = 100 * (1 - sum(pinball(truth, value_cal, level)) / sum(pinball(truth, value_base, level))), - ae_pct = 100 * (1 - sum(abs(truth - value_cal)[level == 0.5]) / sum(abs(truth - value_base)[level == 0.5])), - .groups = "drop" - ) %>% - inner_join(select(geo_map, location, abbrev), by = "location") - -# Best/worst by WIS then AE, skipping a state already taken so four are shown. -pick <- function(ranked, taken) ranked$location[!ranked$location %in% taken][1] -roles <- list( - `best WIS` = arrange(by_state, desc(wis_pct)), - `worst WIS` = arrange(by_state, wis_pct), - `best AE` = arrange(by_state, desc(ae_pct)), - `worst AE` = arrange(by_state, ae_pct) -) -taken <- character(0) -for (r in names(roles)) taken <- c(taken, pick(roles[[r]], taken)) -states <- tibble(location = taken, role = names(roles)) %>% - left_join(by_state, by = "location") %>% - mutate(label = sprintf("%s (%s: WIS %+.1f%%, AE %+.1f%%)", abbrev, role, wis_pct, ae_pct)) -``` - -# Which states, and why - -Per-state change from baseline calibration over both live seasons, all -horizons pooled (positive is an improvement). The four panels below are the -extremes of these two columns. - -```{r state-table} -knitr::kable(states %>% transmute(state = abbrev, role, `WIS change %` = round(wis_pct, 1), `AE change %` = round(ae_pct, 1))) -``` - -# Summary across variants - -WIS % improvement over the base forecast (all seasons, burn-in excluded), per -horizon. - -```{r variant-summary} -purrr::imap(cals, function(cal, nm) { - hub_quantile_loss(cal) %>% transmute(variant = variant_titles[[nm]], horizon, pct = round(pct_improvement, 1)) -}) %>% - bind_rows() %>% - tidyr::pivot_wider(names_from = horizon, values_from = pct, names_prefix = "h") -``` - -## Where in the season the baseline gains and loses - -Baseline variant, pooled over all locations and both live seasons, by calendar -month of the reference date. `WIS change %` is per month and horizon; `share of -base WIS %` is how much of the season's total base WIS that month carries, so a -large percentage loss in April costs little in absolute terms while a small -gain in December or January is worth a lot. `median shift %` is the calibrated -median minus the base median, summed over the month, as a percentage of summed -truth: the direction and size of the correction the tracker was applying. - -```{r by-month} -by_month <- cals$baseline$forecasts %>% - filter(!is_burn_in, !is.na(truth)) %>% - mutate(month = format(reference_date, "%b"), month_n = as.integer(format(reference_date, "%m"))) %>% - group_by(horizon, month, month_n) %>% - summarize( - base = sum(pinball(truth, value_base, level)), - cal = sum(pinball(truth, value_cal, level)), - shift = 100 * sum((value_cal - value_base)[level == 0.5]) / sum(truth[level == 0.5]), - .groups = "drop" - ) %>% - group_by(horizon) %>% - mutate(pct = 100 * (base - cal) / base, share = 100 * base / sum(base)) %>% - ungroup() %>% - mutate(month = factor(month, levels = unique(month[order((month_n - 7) %% 12)]))) - -show_wide <- function(col, digits = 1) { - by_month %>% - select(horizon, month, value = all_of(col)) %>% - mutate(value = round(value, digits)) %>% - tidyr::pivot_wider(names_from = horizon, values_from = value, names_prefix = "h") %>% - arrange(month) -} -knitr::kable(show_wide("pct"), caption = "WIS change % (calibrated vs base)") -knitr::kable(show_wide("share"), caption = "Share of base WIS %") -knitr::kable(show_wide("shift"), caption = "Median shift % of truth") -``` - -```{r helpers} -headline_table <- function(cal) { - bind_rows( - left_join( - hub_coverage_summary(cal), - select(hub_quantile_loss(cal), horizon, loss_base, loss_cal, loss_pct = pct_improvement), - by = "horizon" - ) %>% mutate(season = "all"), - left_join( - hub_coverage_summary(cal, by = "season"), - select(hub_quantile_loss(cal, by = "season"), - season, horizon, loss_base, loss_cal, loss_pct = pct_improvement - ), - by = c("season", "horizon") - ) - ) %>% - transmute( - season, horizon, - `cal err (base)` = round(cal_error_base, 4), - `cal err (calibrated)` = round(cal_error_cal, 4), - `cal err improvement %` = round(pct_improvement, 1), - `WIS (base)` = round(2 * loss_base, 1), - `WIS (calibrated)` = round(2 * loss_cal, 1), - `WIS improvement %` = round(loss_pct, 1) - ) %>% - arrange(season, horizon) -} - -fan_cols <- c(base = "#0072B2", calibrated = "#D55E00") - -# One plot per (state, season): whole-season fan (odd season_rounds only, h 0..3; -# h -1 dropped -- with h -1 in, 5 horizons x ~13 rounds x 2 fans was unreadable) -# plus vintage patchwork strips underneath, plus an eta strip. -season_plot <- function(cal, loc, season) { - rr <- cal$rounds %>% filter(season == !!season, season_round %% 2 == 1) - ref_dates <- sort(rr$round_date) - - fin <- truth %>% - filter(location == loc, target_end_date >= min(ref_dates) - 21L, target_end_date <= max(ref_dates) + 28L) - - fan <- cal$forecasts %>% - filter(location == loc, reference_date %in% ref_dates, horizon %in% 0:3, level %in% c(0.1, 0.5, 0.9)) %>% - mutate( - target_end_date = reference_date + 7L * horizon, - lvl = c(`0.1` = "lo", `0.5` = "med", `0.9` = "hi")[as.character(level)] - ) %>% - select(reference_date, target_end_date, lvl, base = value_base, calibrated = value_cal) %>% - tidyr::pivot_longer(c(base, calibrated), names_to = "which") %>% - tidyr::pivot_wider(names_from = lvl, values_from = value) - - # Vintage patchwork: each plotted round's as-of snapshot paints only the - # fortnight since the previous plotted round (like backtesting.Rmd's vignette). - vint_windows <- tibble(ref = ref_dates) %>% - mutate( - win_start = dplyr::lag(ref, default = ref[1] - 14L) - 3L, - win_end = ref - 3L - ) - vint <- purrr::pmap(vint_windows, function(ref, win_start, win_end) { - snapshots %>% - filter(reference_date == ref, location == loc, time_value >= win_start, time_value <= win_end) - }) %>% bind_rows() - - main <- ggplot() + - geom_line(data = fin, aes(target_end_date, truth), colour = "grey65") + - geom_point(data = fin, aes(target_end_date, truth), colour = "grey65", size = 0.7) + - geom_ribbon( - data = filter(fan, which == "base"), - aes(target_end_date, ymin = lo, ymax = hi, fill = which, group = reference_date), - alpha = 0.2 - ) + - geom_line( - data = filter(fan, which == "calibrated"), - aes(target_end_date, lo, colour = which, group = reference_date), linewidth = 0.3 - ) + - geom_line( - data = filter(fan, which == "calibrated"), - aes(target_end_date, hi, colour = which, group = reference_date), linewidth = 0.3 - ) + - geom_line( - data = fan, - aes(target_end_date, med, colour = which, group = interaction(reference_date, which)) - ) + - geom_line(data = vint, aes(time_value, value), colour = "black") + - geom_point(data = vint, aes(time_value, value), colour = "black", size = 0.6) + - geom_vline(xintercept = ref_dates - 3L, linetype = "dashed", colour = "grey40", linewidth = 0.3) + - scale_fill_manual(values = fan_cols, aesthetics = c("fill", "colour")) + - labs(x = NULL, y = "admissions", colour = NULL, fill = NULL) + - theme(legend.position = "top") - - eta <- cal$series %>% - filter(location == loc, season == !!season, horizon %in% 0:3) %>% - # lr is 0 at rounds with no reveal (gaps); blank those rather than plot dips. - mutate(lr = if_else(lr > 0, lr, NA_real_)) %>% - ggplot(aes(round_date, lr, colour = factor(horizon))) + - geom_line(na.rm = TRUE) + - geom_point(size = 0.6, na.rm = TRUE) + - labs(x = NULL, y = "eta", colour = "horizon") + - theme(legend.position = "top") - - title <- paste0(geo_map$abbrev[match(loc, geo_map$location)], " — ", season) - cowplot::plot_grid( - cowplot::ggdraw() + cowplot::draw_label(title, fontface = "bold"), - cowplot::plot_grid(main, eta, ncol = 1, rel_heights = c(3, 1), align = "v", axis = "lr"), - ncol = 1, rel_heights = c(0.08, 1) - ) -} - -# oh_table.R's internals table, generalized to any location/horizon/season. -oh_table <- function(cal, loc, h, season) { - cal$forecasts %>% - filter(location == loc, horizon == h, season == !!season, level %in% c(0.05, 0.25, 0.5, 0.75, 0.95)) %>% - mutate(level = paste0("q", level)) %>% - select(reference_date, level, lr_level, truth, hidden, offset, value_base, value_cal) %>% - tidyr::pivot_wider( - names_from = level, - values_from = c(hidden, offset, value_base, value_cal, lr_level), - names_glue = "{level}_{.value}" - ) %>% - mutate( - eta = round(q0.5_lr_level, 3), - across(c(truth, matches("hidden|offset|value_base|value_cal")), ~ round(.x)) - ) %>% - select( - reference_date, eta, truth, - matches("^q.*_hidden$"), matches("^q.*_offset$"), - matches("^q.*_value_base$"), matches("^q.*_value_cal$") - ) %>% - arrange(reference_date) -} - -internals_block <- function(cal, season) { - for (i in seq_len(nrow(states))) { - loc <- states$location[i] - cat(sprintf("\n\n
%s internals (h=2)\n\n", states$label[i])) - print(knitr::kable(oh_table(cal, loc, 2L, season))) - cat("\n\n
\n\n") - } -} -``` - -The whole-season fans plot horizons 0–3 only (dropping −1) and only every -other round. Blue ribbon and line: base 80% band and median; orange lines: -calibrated 10%, 50%, 90% quantiles. Black: the NHSN vintage each forecast saw, -painted over the fortnight before its reference date (dashed); grey: finalized -truth. The strip below is eta per horizon (blank where a round revealed no -outcome). Read it as "where do the medians diverge, and does the orange band -contain the grey truth better than the blue", not round-by-round. - -```{r sections, results='asis'} -for (nm in names(cals)) { - cal <- cals[[nm]] - cat(sprintf("\n\n# %s\n\n", variant_titles[[nm]])) - - desc <- switch(nm, - baseline = "The single-term tracker run on sqrt(counts): offsets and eta live in sqrt - units, so a correction learned at the peak is smaller in counts when applied in the - trough. lr_mult = 0.03, lr_window = 20, carry, isotonic projection; offsets update - every round including the spring tail. The eta strip is in sqrt units.", - count_space = "The paper's method as originally run: additive offsets in count space, same - multiplier and window. Kept for reference; the eta strip is in counts.", - warm_static = "No online tracking at all (eta is effectively zero). The offset per (horizon, - level) is the conformal offset implied by the 2023-24 burn-in season pooled over - locations -- the level-quantile of truth minus the base quantile, in sqrt units -- and - it is applied unchanged to both live seasons. Shows how much of the gain is the - persistent low bias alone.", - warm_fast = "The conformal warm start plus the sqrt-space tracker with a 10% per-round leak - on its offset: a correction that is not refreshed by new outcomes decays back toward - the warm-start level instead of persisting through the reveal lag.", - warm_slow_fast = "The current operating point: warm start, a slow term that keeps updating - the persistent level with a tiny step (mult 0.003, expanding window, never decays, - carries across seasons), and the leaky fast tracker on top. The internals table's - hidden column is slow + fast.", - off_feb = "The earlier reference: rounds from February 15 through June play the base - forecast unchanged and take no gradient step; the offset learned through the peak is - frozen and carried into the next season." - ) - cat(desc, "\n\n") - - cat("## Headline\n\n") - print(knitr::kable(headline_table(cal))) - - for (season in seasons) { - cat(sprintf("\n\n## %s\n\n", season)) - for (i in seq_len(nrow(states))) { - loc <- states$location[i] - cat(sprintf("\n\n### %s — %s\n\n", states$label[i], season)) - print(season_plot(cal, loc, season)) - } - } - - cat("\n\n## Internals\n\n") - for (season in seasons) { - cat(sprintf("\n\n### %s\n\n", season)) - internals_block(cal, season) - } -} -``` diff --git a/reports/writeups/calibration_experiments/_load.Rmd b/reports/writeups/calibration_experiments/_load.Rmd new file mode 100644 index 00000000..36f668c3 --- /dev/null +++ b/reports/writeups/calibration_experiments/_load.Rmd @@ -0,0 +1,19 @@ +```{r load-data, cache=FALSE} +# Shared by the experiment notebooks: hub submissions, finalized truth for +# scoring, NHSN vintages for learning, and population for rates. The disease +# comes from `params$disease` (default flu). +suppressPackageStartupMessages(source(here::here("R/load_all.R"))) +disease <- params$disease %||% "flu" +archive <- get_nhsn_data_archive(disease) +truth <- nhsn_read_truth(disease, archive) +vintages <- nhsn_read_vintages(disease, archive) +hub_target <- c(flu = HUB_FLU_TARGET, covid = HUB_COVID_TARGET)[[disease]] +hub_fc <- suppressMessages(hub_read_forecasts(hub_dir = here::here(params$hub_dir), target = hub_target)) +population <- get_population_data() %>% + distinct(state_code, .keep_all = TRUE) %>% + filter(state_code %in% unique(hub_fc$location)) %>% + select(location = state_code, population) +rate_scales <- population %>% transmute(location, from = as.Date("2000-01-01"), scale = population / 1e5) +LIVE <- c("2024-2025", "2025-2026") +run <- function(...) cal_run_cached(hub_fc, truth, ..., vintages = vintages, workers = params$workers) +``` diff --git a/reports/writeups/calibration_experiments/_month_view.Rmd b/reports/writeups/calibration_experiments/_month_view.Rmd new file mode 100644 index 00000000..e3a72b28 --- /dev/null +++ b/reports/writeups/calibration_experiments/_month_view.Rmd @@ -0,0 +1,46 @@ +# Month view: WIS, coverage, median error + + + +Scores by calendar month of the reference date, one figure per season, +**states only (US excluded)**. Rows are the three metrics, columns the +horizons: + +- **WIS reduction %**: `100 × (WIS_base − WIS_cal) / WIS_base` within the + month, so positive = calibrated WIS lower than base. Grey bars are the + month's share of the season's base WIS, so a large swing in a month with a + small bar costs little overall. +- **L1 coverage bias reduction %**: `100 × (bias_base − bias_cal) / + bias_base`, so positive = calibrated coverage bias lower than base. The + bias is computed within the month: the mean over the 23 levels of |share + of outcomes at or below the quantile − level|. A season-pooled bias lets + under-prediction on the ramp cancel over-prediction after the peak; the + monthly one does not. When the base's bias is already small, a small + absolute change shows as a large percentage; the month-average column of + the coverage table below gives the absolute values. +- **Median AE reduction %**: the same as WIS reduction % for the absolute + error of the 0.5 quantile, so positive = calibrated median closer to the + outcome. + +A month holds one to five rounds of about 52 states, and outcomes in the +same round move together, so a single month's coverage bias is noisy and +sits above zero even for a calibrated forecaster. That's why the plot shows +it relative to the base. + +```{r mview-plots, results="asis", fig.height=7.5} +mv_plot_month <- mv$month +if (!is.null(get0("mv_plot_cfgs"))) { + mv_plot_month <- mv_plot_month %>% filter(config %in% mv_plot_cfgs) %>% mutate(config = droplevels(config)) +} +for (s in sort(unique(mv_plot_month$season))) { + cat("\n\n", cal_h(2), " ", s, "\n\n", sep = "") + print(cal_month_plot(mv_plot_month, s, colours = get0("mv_cols"), linetype = get0("mv_linetype"), colour_by = get0("mv_colour_by") %||% "config")) +} +``` + +```{r mview-tables, results="asis"} +cat("\n\n", cal_h(2), " By horizon\n\n", sep = "") +cat("WIS reduction % (positive = calibrated WIS lower than base), states only; `both` pools the seasons.\n\n") +print(knitr::kable(cal_month_wis_table(mv))) +print(knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION)) +``` diff --git a/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd b/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd new file mode 100644 index 00000000..1d6bd160 --- /dev/null +++ b/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd @@ -0,0 +1,117 @@ +--- +title: "E00: learning from vintages instead of finalized truth" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true +params: + hub_dir: "../FluSight-forecast-hub" + workers: 12 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +# cache = TRUE chunks are keyed on chunk code, not on functions in R/. After +# changing calibration code, clear cache/calibration/experiments/knitr_e00/. +knitr::opts_chunk$set( + fig.align = "center", fig.width = 10, fig.height = 5, + message = FALSE, warning = FALSE, + cache.path = here::here("cache/calibration/experiments/knitr_e00/") +) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw()) +``` + +Question: how do results change when the tracker learns from NHSN data as +published, rather than from finalized values? + +Setup: + +- Disease: flu. Base forecaster: the submitted hub ensemble (spoiled + submissions dropped). +- References: REF-paper (count scale, one offset per level) and REF-op (sqrt + scale, leak 0.1, slow term with a burn-in-quantile warm start); see the index. +- Burn-in: 2023-24, used only to warm up the learning rate. Offsets start at + zero at the first 2024-25 round (REF-op's slow term starts from its warm + start). +- Seasons scored: 2024-25 and 2025-26, burn-in rounds excluded. Horizons + h−1 to h3. States only (US excluded). +- Scores are always against finalized truth. + +Configs are `ref/settle_days/learn`: + +| axis | values | +|---|---| +| `ref` | `paper`, `op` | +| `settle_days` | `14` (one week of extra delay, the reference) or `7` (none) | +| `learn` | `final`: latest vintage. `exact`: re-run from the first round at every round on the data published by then, as prod does. `vintage`: one run where each outcome keeps the value it had at the round that revealed it (a stateful tracker that never revisits a step) | + +A round with reference date `d` sees data published by `d - 3`. Vintages +start 2024-11-19; earlier as-of dates use that first vintage. "WIS reduction +%" is 100 × (base WIS − calibrated WIS) / base WIS, so positive = calibrated +WIS lower than base. "L1 coverage bias" is lower-is-better. + +```{r libs, cache=FALSE} +suppressPackageStartupMessages(source(here::here("R/load_all.R"))) +``` + +```{r load, cache=TRUE} +archive <- get_nhsn_data_archive("flu") +truth <- nhsn_read_truth("flu", archive) +vintages <- nhsn_read_vintages("flu", archive) +hub_fc <- hub_read_forecasts(hub_dir = here::here(params$hub_dir)) +``` + +```{r runs, cache=TRUE, dependson="load"} +grid <- tidyr::expand_grid(ref = c("paper", "op"), settle_days = c(7L, 14L), learn = c("final", "vintage", "exact")) +cals <- purrr::pmap(grid, function(ref, settle_days, learn) { + cal_run(hub_fc, truth, ref, settle_days = settle_days, learn = learn, vintages = vintages, workers = params$workers) +}) +names(cals) <- with(grid, paste(ref, settle_days, learn, sep = "/")) +``` + +```{r headline} +LIVE <- c("2024-2025", "2025-2026") +first_version <- min(vintages$version) +n_clamped <- sum(unique(hub_fc$reference_date) - 3L < first_version & + hub_label_seasons(unique(hub_fc$reference_date))$season %in% LIVE) +mv <- cal_month_view(cals, seasons = LIVE, horizons = NULL) +# Runs drawn in the month view; the tables keep all twelve. +mv_plot_cfgs <- c("op/14/final", "op/14/exact", "op/7/vintage", "op/7/exact", "paper/14/final", "paper/14/exact") +``` + +`r n_clamped` live rounds are issued before the first vintage (2024-11-19); only +burn-in rounds use that vintage in place of their own. Configs are +`ref/settle_days/learn`. + +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +``` + +# Vintage vs exact: difference in the calibrated median + +Median over locations of |vintage − exact| in the calibrated median, +relative to the base median, by month, one panel per `ref/settle_days`: + +```{r drift, fig.height=6} +drift <- purrr::map(unique(paste(grid$ref, grid$settle_days, sep = "/")), function(cfg) { + pick <- function(mode) { + cals[[paste(cfg, mode, sep = "/")]]$forecasts %>% + filter(level == 0.5, !is_burn_in, !is.na(value_base)) %>% + select(location, horizon, reference_date, value_base, value_cal) + } + inner_join(pick("vintage"), pick("exact"), by = c("location", "horizon", "reference_date", "value_base"), suffix = c("_cheap", "_exact")) %>% + mutate(config = cfg) +}) %>% + bind_rows() %>% + mutate(month = lubridate::floor_date(reference_date, "month")) %>% + group_by(config, horizon, month) %>% + summarize(rel = median(abs(value_cal_cheap - value_cal_exact) / pmax(value_base, 1)), .groups = "drop") +ggplot(drift, aes(month, 100 * rel, colour = factor(horizon), group = horizon)) + + geom_line() + + geom_point(size = 0.8) + + facet_wrap(~config) + + labs(x = NULL, y = "median |vintage − exact| / base median (%)", colour = "horizon") +``` diff --git a/reports/writeups/calibration_experiments/e01_eta_settings.Rmd b/reports/writeups/calibration_experiments/e01_eta_settings.Rmd new file mode 100644 index 00000000..5288a51b --- /dev/null +++ b/reports/writeups/calibration_experiments/e01_eta_settings.Rmd @@ -0,0 +1,199 @@ +--- +title: "E01: adaptive learning-rate settings" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +Question: how do the adaptive learning-rate settings change WIS and +coverage? The paper's tracker sets +`eta = mult × 90th percentile of |outcome − forecast|` over the last +`lr_window` observed rounds; the settings are the multiplier (`lr_mult`), the +window (`lr_window`), and the summer policy (`season_policy`: carry offsets +into the next season, or reset them to zero). + +This re-asks the original E01 (a pre-fix, count-scale sweep on the submitted +ensemble with finalized truth, since deleted) on the current setup. + +Setup: + +- Disease: flu. Base forecaster: the clean `windowed_seasonal` replay (all 53 + locations, the 55 hub rounds; 27 in 2024-25, 28 in 2025-26). Horizons h0–h3. +- Learning truth: exact (re-run at every round on the data published by then; + E00), one week of extra delay. +- Cold start: no burn-in season, offsets start at zero in November 2024. With + no burn-in there is one summer to cross (2025), so the season policy only + changes 2025-26. +- Tracker: REF-paper (single term, adaptive rate) on the square-root scale, + learning-rate floor 0.001. +- Grid: `lr_mult` {0.3, 0.1, 0.03, 0.01} × `lr_window` {8, 20, Inf} × + `season_policy` {carry, reset}, 24 runs, plus one count-scale run at mult + 0.03, window 20, carry. +- Scores: states only (US excluded). Each season is scored on its + own rounds. + +WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated +WIS lower than base. L1 coverage bias: lower is better. + +```{r load} +source(here::here("scripts/calibration/calibration_ws_replay.R")) +wi <- ws_inputs("flu") +MULTS <- c(0.3, 0.1, 0.03, 0.01) +WINDOWS <- c(8, 20, Inf) +POLICIES <- c("carry", "reset") +# The sqrt single-term tracker, cold; E02 builds its configs the same way so runs are shared. +sqrt_cfg <- function(mult = 0.03, window = 20, policy = "carry") { + list( + ref = "paper", transform = "sqrt", lr_args = list(mult = mult, floor = 1e-3), + burn_in_seasons = character(0), slow_init = NULL, lr_window = window, season_policy = policy + ) +} +run <- function(cfg) { + do.call(cal_run_cached, c(list(wi$fc, wi$truth), cfg, list(learn = "exact", vintages = wi$vintages, workers = params$workers))) +} +grid <- tidyr::expand_grid(mult = MULTS, window = WINDOWS, policy = POLICIES) +sweep <- purrr::pmap(grid, \(mult, window, policy) run(sqrt_cfg(mult, window, policy))) +count_ref <- run(list(ref = "paper", burn_in_seasons = character(0), slow_init = NULL)) +``` + +```{r metrics} +GEOS <- c(states = "states only (US excluded)") +SEASONS <- c(LIVE, "both") +score <- function(cal) { + fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) + purrr::imap(WS_GEOS[names(GEOS)], function(keep, g) ws_scores(fc %>% filter(keep(location))) %>% mutate(geos = g)) %>% + bind_rows() %>% + select(geos, season, horizon, which, wis, cal_err) %>% + tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err)) %>% + mutate(wis_pct = 100 * (wis_base - wis_cal) / wis_base) +} +label <- function(df) { + df %>% mutate( + geos = factor(geos, levels = names(GEOS), labels = GEOS), + season = factor(season, levels = SEASONS, labels = c(LIVE, "both seasons")), + h = factor(paste0("h", horizon), levels = paste0("h", 0:3)) + ) +} +sweep_m <- purrr::map2(sweep, seq_len(nrow(grid)), \(cal, i) score(cal) %>% mutate(grid[i, ])) %>% + bind_rows() %>% + label() %>% + mutate(window_f = factor(as.character(window), levels = as.character(WINDOWS)), policy = factor(policy, levels = POLICIES)) +count_m <- score(count_ref) %>% label() +WINDOW_COLS <- c(`8` = "#E69F00", `20` = "#0072B2", `Inf` = "#009E73") +``` + +# 1. Coverage bias vs WIS over the multiplier + +One line per window and season policy, tracing the multiplier from 0.01 +(near the base, the open circle) to 0.3. Labels mark the multiplier on the +window-20 carry line. The cross is the count-scale run at mult 0.03, window +20, carry. The y axis is clipped at −15%. + +```{r curve-fn} +curve_plot <- function(g) { + d <- sweep_m %>% filter(geos == g) %>% arrange(mult) + base <- d %>% distinct(h, season, cal_err_base) + ggplot(d, aes(cal_err_cal, wis_pct, colour = window_f)) + + geom_hline(yintercept = 0, colour = "grey60", linewidth = 0.3) + + geom_path(aes(linetype = policy, group = interaction(window_f, policy)), linewidth = 0.7) + + geom_point(size = 1) + + geom_text(data = d %>% filter(window == 20, policy == "carry"), aes(label = mult), + size = 2.4, vjust = -0.8, show.legend = FALSE) + + geom_point(data = base, aes(cal_err_base, 0, shape = "base (uncalibrated)"), colour = "black", fill = "white", size = 2.2) + + geom_point(data = count_m %>% filter(geos == g), aes(cal_err_cal, wis_pct, shape = "count scale, mult 0.03"), colour = "black", size = 2.4) + + scale_shape_manual(values = c(`base (uncalibrated)` = 21, `count scale, mult 0.03` = 4), name = NULL) + + scale_colour_manual(values = WINDOW_COLS, name = "lr_window") + + scale_linetype_manual(values = c(carry = "solid", reset = "22"), name = "season policy") + + facet_grid(season ~ h) + + coord_cartesian(ylim = c(max(-15, min(d$wis_pct)), max(d$wis_pct) + 0.5)) + + labs(x = "L1 coverage bias (lower is better)", y = "WIS reduction % (positive = calibrated WIS lower than base)", title = g) + + theme(legend.position = "bottom") +} +``` + +```{r curve-states, fig.height=8} +curve_plot(GEOS[["states"]]) +``` + +# 2. Which setting matters? + +For each season and horizon, how far WIS reduction % moves when one setting +changes and the other two stay fixed: the range over the four multipliers, +over the three windows, and the carry-vs-reset gap, each averaged over the +settings held fixed. States only. The policy gap is zero in 2024-25 by +construction (no summer crossed yet). + +```{r spread} +st <- sweep_m %>% filter(geos == GEOS[["states"]]) +spread <- bind_rows( + st %>% group_by(season, h, window, policy) %>% summarize(r = diff(range(wis_pct)), .groups = "drop") %>% + group_by(season, h) %>% summarize(v = mean(r), .groups = "drop") %>% mutate(setting = "multiplier"), + st %>% group_by(season, h, mult, policy) %>% summarize(r = diff(range(wis_pct)), .groups = "drop") %>% + group_by(season, h) %>% summarize(v = mean(r), .groups = "drop") %>% mutate(setting = "window"), + st %>% group_by(season, h, mult, window) %>% summarize(r = diff(range(wis_pct)), .groups = "drop") %>% + group_by(season, h) %>% summarize(v = mean(r), .groups = "drop") %>% mutate(setting = "season policy") +) +spread %>% + transmute(setting = factor(setting, levels = c("multiplier", "window", "season policy")), col = paste(season, h), v = round(v, 1)) %>% + tidyr::pivot_wider(names_from = col, values_from = v) %>% + arrange(setting) %>% + knitr::kable(caption = "Mean range of WIS reduction % (points; positive = calibrated WIS lower than base) when one setting varies, states only") +``` + +# 3. Each metric against the multiplier + +Top: WIS reduction % (positive = calibrated WIS lower than base), clipped at +−30% (mult 0.3 goes far below). Bottom: L1 coverage bias (lower is better), each season's base as a thin line. Carry only, states only; +colour is the window. + +```{r vs-mult, fig.height=6.5} +long <- st %>% + filter(policy == "carry", season != "both seasons") %>% + transmute(h, season, mult, window_f, `WIS reduction % (positive = calibrated WIS lower than base)` = pmax(wis_pct, -30), `L1 coverage bias (lower is better)` = cal_err_cal) %>% + tidyr::pivot_longer(c(`WIS reduction % (positive = calibrated WIS lower than base)`, `L1 coverage bias (lower is better)`), names_to = "metric") %>% + mutate(metric = factor(metric, levels = c("WIS reduction % (positive = calibrated WIS lower than base)", "L1 coverage bias (lower is better)"))) +base_lines <- st %>% + filter(season != "both seasons") %>% + distinct(h, season, value = cal_err_base) %>% + mutate(metric = factor("L1 coverage bias (lower is better)", levels = levels(long$metric))) +ggplot(long, aes(mult, value, colour = window_f, linetype = season)) + + geom_hline(data = base_lines, aes(yintercept = value, linetype = season), colour = "grey40", linewidth = 0.3) + + geom_line() + + geom_point(size = 1) + + scale_x_log10(breaks = MULTS) + + scale_colour_manual(values = WINDOW_COLS, name = "lr_window") + + scale_linetype_manual(values = c(`2024-2025` = "solid", `2025-2026` = "22"), name = NULL) + + facet_grid(metric ~ h, scales = "free_y", labeller = labeller(metric = label_wrap_gen(28))) + + labs(x = "lr_mult", y = NULL) + + theme(legend.position = "bottom") +``` + +```{r month-view-cfgs} +# The plots show the multipliers around the elbow and the season policy at window 20, plus the count scale; the tables cover every config. +mv_cals <- c( + setNames(sweep, with(grid, sprintf("sqrt mult %s, window %s, %s", mult, as.character(window), policy))), + list(`count mult 0.03, window 20, carry` = count_ref) +) +mv <- cal_month_view(mv_cals, seasons = LIVE) +mv_plot_cfgs <- c( + "sqrt mult 0.1, window 20, carry", "sqrt mult 0.03, window 20, carry", "sqrt mult 0.01, window 20, carry", + "sqrt mult 0.03, window 20, reset", "count mult 0.03, window 20, carry" +) +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +``` diff --git a/reports/writeups/calibration_experiments/e02_eta_variants.Rmd b/reports/writeups/calibration_experiments/e02_eta_variants.Rmd new file mode 100644 index 00000000..6777f433 --- /dev/null +++ b/reports/writeups/calibration_experiments/e02_eta_variants.Rmd @@ -0,0 +1,99 @@ +--- +title: "E02: switch-off date and learning-rate variants" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +Question: how do three variants of the adaptive tracker, each changed alone +from a reference, change WIS and coverage? + +- **Switch off in spring** (`off_after`): from a calendar date until July, + play the base forecast and stop learning; the offsets carry unchanged into + the next season. Dates: 1 April, 1 March, 15 February. +- **Per-level learning rate** (`lr_args$per_level`): each of the 23 quantile + levels gets its own eta, instead of one eta pooled over levels. +- **Seasonal window** (`lr_seasonal`): eta is sized from the last 10 rounds + plus the rounds within 5 weeks of the same point in earlier seasons. Run + with carry and with reset at the summer. This also shortens the window + from 20 to 10, so it changes two things at once. + +This re-asks the original E02 (a pre-fix, count-scale comparison on the +submitted ensemble with finalized truth, since deleted) on the current setup. + +Setup: + +- Disease: flu. Base forecaster: the clean `windowed_seasonal` replay (all 53 + locations, the 55 hub rounds). Horizons h0–h3. +- Learning truth: exact (re-run at every round on the data published by then; + E00), one week of extra delay. +- Reference: E01's center point, the sqrt single-term tracker with the + adaptive rate (mult 0.03, floor 0.001, window 20, carry), cold start (no + burn-in). +- The seasonal window needs an earlier season to look back on, so it runs + with the 2023-24 burn-in (the HHS-stitched backfill from E13), which only + warms eta. The reference is re-run with the same burn-in + (`reference + burn-in`) for comparison. +- Scores: states only (US excluded). + +WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated +WIS lower than base. L1 coverage bias: lower is better. + +```{r load} +source(here::here("scripts/calibration/calibration_ws_replay.R")) +wi <- ws_inputs("flu") +fc_burn_in <- bind_rows(ws_burn_in_forecasts("flu"), wi$fc) +# Same builder as E01, so the reference run is shared. +sqrt_cfg <- function(mult = 0.03, window = 20, policy = "carry") { + list( + ref = "paper", transform = "sqrt", lr_args = list(mult = mult, floor = 1e-3), + burn_in_seasons = character(0), slow_init = NULL, lr_window = window, season_policy = policy + ) +} +ref <- sqrt_cfg() +burn <- list(burn_in_seasons = "2023-2024") +seasonal <- list(lr_window = 10, lr_seasonal = list(half_width_weeks = 5)) +CFGS <- list( + reference = list(cfg = ref), + `off after 1 Apr` = list(cfg = c(ref, list(off_after = "04-01"))), + `off after 1 Mar` = list(cfg = c(ref, list(off_after = "03-01"))), + `off after 15 Feb` = list(cfg = c(ref, list(off_after = "02-15"))), + `per-level eta` = list(cfg = utils::modifyList(ref, list(lr_args = list(mult = 0.03, floor = 1e-3, per_level = TRUE)))), + `reference + burn-in` = list(cfg = utils::modifyList(ref, burn), burn_in = TRUE), + `seasonal window, carry` = list(cfg = utils::modifyList(ref, c(burn, seasonal)), burn_in = TRUE), + `seasonal window, reset` = list(cfg = utils::modifyList(ref, c(burn, seasonal, list(season_policy = "reset"))), burn_in = TRUE) +) +run <- function(spec) { + fc <- if (isTRUE(spec$burn_in)) fc_burn_in else wi$fc + do.call(cal_run_cached, c(list(fc, wi$truth), spec$cfg, list(learn = "exact", vintages = wi$vintages, workers = params$workers))) +} +cals <- purrr::map(CFGS, run) +``` + +```{r month-view-cfgs} +CFG_COLS <- setNames(c("black", "#56B4E9", "#0072B2", "#003F6B", "#CC79A7", "grey55", "#E69F00", "#D55E00"), names(CFGS)) +mv <- cal_month_view(cals, seasons = LIVE) +# The plots leave out the burn-in configs except the seasonal window with reset; the tables keep every config. +mv_plot_cfgs <- c("reference", "off after 1 Mar", "off after 15 Feb", "per-level eta", "seasonal window, reset") +mv_cols <- CFG_COLS +``` + +The seasonal-window configs compare with "reference + burn-in", the others +with "reference". + +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +``` diff --git a/reports/writeups/calibration_experiments/e03_scale.Rmd b/reports/writeups/calibration_experiments/e03_scale.Rmd new file mode 100644 index 00000000..81dab88a --- /dev/null +++ b/reports/writeups/calibration_experiments/e03_scale.Rmd @@ -0,0 +1,65 @@ +--- +title: "E03: working scale (count, sqrt, rate)" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + hub_dir: "../FluSight-forecast-hub" + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 10, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_load.Rmd")} +``` + +Question: does the scale the tracker works on (count, sqrt, rate) change +WIS and coverage? + +Setup: + +- Disease: flu. Base forecaster: the submitted hub ensemble (spoiled + submissions dropped). +- Reference: REF-paper. One offset per level; adaptive learning rate of 0.03 + × the 90th percentile of the last 20 rounds' absolute errors; offsets + carried across seasons; one week of extra delay (`settle_days = 14`). +- Burn-in: 2023-24, used only to warm up the learning rate. Offsets start at + zero at the first 2024-25 round (cold start). +- Learning truth: exact (re-run at every round on the data published by then, + as prod does; E00). Scores against finalized truth. +- Seasons scored: 2024-25 and 2025-26. Horizons h−1 to h3. States only + (US excluded). + +| config | change from REF-paper | +|---|---| +| `count` | none (additive offsets in admissions) | +| `sqrt` | offsets on the square root of admissions, mapped back; learning-rate floor 0.001 | +| `rate` | offsets in admissions per 100k; learning-rate floor 0.001 | + +The learning-rate floor (0.1 by default) is in the tracker's working units, +so the sqrt and rate configs use 0.001. + +```{r runs} +cals <- list( + count = run("paper"), + sqrt = run("paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3)), + rate = run("paper", scales = rate_scales, lr_args = list(mult = 0.03, floor = 1e-3)) +) +``` + +```{r month-view-data} +mv <- cal_month_view(cals, seasons = LIVE, horizons = NULL) +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +``` diff --git a/reports/writeups/calibration_experiments/e07_across_eras.Rmd b/reports/writeups/calibration_experiments/e07_across_eras.Rmd new file mode 100644 index 00000000..7ea83037 --- /dev/null +++ b/reports/writeups/calibration_experiments/e07_across_eras.Rmd @@ -0,0 +1,172 @@ +--- +title: "E07: the submitted ensemble vs one forecaster" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + hub_dir: "../FluSight-forecast-hub" + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +Question: does calibration change WIS and coverage the same way for the +submitted ensemble as for one fixed forecaster? + +**Setup.** Two forecasters, everything else shared: + +- **A**: the submitted flu ensemble, read from the hub as in E00 and + E03–E05. +- **B**: the clean `windowed_seasonal` replay (E12–E17), with the 2023-24 + HHS backfill (E13) as its burn-in season. + +| | | +|---|---| +| rows | identical (round, location, horizon, level) rows for A and B: hub submission rounds, submissions excluded from the hub reads dropped from both | +| seasons | 2023-24 burn-in (27 rounds), 2024-25 (27) and 2025-26 (28) scored | +| locations | all 53 learned; scores are states only | +| horizons | h0–h3 (the ensemble's h−1 is a different model) | +| learning truth | NHSN as published each week, re-run from scratch each week, one week of extra revision delay | +| scoring truth | the store's NHSN truth | + +Trackers (section 2), all sqrt scale, adaptive learning rate (mult 0.03, +floor 0.001, window 20, carry): + +- **single**: REF-paper at sqrt. +- **leaky**: single with the fast offset shrinking 10% per round. +- **operating**: REF-op (leaky plus a slow offset and a warm start from the + burn-in season's errors). + +This rebuilds the original E07 (a script since deleted), +which learned from finalized data, used the replay with prod's data +substitutions and included h−1. + +"WIS reduction %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so +**positive = calibrated WIS lower than base**. "L1 coverage bias" (lower is +better) is the mean over the 23 levels of |empirical coverage − level|. + +```{r load} +source(here::here("scripts/calibration/calibration_ws_replay.R")) +wi <- ws_inputs("flu") +hub_fc <- suppressMessages(hub_read_forecasts(hub_dir = here::here(params$hub_dir), target = HUB_FLU_TARGET)) %>% + filter(horizon %in% 0:3) +ws_fc <- bind_rows(ws_burn_in_forecasts("flu"), wi$fc) +# Keep only (round, location, horizon, level) rows both forecasters have. +key <- c("reference_date", "location", "horizon", "level") +common <- inner_join(distinct(hub_fc, across(all_of(key))), distinct(ws_fc, across(all_of(key))), by = key) +FCS <- list( + `A ensemble` = semi_join(hub_fc, common, by = key), + `B windowed_seasonal` = semi_join(ws_fc, common, by = key) +) +sqrt_single <- list(ref = "paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3), burn_in_seasons = "2023-2024") +TRACKERS <- list( + single = sqrt_single, + leaky = c(sqrt_single, list(fast_decay = 0.1)), + operating = list(ref = "op") +) +grid <- tidyr::expand_grid(forecaster = names(FCS), tracker = names(TRACKERS)) +cals <- purrr::pmap(grid, function(forecaster, tracker) { + do.call(cal_run_cached, c(list(FCS[[forecaster]], wi$truth), TRACKERS[[tracker]], list(learn = "exact", vintages = wi$vintages, workers = params$workers))) +}) +``` + +```{r rows} +purrr::imap(FCS, \(fc, nm) fc %>% mutate(season = season_of(reference_date)) %>% count(season, name = nm)) %>% + purrr::reduce(full_join, by = "season") %>% + left_join(FCS[[1]] %>% mutate(season = season_of(reference_date)) %>% group_by(season) %>% summarize(rounds = n_distinct(reference_date)), by = "season") %>% + knitr::kable() +``` + +```{r metrics} +GEOS <- c(states = "states only (US excluded)") +SEASONS <- c(LIVE, "both") +score <- function(cal) { + fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) + purrr::imap(WS_GEOS[names(GEOS)], function(keep, g) ws_scores(fc %>% filter(keep(location))) %>% mutate(geos = g)) %>% + bind_rows() %>% + select(geos, season, horizon, which, wis, cal_err) %>% + tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err)) %>% + mutate(wis_pct = 100 * (wis_base - wis_cal) / wis_base) +} +m <- purrr::map2(cals, seq_len(nrow(grid)), \(cal, i) score(cal) %>% mutate(grid[i, ])) %>% + bind_rows() %>% + mutate( + forecaster = factor(forecaster, levels = names(FCS)), tracker = factor(tracker, levels = names(TRACKERS)), + geos = factor(geos, levels = names(GEOS), labels = GEOS), + season = factor(season, levels = SEASONS, labels = c(LIVE, "both seasons")), + h = factor(paste0("h", horizon), levels = paste0("h", 0:3)) + ) +FC_COLS <- c(`A ensemble` = "#0072B2", `B windowed_seasonal` = "#D55E00") +``` + +# 1. The two bases + +## Base WIS + +Uncalibrated WIS (2 × mean pinball, in admissions) on identical rows, and +the ratio B/A (below 1: `windowed_seasonal` is better). + +```{r base-wis} +m %>% + filter(tracker == "single") %>% + select(season, h, forecaster, wis_base) %>% + tidyr::pivot_wider(names_from = forecaster, values_from = wis_base) %>% + transmute(season, h, A = .data[[names(FCS)[1]]], B = .data[[names(FCS)[2]]], `B/A` = B / A) %>% + arrange(season, h) %>% + knitr::kable(digits = c(0, 0, 1, 1, 3), caption = "Base WIS, A ensemble vs B windowed_seasonal, states only") +``` + +## Base bias by month + +Share of outcomes above the base median and L1 coverage bias, by calendar +month of the reference date, states only. + +```{r base-month, fig.height=7} +base_fc <- purrr::map(which(grid$tracker == "single"), \(i) cals[[i]]$forecasts %>% mutate(forecaster = grid$forecaster[i])) %>% + bind_rows() %>% + mutate(season = season_of(reference_date)) %>% + filter(season %in% LIVE, !is.na(truth), !is.na(value_base)) +month_base <- function(d) { + d %>% + group_by(across(c(any_of(c("forecaster", "season", "horizon", "month")), level))) %>% + summarize(gap = mean(truth <= value_base) - first(level), above = mean(truth > value_base), n = n(), .groups = "drop") %>% + group_by(across(any_of(c("forecaster", "season", "horizon", "month")))) %>% + summarize(above = above[level == 0.5], cal_err = mean(abs(gap)), n = first(n), .groups = "drop") +} +base_fc_states <- base_fc %>% + filter(location != "US") %>% + mutate(month = factor(format(reference_date, "%b"), levels = month.abb[c(8:12, 1:7)])) +mb <- month_base(base_fc_states) %>% mutate(h = factor(paste0("h", horizon), levels = paste0("h", 0:3))) +mb %>% + tidyr::pivot_longer(c(above, cal_err), names_to = "metric") %>% + mutate(metric = factor(metric, levels = c("above", "cal_err"), labels = c("share above median", "L1 coverage bias (lower is better)"))) %>% + ggplot(aes(month, value, colour = forecaster, group = forecaster)) + + geom_hline(data = data.frame(metric = factor("share above median", levels = c("share above median", "L1 coverage bias (lower is better)")), y = 0.5), aes(yintercept = y), colour = "grey50") + + geom_line() + + geom_point(size = 1.1) + + facet_grid(metric + season ~ h, scales = "free", space = "free_x", labeller = labeller(metric = label_wrap_gen(20))) + + scale_colour_manual(values = FC_COLS, name = NULL) + + labs(x = "month of reference date", y = NULL) + + theme(legend.position = "bottom") +``` + +# 2. What calibration does to each + +WIS reduction % (positive = calibrated WIS lower than base) and L1 coverage +bias (lower is better), states only. + +```{r table-states} +mv_trk <- cal_month_view(setNames(cals, paste(grid$forecaster, grid$tracker, sep = ", ")), seasons = LIVE) +knitr::kable(cal_month_wis_table(mv_trk)) +knitr::kable(cal_month_cov_table(mv_trk), caption = CAL_MONTH_COV_CAPTION) +``` diff --git a/reports/writeups/calibration_experiments/e09_lr_delay.Rmd b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd new file mode 100644 index 00000000..09f98977 --- /dev/null +++ b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd @@ -0,0 +1,282 @@ +--- +title: "E09: constant learning rate × revision delay (rate scale)" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true +params: + hub_dir: "../FluSight-forecast-hub" + workers: 14 + wis_max: 20 + smoke: false +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set( + fig.align = "center", fig.width = 11, fig.height = 11, + message = FALSE, warning = FALSE, + cache.path = here::here("cache/calibration/experiments/knitr_e09/") +) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +Question: for the plain quantile tracker on the rate scale (admissions per +100k) with a constant learning rate, how do WIS and coverage change with +the learning rate and with the extra delay before an outcome is learned? + +**Setup.** + +| | | +|---|---| +| disease, base | flu, the submitted hub ensemble (`CMU-TimeSeries`) | +| seasons | 2023-24 burn-in (its outcomes take no steps; offsets start at zero); 2024-25 and 2025-26 scored | +| locations, horizons | learns on all 53 hub locations; scores are states only (US excluded), h−1 to h3 | +| scoring truth | finalized NHSN | +| tracker | REF-paper (one offset per level, carried across seasons) on the rate scale | +| learning rate | constant, 0.001 to about 3.2 per 100k per step (15 values, a quarter-decade apart) | +| extra delay | 0, 1, 2 or 3 weeks (`settle_days` 7/14/21/28) | +| learning truth | two designs, below | + +**The tracker.** For each location, horizon and quantile level, the offset +moves by `lr × (1{y > q} − (1 − τ))` when an outcome is learned. With a +constant `lr` the step does not depend on the forecast errors. The +reference configs use the adaptive rate instead: 0.03 × the 90th +percentile of the location's absolute base errors over its last 20 +observed rounds. + +**Extra delay.** The first NHSN report of the week ending Saturday `X` is +published Wednesday `X + 4`, in time for the round with reference date +`X + 7`. Learning from it at that round is zero extra delay; each further +week of waiting is one week of extra delay. The reference configs use one +week. + +**Learning truth designs.** + +- **re-run** (prod): each week the tracker is re-run from the first round + on that week's vintage, so an outcome is learned from its latest value. +- **stateful**: the tracker runs once and learns each outcome from the + value published at the round it becomes available. + +**Reference configs** (marked on the plots): + +- **REF-paper**: the paper's tracker on counts, adaptive learning rate. +- **REF-op**: prod's config: REF-paper on the sqrt scale, plus a 10% + per-round leak, a slow offset and a warm start from the burn-in season. + +"WIS reduction %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so positive = +calibrated WIS lower than base. "L1 coverage bias" (lower is better) is the +mean over the 23 quantile levels of |empirical coverage − level|. "Knee" +below is the constant rate 0.056. + +```{r libs, cache=FALSE} +suppressPackageStartupMessages(source(here::here("R/load_all.R"))) +LRS <- 10^seq(-3, 0.5, by = 0.25) +SETTLE <- c(7L, 14L, 21L, 28L) +# smoke = TRUE checks the notebook end to end in a few minutes: a small grid, +# and the re-run design computed with the fast stateful mode. +if (params$smoke) { + LRS <- LRS[c(1, 9, 13)] + SETTLE <- c(7L, 14L) +} +LEARN_LABELS <- c(exact = "re-run (prod)", vintage = "stateful") +``` + +```{r load, cache=TRUE} +archive <- get_nhsn_data_archive("flu") +truth <- nhsn_read_truth("flu", archive) +vintages <- nhsn_read_vintages("flu", archive) +hub_fc <- hub_read_forecasts(hub_dir = here::here(params$hub_dir)) +rate_scales <- get_population_data() %>% + distinct(state_code, .keep_all = TRUE) %>% + filter(state_code %in% unique(hub_fc$location)) %>% + transmute(location = state_code, from = as.Date("2000-01-01"), scale = population / 1e5) +stopifnot(setequal(rate_scales$location, unique(hub_fc$location))) +``` + +```{r runs, cache=FALSE} +# Each run is cached to its own file, so a change to the grid only computes the +# new runs. Clear cache/calibration/experiments/e09/ after changing R/calibration/. +run_dir <- here::here("cache/calibration/experiments", if (params$smoke) "e09_smoke" else "e09") +dir.create(run_dir, showWarnings = FALSE, recursive = TRUE) +cached_run <- function(key, ..., learn) { + path <- file.path(run_dir, paste0(key, ".rds")) + if (file.exists(path)) return(readRDS(path)) + if (params$smoke) learn <- "vintage" + cal <- cal_run(hub_fc, truth, ..., learn = learn, vintages = vintages, workers = params$workers) + # Only what the metrics need, to keep the cache small. + cal <- list(forecasts = cal$forecasts %>% + select(location, horizon, reference_date, season, level, value_base, value_cal, truth, is_burn_in)) + saveRDS(cal, path) + cal +} +rate <- list(transform = "identity", scales = rate_scales) + +sweep_grid <- tidyr::expand_grid(learn = c("exact", "vintage"), settle_days = SETTLE, lr = LRS) +sweep <- purrr::pmap(sweep_grid, function(learn, settle_days, lr) { + key <- sprintf("sweep_%s_s%d_lr%.4f", learn, settle_days, lr) + do.call(cached_run, c(list(key, "paper", settle_days = settle_days, lr = lr, learn = learn), rate)) +}) + +# Reference points per (learn, settle_days). +ref_grid <- tidyr::expand_grid(learn = c("exact", "vintage"), settle_days = SETTLE) +refs <- purrr::pmap(ref_grid, function(learn, settle_days) { + s <- sprintf("%s_s%d", learn, settle_days) + list( + `REF-paper (count, adaptive)` = cached_run(paste0("paper_", s), "paper", settle_days = settle_days, learn = learn), + `REF-op` = cached_run(paste0("op_", s), "op", settle_days = settle_days, learn = learn) + ) +}) +``` + +```{r metrics, cache=FALSE} +pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) +# Per (location, horizon) and pooled over states: WIS reduction % and L1 +# coverage bias, for the calibrated forecast and the base. +metrics <- function(cal) { + fc <- cal$forecasts %>% filter(!is_burn_in, !is.na(truth), !is.na(value_base), location != "US") + one <- function(df, by) { + df %>% + group_by(across(all_of(c(by, "horizon", "level")))) %>% + summarize( + cov_base = mean(truth <= value_base), cov_cal = mean(truth <= value_cal), + wis_base = sum(pinball(truth, value_base, level)), wis_cal = sum(pinball(truth, value_cal, level)), + .groups = "drop" + ) %>% + group_by(across(all_of(c(by, "horizon")))) %>% + summarize( + bias_base = mean(abs(cov_base - level)), bias_cal = mean(abs(cov_cal - level)), + wis_pct = 100 * (1 - sum(wis_cal) / sum(wis_base)), + .groups = "drop" + ) + } + bind_rows(one(fc, "location"), one(fc, character(0)) %>% mutate(location = "pooled")) +} +sweep_m <- purrr::map2(sweep, seq_along(sweep), function(cal, i) metrics(cal) %>% bind_cols(sweep_grid[i, ])) %>% + bind_rows() +ref_m <- purrr::map2(refs, seq_along(refs), function(r, i) { + purrr::imap(r, function(cal, nm) metrics(cal) %>% mutate(ref = nm)) %>% bind_rows() %>% bind_cols(ref_grid[i, ]) +}) %>% + bind_rows() %>% + filter(location == "pooled") +label_panels <- function(df) { + df %>% mutate( + extra = factor(sprintf("+%d wk extra delay", (settle_days - 7L) %/% 7L)), + horizon = factor(paste0("h", horizon), levels = paste0("h", -1:3)) + ) +} +sweep_m <- label_panels(sweep_m) +ref_m <- label_panels(ref_m) +``` + +# Coverage bias vs WIS, per location + +Each thin grey line is one location traced across learning rates; the bold +line is the pooled result, labelled with the learning rate at each decade. Learning rates whose pooled WIS is more than +`r params$wis_max`% worse than the base in a panel are dropped from that panel; the axes are +clipped to the bulk of the per-location points, so a few location lines run +off the edge. The open circle is the base forecast (no calibration) and the +triangles are the reference configs at the same delay. + +Per-location coverage comes from about 50 forecasts per horizon, so a +location's L1 bias has a noise floor of a few points even for a perfectly +calibrated forecaster. The grey lines show the spread across locations. + +```{r curve-fn} +ACCENT <- "#1f5fa8" +curve_plot <- function(which_learn) { + d <- sweep_m %>% filter(learn == which_learn) + keep <- d %>% + filter(location == "pooled", wis_pct >= -params$wis_max) %>% + select(horizon, extra, lr) + d <- d %>% semi_join(keep, by = c("horizon", "extra", "lr")) %>% arrange(lr) + pooled <- d %>% filter(location == "pooled") + locs <- d %>% filter(location != "pooled") + base <- pooled %>% distinct(horizon, extra, bias_base) + marks <- ref_m %>% filter(learn == which_learn, ref %in% c("REF-paper (count, adaptive)", "REF-op")) + lab <- pooled %>% filter(abs(log10(lr) - round(log10(lr))) < 1e-9) + xlim <- c(0, max(stats::quantile(locs$bias_cal, 0.97), base$bias_base) * 1.05) + ylim <- range(stats::quantile(locs$wis_pct, c(0.03, 0.97)), pooled$wis_pct, marks$wis_pct, 0) + ggplot() + + geom_hline(yintercept = 0, colour = "grey60", linewidth = 0.3) + + geom_path(data = locs, aes(bias_cal, wis_pct, group = location), colour = "grey45", alpha = 0.25, linewidth = 0.3) + + geom_path(data = pooled, aes(bias_cal, wis_pct), colour = ACCENT, linewidth = 1.1) + + geom_point(data = pooled, aes(bias_cal, wis_pct), colour = ACCENT, size = 1.2) + + geom_text(data = lab, aes(bias_cal, wis_pct, label = format(lr, scientific = FALSE, drop0trailing = TRUE)), + colour = "grey15", size = 2.6, vjust = -0.9) + + geom_point(data = base, aes(bias_base, 0), shape = 21, fill = "white", colour = "black", size = 2.2) + + geom_point(data = marks, aes(bias_cal, wis_pct, shape = ref), colour = "black", size = 2) + + scale_shape_manual(values = c(`REF-paper (count, adaptive)` = 2, `REF-op` = 17), name = NULL) + + facet_grid(horizon ~ extra) + + coord_cartesian(xlim = xlim, ylim = ylim) + + labs(x = "L1 coverage bias (lower is better)", y = "WIS reduction % (positive = calibrated WIS lower than base)") + + theme(legend.position = "bottom") +} +``` + +## Re-run design (prod) + +```{r curve-exact} +curve_plot("exact") +``` + +## Stateful design + +```{r curve-vintage} +curve_plot("vintage") +``` + +# Pooled curves by delay + +The bold lines from above, one colour per extra delay (darker is longer), so +the delays can be compared directly. Solid is the re-run design, dashed the +stateful one. + +```{r pooled-overlay, fig.height=6} +DELAY_COLS <- setNames(RColorBrewer::brewer.pal(9, "Blues")[c(4, 6, 8, 9)], levels(sweep_m$extra)) +pooled_all <- sweep_m %>% + filter(location == "pooled") %>% + semi_join(sweep_m %>% filter(location == "pooled", wis_pct >= -params$wis_max) %>% select(horizon, extra, lr, learn), + by = c("horizon", "extra", "lr", "learn")) %>% + arrange(lr) %>% + mutate(design = factor(LEARN_LABELS[learn], levels = LEARN_LABELS)) +ggplot(pooled_all, aes(bias_cal, wis_pct, colour = extra, linetype = design, group = interaction(extra, design))) + + geom_hline(yintercept = 0, colour = "grey60", linewidth = 0.3) + + geom_path(linewidth = 0.8) + + geom_point(data = pooled_all %>% distinct(horizon, bias_base), aes(bias_base, 0), inherit.aes = FALSE, + shape = 21, fill = "white", colour = "black", size = 2) + + scale_colour_manual(values = DELAY_COLS, name = NULL) + + scale_linetype_manual(values = c(`re-run (prod)` = "solid", stateful = "22"), name = NULL) + + facet_wrap(~horizon, nrow = 1) + + labs(x = "L1 coverage bias (lower is better)", y = "WIS reduction % (positive = calibrated WIS lower than base)") + + theme(legend.position = "bottom", legend.box = "vertical") +``` + +```{r month-view-cfgs} +sweep_run <- function(learn, settle_days, lr) { + sweep[[which(sweep_grid$learn == learn & sweep_grid$settle_days == settle_days & abs(log10(sweep_grid$lr / lr)) < 1e-6)]] +} +ref_run <- function(learn, settle_days, nm) refs[[which(ref_grid$learn == learn & ref_grid$settle_days == settle_days)]][[nm]] +mv_cals <- list( + `REF-paper, re-run +1 wk` = ref_run("exact", 14L, "REF-paper (count, adaptive)"), + `REF-op, re-run +1 wk` = ref_run("exact", 14L, "REF-op"), + `0.032, re-run +1 wk` = sweep_run("exact", 14L, 10^-1.5), + `knee 0.056, re-run +0 wk` = sweep_run("exact", 7L, 10^-1.25), + `knee 0.056, re-run +1 wk` = sweep_run("exact", 14L, 10^-1.25), + `knee 0.056, re-run +2 wk` = sweep_run("exact", 21L, 10^-1.25), + `0.1, re-run +1 wk` = sweep_run("exact", 14L, 10^-1), + `0.18, re-run +1 wk` = sweep_run("exact", 14L, 10^-0.75), + `knee 0.056, stateful +0 wk` = sweep_run("vintage", 7L, 10^-1.25), + `knee 0.056, stateful +1 wk` = sweep_run("vintage", 14L, 10^-1.25) +) +mv <- cal_month_view(mv_cals, seasons = c("2024-2025", "2025-2026"), horizons = NULL) +# The plots show REF-op and the constant rates around the knee at one week of delay; the tables also cover the other delays and designs. +mv_plot_cfgs <- c("REF-op, re-run +1 wk", "0.032, re-run +1 wk", "knee 0.056, re-run +1 wk", "0.1, re-run +1 wk", "0.18, re-run +1 wk") +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +``` diff --git a/reports/writeups/calibration_experiments/e10_covid.Rmd b/reports/writeups/calibration_experiments/e10_covid.Rmd new file mode 100644 index 00000000..614756f8 --- /dev/null +++ b/reports/writeups/calibration_experiments/e10_covid.Rmd @@ -0,0 +1,203 @@ +--- +title: "E10: the finalists on flu and covid" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + disease: "covid" + hub_dir: "../covid19-forecast-hub" + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 10, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +# The month view is knit several times, so its chunk labels repeat. +options(knitr.duplicate.label = "allow") +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_load.Rmd")} +``` + +Question: how do the two finalists, REF-op and sqrt constant 0.018, do on +flu and covid, on the submitted ensemble and on the clean replay? + +Part 1 runs them on both diseases and both bases, h0–h3, on identical rows. +Part 1 also checks whether the finalists rank the same way on both bases. +Part 2 is the covid ensemble at h−1–h3. + +# Part 1: both diseases, both bases + +Setup: + +- Bases: the submitted `CMU-TimeSeries` ensemble (hub) and the clean + `windowed_seasonal` replay, on the (round, location, horizon, level) rows + both have, h0–h3 (the ensemble's h−1 is a different model). +- Learning truth: exact (re-run every round on the store's NHSN vintages), + one week of extra delay. Scores against the store's finalized NHSN, states + only, the two live seasons (August–July). +- Configs: REF-op warm (flu prod) and cold (covid prod); sqrt constant 0.018 + per 100k (E16), cold and warm (`slow_init = "burn_in_quantile"`). Warm + runs see a 2023-24 burn-in season: for flu the rows both bases have (hub + submissions and the HHS-stitched backfill, as E07); for covid only the + replay's backfill, since covid submissions start 2024-11-23. So the covid + ensemble has no warm runs. Cold runs see the live seasons only. + +```{r fin-runs} +source(here::here("scripts/calibration/calibration_ws_replay.R")) +FIN <- purrr::map(c(flu = "flu", covid = "covid"), \(d) fin_runs(d, workers = params$workers)) +``` + +Per disease, a month view for each base. The ensemble and replay rows are +the same (round, location, horizon, level) cells, so the two bases' tables +compare like for like. + +```{r fin-month, results="asis"} +for (d in names(FIN)) { + cat("\n\n", cal_h(2), " ", d, "\n\n", sep = "") + for (b in c("ensemble", "replay")) { + mv <- cal_month_view(FIN[[d]]$cals[startsWith(names(FIN[[d]]$cals), paste0(b, ", "))], seasons = LIVE) + cal_month_section(paste(d, b), level = 3L) + } +} +``` + +## WIS by season with bootstrap intervals + +Every cell carries a 90% interval from a round bootstrap: the season's +rounds are resampled with replacement 1000 times and the WIS reduction +recomputed on each draw (`cal_wis_boot()`), the same draws for every +config. The paired-gap tables give the finalist's gap to prod's config on +each base with its paired interval. The round is the resampling unit, which +ignores dependence between neighbouring rounds, so the intervals are if +anything too narrow. + +```{r fin-boot, results="asis"} +FIN_PAIRS <- list( + c("sqrt 0.018 warm + leak", "REF-op warm"), + c("sqrt 0.018 warm + leak", "sqrt 0.018 warm"), + c("sqrt 0.018", "REF-op cold") +) +for (d in names(FIN)) { + cat("\n\n", cal_h(3), " ", d, "\n\n", sep = "") + bt <- cal_wis_boot(FIN[[d]]$cals, seasons = LIVE) + cal_wis_boot_table(bt) %>% + knitr::kable(caption = paste0(d, ": ", cal_wis_boot_caption(bt))) %>% + print() + gaps <- purrr::map(c("ensemble", "replay"), function(b) { + purrr::map(FIN_PAIRS, function(pr) { + a <- paste(b, pr[1], sep = ", "); bb <- paste(b, pr[2], sep = ", ") + if (!all(c(a, bb) %in% names(FIN[[d]]$cals))) return(NULL) + cal_wis_boot_diff(bt, a, bb) %>% mutate(base = b, pair = paste(pr[1], "−", pr[2]), .before = 1) + }) %>% bind_rows() + }) %>% bind_rows() + if (nrow(gaps) > 0) { + gaps %>% + transmute(base, pair, col = paste(season, h), cell = sprintf("%+.1f [%+.1f, %+.1f]", diff, lo, hi)) %>% + tidyr::pivot_wider(names_from = col, values_from = cell) %>% + knitr::kable(caption = paste0(d, ": gap in WIS reduction % (first − second) with its 90% paired bootstrap interval")) %>% + print() + } +} +``` + +## Do the finalists rank the same way on both bases? + +Per disease, the candidates run on both bases (covid has no warm runs on the +ensemble), ordered by mean WIS reduction % (positive = calibrated WIS lower +than base) over h0–h3, separately on each base, states only. `rank` 1 is the +largest WIS reduction; `cov rank` 1 is the lowest month-avg L1 coverage +bias. Spearman ρ compares the two bases over all (candidate, horizon) cells +of a season. + +```{r fin-rank, results="asis"} +for (d in names(FIN)) { + mv_all <- cal_month_view(FIN[[d]]$cals, seasons = LIVE) + wis <- bind_rows(mv_all$month, mv_all$month %>% mutate(season = "both")) %>% + group_by(config, season, horizon) %>% + summarize(wis = 100 * (sum(wis_base) - sum(wis_cal)) / sum(wis_base), .groups = "drop") + cells <- mv_all$cov %>% + transmute(config, season = as.character(season), horizon, cov = mavg_cal) %>% + inner_join(wis, by = c("config", "season", "horizon")) %>% + tidyr::separate(config, c("base", "cand"), sep = ", ", extra = "merge") + shared <- cells %>% distinct(base, cand) %>% count(cand) %>% filter(n == 2) %>% pull(cand) + cells <- cells %>% filter(cand %in% shared) + cat("\n\n", cal_h(3), " ", d, "\n\n", sep = "") + cells %>% + group_by(base, season, cand) %>% + summarize(wis = mean(wis), cov = mean(cov), .groups = "drop") %>% + group_by(base, season) %>% + mutate(rank = rank(-wis), cov_rank = rank(cov)) %>% + ungroup() %>% + tidyr::pivot_wider(names_from = base, values_from = c(wis, rank, cov, cov_rank)) %>% + arrange(season, rank_ensemble) %>% + transmute(season, cand, + `WIS % ensemble` = round(wis_ensemble, 2), `rank ensemble` = rank_ensemble, + `WIS % replay` = round(wis_replay, 2), `rank replay` = rank_replay, + `cov ensemble` = round(cov_ensemble, 3), `cov rank ensemble` = cov_rank_ensemble, + `cov replay` = round(cov_replay, 3), `cov rank replay` = cov_rank_replay + ) %>% + knitr::kable(caption = "Mean over h0–h3, states only. Coverage is month-avg L1 coverage bias (lower is better)") %>% + print() + cells %>% + select(season, cand, horizon, base, wis, cov) %>% + tidyr::pivot_wider(names_from = base, values_from = c(wis, cov)) %>% + group_by(season) %>% + summarize( + `ρ WIS cells` = cor(wis_ensemble, wis_replay, method = "spearman"), + `ρ coverage cells` = cor(cov_ensemble, cov_replay, method = "spearman") + ) %>% + knitr::kable(digits = 2, caption = "Spearman correlation of ensemble vs replay over (candidate, horizon) cells") %>% + print() +} +``` + +# Part 2: covid ensemble, h−1–h3 + +- Disease: covid. Base forecaster: the submitted `CMU-TimeSeries` ensemble. +- Learning truth: NHSN covid vintages, re-run at every round as prod does + (E00). Scores against finalized covid NHSN. +- No burn-in season (covid submissions start 2024-11-23); both trackers start + from zero offsets. REF-op runs without its warm start, as in covid prod. +- Rounds 2024-11-30 to 2026-09-19. The 2024-11-23 round is excluded (made + from a bad NHSN release corrected the next day). +- Covid is submitted year-round; per-season views use August–July seasons. +- Horizons h−1 to h3. Scores: states only (US excluded), the two live + seasons. + +| config | tracker | +|---|---| +| `REF-op cold` | covid prod: sqrt scale, adaptive learning rate with floor, slow term 0.003, leak 0.1; no warm start | +| `sqrt 0.018` | sqrt of the per-100k rate, constant learning rate 0.018 (E16), single term | + +WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated +WIS lower than base. L1 coverage bias: lower is better. + +```{r runs} +no_burn_in <- list(burn_in_seasons = character(0), slow_init = NULL) +cals <- list( + `REF-op cold` = do.call(run, c(list("op"), no_burn_in)), + `sqrt 0.018` = do.call(run, c(list("paper", transform = "sqrt", lr = 0.018, scales = rate_scales), no_burn_in)) +) +season_of <- function(d) { + y <- as.integer(format(d, "%Y")) - (as.integer(format(d, "%m")) < 8) + paste0(y, "-", y + 1) +} +cals <- purrr::map(cals, function(cal) { + cal$forecasts <- cal$forecasts %>% mutate(season = season_of(reference_date)) + cal +}) +LIVE <- c("2024-2025", "2025-2026") +``` + +```{r p2-month-view, results="asis"} +mv <- cal_month_view(cals, seasons = LIVE, horizons = NULL) +cal_month_section("Month view", level = 2L) +``` diff --git a/reports/writeups/calibration_experiments/e15_wis_sources.Rmd b/reports/writeups/calibration_experiments/e15_wis_sources.Rmd new file mode 100644 index 00000000..d2a74b6e --- /dev/null +++ b/reports/writeups/calibration_experiments/e15_wis_sources.Rmd @@ -0,0 +1,194 @@ +--- +title: "E15: where the WIS gains come from" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 10, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +Question: for each finalist, how does the WIS change split across quantile +levels, how large are the offsets next to the base forecast's spread, and +how much of the change comes from moving the median versus changing widths? + +Setup: + +- Diseases: flu and covid, one section each. +- Base forecaster: the clean `windowed_seasonal` replay, on the (round, + location, horizon, level) rows the submitted ensemble also has, h0–h3 + (E10's replay runs). +- Learning truth: exact (re-run at every round on the data published by then; + E00), one week of extra delay. +- Scores against finalized truth over 2024-25 and 2025-26, pooled; states + only (US excluded). +- Configs, E10's finalists: + - **REF-op warm**: flu prod; warm start from the 2023-24 burn-in season. + - **REF-op cold**: covid prod; no burn-in season. + - **sqrt 0.018**: constant learning rate 0.018 on the square root of + admissions per 100k (E16), cold. + - **sqrt 0.018 warm**: the same with the burn-in season and a warm start. + +WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated +WIS lower than base. + +**Three ways to cut the WIS reduction.** + +1. **By quantile level.** WIS is the average pinball loss over the 23 + quantile levels, so the total change splits exactly into one piece per + level. A level's piece is its change in pinball loss as a share of the + base WIS; the pieces add up to the total. +2. **Offset size against the forecast's own spread.** For each forecast, + the offset at a level is `calibrated quantile − base quantile` (in + admissions, after every projection step). It is compared with the base + forecast's 90% interval width (q95 − q05) and 50% interval width + (q75 − q25). A ratio of 0.1 means the quantile moved by a tenth of the + interval width. +3. **Median shift vs width change.** Each calibrated forecast is split into + two parts and each part is scored on its own: + - **median shift only**: every base quantile moves by the same amount, + the calibrated median minus the base median. The base forecast's + shape and widths are kept. + - **width change only**: the calibrated forecast is moved so that its + median sits on the base median. The calibrated widths and shape are + kept, the base centre is kept. + + The interaction is the full change minus the two parts. Both parts are + floored at zero, as the calibrated forecast is. + +```{r load} +source(here::here("scripts/calibration/calibration_ws_replay.R")) +pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) +FIN_COLS <- c(`REF-op warm` = "#000000", `REF-op cold` = "#D55E00", `sqrt 0.018` = "#0072B2", `sqrt 0.018 warm` = "#56B4E9") +fin_fc <- function(d) { + f <- fin_runs(d, bases = "replay", workers = params$workers) + purrr::imap(f$cals, \(cal, nm) cal$forecasts %>% mutate(config = sub("^replay, ", "", nm))) %>% + bind_rows() %>% + mutate(season = season_of(reference_date), config = factor(config, levels = names(FIN_COLS))) %>% + filter(season %in% LIVE, !is_burn_in, !is.na(truth), !is.na(value_base), location != "US") +} +FC <- purrr::map(c(flu = "flu", covid = "covid"), fin_fc) +``` + +```{r fns} +# 1. Each level's change in pinball loss as a share of the base WIS. +by_level <- function(fc) { + fc %>% + group_by(config, horizon, level) %>% + summarize(base = sum(pinball(truth, value_base, level)), cal = sum(pinball(truth, value_cal, level)), .groups = "drop") %>% + group_by(config, horizon) %>% + mutate(contrib = 100 * (base - cal) / sum(base), total = sum(contrib)) %>% + ungroup() +} +level_plot <- function(bl) { + d <- bl %>% mutate(panel = sprintf("h%d, total %+.1f%%", horizon, total)) + ggplot(d, aes(factor(level), contrib, fill = config)) + + geom_hline(yintercept = 0, colour = "grey50") + + geom_col(width = 0.75) + + facet_wrap(~ paste(config, panel, sep = ": "), ncol = 4) + + scale_fill_manual(values = FIN_COLS, guide = "none") + + scale_x_discrete(labels = function(x) ifelse(x %in% c("0.01", "0.05", "0.25", "0.5", "0.75", "0.95", "0.99"), x, "")) + + labs(x = "quantile level", y = "share of WIS reduction, % of base WIS\n(positive = calibrated WIS lower than base)") + + theme(axis.text.x = element_text(angle = 90, vjust = 0.5, size = 7)) +} +group_tbl <- function(bl) { + grouped <- bl %>% + mutate( + band = case_when(level <= 0.05 | level >= 0.95 ~ "tails", level <= 0.3 | level >= 0.7 ~ "shoulders", TRUE ~ "centre"), + side = case_when(level < 0.5 ~ "lower", level > 0.5 ~ "upper", TRUE ~ "median") + ) + bind_rows( + grouped %>% group_by(config, horizon, part = band) %>% summarize(v = sum(contrib), .groups = "drop"), + grouped %>% group_by(config, horizon, part = paste(side, "side")) %>% summarize(v = sum(contrib), .groups = "drop"), + grouped %>% group_by(config, horizon) %>% summarize(v = sum(contrib), .groups = "drop") %>% mutate(part = "total") + ) %>% + mutate(part = factor(part, levels = c("tails", "shoulders", "centre", "lower side", "median side", "upper side", "total"), + labels = c("tails", "shoulders", "centre", "below the median", "the median level", "above the median", "total"))) %>% + transmute(config, part, horizon = paste0("h", horizon), v = sprintf("%+.2f", v)) %>% + tidyr::pivot_wider(names_from = horizon, values_from = v) %>% + arrange(config, part) +} +# 2. Signed offset ÷ the base 90% interval width, per forecast and level. +offset_plot <- function(fc) { + ref <- levels(droplevels(fc$config))[1] + widths <- fc %>% + filter(config == ref, level %in% c(0.05, 0.95)) %>% + select(location, horizon, reference_date, level, value_base) %>% + tidyr::pivot_wider(names_from = level, values_from = value_base, names_prefix = "q") %>% + transmute(location, horizon, reference_date, w90 = q0.95 - q0.05) + off_q <- fc %>% + inner_join(widths, by = c("location", "horizon", "reference_date")) %>% + filter(w90 > 0) %>% + mutate(r90 = (value_cal - value_base) / w90) %>% + group_by(config, horizon, level) %>% + summarize(med = median(r90), lo = quantile(r90, 0.1), hi = quantile(r90, 0.9), .groups = "drop") + ggplot(off_q, aes(level, med, colour = config, fill = config)) + + geom_hline(yintercept = 0, colour = "grey50") + + geom_ribbon(aes(ymin = lo, ymax = hi), alpha = 0.12, colour = NA) + + geom_line() + + geom_point(size = 0.8) + + facet_wrap(~ paste0("h", horizon), nrow = 2) + + scale_colour_manual(values = FIN_COLS, name = NULL, aesthetics = c("colour", "fill")) + + labs(x = "quantile level", y = "offset / base (q95 − q05)") + + theme(legend.position = "bottom") +} +# 3. Score the median shift and the width change on their own. +split_tbl <- function(fc) { + med <- fc %>% + filter(level == 0.5) %>% + select(config, location, horizon, reference_date, med_base = value_base, med_cal = value_cal) + scores <- fc %>% + left_join(med, by = c("config", "location", "horizon", "reference_date")) %>% + mutate(median_only = pmax(value_base + (med_cal - med_base), 0), width_only = pmax(med_base + (value_cal - med_cal), 0)) %>% + group_by(config, horizon) %>% + summarize( + base = sum(pinball(truth, value_base, level)), + full = sum(pinball(truth, value_cal, level)), + `median shift only` = sum(pinball(truth, median_only, level)), + `width change only` = sum(pinball(truth, width_only, level)), + .groups = "drop" + ) %>% + tidyr::pivot_longer(c(full, `median shift only`, `width change only`), names_to = "variant", values_to = "wis") %>% + mutate(pct = 100 * (base - wis) / base) + bind_rows( + scores %>% select(config, variant, horizon, pct), + scores %>% + group_by(config, horizon) %>% + summarize(pct = pct[variant == "full"] - pct[variant == "median shift only"] - pct[variant == "width change only"], .groups = "drop") %>% + mutate(variant = "interaction (full − the two parts)") + ) %>% + mutate(variant = factor(variant, levels = c("full", "median shift only", "width change only", "interaction (full − the two parts)"))) %>% + transmute(config, variant, horizon = paste0("h", horizon), pct = sprintf("%+.2f", pct)) %>% + tidyr::pivot_wider(names_from = horizon, values_from = pct) %>% + arrange(config, variant) +} +``` + +```{r sections, results="asis", fig.height=9} +for (d in names(FC)) { + fc <- FC[[d]] + bl <- by_level(fc) + cat("\n\n# ", d, "\n\n", sep = "") + cat("\n\n## WIS reduction by quantile level\n\n") + cat("Each bar is one level's share of the total WIS reduction %; the bars in a panel add up to the panel's total, printed in the strip.\n\n") + print(level_plot(bl)) + cat("\n\nThe same pieces summed over groups of levels: **tails** are the six levels at or beyond 0.05/0.95, **shoulders** the ten levels 0.1–0.3 and 0.7–0.9, **centre** the seven levels 0.35–0.65. Below and above the median are split, to show whether the gain comes from moving the lower or the upper side.\n\n") + print(knitr::kable(group_tbl(bl), caption = paste0(d, ": WIS reduction % by group of levels (positive = calibrated WIS lower than base)"))) + cat("\n\n## Offset size against the forecast's spread\n\n") + cat("Offsets are signed: positive means the calibrated quantile is above the base one. The line is the median over forecasts (location × round) and the band the middle 80%, offset ÷ base 90% interval width. Forecasts with a zero-width base interval are left out.\n\n") + print(offset_plot(fc)) + cat("\n\n## Median shift vs width change\n\n") + print(knitr::kable(split_tbl(fc), caption = paste0(d, ": WIS reduction % (positive = calibrated WIS lower than base)"))) +} +``` diff --git a/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd b/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd new file mode 100644 index 00000000..04a84bc7 --- /dev/null +++ b/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd @@ -0,0 +1,246 @@ +--- +title: "E16: constant learning rate by season, sqrt and rate scales" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +Question: how do WIS and coverage change with a constant learning rate, +season by season, on the square-root scale and on the rate scale, and how do +they compare with REF-op cold? + +Setup: + +- Disease: flu. Base forecaster: the clean `windowed_seasonal` replay (all 53 + locations, the 55 hub rounds). Horizons h0–h3. +- Learning truth: exact (re-run at every round on the data published by then; + E00), one week of extra delay. Cold start, no burn-in season. +- The tracker runs over both seasons, so 2025-26 starts from the offsets + 2024-25 left behind; each season is scored on its own rounds. Scores: + states only (US excluded). +- Configs: + - `sqrt `: the plain tracker with a constant learning rate on the + square root of admissions per 100k; 11 rates from 0.001 to 0.32, a + quarter decade apart, in sqrt(per-100k) units (not comparable number for + number with the rate scale). The grid was set after a first look at + results. + - `rate `: the same tracker on admissions per 100k; 12 rates from + 0.0032 to 1.8, a quarter decade apart. 0.1 is E11's config, though E11 + used the submitted ensemble. + - `REF-op cold`: prod's tracker without its warm start. + - `sqrt, adaptive`: REF-paper on the sqrt scale with its adaptive learning + rate (E05's `single`, cold). +- A sqrt rate is selected below as the highest mean gain with no loss in any + season × horizon. The selection is in-sample. + +WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated +WIS lower than base. L1 coverage bias: lower is better. + +```{r load} +source(here::here("scripts/calibration/calibration_ws_replay.R")) +wi <- ws_inputs("flu") +RATE_LRS <- signif(10^seq(-2.5, 0.25, by = 0.25), 2) +SQRT_LRS <- signif(10^seq(-3, -0.5, by = 0.25), 2) +all_cfgs <- ws_configs(wi$rate_scales, burn_in = FALSE) +rate_cfg <- function(lr) utils::modifyList(all_cfgs[["E11 constant 0.1"]], list(lr = lr)) +sqrt_cfg <- function(lr) utils::modifyList(all_cfgs[["E11 constant 0.1"]], list(lr = lr, transform = "sqrt")) +# Cached under cache/calibration/experiments/runs; E17, E18 and E07 reuse some of these runs. +run <- function(cfg) { + do.call(cal_run_cached, c(list(wi$fc, wi$truth), cfg, list(learn = "exact", vintages = wi$vintages, workers = params$workers))) +} +rate_runs <- purrr::map(RATE_LRS, \(lr) run(rate_cfg(lr))) +sqrt_runs <- purrr::map(SQRT_LRS, \(lr) run(sqrt_cfg(lr))) +refop <- run(all_cfgs[["REF-op cold"]]) +sqrt_adaptive <- run(ws_configs(wi$rate_scales, burn_in = TRUE)[["E05 single cold"]]) +``` + +```{r metrics} +GEOS <- c(states = "states only (US excluded)") +SEASONS <- c(LIVE, "both") +SEASON_COLS <- c(`2024-2025` = "#0072B2", `2025-2026` = "#D55E00", `both seasons` = "grey45") +score <- function(cal) { + fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) + purrr::imap(WS_GEOS[names(GEOS)], function(keep, g) ws_scores(fc %>% filter(keep(location))) %>% mutate(geos = g)) %>% + bind_rows() %>% + select(geos, season, horizon, which, wis, cal_err) %>% + tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err)) %>% + mutate(wis_pct = 100 * (wis_base - wis_cal) / wis_base) +} +label <- function(df) { + df %>% mutate( + geos = factor(geos, levels = names(GEOS), labels = GEOS), + season = factor(season, levels = SEASONS, labels = c(LIVE, "both seasons")), + h = factor(paste0("h", horizon), levels = paste0("h", 0:3)) + ) +} +sweep_m <- bind_rows( + purrr::map2(rate_runs, RATE_LRS, \(cal, lr) score(cal) %>% mutate(scale = "rate", lr = lr)), + purrr::map2(sqrt_runs, SQRT_LRS, \(cal, lr) score(cal) %>% mutate(scale = "sqrt", lr = lr)) +) %>% label() +ref_m <- bind_rows( + score(refop) %>% mutate(ref = "REF-op cold"), + score(sqrt_adaptive) %>% mutate(ref = "sqrt, adaptive") +) %>% label() +# Best sqrt rate: the highest mean WIS reduction among rates that improve WIS in both seasons at h0–h3 (states only). +best_tbl <- sweep_m %>% + filter(scale == "sqrt", geos == GEOS[["states"]], season %in% LIVE) %>% + group_by(lr) %>% + summarize(worst = min(wis_pct), mean = mean(wis_pct), .groups = "drop") +BEST_SQRT <- best_tbl %>% filter(worst > 0) %>% slice_max(mean, n = 1) %>% pull(lr) +``` + +# 1. Coverage bias vs WIS, by season and scale + +One curve per season and scale, tracing the learning rate from smallest +(near the base, the open circle) to largest. Solid lines are the rate scale, +dashed lines the sqrt scale. Labels mark a few rates on each curve. +Triangles are REF-op cold, squares the adaptive sqrt tracker. The y axis is +clipped at −10%. The sqrt rate +selected by the rule below is **`r BEST_SQRT`**. + +```{r curve-fn} +LABELLED <- list(rate = c(0.01, 0.032, 0.1, 0.32), sqrt = c(0.0032, 0.01, 0.032, 0.1)) +curve_plot <- function(g) { + d <- sweep_m %>% filter(geos == g, season != "both seasons") %>% arrange(scale, lr) + r <- ref_m %>% filter(geos == g, season != "both seasons") + lab <- d %>% filter((scale == "rate" & lr %in% LABELLED$rate) | (scale == "sqrt" & lr %in% LABELLED$sqrt)) + base <- d %>% distinct(h, season, cal_err_base) + ggplot(d, aes(cal_err_cal, wis_pct, colour = season)) + + geom_hline(yintercept = 0, colour = "grey60", linewidth = 0.3) + + geom_path(aes(linetype = scale, group = interaction(season, scale)), linewidth = 0.7) + + geom_point(aes(shape = scale), size = 1.1) + + geom_text(data = lab, aes(label = paste0(ifelse(scale == "sqrt", "s", "r"), format(lr, drop0trailing = TRUE))), + size = 2.3, vjust = -0.8, show.legend = FALSE) + + geom_point(data = base, aes(cal_err_base, 0), shape = 21, fill = "white", size = 2.2) + + geom_point(data = r, aes(shape = ref), size = 2.6) + + scale_colour_manual(values = SEASON_COLS, name = NULL) + + scale_linetype_manual(values = c(rate = "solid", sqrt = "22"), name = "scale") + + scale_shape_manual(values = c(rate = 16, sqrt = 1, `REF-op cold` = 17, `sqrt, adaptive` = 15), name = NULL) + + facet_wrap(~h, nrow = 1) + + coord_cartesian(ylim = c(max(-10, min(d$wis_pct)), max(c(d$wis_pct, r$wis_pct)) + 0.5)) + + labs(x = "L1 coverage bias (lower is better)", y = "WIS reduction % (positive = calibrated WIS lower than base)", title = g, + caption = "labels: r = rate-scale learning rate (per 100k), s = sqrt-scale learning rate") + + theme(legend.position = "bottom") +} +``` + +```{r curve-states} +curve_plot(GEOS[["states"]]) +``` + +## Each metric against the learning rate + +One panel set per scale; the two scales' rates are in different units. +Top: WIS reduction % (positive = calibrated WIS lower than base). Bottom: L1 +coverage bias (lower is better), each season's base dotted. States only. + +```{r vs-lr-fn} +vs_lr_plot <- function(sc, breaks, xlab) { + long <- sweep_m %>% + filter(scale == sc, geos == GEOS[["states"]], season != "both seasons") %>% + transmute(h, season, lr, `WIS reduction % (positive = calibrated WIS lower than base)` = wis_pct, `L1 coverage bias (lower is better)` = cal_err_cal) %>% + tidyr::pivot_longer(c(`WIS reduction % (positive = calibrated WIS lower than base)`, `L1 coverage bias (lower is better)`), names_to = "metric") %>% + mutate(metric = factor(metric, levels = c("WIS reduction % (positive = calibrated WIS lower than base)", "L1 coverage bias (lower is better)"))) + base_lines <- sweep_m %>% + filter(scale == sc, geos == GEOS[["states"]], season != "both seasons") %>% + distinct(h, season, value = cal_err_base) %>% + mutate(metric = factor("L1 coverage bias (lower is better)", levels = levels(long$metric))) + zero <- tibble(metric = factor("WIS reduction % (positive = calibrated WIS lower than base)", levels = levels(long$metric)), value = 0) + ggplot(long, aes(lr, value, colour = season)) + + geom_hline(data = zero, aes(yintercept = value), colour = "grey60", linewidth = 0.3) + + geom_hline(data = base_lines, aes(yintercept = value, colour = season), linetype = "dotted") + + geom_line() + + geom_point(size = 1) + + scale_x_log10(breaks = breaks, labels = format(breaks, drop0trailing = TRUE)) + + scale_colour_manual(values = SEASON_COLS, name = NULL) + + facet_grid(metric ~ h, scales = "free_y", labeller = labeller(metric = label_wrap_gen(28))) + + labs(x = xlab, y = NULL, title = paste(sc, "scale")) + + theme(legend.position = "bottom") +} +``` + +```{r vs-lr-sqrt, fig.height=6.5} +vs_lr_plot("sqrt", c(0.001, 0.003, 0.01, 0.03, 0.1, 0.3), "constant learning rate (sqrt of per-100k units per step)") +``` + +```{r vs-lr-rate, fig.height=6.5} +vs_lr_plot("rate", c(0.003, 0.01, 0.03, 0.1, 0.3, 1), "constant learning rate (per 100k per step)") +``` + +## Selecting a sqrt rate + +For each sqrt rate, the worst and the mean WIS reduction % (positive = calibrated WIS lower than base) over the two seasons +and h0–h3 (states only). The selected rate has the highest mean among those +whose worst case is still a gain. + +```{r best} +knitr::kable(best_tbl %>% mutate(across(c(worst, mean), \(x) round(x, 2)))) +``` + +```{r month-view-cfgs} +lr_lab <- function(lr) vapply(lr, \(x) format(x, drop0trailing = TRUE), character(1)) +mv <- cal_month_view(c( + setNames(sqrt_runs, paste("sqrt", lr_lab(SQRT_LRS))), + setNames(rate_runs, paste("rate", lr_lab(RATE_LRS))), + list(`REF-op cold` = refop, `sqrt, adaptive` = sqrt_adaptive) +), seasons = LIVE) +# The plots show the sqrt rates around the elbow, rate 0.032 and REF-op cold; the tables cover every rate on both scales. +MONTH_COLS <- c(`sqrt 0.01` = "#56B4E9", `sqrt 0.018` = "#CC79A7", `sqrt 0.032` = "#0072B2", `rate 0.032` = "#009E73", `REF-op cold` = "#D55E00") +mv_plot_cfgs <- names(MONTH_COLS) +mv_cols <- MONTH_COLS +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +``` + +# Mean offset at the median over time + +The calibrated minus base median, in admissions per 100k, averaged over +states at each round, h0 and h2. The bottom row is the mean base median per +100k at h0, to place the season peak. + +```{r offset-data} +OFFSET_CFGS <- list(`sqrt 0.018` = sqrt_runs[[which(SQRT_LRS == 0.018)]], `rate 0.032` = rate_runs[[which(RATE_LRS == 0.032)]], `REF-op cold` = refop) +fc_pick <- purrr::imap(OFFSET_CFGS, \(cal, nm) cal$forecasts %>% mutate(config = nm)) %>% + bind_rows() %>% + mutate(season = season_of(reference_date), config = factor(config, levels = names(OFFSET_CFGS))) %>% + filter(season %in% LIVE, location != "US", !is.na(value_base)) +``` + +```{r offset-time, fig.height=7} +pop <- wi$rate_scales %>% select(location, scale) +med <- fc_pick %>% + filter(level == 0.5, horizon %in% c(0, 2), config %in% names(OFFSET_CFGS)) %>% + left_join(pop, by = "location") %>% + group_by(config, season, horizon, reference_date) %>% + summarize(off = mean((value_cal - value_base) / scale), base = mean(value_base / scale), truth = mean(truth / scale, na.rm = TRUE), .groups = "drop") +off_long <- bind_rows( + med %>% transmute(config, season, reference_date, row = paste0("offset, h", horizon), value = off), + med %>% filter(horizon == 0, config == names(OFFSET_CFGS)[1]) %>% transmute(config = "base median", season, reference_date, row = "base median h0", value = base), + med %>% filter(horizon == 0, config == names(OFFSET_CFGS)[1]) %>% transmute(config = "truth", season, reference_date, row = "base median h0", value = truth) +) %>% + mutate(config = factor(config, levels = c(names(OFFSET_CFGS), "base median", "truth")), row = factor(row, levels = c("offset, h0", "offset, h2", "base median h0"))) +ggplot(off_long, aes(reference_date, value, colour = config)) + + geom_hline(yintercept = 0, colour = "grey60") + + geom_line() + + geom_point(size = 0.7) + + facet_grid(row ~ season, scales = "free") + + scale_colour_manual(values = c(MONTH_COLS[names(OFFSET_CFGS)], `base median` = "grey30", truth = "grey65"), name = NULL) + + scale_x_date(date_breaks = "1 month", date_labels = "%b") + + labs(x = "reference date", y = "admissions per 100k (mean over states)") + + theme(legend.position = "bottom") +``` diff --git a/reports/writeups/calibration_experiments/e17_late_decay.Rmd b/reports/writeups/calibration_experiments/e17_late_decay.Rmd new file mode 100644 index 00000000..f3c506ce --- /dev/null +++ b/reports/writeups/calibration_experiments/e17_late_decay.Rmd @@ -0,0 +1,113 @@ +--- +title: "E17: shrinking the offset late in the season" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +Question: does shrinking the offset from 1 March, or switching calibration +off from 1 March, change WIS and coverage compared with the same tracker +without it? + +Setup: + +- Disease: flu. Base forecaster: the clean `windowed_seasonal` replay (all 53 + locations, the 55 hub rounds). Horizons h0–h3. +- Learning truth: exact (re-run at every round on the data published by then; + E00), one week of extra delay. Cold start, no burn-in season. States only. +- Three constant-rate trackers: rate 0.018 and rate 0.032 per 100k (E16), + and sqrt 0.018 (E16). Each crossed with five settings: + - **no decay**. + - **`late_decay`** with factor 0.85, 0.7 or 0.5: from 1 March to 30 June + the stored offset is multiplied by the factor at each round before it is + played. Learning continues, and because offsets carry across seasons, + the shrink carries into the next season. + - **`off_after = "03-01"`**: from 1 March to 30 June the base is played, + those rounds produce no updates, and the offset carries unchanged into + the next season. +- The 1 March date was chosen after E16, on these same seasons. + (`fast_decay` and `season_policy` are other ways to shrink offsets; they + are not run here.) + +WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated +WIS lower than base. L1 coverage bias: lower is better. + +```{r load} +source(here::here("scripts/calibration/calibration_ws_replay.R")) +wi <- ws_inputs("flu") +base_cfg <- ws_configs(wi$rate_scales, burn_in = FALSE)[["E11 constant 0.1"]] +SQRT_BEST <- 0.018 +RATES <- list( + `rate 0.018` = list(lr = 0.018), + `rate 0.032` = list(lr = 0.032), + sqrt = list(lr = SQRT_BEST, transform = "sqrt") +) +names(RATES)[3] <- paste("sqrt", SQRT_BEST) +DECAYS <- list( + `no decay` = list(), + `decay 0.85` = list(late_decay = list(from = "03-01", factor = 0.85)), + `decay 0.7` = list(late_decay = list(from = "03-01", factor = 0.7)), + `decay 0.5` = list(late_decay = list(from = "03-01", factor = 0.5)), + `off after 1 Mar` = list(off_after = "03-01") +) +grid <- tidyr::expand_grid(rate = names(RATES), decay = names(DECAYS)) +# Cached under cache/calibration/experiments/runs; the no-decay runs are E16's. +cals <- purrr::pmap(grid, function(rate, decay) { + cfg <- c(utils::modifyList(base_cfg, RATES[[rate]]), DECAYS[[decay]]) + do.call(cal_run_cached, c(list(wi$fc, wi$truth), cfg, list(learn = "exact", vintages = wi$vintages, workers = params$workers))) +}) +fc <- purrr::map2(cals, seq_len(nrow(grid)), function(cal, i) { + cal$forecasts %>% mutate(rate = grid$rate[i], decay = grid$decay[i]) +}) %>% + bind_rows() %>% + mutate(season = season_of(reference_date), rate = factor(rate, levels = names(RATES)), decay = factor(decay, levels = names(DECAYS))) %>% + filter(season %in% LIVE, location != "US") +DECAY_COLS <- setNames(c("black", "#56B4E9", "#0072B2", "#D55E00", "#009E73"), names(DECAYS)) +``` + +```{r month-view-cfgs} +mv <- cal_month_view(setNames(cals, paste(grid$rate, grid$decay, sep = ", ")), seasons = LIVE) +# The plots show sqrt 0.018 with each decay setting; the tables cover all three trackers. +mv_plot_cfgs <- paste(names(RATES)[3], names(DECAYS), sep = ", ") +mv_cols <- setNames(DECAY_COLS, mv_plot_cfgs) +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +``` + +# Mean offset at the median over time + +Calibrated minus base median in admissions per 100k, averaged over states at +each round, h2. + +```{r offset-time, fig.height=7} +pop <- wi$rate_scales %>% select(location, scale) +fc %>% + filter(level == 0.5, horizon == 2) %>% + left_join(pop, by = "location") %>% + group_by(rate, decay, season, reference_date) %>% + summarize(off = mean((value_cal - value_base) / scale), .groups = "drop") %>% + ggplot(aes(reference_date, off, colour = decay)) + + geom_hline(yintercept = 0, colour = "grey60") + + geom_line() + + geom_point(size = 0.6) + + facet_grid(rate ~ season, scales = "free") + + scale_colour_manual(values = DECAY_COLS, name = NULL) + + scale_x_date(date_breaks = "1 month", date_labels = "%b") + + labs(x = "reference date", y = "mean offset at the median, per 100k") + + theme(legend.position = "bottom") +``` diff --git a/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd b/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd new file mode 100644 index 00000000..cff2c1af --- /dev/null +++ b/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd @@ -0,0 +1,207 @@ +--- +title: "E18: a bridge from REF-op to sqrt constant 0.018" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +Question: REF-op (prod's tracker) and sqrt constant 0.018 (E16) differ in +several settings. How do WIS and coverage change as one setting at a time is +changed from one to the other? + +Setup: + +- Disease: flu. Base forecaster: the clean `windowed_seasonal` replay (53 + locations, the 55 hub rounds). Horizons h0–h3. +- Learning truth: exact (re-run at every round on the data published by then; + E00), one week of extra delay. +- Steps 0–3 run with the 2023-24 season present (the HHS-stitched backfill + from E13); steps 4–6 and the references are cold. +- Scores over the two live seasons, each on its own rounds; states only (US + excluded). + +Each row changes one setting from the row above: + +| step | change | config after the change | +|---|---|---| +| 0 REF-op | — | sqrt, adaptive mult 0.03 floor 1e-3, window 20, carry; slow term (mult 0.003) warm-started from the 2023-24 burn-in; leak 0.1 on the fast term | +| 1 no warm start | `slow_init = NULL` | the slow term starts at zero | +| 2 no slow term | `lr_slow = NULL` | single fast term with the leak | +| 3 no leak | `fast_decay = 0` | sqrt, adaptive, mult 0.03, window 20, carry, with the 2023-24 season present | +| 4 no burn-in season | drop the 2023-24 rounds | as 3, cold: the eta pool starts empty | +| 5 per-100k units | `scales` = population / 100k | as 4, on the square root of admissions per 100k | +| 6 sqrt constant 0.018 (E16) | `lr = 0.018` | sqrt per 100k, constant 0.018 | + +The 2023-24 season does two things: with `slow_init` it sets the slow term's +starting value (step 1 removes this), and its residuals fill the adaptive +learning rate's window, so eta is sized from the first live round (step 4 +removes this). Burn-in outcomes never step the offsets. Step 5 changes only +units, so that step 6 changes only the learning rate. + +Outside the chain: a branch, sqrt 0.018 warm, which is step 6 with the 2023-24 season present and `slow_init = +"burn_in_quantile"` (the warm start of step 0, here on the single term); and +REF-op cold (prod's covid config: step 0 without the warm start or the +2023-24 season), so the state scatters below can pair warm with warm and +cold with cold. + +Steps 0, 3, 4 and 6 are the cached runs of E13 (REF-op warm), E02 +(`reference + burn-in`), E01/E02 (`reference`) and E16 (sqrt 0.018), so they +are not independent checks of those notebooks. + +WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated +WIS lower than base. L1 coverage bias: lower is better. "Month-avg" coverage +bias is computed within each calendar month and then averaged. + +```{r load} +source(here::here("scripts/calibration/calibration_ws_replay.R")) +wi <- ws_inputs("flu") +fc_burn_in <- bind_rows(ws_burn_in_forecasts("flu"), wi$fc) +# Each config is written as the experiment that first ran it writes it, so cached runs are reused. +refop_warm <- ws_configs(wi$rate_scales, burn_in = TRUE)[["REF-op warm"]] +sqrt_adaptive <- list( + ref = "paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3), + burn_in_seasons = character(0), slow_init = NULL, lr_window = 20, season_policy = "carry" +) +const_cfg <- ws_configs(wi$rate_scales, burn_in = FALSE)[["E11 constant 0.1"]] +STEPS <- list( + `0 REF-op` = list(cfg = refop_warm, burn_in = TRUE), + `1 no warm start` = list(cfg = list(ref = "op", burn_in_seasons = "2023-2024", slow_init = NULL), burn_in = TRUE), + `2 no slow term` = list(cfg = list(ref = "op", burn_in_seasons = "2023-2024", slow_init = NULL, lr_slow = NULL), burn_in = TRUE), + `3 no leak` = list(cfg = utils::modifyList(sqrt_adaptive, list(burn_in_seasons = "2023-2024")), burn_in = TRUE), + `4 no burn-in season` = list(cfg = sqrt_adaptive, burn_in = FALSE), + `5 per-100k units` = list(cfg = c(sqrt_adaptive, list(scales = wi$rate_scales)), burn_in = FALSE), + `6 sqrt constant 0.018 (E16)` = list(cfg = utils::modifyList(const_cfg, list(lr = 0.018, transform = "sqrt")), burn_in = FALSE) +) +# Outside the chain: step 6 with a warm start, and REF-op without one (E16's cached run). +REFS <- list( + `branch: sqrt 0.018 warm` = list(cfg = utils::modifyList(STEPS[[7]]$cfg, list(burn_in_seasons = "2023-2024", slow_init = "burn_in_quantile")), burn_in = TRUE), + `REF-op cold` = list(cfg = ws_configs(wi$rate_scales, burn_in = FALSE)[["REF-op cold"]], burn_in = FALSE) +) +run <- function(spec) { + fc <- if (spec$burn_in) fc_burn_in else wi$fc + do.call(cal_run_cached, c(list(fc, wi$truth), spec$cfg, list(learn = "exact", vintages = wi$vintages, workers = params$workers))) +} +cals <- purrr::map(c(STEPS, REFS), run) +``` + +```{r metrics} +SEASONS <- c(LIVE, "both") +STEP_COLS <- setNames(c("#000000", "#E69F00", "#56B4E9", "#009E73", "#CC79A7", "#0072B2", "#D55E00", "#AA4499", "#999999"), names(c(STEPS, REFS))) +step_m <- purrr::imap(cals, function(cal, nm) { + fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) %>% filter(location != "US") + ws_scores(fc) %>% + select(season, horizon, which, wis, cal_err) %>% + tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err)) %>% + mutate(step = nm, wis_pct = 100 * (wis_base - wis_cal) / wis_base) +}) %>% + bind_rows() %>% + mutate( + step = factor(step, levels = names(c(STEPS, REFS))), + season = factor(season, levels = SEASONS, labels = c(LIVE, "both seasons")), + h = factor(paste0("h", horizon), levels = paste0("h", 0:3)) + ) +mv <- cal_month_view(cals, seasons = LIVE) +``` + +```{r month-view-cfgs} +# Five configs, so the month plots stay readable; the tables below them cover every config. +mv_plot_cfgs <- c("0 REF-op", "1 no warm start", "4 no burn-in season", "6 sqrt constant 0.018 (E16)", "branch: sqrt 0.018 warm") +mv_cols <- STEP_COLS +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +``` + +# What each step changes + +The change each step makes to the step before it, both seasons pooled, +states only. WIS: change in WIS reduction % (positive = the step lowers +WIS). Coverage: change in L1 coverage bias, season-pooled and month-average +(negative = the step improves coverage). + +```{r deltas} +wis_both <- step_m %>% + filter(season == "both seasons", step %in% names(STEPS)) %>% + transmute(step, h, v = wis_pct) +cov_both <- mv$cov %>% + filter(season == "both") %>% + transmute(step = config, h = factor(paste0("h", horizon), levels = paste0("h", 0:3)), pooled = pooled_cal, mavg = mavg_cal) +delta <- wis_both %>% + left_join(cov_both, by = c("step", "h")) %>% + arrange(h, step) %>% + group_by(h) %>% + mutate(d_wis = v - lag(v), d_pooled = pooled - lag(pooled), d_mavg = mavg - lag(mavg)) %>% + ungroup() %>% + filter(step != names(STEPS)[1]) +knitr::kable( + delta %>% + transmute(step, h, cell = sprintf("%+.1f / %+.3f / %+.3f", d_wis, d_pooled, d_mavg)) %>% + tidyr::pivot_wider(names_from = h, values_from = cell) %>% + arrange(step), + caption = "Both seasons, states only: change from the previous step in WIS reduction % / season-pooled L1 coverage bias / month-avg L1 coverage bias" +) +``` + +# State by state: REF-op against sqrt 0.018, matched on the warm start + +Per state, pooled over h0–h3: WIS reduction % of REF-op against sqrt 0.018 +(above the line, REF-op is better). Three pairs: the chain's ends (REF-op +warm vs sqrt 0.018 cold), then warm vs warm and cold vs cold. Cold configs +play the base through Nov–Dec 2024 (35–40% of that season's base WIS), so +only the matched pairs compare the trackers rather than the start. + +```{r state-scatter, fig.height=4.5} +PAIRS <- list( + `chain ends: REF-op warm vs sqrt 0.018 cold` = c("0 REF-op", "6 sqrt constant 0.018 (E16)"), + `warm vs warm` = c("0 REF-op", "branch: sqrt 0.018 warm"), + `cold vs cold` = c("REF-op cold", "6 sqrt constant 0.018 (E16)") +) +state_pct <- cal_state_wis(cals[unique(unlist(PAIRS))]) %>% + group_by(config, season, location) %>% + summarize(pct = 100 * (sum(wis_base) - sum(wis_cal)) / sum(wis_base), .groups = "drop") +pair_pts <- purrr::imap(PAIRS, function(p, nm) { + state_pct %>% + filter(config %in% p) %>% + mutate(which = ifelse(config == p[1], "refop", "sqrt")) %>% + select(-config) %>% + tidyr::pivot_wider(names_from = which, values_from = pct) %>% + mutate(pair = nm) +}) %>% + bind_rows() %>% + mutate(pair = factor(pair, levels = names(PAIRS))) +ggplot(pair_pts, aes(sqrt, refop)) + + geom_abline(slope = 1, intercept = 0, colour = "grey50") + + geom_hline(yintercept = 0, colour = "grey85") + + geom_vline(xintercept = 0, colour = "grey85") + + geom_point(alpha = 0.8) + + facet_grid(season ~ pair) + + labs(x = "sqrt 0.018: WIS reduction %, h0–h3", y = "REF-op: WIS reduction %, h0–h3") +``` + +States where REF-op has the lower WIS, out of the states scored, per pair +and season. + +```{r state-counts} +knitr::kable( + pair_pts %>% + group_by(pair, season) %>% + summarize(refop_better = sum(refop > sqrt), states = n(), .groups = "drop") %>% + transmute(pair, season, cell = sprintf("%d / %d", refop_better, states)) %>% + tidyr::pivot_wider(names_from = season, values_from = cell), + caption = "States (of those scored) where REF-op beats sqrt 0.018, h0–h3 pooled" +) +``` diff --git a/reports/writeups/calibration_experiments/e19_tracker_internals.Rmd b/reports/writeups/calibration_experiments/e19_tracker_internals.Rmd new file mode 100644 index 00000000..eee19776 --- /dev/null +++ b/reports/writeups/calibration_experiments/e19_tracker_internals.Rmd @@ -0,0 +1,242 @@ +--- +title: "E19: tracker internals by month, and carry vs reset, on both anchors" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + workers: 14 + disease: flu +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +Question: what does each tracker do month by month? How large is its step, +how large is the offset it plays, and does the WIS change in a month follow +from those together with the base's bias in that month? And how much of the +second season's result is the offset carried in from the first? + +Setup: + +- Disease: `r params$disease`. Base forecaster: the clean `windowed_seasonal` + replay (53 locations, the hub rounds). Horizons h0–h3. +- Learning truth: exact (re-run at every round on the data published by then; + E00), one week of extra delay. Each round's offsets come from that round's + own re-run, so they are the state that played. The step size is taken + from one pass on final truth per config, because in a round's own re-run + that round's outcomes have not yet stepped the tracker. +- Configs: the E20 grid ({adaptive 0.03, constant 0.018} × {cold, warm} × + {no leak, leak 0.1}), prod's REF-op warm and REF-op cold, and two + `season_policy = "reset"` runs (each anchor, cold, no leak), which zero the + offsets at the start of 2025-26. +- Scores over the two live seasons, each on its own rounds; states only (US + excluded). + +Internals are unit-free: the learning rate and the median-level offset are +divided by the base's 90% interval width, the rate in the config's working +units (sqrt of counts, or of counts per 100k) and the offset in counts. Each +value is the median over states for a (horizon, round), then averaged within +the calendar month. "Share above base median" is the fraction of state +outcomes above the base's median, the base's own bias that month. + +```{r load} +source(here::here("scripts/calibration/calibration_ws_replay.R")) +wi <- ws_inputs(params$disease) +fc_burn_in <- bind_rows(ws_burn_in_forecasts(params$disease), wi$fc) +grid <- ws_grid_configs(wi$rate_scales) +cold_no_leak <- grid[c("adaptive 0.03 | cold | no leak", "constant 0.018 | cold | no leak")] +RESETS <- purrr::map(cold_no_leak, function(s) list(cfg = utils::modifyList(s$cfg, list(season_policy = "reset")), burn_in = FALSE)) +names(RESETS) <- sub(" \\| no leak$", " | reset", names(RESETS)) +SPECS <- c(grid, ws_prod_specs(wi$rate_scales), RESETS) +# Internals are not in the standard cache, so cached runs without them are re-run here. +cals <- purrr::map(SPECS, ws_run_spec, wi = wi, fc_burn_in = fc_burn_in, workers = params$workers, internals = TRUE) +# An exact run's round-t rows come from the re-run at t, before round t's outcomes step the tracker, +# so their step size is zero. The step size comes from one pass on final truth per config instead. +cals_final <- purrr::map(SPECS, function(spec) { + fc <- if (spec$burn_in) fc_burn_in else wi$fc + do.call(cal_run, c(list(fc, wi$truth), spec$cfg, list(learn = "final"))) +}) +ANCHOR_COLS <- c(`adaptive 0.03` = "#E69F00", `constant 0.018` = "#0072B2", `REF-op` = "#000000") +anchor_of <- function(config) dplyr::case_when( + grepl("^adaptive", config) ~ "adaptive 0.03", + grepl("^constant", config) ~ "constant 0.018", + TRUE ~ "REF-op" +) +``` + +```{r internals} +internals <- purrr::imap(SPECS, function(spec, nm) { + eta <- ws_internals(cals_final[[nm]], spec$cfg, wi$rate_scales) %>% select(horizon, reference_date, eta_rel) + ws_internals(cals[[nm]], spec$cfg, wi$rate_scales) %>% + select(-eta_rel) %>% + left_join(eta, by = c("horizon", "reference_date")) %>% + mutate(config = nm) +}) %>% + bind_rows() %>% + mutate( + config = factor(config, levels = names(SPECS)), + season = season_of(reference_date), + month = factor(format(reference_date, "%b"), levels = month.abb[c(8:12, 1:7)]), + h = factor(paste0("h", horizon), levels = paste0("h", 0:3)) + ) %>% + filter(season %in% LIVE) +mv <- cal_month_view(cals, seasons = LIVE) +by_month <- internals %>% + group_by(config, season, month, h, horizon) %>% + summarize(across(c(eta_rel, offset_rel, offset_abs_rel, above_base_median), \(x) mean(x, na.rm = TRUE)), .groups = "drop") %>% + left_join(mv$month %>% select(config, season, month, horizon, wis_pct, share), by = c("config", "season", "month", "horizon")) %>% + mutate(anchor = anchor_of(as.character(config))) +# Internals are only shown for the no-leak grid, prod and the resets; the leak runs' offsets are in the tables. +PLOT_CFGS <- c(names(grid)[c(1, 3, 5, 7)], "REF-op warm", "REF-op cold") +``` + +# Per round + +Median over states of the step size and the played offset, both relative to +the base's 90% width, per round. Warm configs start with an offset; cold +ones at zero. + +```{r per-round, fig.height=8} +long <- internals %>% + filter(config %in% PLOT_CFGS) %>% + select(config, h, reference_date, `eta / width` = eta_rel, `median-level offset / width` = offset_rel, `mean |offset| / width` = offset_abs_rel) %>% + tidyr::pivot_longer(-c(config, h, reference_date)) +ggplot(long, aes(reference_date, value, colour = config)) + + geom_hline(yintercept = 0, colour = "grey60") + + geom_line() + + facet_grid(name ~ h, scales = "free_y") + + labs(x = NULL, y = "median over states", colour = NULL) + + theme(legend.position = "bottom") + + guides(colour = guide_legend(nrow = 2)) +``` + +# By month + +One row per config, horizon and season-month: step size, offset, the base's +bias that month, and the month's WIS change. The grey bar is the month's +share of the season's base WIS. + +```{r by-month-plot, fig.height=9} +pm <- by_month %>% filter(config %in% PLOT_CFGS) +for (s in LIVE) { + d <- pm %>% filter(season == s) + long <- d %>% + select(config, h, month, `eta / width` = eta_rel, `offset / width` = offset_rel, `share above base median` = above_base_median, `WIS reduction %` = wis_pct) %>% + tidyr::pivot_longer(-c(config, h, month)) %>% + mutate(name = factor(name, levels = c("eta / width", "offset / width", "share above base median", "WIS reduction %"))) + bars <- d %>% distinct(h, month, share) %>% mutate(name = factor("WIS reduction %", levels = levels(long$name))) + print( + ggplot(long, aes(month, value, colour = config, group = config)) + + geom_col(data = bars, aes(month, share), inherit.aes = FALSE, fill = "grey85", width = 0.7) + + geom_hline(data = tibble(name = factor("share above base median", levels = levels(long$name)), y = 0.5), aes(yintercept = y), colour = "grey60") + + geom_hline(data = tibble(name = factor(c("offset / width", "WIS reduction %"), levels = levels(long$name)), y = 0), aes(yintercept = y), colour = "grey60") + + geom_line() + + geom_point(size = 1) + + facet_grid(name ~ h, scales = "free_y") + + labs(title = s, x = NULL, y = NULL, colour = NULL) + + theme(legend.position = "bottom") + + guides(colour = guide_legend(nrow = 2)) + ) +} +``` + +## Tables + +```{r by-month-tables, results="asis"} +month_cell <- function(d, col, fmt) { + d %>% + transmute(config, h, col = paste(season, month), v = sprintf(fmt, .data[[col]])) %>% + tidyr::pivot_wider(names_from = col, values_from = v) %>% + arrange(h, config) +} +for (spec in list( + c("eta_rel", "%.3f", "Step size / base 90% width (mean over rounds of the state median)"), + c("offset_rel", "%+.3f", "Median-level offset / base 90% width"), + c("wis_pct", "%+.1f", "WIS reduction % within the month (positive = calibrated WIS lower than base)") +)) { + cat("\n\n", cal_h(3), " ", spec[3], "\n\n", sep = "") + print(knitr::kable(month_cell(by_month, spec[1], spec[2]))) +} +cat("\n\n", cal_h(3), " Share of state outcomes above the base median (the base's bias; same for every config)\n\n", sep = "") +print(knitr::kable( + by_month %>% + filter(config == names(SPECS)[1]) %>% + transmute(h, col = paste(season, month), v = sprintf("%.2f", above_base_median)) %>% + tidyr::pivot_wider(names_from = col, values_from = v) +)) +``` + +# Offset against WIS, month by month + +Each point is a config in one season-month at one horizon: the median-level +offset it played against the base's bias that month, coloured by the +month's WIS change. If a positive offset only pays when the base +under-predicts, the gains sit in the upper right and lower left. + +```{r offset-vs-bias, fig.height=4.5} +ggplot(by_month %>% filter(config %in% names(grid)), aes(above_base_median, offset_rel, colour = wis_pct)) + + geom_hline(yintercept = 0, colour = "grey60") + + geom_vline(xintercept = 0.5, colour = "grey60") + + geom_point(alpha = 0.8) + + facet_grid(season ~ h) + + scale_colour_gradient2(low = "#D55E00", mid = "grey80", high = "#009E73", midpoint = 0) + + labs(x = "share of outcomes above the base median", y = "median-level offset / width", colour = "WIS reduction %") +``` + +# Carry vs reset + +The offset each cold, no-leak tracker brings into 2025-26, and what resetting +it to zero at the season boundary does to the season's WIS. E01 did this for +the adaptive rate; the constant one is new. + +```{r carry-reset} +pairs <- tibble::tibble( + anchor = c("adaptive 0.03", "constant 0.018"), + carry = names(cold_no_leak), + reset = names(RESETS) +) +m <- ws_wis_by_season(cals[c(pairs$carry, pairs$reset)]) +cr <- pairs %>% + tidyr::pivot_longer(c(carry, reset), names_to = "policy", values_to = "config") %>% + left_join(m %>% mutate(config = as.character(config)), by = "config") +knitr::kable( + cr %>% + transmute(anchor, policy, col = paste(season, h), v = sprintf("%+.1f", wis_pct)) %>% + tidyr::pivot_wider(names_from = col, values_from = v), + caption = "WIS reduction %, states only, by season and horizon" +) +first_live <- internals %>% + filter(season == LIVE[2], config %in% pairs$carry) %>% + group_by(config, h) %>% + slice_min(reference_date, n = 1, with_ties = FALSE) %>% + ungroup() +knitr::kable( + first_live %>% + transmute(config, h, v = sprintf("%+.3f", offset_rel)) %>% + tidyr::pivot_wider(names_from = h, values_from = v), + caption = sprintf("Median-level offset / width at the first %s round, carried in from %s", LIVE[2], LIVE[1]) +) +``` + +```{r carry-reset-month, fig.height=4} +cr_month <- by_month %>% + filter(config %in% c(pairs$carry, pairs$reset), season == LIVE[2]) %>% + mutate(policy = ifelse(config %in% pairs$reset, "reset", "carry")) +ggplot(cr_month, aes(month, wis_pct, colour = anchor, linetype = policy, group = config)) + + geom_hline(yintercept = 0, colour = "grey60") + + geom_line() + + geom_point(size = 1) + + facet_wrap(~h, nrow = 1) + + scale_colour_manual(values = ANCHOR_COLS) + + labs(title = paste(LIVE[2], "by month"), x = NULL, y = "WIS reduction %", colour = NULL, linetype = NULL) + + theme(legend.position = "bottom") +``` diff --git a/reports/writeups/calibration_experiments/e20_structure_grid.Rmd b/reports/writeups/calibration_experiments/e20_structure_grid.Rmd new file mode 100644 index 00000000..7bb1be9d --- /dev/null +++ b/reports/writeups/calibration_experiments/e20_structure_grid.Rmd @@ -0,0 +1,239 @@ +--- +title: "E20: warm start and leak on both learning-rate anchors, flu and covid" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 3 +params: + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +# The month view is knit once per disease, so its chunk labels repeat. +options(knitr.duplicate.label = "allow") +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +Question: do the warm start and the leak change WIS and coverage in the same +way on the adaptive learning rate and on the constant one, and on both +diseases? E04 and E05 asked this on the flu ensemble with the adaptive rate +only; E18 found that the leak's effect depends on how large the adaptive +step gets late in the season, so the two anchors may answer differently. + +Setup: + +- Diseases: flu and covid. Base forecaster: the clean `windowed_seasonal` + replay (53 locations, the hub rounds). Horizons h0–h3. +- Learning truth: exact (re-run at every round on the data published by then; + E00), one week of extra delay. +- Warm configs run with the 2023-24 season present (the HHS-stitched backfill + from E13) and start the offset from that season's error quantiles; cold + configs start at zero with no burn-in rounds, so they play the base through + the first live rounds. +- Scores over the two live seasons, each on its own rounds; states only (US + excluded). + +The grid is 2 × 2 × 2 (E18 step 2 shows the slow term does nothing, so it is +left out), plus prod's two configs: + +| factor | levels | +|---|---| +| anchor | `adaptive 0.03`: sqrt scale, eta = 0.03 × the 90th percentile of the last 20 rounds' absolute errors, floor 0.001 (E01's elbow). `constant 0.018`: sqrt of admissions per 100k, eta = 0.018 (E16's elbow, picked on flu) | +| start | `cold`: offsets start at zero. `warm`: `slow_init = "burn_in_quantile"` from the 2023-24 season | +| leak | `no leak`. `leak 0.1`: the offset shrinks by 10% each round | +| references | `REF-op warm` (flu prod: adaptive, warm, leak, plus a slow term); `REF-op cold` (covid prod) | + +WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated +WIS lower than base. L1 coverage bias: lower is better. "Month-avg" coverage +bias is computed within each calendar month and then averaged. + +Every WIS cell carries a 90% interval from a round bootstrap: the season's +rounds are resampled with replacement 1000 times and the WIS reduction +recomputed on each draw (`cal_wis_boot()`). The same draws score every +config, so a gap between two configs has a paired interval; the "Paired +gaps" tables show it for the cells the ledger compares. Outcomes within a +round move together, so the round is the resampling unit; this ignores +dependence between neighbouring rounds, so the intervals are if anything +too narrow. + +```{r load} +source(here::here("scripts/calibration/calibration_ws_replay.R")) +DISEASES <- c(flu = "flu", covid = "covid") +RUNS <- purrr::map(DISEASES, function(d) { + wi <- ws_inputs(d) + fc_burn_in <- bind_rows(ws_burn_in_forecasts(d), wi$fc) + specs <- c(ws_grid_configs(wi$rate_scales), ws_prod_specs(wi$rate_scales)) + list(wi = wi, specs = specs, cals = purrr::map(specs, ws_run_spec, wi = wi, fc_burn_in = fc_burn_in, workers = params$workers)) +}) +grid_names <- names(ws_grid_configs(RUNS$flu$wi$rate_scales)) +split_name <- function(d) { + d %>% + mutate(config = as.character(config)) %>% + tidyr::separate_wider_delim(config, " | ", names = c("anchor", "start", "leak"), cols_remove = FALSE, too_few = "align_start") +} +ANCHOR_COLS <- c(`adaptive 0.03` = "#E69F00", `constant 0.018` = "#0072B2") +h_levels <- paste0("h", 0:3) +``` + +```{r metrics} +M <- purrr::map(RUNS, function(r) { + m <- ws_wis_by_season(r$cals) + mv <- cal_month_view(r$cals, seasons = LIVE) + cov <- mv$cov %>% transmute(config, season, h = factor(paste0("h", horizon), levels = h_levels), pooled = pooled_cal, mavg = mavg_cal) + bt <- cal_wis_boot(r$cals, seasons = LIVE) + list(m = m, mv = mv, cov = cov, bt = bt) +}) +``` + +# WIS by season and horizon + +```{r wis-table, results="asis"} +for (d in names(M)) { + cat("\n\n", cal_h(2), " ", d, "\n\n", sep = "") + cal_wis_boot_table(M[[d]]$bt) %>% + knitr::kable(caption = paste0(d, ": ", cal_wis_boot_caption(M[[d]]$bt))) %>% + print() +} +``` + +# Paired gaps + +The gap in WIS reduction % between two cells, with its paired bootstrap +interval (same draws on both sides). An interval that excludes zero in a +season means the ranking of the pair survived resampling that season's +rounds. + +```{r paired-gaps, results="asis"} +PAIRS <- list( + flu = list( + c("constant 0.018 | warm | leak 0.1", "constant 0.018 | warm | no leak"), + c("constant 0.018 | warm | leak 0.1", "REF-op warm"), + c("constant 0.018 | warm | leak 0.1", "adaptive 0.03 | warm | leak 0.1"), + c("constant 0.018 | warm | leak 0.1", "constant 0.018 | cold | leak 0.1") + ), + covid = list( + c("constant 0.018 | cold | leak 0.1", "REF-op cold"), + c("constant 0.018 | cold | leak 0.1", "constant 0.018 | cold | no leak"), + c("constant 0.018 | cold | leak 0.1", "constant 0.018 | warm | leak 0.1") + ) +) +for (d in names(M)) { + cat("\n\n", cal_h(2), " ", d, "\n\n", sep = "") + purrr::map(PAIRS[[d]], function(pr) { + cal_wis_boot_diff(M[[d]]$bt, pr[1], pr[2]) %>% mutate(pair = paste(pr[1], "−", pr[2]), .before = 1) + }) %>% + bind_rows() %>% + transmute(pair, col = paste(season, h), cell = sprintf("%+.1f [%+.1f, %+.1f]", diff, lo, hi)) %>% + tidyr::pivot_wider(names_from = col, values_from = cell) %>% + knitr::kable(caption = paste0(d, ": gap in WIS reduction % (first − second) with its 90% paired bootstrap interval")) %>% + print() +} +``` + +# What each factor changes, per anchor + +The change from adding the warm start (warm − cold) at each leak level, and +from adding the leak (leak − no leak) at each start, per anchor. WIS: change +in WIS reduction % (positive = the factor lowers WIS). Coverage: change in +month-avg L1 coverage bias (negative = the factor improves coverage). + +```{r factor-deltas, results="asis"} +contrast <- function(d, factor, hi, lo, held) { + d %>% + select(anchor, start, leak, season, h, wis_pct, mavg) %>% + tidyr::pivot_wider(names_from = all_of(factor), values_from = c(wis_pct, mavg)) %>% + transmute( + anchor, season, h, + factor = paste0("+ ", hi, " (", .data[[held]], ")"), + d_wis = .data[[paste0("wis_pct_", hi)]] - .data[[paste0("wis_pct_", lo)]], + d_mavg = .data[[paste0("mavg_", hi)]] - .data[[paste0("mavg_", lo)]] + ) +} +DELTAS <- purrr::imap(M, function(x, d) { + g <- x$m %>% + filter(config %in% grid_names) %>% + select(config, season, h, wis_pct) %>% + left_join(x$cov, by = c("config", "season", "h")) %>% + split_name() + bind_rows(contrast(g, "start", "warm", "cold", "leak"), contrast(g, "leak", "leak 0.1", "no leak", "start")) %>% + mutate(disease = d) +}) +for (d in names(DELTAS)) { + cat("\n\n", cal_h(2), " ", d, "\n\n", sep = "") + DELTAS[[d]] %>% + transmute(anchor, factor, col = paste(season, h), cell = sprintf("%+.1f / %+.3f", d_wis, d_mavg)) %>% + tidyr::pivot_wider(names_from = col, values_from = cell) %>% + arrange(anchor, factor) %>% + knitr::kable(caption = paste0(d, ": change in WIS reduction % / month-avg L1 coverage bias from adding the factor, per anchor and season")) %>% + print() +} +``` + +```{r factor-plot, fig.height=8} +ggplot(bind_rows(DELTAS) %>% filter(season != "both"), aes(h, d_wis, colour = anchor, group = anchor)) + + geom_hline(yintercept = 0, colour = "grey60") + + geom_line() + + geom_point() + + facet_grid(disease + season ~ factor) + + scale_colour_manual(values = ANCHOR_COLS) + + labs(x = NULL, y = "change in WIS reduction %", colour = NULL) + + theme(legend.position = "bottom") +``` + +# Anchor against anchor, state by state + +Per state, pooled over h0–h3: WIS reduction % of the adaptive anchor against +the constant one at the same start and leak (above the line, adaptive is +better). + +```{r state-scatter, results="asis", fig.height=5.5} +STATE <- purrr::map(RUNS, function(r) { + cal_state_wis(r$cals[grid_names]) %>% + group_by(config, season, location) %>% + summarize(pct = 100 * (sum(wis_base) - sum(wis_cal)) / sum(wis_base), .groups = "drop") %>% + split_name() %>% + select(-config) %>% + tidyr::pivot_wider(names_from = anchor, values_from = pct) %>% + mutate(cell = paste(start, leak, sep = ", ")) +}) +for (d in names(STATE)) { + cat("\n\n", cal_h(2), " ", d, "\n\n", sep = "") + print( + ggplot(STATE[[d]], aes(`constant 0.018`, `adaptive 0.03`)) + + geom_abline(slope = 1, intercept = 0, colour = "grey50") + + geom_hline(yintercept = 0, colour = "grey85") + + geom_vline(xintercept = 0, colour = "grey85") + + geom_point(alpha = 0.8) + + facet_grid(season ~ cell) + + labs(title = d, x = "constant 0.018: WIS reduction %, h0–h3", y = "adaptive 0.03: WIS reduction %, h0–h3") + ) + STATE[[d]] %>% + group_by(cell, season) %>% + summarize(adaptive_better = sum(`adaptive 0.03` > `constant 0.018`), states = n(), .groups = "drop") %>% + transmute(cell, season, v = sprintf("%d / %d", adaptive_better, states)) %>% + tidyr::pivot_wider(names_from = season, values_from = v) %>% + knitr::kable(caption = paste0(d, ": states (of those scored) where the adaptive anchor beats the constant one, h0–h3 pooled")) %>% + print() +} +``` + +# Month views + +The month plots show the no-leak grid and flu prod; the tables keep every +config. + +```{r month-views, results="asis"} +mv_plot_cfgs <- c(grid_names[c(1, 3, 5, 7)], "REF-op warm") +mv_cols <- setNames(c("#E69F00", "#D55E00", "#0072B2", "#56B4E9", "#000000"), mv_plot_cfgs) +for (d in names(M)) { + mv <- M[[d]]$mv + cal_month_section(d, level = 2L) +} +``` diff --git a/reports/writeups/calibration_experiments/e21_data_by_season.Rmd b/reports/writeups/calibration_experiments/e21_data_by_season.Rmd new file mode 100644 index 00000000..3afeea2d --- /dev/null +++ b/reports/writeups/calibration_experiments/e21_data_by_season.Rmd @@ -0,0 +1,268 @@ +--- +title: "E21: the two live seasons' data" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + disease: flu +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +Question: how do 2024-25 and 2025-26 differ as data? Most tracker rankings +flip between the two seasons (ledger), so before reading a ranking as a +property of the tracker, this notebook describes what each season gave the +tracker to learn from: the shape of the season, how much the published +values were later revised, how far the value the tracker learns from sits +from the final one, and how the HHS-stitched 2023-24 burn-in compares with +NHSN. + +Setup: + +- Disease: `r params$disease`. Truth and vintages are the replay's + (`ws_inputs()`): the NHSN archive on hub coordinates, final values for + scoring, every vintage for learning. +- Seasons are August–July years. Hub rounds are the CMU-TimeSeries + submission rounds; states only (US excluded) unless stated. + +```{r load} +source(here::here("scripts/calibration/calibration_ws_replay.R")) +wi <- ws_inputs(params$disease) +states <- function(d) d %>% filter(location != "US") +truth <- states(wi$truth) %>% mutate(season = season_of(target_end_date)) +vint <- states(wi$vintages) %>% mutate(season = season_of(target_end_date)) +rounds <- wi$fc %>% distinct(reference_date) %>% mutate(season = season_of(reference_date)) %>% filter(season %in% LIVE) +pop <- wi$rate_scales %>% select(location, scale) +SEASON_COLS <- c(`2024-2025` = "#D55E00", `2025-2026` = "#0072B2") +``` + +# Season shape + +```{r shape-table} +natl <- truth %>% + filter(season %in% LIVE) %>% + group_by(season, target_end_date) %>% + summarize(v = sum(truth, na.rm = TRUE), .groups = "drop") +peaks <- natl %>% group_by(season) %>% slice_max(v, n = 1, with_ties = FALSE) %>% select(season, peak_week = target_end_date, peak = v) +knitr::kable( + rounds %>% + group_by(season) %>% + summarize(rounds = n(), first_round = min(reference_date), last_round = max(reference_date), .groups = "drop") %>% + left_join(peaks, by = "season") %>% + left_join(natl %>% group_by(season) %>% summarize(total = sum(v), .groups = "drop"), by = "season") %>% + mutate(across(c(peak, total), \(x) round(x))), + caption = "Hub rounds, peak week and size (states summed), and season total admissions" +) +``` + +```{r shape-plot, fig.height=4} +natl_wk <- natl %>% + group_by(season) %>% + mutate(week = as.integer(target_end_date - min(target_end_date)) %/% 7L + 1L) %>% + ungroup() +round_wk <- rounds %>% + left_join(natl %>% group_by(season) %>% summarize(start = min(target_end_date), .groups = "drop"), by = "season") %>% + mutate(week = as.integer(reference_date - start) %/% 7L + 1L) +ggplot(natl_wk, aes(week, v, colour = season)) + + geom_rug(data = round_wk, aes(week, colour = season), inherit.aes = FALSE, sides = "b", alpha = 0.6) + + geom_line() + + scale_colour_manual(values = SEASON_COLS) + + labs(x = "week of the season (from the first August week)", y = "admissions, states summed", colour = NULL, + subtitle = "Ticks along the bottom are hub rounds") + + theme(legend.position = "bottom") +``` + +Per state: when each state peaked relative to the national peak, and how +large the peak was per 100k. + +```{r state-peaks, fig.height=4} +state_pk <- truth %>% + filter(season %in% LIVE) %>% + group_by(season, location) %>% + slice_max(truth, n = 1, with_ties = FALSE) %>% + ungroup() %>% + left_join(peaks, by = "season") %>% + left_join(pop, by = "location") %>% + mutate(weeks_from_national = as.integer(target_end_date - peak_week) %/% 7L, peak_per_100k = truth / scale) +knitr::kable( + state_pk %>% + group_by(season) %>% + summarize( + `peak offset, weeks (10th / 50th / 90th pct)` = sprintf("%+d / %+d / %+d", quantile(weeks_from_national, 0.1) %/% 1, median(weeks_from_national) %/% 1, quantile(weeks_from_national, 0.9) %/% 1), + `states within ±1 week of national` = sum(abs(weeks_from_national) <= 1), + `peak per 100k (median)` = round(median(peak_per_100k, na.rm = TRUE), 1), + .groups = "drop" + ), + caption = "State peaks relative to the national peak week" +) +ggplot(state_pk, aes(weeks_from_national, fill = season)) + + geom_bar(position = "dodge") + + scale_fill_manual(values = SEASON_COLS) + + labs(x = "state peak week minus national peak week", y = "states", fill = NULL) + + theme(legend.position = "bottom") +``` + +# Revisions + +Every vintage of each (state, week) in the season, summarized by +`epiprocess::revision_analysis()`: how many times the value changed, the +spread between its smallest and largest published value relative to the +largest, and how long it took to settle within 10% of the final value. Weeks +before the first archive version (2024-11-19) have no revision history and +are dropped. + +```{r revisions} +archive_of <- function(d) { + d %>% + transmute(geo_value = location, time_value = target_end_date, version, value = truth) %>% + epiprocess::as_epi_archive(compactify = TRUE) +} +first_version <- min(vint$version) +rev <- purrr::map(LIVE, function(s) { + a <- vint %>% filter(season == s, target_end_date >= first_version) %>% archive_of() + a$time_type <- "day" + epiprocess::revision_analysis(a, value, min_waiting_period = NULL, return_only_tibble = TRUE) %>% mutate(season = s) +}) %>% bind_rows() +knitr::kable( + rev %>% + group_by(season) %>% + summarize( + keys = n(), + `no revision` = sprintf("%.0f%%", 100 * mean(n_revisions == 0)), + `revisions (median / 90th pct)` = sprintf("%d / %d", median(n_revisions), quantile(n_revisions, 0.9)), + `rel. spread (median / 90th pct)` = sprintf("%.3f / %.3f", median(rel_spread, na.rm = TRUE), quantile(rel_spread, 0.9, na.rm = TRUE)), + `rel. spread > 10%` = sprintf("%.0f%%", 100 * mean(rel_spread > 0.1, na.rm = TRUE)), + `days to within 10% of final (median / 90th pct)` = sprintf("%s / %s", median(as.numeric(lag_near_latest), na.rm = TRUE), quantile(as.numeric(lag_near_latest), 0.9, na.rm = TRUE)), + .groups = "drop" + ), + caption = "Revision behaviour of (state, week) values, per season" +) +``` + +```{r revisions-plot, fig.height=4} +ggplot(rev %>% filter(!is.na(rel_spread)), aes(rel_spread, colour = season)) + + stat_ecdf() + + coord_cartesian(xlim = c(0, 0.5)) + + scale_colour_manual(values = SEASON_COLS) + + labs(x = "relative spread of published values", y = "share of (state, week) keys", colour = NULL) + + theme(legend.position = "bottom") +``` + +# The value the tracker learns from + +The tracker learns from the value as published, with one week of extra +delay (`settle_days = 14`), so a week's outcome enters about two Wednesdays +after the week ends. Here, for each (state, week), the value as of the +Wednesday L weeks after the week end against the final value: the signed +relative gap (negative = the early value was below the final one), per +season and lag, and by month at L = 2. + +```{r learn-gap} +as_of_lag <- function(L) { + vint %>% + filter(season %in% LIVE, version <= target_end_date + 7L * L - 3L) %>% + group_by(season, location, target_end_date) %>% + slice_max(version, n = 1, with_ties = FALSE) %>% + ungroup() %>% + select(season, location, target_end_date, early = truth) %>% + mutate(lag = L) +} +gap <- purrr::map(1:4, as_of_lag) %>% + bind_rows() %>% + inner_join(truth %>% select(location, target_end_date, final = truth), by = c("location", "target_end_date")) %>% + filter(target_end_date >= first_version, final > 0) %>% + mutate(rel = (early - final) / final, month = factor(format(target_end_date, "%b"), levels = month.abb[c(8:12, 1:7)])) +knitr::kable( + gap %>% + group_by(season, lag) %>% + summarize( + keys = n(), + `not yet published` = sprintf("%.0f%%", 100 * mean(is.na(early))), + `signed gap (10th / 50th / 90th pct)` = sprintf("%+.3f / %+.3f / %+.3f", quantile(rel, 0.1, na.rm = TRUE), median(rel, na.rm = TRUE), quantile(rel, 0.9, na.rm = TRUE)), + `|gap| > 10%` = sprintf("%.0f%%", 100 * mean(abs(rel) > 0.1, na.rm = TRUE)), + .groups = "drop" + ), + caption = "Value as of the Wednesday L weeks after the week end, relative to the final value" +) +``` + +```{r learn-gap-month, fig.height=4} +gm <- gap %>% + filter(lag == 2, !is.na(rel)) %>% + group_by(season, month) %>% + summarize(p10 = quantile(rel, 0.1), p50 = median(rel), p90 = quantile(rel, 0.9), n = n(), .groups = "drop") +ggplot(gm, aes(month, p50, colour = season, group = season)) + + geom_hline(yintercept = 0, colour = "grey60") + + geom_ribbon(aes(ymin = p10, ymax = p90, fill = season), alpha = 0.15, colour = NA) + + geom_line() + + geom_point() + + scale_colour_manual(values = SEASON_COLS) + + scale_fill_manual(values = SEASON_COLS) + + coord_cartesian(ylim = c(-0.3, 0.3)) + + labs(x = NULL, y = "(value at lag 2 − final) / final", colour = NULL, fill = NULL, subtitle = "Median with the 10th–90th percentile band over (state, week)") + + theme(legend.position = "bottom") +``` + +# The 2023-24 burn-in + +Warm configs read the 2023-24 season from the HHS archive stitched in before +NHSN's first version (E13). NHSN's final values also cover 2023-24, so the +two sources can be compared week by week: the latest HHS value against the +NHSN final, by month (states summed) and per state. + +```{r burn-in} +hhs <- readRDS(file.path(CH_CACHE_DIR, "hhs_weekly_archive_2020_2024.rds"))[[params$disease]] %>% + as_tibble() %>% + filter(!is.na(value)) %>% + group_by(geo_value, time_value) %>% + slice_max(version, n = 1, with_ties = FALSE) %>% + ungroup() %>% + # HHS weeks are dated by the Wednesday; NHSN by the Saturday week end. + transmute(geo_value, target_end_date = time_value + 3L, hhs = value) %>% + nhsn_to_hub_locations() %>% + states() %>% + filter(season_of(target_end_date) == "2023-2024") +cmp <- hhs %>% + inner_join(truth %>% select(location, target_end_date, nhsn = truth), by = c("location", "target_end_date")) %>% + mutate(month = factor(format(target_end_date, "%b"), levels = month.abb[c(8:12, 1:7)])) +knitr::kable( + cmp %>% + group_by(month) %>% + summarize(weeks = n_distinct(target_end_date), states = n_distinct(location), hhs = sum(hhs), nhsn = sum(nhsn), .groups = "drop") %>% + mutate(`hhs / nhsn` = sprintf("%.3f", hhs / nhsn), across(c(hhs, nhsn), \(x) round(x))), + caption = "2023-24, states summed: latest HHS value against the NHSN final" +) +# States with no NHSN admissions over the season have no ratio. +state_ratio <- cmp %>% + group_by(location) %>% + summarize(ratio = sum(hhs) / sum(nhsn), .groups = "drop") %>% + filter(is.finite(ratio)) +knitr::kable( + state_ratio %>% + summarize( + `ratio (10th / 50th / 90th pct)` = sprintf("%.3f / %.3f / %.3f", quantile(ratio, 0.1), median(ratio), quantile(ratio, 0.9)), + `states below 0.9` = sum(ratio < 0.9), + `states above 1.1` = sum(ratio > 1.1) + ), + caption = "Per-state ratio of HHS to NHSN over the 2023-24 season" +) +``` + +```{r burn-in-plot, fig.height=4} +ggplot(cmp %>% group_by(target_end_date) %>% summarize(hhs = sum(hhs), nhsn = sum(nhsn), .groups = "drop") %>% tidyr::pivot_longer(-target_end_date), + aes(target_end_date, value, colour = name)) + + geom_line() + + scale_colour_manual(values = c(hhs = "#E69F00", nhsn = "#000000")) + + labs(x = NULL, y = "admissions, states summed", colour = NULL) + + theme(legend.position = "bottom") +``` diff --git a/reports/writeups/calibration_experiments/index.Rmd b/reports/writeups/calibration_experiments/index.Rmd new file mode 100644 index 00000000..4e52cec8 --- /dev/null +++ b/reports/writeups/calibration_experiments/index.Rmd @@ -0,0 +1,160 @@ +--- +title: "Calibration experiments" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + toc: true + toc_float: true +--- + +```{r setup, include=FALSE} +knitr::opts_knit$set(root.dir = here::here()) +``` + +These notebooks test a post-hoc calibration step for our FluSight and covid +hub submissions. The step adjusts each submitted quantile forecast by an +additive offset that an online quantile tracker (MultiQT; Ding, Gibbs & +Tibshirani, 2025) learns from past coverage misses. Each notebook changes one +setting of that tracker and compares the result with the uncalibrated +forecast. + +There are no uncertainty estimates, so a WIS gap under 1 point or a +month-avg coverage gap under 0.01 is read as no clear difference. +Conclusions are kept in `notes/calibration-ledger.md`. + +```{r rows, include=FALSE} +out_dir <- here::here("reports/calibration_experiments") +link <- function(id, file) { + html <- paste0(file, ".html") + if (!file.exists(file.path(out_dir, html))) return(paste(id, "(not rendered)")) + stamp <- format(file.mtime(file.path(out_dir, html)), "%m-%d", tz = "America/Los_Angeles") + sprintf("[%s](%s) (%s)", id, html, stamp) +} +``` + +# Notebooks + +One notebook per experiment, grouped by topic; the date after each link is +when it was last rendered. The ids match `notes/calibration-ledger.md`. + +## Finalists + +| | base | question | +|---|---|---| +| `r link("E10", "e10_covid")` | both diseases, both bases | How do the finalists do on flu and covid, and do they rank the same way on the ensemble and the replay? (Part 2: covid ensemble at h−1–h3.) | +| `r link("E18", "e18_ref_op_bridge")` | replay | Which setting accounts for the gap between REF-op and sqrt constant 0.018, stepping from one to the other a setting at a time? | +| `r link("E15", "e15_wis_sources")` | replay, both diseases | Where does each finalist's WIS change come from? | + +## Prod configs + +`r link("Prod views", "prod_views")`: per-state forecast fans and a gallery +of the largest gains and losses, for flu REF-op warm and covid REF-op cold +on the submitted ensemble. + +## Bases + +| | base | question | +|---|---|---| +| `r link("E07", "e07_across_eras")` | both | Does calibration act the same on the ensemble and the clean replay? | + +## Learning rate + +| | base | question | +|---|---|---| +| `r link("E01", "e01_eta_settings")` | replay, cold | How do the adaptive learning-rate settings (multiplier, window, carry or reset) change WIS and coverage? | +| `r link("E09", "e09_lr_delay")` | ensemble | With a constant learning rate on the rate scale, how do WIS and coverage change with the rate and with extra learning delay? | +| `r link("E16", "e16_sqrt_constant_lr")` | replay, cold | How do WIS and coverage change with a constant learning rate, season by season, on the square-root and rate scales? | +| `r link("E19", "e19_tracker_internals")` | replay | Month by month, how large a step does each tracker take, how large an offset does it play, and does the month's WIS change follow from those and the base's bias? How much of 2025-26 is the offset carried in from 2024-25? | + +The two learning-rate anchors used by the structure notebooks are E01's +adaptive multiplier 0.03 and E16's constant 0.018 on the sqrt per-100k +scale. + +## Tracker structure + +E20 asks the warm-start and leak questions on both anchors and both +diseases, on the replay. E03 is the only scale comparison: every E20 cell +runs on the square-root scale. (E04, E05 and E11 asked the leak, warm-start +and constant-rate questions on the ensemble with the adaptive rate; E20 +reproduces their directions on the replay, so they were retired on +2026-10-07. Their numbers stay in the ledger.) + +| | base | question | +|---|---|---| +| `r link("E20", "e20_structure_grid")` | replay, both diseases | Do the warm start and the leak change WIS and coverage in the same way on the adaptive and the constant learning rate, for flu and for covid? State by state, which anchor wins at the same start and leak? | +| `r link("E03", "e03_scale")` | ensemble | Which scale should the offsets live on: counts, square root, or rate per 100k? | + +## Late season + +| | base | question | +|---|---|---| +| `r link("E02", "e02_eta_variants")` | replay, cold | Do a spring switch-off, a per-level learning rate or a seasonal window change WIS and coverage? | +| `r link("E17", "e17_late_decay")` | replay, cold | Does shrinking or switching off the offset from 1 March change WIS and coverage? | + +## Learning truth + +| | base | question | +|---|---|---| +| `r link("E00", "e00_vintage_backtest")` | ensemble | Should the tracker learn from finalized data or from data as published? | + +## Data + +| | base | question | +|---|---|---| +| `r link("E21", "e21_data_by_season")` | replay data | How do 2024-25 and 2025-26 differ as data: season shape, revisions, the gap between the value the tracker learns from and the final one, and the HHS-stitched 2023-24 burn-in against NHSN? | + +E12 and E13 (the clean replay without and with a 2023-24 burn-in) have no +notebook; they are `scripts/calibration/calibration_ws_replay.R`. E14 is +part of E16, and E07b's ranking is part of E10. + +# How to read the results + +- **Horizons** h−1 to h3 are weeks relative to the forecast's reference date. + h−1 is the latest week NHSN has reported, still open to revision; h0 and + later have not been reported yet. +- **WIS reduction %** is `100 × (WIS_base − WIS_cal) / WIS_base`, the + calibrated forecast's weighted interval score against the uncalibrated + forecast's, pooled over locations. Positive = calibrated WIS lower than + base. +- **L1 coverage bias** (lower is better) is the mean over the 23 quantile + levels of |empirical coverage − level|. **Month-avg** computes it within + each calendar month and averages; season-pooled lets under-prediction on + the ramp cancel over-prediction after the peak, and usually looks much + better. Month-level noise is about 0.01. +- **Two bases.** "Ensemble" notebooks calibrate our submitted forecasts + (h−1 to h3). "Replay" notebooks calibrate a clean weekly replay of + `windowed_seasonal` with no hand edits (h0–h3). +- **Scoring.** Every score is states only (US excluded), against finalized + NHSN data over the live seasons (2024-25 and 2025-26). A burn-in season (2023-24 for flu) only warms + up the learning rate and is not scored. +- **Cold start.** Without a warm start, offsets start at zero, and every + config equals the base through Nov–Dec 2024, so 2024-25 gains are diluted. +- **In-sample choices.** The constant rates and switch-off dates in later + notebooks were picked on the same two seasons they are scored on. +- **Excluded submissions.** A few submissions were broken by pipeline bugs + or bad data and are left out, both from what the tracker learns and from + the scores. Flu: h−1 of the 2025-11-22 round (every quantile was 0) and + the US forecasts of the 2024-12-14, 2024-12-21 and 2025-01-04 rounds + (about half the sum of the states). Covid: the whole 2024-11-23 round + (made from a bad NHSN release that was corrected the next day) and h−1 of + the 2026-04-25 round (near zero in many states, most likely from a stale + data fetch). +- **Extra delay.** The tracker can learn from a week's outcome as soon as + NHSN first reports it (zero extra delay) or wait for revisions. The + reference configs wait one extra week. +- **Re-run vs stateful.** Prod re-runs the tracker from the first round each + week on that week's data, so early reports are replaced by their revisions. + A stateful tracker instead learns each outcome once, from the first value + it sees. All notebooks use the re-run design; E00 and E09 also show the + stateful one. + +## Reference configs + +- **REF-paper**: the paper's tracker on counts. One offset per quantile level, + updated with an adaptive learning rate (0.03 × the 90th percentile of the + last 20 rounds' absolute errors), carried across seasons, with one week of + extra delay. +- **REF-op**: the configuration running in prod. REF-paper on the + square-root scale, with the offset shrinking 10% per round, a second + slow-moving offset, and a starting offset taken from the burn-in season's + errors. diff --git a/reports/writeups/calibration_experiments/prod_views.Rmd b/reports/writeups/calibration_experiments/prod_views.Rmd new file mode 100644 index 00000000..71db3d65 --- /dev/null +++ b/reports/writeups/calibration_experiments/prod_views.Rmd @@ -0,0 +1,106 @@ +--- +title: "Prod configs: forecasts by state" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + flu_hub_dir: "../FluSight-forecast-hub" + covid_hub_dir: "../covid19-forecast-hub" + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 10, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +What the calibrated forecasts look like next to the submitted ones, for the +configs running in prod: + +- **Flu**: REF-op warm (warm start from the 2023-24 burn-in season). +- **Covid**: REF-op cold (no burn-in season; covid submissions start + 2024-11-23). + +Both calibrate the submitted `CMU-TimeSeries` ensemble, learn from NHSN data +as published (re-run every round, as prod does; E00), and are drawn against +finalized truth. Spoiled submissions are dropped. States only (US excluded). Seasons run August–July; +only the two live seasons (2024-25 and 2025-26) are shown. + +```{r load} +suppressPackageStartupMessages(source(here::here("R/load_all.R"))) +LIVE <- c("2024-2025", "2025-2026") +# Same inputs and arguments as the E05/E11 (flu) and E10 (covid) runs, so the cached runs are reused. +load_prod <- function(disease, hub_dir, run_args) { + archive <- get_nhsn_data_archive(disease) + truth <- nhsn_read_truth(disease, archive) + vintages <- nhsn_read_vintages(disease, archive) + hub_target <- c(flu = HUB_FLU_TARGET, covid = HUB_COVID_TARGET)[[disease]] + hub_fc <- suppressMessages(hub_read_forecasts(hub_dir = here::here(hub_dir), target = hub_target)) + population <- get_population_data() %>% + distinct(state_code, .keep_all = TRUE) %>% + filter(state_code %in% unique(hub_fc$location)) %>% + select(location = state_code, population) + cal <- do.call(cal_run_cached, c(list(hub_fc, truth, "op"), run_args, list(vintages = vintages, workers = params$workers))) + cal$forecasts <- cal$forecasts %>% mutate(season = cal_season_of(reference_date)) + list(cal = cal, truth = truth, vintages = vintages, population = population) +} +PROD <- list( + flu = load_prod("flu", params$flu_hub_dir, list()), + covid = load_prod("covid", params$covid_hub_dir, list(burn_in_seasons = character(0), slow_init = NULL)) +) +abbrev <- setNames(toupper(get_population_data()$state_id), get_population_data()$state_code) +STATES <- CAL_STATES[names(CAL_STATES) != "US"] +``` + +```{r sections, results="asis", fig.height=4} +for (d in names(PROD)) { + p <- PROD[[d]] + cat("\n\n# ", d, "\n\n", sep = "") + + cat("\n\n## Fixed states {.tabset}\n\n") + cat( + "The same states (CA, GA, MN, VT) for both diseases. Every other", + "round's fan at h−1–h3: blue ribbon and line are the base 80% band and", + "median, orange lines the calibrated 10/50/90% quantiles, grey the", + "finalized truth. The black line is the last two reported weeks of NHSN", + "data as published when that round was forecast; its dot is normally the", + "round's h−1 week, where its fan starts. Black sits below grey where first", + "reports were later revised up. A black line with no fan in front of it is", + "a round with no submission.\n\n" + ) + for (nm in names(STATES)) { + cat("\n\n### ", nm, "\n\n", sep = "") + for (s in LIVE) print(cal_state_panel(p$cal, STATES[[nm]], s, p$truth, p$vintages, title = paste(d, nm, s))) + } + + cat("\n\n## Gallery {.tabset}\n\n") + cat( + "For each season phase and location size (population above or below the", + "median), the forecast with the largest WIS gain and the one with the", + "largest loss, each from a different location. Forecasts are ranked by", + "their WIS reduction as a share of their own location-season's base WIS,", + "so small states late in the season, where the base is tiny, do not crowd", + "the gallery. Ribbons are the 50% and 90% bands, the dashed line is the", + "forecast date.\n\n" + ) + cal_live <- p$cal + cal_live$forecasts <- cal_live$forecasts %>% filter(season %in% LIVE, location != "US") + picks <- cal_gallery_pick(cal_live, p$population %>% filter(location != "US")) + for (i in seq_len(nrow(picks))) { + pk <- picks[i, ] + if (i == 1L || pk$phase != picks$phase[i - 1L]) cat("\n\n### ", as.character(pk$phase), "\n\n", sep = "") + sub <- sprintf( + "%s (%s), %s, %s: WIS reduction %+.1f%% (%+.1f%% of its season's base WIS; positive = calibrated WIS lower)", + abbrev[[pk$location]], pk$size, pk$reference_date, pk$kind, pk$wis_pct, pk$share + ) + print(cal_gallery_panel(p$cal, pk$location, pk$reference_date, p$truth, p$vintages, subtitle = sub)) + } +} +``` diff --git a/scripts/calibration/calibration_export_series.R b/scripts/calibration/calibration_export_series.R index 9d2a9df8..b5436c26 100644 --- a/scripts/calibration/calibration_export_series.R +++ b/scripts/calibration/calibration_export_series.R @@ -7,7 +7,7 @@ # the identical problem otherwise. # # Usage: -# Rscript scripts/calibration_export_series.R [outdir] [n_series] +# Rscript scripts/calibration/calibration_export_series.R [outdir] [n_series] suppressPackageStartupMessages(source(here::here("R/load_all.R"))) @@ -18,19 +18,7 @@ n_series <- if (length(args) >= 2) as.integer(args[[2]]) else 4L dir.create(outdir, showWarnings = FALSE, recursive = TRUE) forecasts <- hub_read_forecasts() -truth <- { - arch <- get_nhsn_data_archive("flu") - arch %>% - epix_as_of(arch$versions_end) %>% - mutate(geo_value = ifelse(geo_value == "usa", "us", geo_value)) %>% - left_join( - get_population_data() %>% select("state_id", location = "state_code"), - by = c("geo_value" = "state_id") - ) %>% - select(target_end_date = time_value, location, truth = value) %>% - filter(!is.na(truth)) %>% - arrange(location, target_end_date) -} +truth <- nhsn_read_truth("flu") rounds <- hub_label_seasons(unique(forecasts$reference_date)) round_date <- rounds$round_date diff --git a/scripts/calibration/calibration_harness.R b/scripts/calibration/calibration_harness.R index 4ccdf1a4..2d8bdfd8 100644 --- a/scripts/calibration/calibration_harness.R +++ b/scripts/calibration/calibration_harness.R @@ -17,8 +17,12 @@ # submitted ensemble) renormalizes to ~86% windowed_seasonal_extra_sources, # ~14% windowed_seasonal, ~0.03% climate_linear. # +# Other targets: ch_use("flu_windowed_seasonal") or ch_use("covid_windowed_seasonal") +# switch every CH_* setting to that forecaster's clean evaluation replay (see +# CH_CONFIGS); ch_read_store() then reads its forecasts instead of backfilling. +# # Usage, from the repo root: -# source("scripts/calibration_harness.R") +# source("scripts/calibration/calibration_harness.R") # inp <- ch_inputs() # archives, geos dropped # raw <- ch_backfill(inp) # ~6 min cold, instant warm (cached) # ch_verify_against_store(raw) # prove the harness == the DAG @@ -72,6 +76,87 @@ CH_SPEC <- list( join_extra_data = TRUE, sort_quantiles = TRUE ) +# What the store's forecasts were built with, for ch_verify_against_store(): the +# substitutions csv (NULL for a replay run with EVALUATION_SUBSTITUTIONS=false) +# and the grid row's output-level `excluded_geos`. +CH_STORE_SUBSTITUTIONS <- here::here("pipelines/covid_data_substitutions.csv") +CH_EXCLUDED_GEOS <- c("mo", "wy") +# Holiday/outage weeks where the evaluation replay ran after the nominal date, +# from the schedule in pipelines/_hosp_prod.R. +CH_SCHEDULE_DELAYS <- c( + "2024-12-25" = "2024-12-26", + "2025-01-01" = "2025-01-02", + "2025-12-24" = "2025-12-29" +) + +# Alternative targets for ch_use(). Each entry overrides the CH_* globals above +# (names are the globals without the prefix); spec values are copied from the +# grid row in pipelines/_hosp_prod.R, split as make_forecaster_grid() +# splits it. The replays they point at were run with substitutions off, and +# windowed_seasonal excludes no geos, so nothing is dropped. +# windowed_seasonal is calibrated at h0 and up; h−1 is left to the revision-aware methods. +CH_CONFIGS <- list( + flu_windowed_seasonal = list( + STORE = "flu_hosp_evaluation", + DISEASE = "flu", + TARGET = "wk inc flu hosp", + FORECASTER_ID = "windowed_seasonal", + AHEADS = 0:3, + DROP_GEOS = character(0), + EXCLUDED_GEOS = NULL, + STORE_SUBSTITUTIONS = NULL, + SCHEDULE_DELAYS = c("2024-11-20" = "2024-11-21", CH_SCHEDULE_DELAYS), + SPEC = list( + forecaster = "scaled_pop_seasonal", + params = list( + outcome = "value", + trainer = epipredict::quantile_reg(), + seasonal_method = "window", + pop_scaling = FALSE, + lags = c(0, 7), + keys_to_ignore = list(list(c("source"), c("flusurv", "ILI+"))) + ), + ahead_multiplier = 7L, + target_date_shift = 3L, + join_extra_data = FALSE, + sort_quantiles = TRUE + ) + ), + covid_windowed_seasonal = list( + STORE = "covid_hosp_evaluation", + DISEASE = "covid", + TARGET = "wk inc covid hosp", + FORECASTER_ID = "windowed_seasonal", + AHEADS = 0:3, + DROP_GEOS = character(0), + EXCLUDED_GEOS = NULL, + STORE_SUBSTITUTIONS = NULL, + SCHEDULE_DELAYS = CH_SCHEDULE_DELAYS, + SPEC = list( + forecaster = "scaled_pop_seasonal", + params = list( + outcome = "value", + trainer = epipredict::quantile_reg(), + seasonal_method = "none", + drop_non_seasons = TRUE, + pop_scaling = FALSE, + lags = c(0, 7) + ), + ahead_multiplier = 7L, + target_date_shift = 3L, + join_extra_data = FALSE, + sort_quantiles = TRUE + ) + ) +) + +#' Point every CH_* setting at one entry of CH_CONFIGS. +ch_use <- function(name) { + config <- CH_CONFIGS[[rlang::arg_match(name, names(CH_CONFIGS))]] + for (nm in names(config)) assign(paste0("CH_", nm), config[[nm]], envir = globalenv()) + cli::cli_alert_info("Harness target: {.field {CH_FORECASTER_ID}} ({CH_DISEASE}), store {.path {CH_STORE}}") + invisible(config) +} # ================================ INPUTS ================================== @@ -119,6 +204,32 @@ ch_inputs <- function(store = CH_STORE, drop_geos = CH_DROP_GEOS) { inputs } +#' ch_inputs() with the HHS weekly archive stitched in before NHSN's first version. +#' +#' HHS rows (from scripts/one_offs/hhs_2023_24_archive.R, run from 2020-08) are +#' relabelled `source = "nhsn"` and kept only at versions before NHSN's first, +#' so a 2023-24 snapshot sees HHS alone and later snapshots see only NHSN. +#' HHS levels are used as-is (ROADMAP 1b). +ch_inputs_with_burn_in <- function( + hhs_path = file.path(CH_CACHE_DIR, "hhs_weekly_archive_2020_2024.rds"), + inputs = ch_inputs() +) { + train <- as_tibble(inputs$nhsn_train$DT) + nhsn <- train %>% filter(source == "nhsn") + first_version <- min(nhsn$version) + hhs <- as_tibble(readRDS(hhs_path)[[CH_DISEASE]]) %>% + filter(version < first_version, geo_value %in% nhsn$geo_value) %>% + mutate(source = "nhsn") %>% + select(all_of(names(train))) + # Every HHS key must be overwritten at NHSN's first version, or HHS would leak past it. + stray <- anti_join(hhs, nhsn %>% filter(version == first_version), by = c("geo_value", "time_value")) + if (nrow(stray) > 0) cli::cli_abort("{nrow(stray)} HHS row{?s} have no NHSN row at {first_version}.") + inputs$nhsn_train <- bind_rows(train, hhs) %>% + as_epi_archive(other_keys = "source", compactify = TRUE) + cli::cli_alert_success("Stitched {nrow(hhs)} HHS rows before NHSN version {.val {format(first_version)}}") + inputs +} + #' Truth as it was published at `version`, on the forecasts' target_end_date key. #' #' This is the honest input for an online update at time `version`: it contains @@ -138,13 +249,7 @@ ch_truth_asof <- function(truth_archive, version) { #' than as the two parallel seq.Date() vectors prod uses. That is exactly #' equivalent for the dates prod covers but can't silently misalign the two #' columns if the window is changed. -ch_schedule <- function(through, from = as.Date("2024-11-20")) { - # Holiday/outage weeks where the forecast actually ran after its nominal date. - delays <- c( - "2024-12-25" = "2024-12-26", - "2025-01-01" = "2025-01-02", - "2025-12-24" = "2025-12-29" - ) +ch_schedule <- function(through, from = as.Date("2024-11-20"), delays = CH_SCHEDULE_DELAYS) { through <- as.Date(through) # Back up to the last Wednesday so the weekly sequence stays on schedule. through <- through - ((as.integer(format(through, "%u")) - 3) %% 7) @@ -227,7 +332,7 @@ ch_backfill <- function( params = CH_SPEC$params, id = CH_FORECASTER_ID, target_date_shift = CH_SPEC$target_date_shift, - join_extra_data = CH_SPEC$join_extra_data, + join_extra_data = CH_SPEC$join_extra_data %||% FALSE, extra_data = extra, excluded_geos = excluded_geos, sort_quantiles = CH_SPEC$sort_quantiles @@ -272,6 +377,42 @@ ch_backfill <- function( } +#' Read this forecaster's forecasts straight from the store's `forecast_nhsn_full`. +#' +#' Same shape as ch_backfill() output. Only equivalent to a backfill when the +#' store was built the way the backfill runs (CH_STORE_SUBSTITUTIONS NULL, no +#' DROP_GEOS), which is the case for the CH_CONFIGS replays; +#' ch_verify_against_store() checks the plumbing on a few dates. +ch_read_store <- function(store = CH_STORE, aheads = CH_AHEADS) { + fc <- targets::tar_read(forecast_nhsn_full, store = store) %>% + filter(forecaster == CH_FORECASTER_ID) %>% + mutate(ahead = as.integer(target_end_date - forecast_date - CH_SPEC$target_date_shift) %/% + CH_SPEC$ahead_multiplier) %>% + filter(ahead %in% aheads) + if (nrow(fc) == 0) cli::cli_abort("No {.field {CH_FORECASTER_ID}} forecasts in {.path {store}}.") + fc %>% + select(geo_value, forecast_date, target_end_date, quantile, value, ahead) %>% + left_join(ch_schedule(through = max(fc$forecast_date)), by = "forecast_date") %>% + arrange(forecast_date, ahead, geo_value, quantile) +} + +#' Harness forecasts in hub schema, for calibrate_hub_forecasts() and cal_run(). +ch_to_hub <- function(forecasts, disease = CH_DISEASE) { + out <- internal_to_hub_forecasts(forecasts, disease) + if (anyNA(out$location)) cli::cli_abort("Some geos have no hub location code.") + out +} + +#' Finalized truth and every vintage of the store's NHSN archive, on hub +#' coordinates: the `truth` and `vintages` arguments of cal_run(). +ch_hub_truth <- function(inputs, disease = CH_DISEASE) { + list( + truth = nhsn_read_truth(disease, inputs$truth), + vintages = nhsn_read_vintages(disease, inputs$truth) + ) +} + + # ============================== CALIBRATION =============================== #' Placeholder online calibrator: an additive per-(geo, ahead, quantile) offset @@ -422,8 +563,10 @@ ch_summarize <- function(scores) { #' Forecast dates for which the store has a cached prod forecast target. ch_cached_prod_dates <- function(store = CH_STORE) { + # The date must follow the id directly, so `windowed_seasonal` does not also + # match `windowed_seasonal_extra_sources`. prefix <- glue::glue("forecast_nhsn_{CH_FORECASTER_ID}_") - list.files(file.path(store, "objects"), pattern = glue::glue("^{prefix}")) %>% + list.files(file.path(store, "objects"), pattern = glue::glue("^{prefix}\\d{{4}}\\.")) %>% stringr::str_extract("\\d{4}\\.\\d{2}\\.\\d{2}") %>% unique() %>% stats::na.omit() %>% @@ -445,7 +588,9 @@ ch_cached_prod_dates <- function(store = CH_STORE) { #' configuration (all geos in the archive, prod's substitutions, output-level geo #' exclusion) over a couple of the cached dates and requires exact equality #' there. What it proves is that the plumbing is identical; the configured -#' deviations are then the only difference that remains. +#' deviations are then the only difference that remains. The store's own +#' substitutions and output-level exclusions come from CH_STORE_SUBSTITUTIONS +#' and CH_EXCLUDED_GEOS. ch_verify_against_store <- function(store = CH_STORE, n_dates = 2, tolerance = 0) { dates <- ch_cached_prod_dates(store) if (length(dates) == 0) { @@ -455,14 +600,13 @@ ch_verify_against_store <- function(store = CH_STORE, n_dates = 2, tolerance = 0 cli::cli_alert_info("Verifying against prod on {.val {format(dates)}}") faithful <- ch_inputs(store = store, drop_geos = character(0)) - # Prod's aheads are -1:3; overlap on the non-negative ones is enough, and the - # harness does not model ahead -1 (see CH_AHEADS). + # Prod's aheads are -1:3; compare on the ones the harness also models. mine <- ch_backfill( faithful, - aheads = intersect(CH_AHEADS, 0:3), + aheads = intersect(CH_AHEADS, -1:3), schedule = ch_schedule(through = max(dates)) %>% filter(forecast_date %in% dates), - substitutions = here::here("pipelines/covid_data_substitutions.csv"), - excluded_geos = CH_DROP_GEOS, + substitutions = CH_STORE_SUBSTITUTIONS, + excluded_geos = CH_EXCLUDED_GEOS, cache_dir = file.path(CH_CACHE_DIR, "verify") ) @@ -484,7 +628,7 @@ ch_verify_against_store <- function(store = CH_STORE, n_dates = 2, tolerance = 0 cli::cli_abort( "Harness diverges from the prod DAG: max abs diff {.val {worst}} over {.val {nrow(joined)}} overlapping rows. CH_SPEC is out of sync with - scripts/covid_hosp_prod.R, or the shared runner changed." + pipelines/{CH_DISEASE}_hosp_prod.R, or the shared runner changed." ) } cli::cli_alert_success( diff --git a/scripts/calibration/calibration_ili_backfill.R b/scripts/calibration/calibration_ili_backfill.R index 700c0aea..344b9c15 100644 --- a/scripts/calibration/calibration_ili_backfill.R +++ b/scripts/calibration/calibration_ili_backfill.R @@ -4,8 +4,8 @@ # that calibrate_hub_forecasts() consumes. ILI+ is the same augmentation source # the prod forecasters train on (see scripts/flu_hosp_prod.R `nhsn_prod_archive`), # so this is "what would our seasonal forecaster have said" for 2010-2024, on the -# ILI+ percent scale rather than NHSN counts. See notes/CALIBRATION.md, "ILI+ -# burn-in", for how the scale mismatch is handled downstream. +# ILI+ percent scale rather than NHSN counts. The `scales` option of +# calibrate_hub_forecasts() handles the scale mismatch downstream. # # ILI+ is faux-versioned (version = time_value), so every snapshot is finalized # data cut at the forecast date; there is no revision realism here. To mimic @@ -14,7 +14,7 @@ # forecast date, so hub horizon 0 is one week past the last observation. # # Usage, from the repo root (~30 min cold, cached in cache/calibration/): -# distrobox enter rocker -- Rscript scripts/calibration_ili_backfill.R +# distrobox enter rocker -- Rscript scripts/calibration/calibration_ili_backfill.R # Then in R: ili <- ili_read_pseudo_hub() # list(forecasts, truth) suppressPackageStartupMessages(source(here::here("R/load_all.R"))) @@ -72,10 +72,21 @@ ili_schedule <- function(archive) { ili_forecast_one <- function(archive, fd, geo_map) { snapshot <- make_forecast_snapshot(archive, forecast_date = fd, generation_date = fd, as_of_policy = "asof") %>% filter(time_value <= fd - 7L) + # extend_lags uses the worst latency across geos, so one state whose ILI+ + # series ended (DC stops in 2015) would push every lag years back. Leave + # such states out of the latency check; they still train, they just get no forecast. + stale <- snapshot %>% + as_tibble() %>% + group_by(geo_value) %>% + summarize(last = max(time_value)) %>% + filter(last < max(snapshot$time_value)) %>% + pull(geo_value) + params <- ILI_PARAMS + if (length(stale) > 0) params$keys_to_ignore <- list(list("geo_value", stale)) purrr::map(ILI_HORIZONS, function(h) { run_forecaster( snapshot = snapshot, forecaster = scaled_pop_seasonal, aheads = h * 7L, - params = ILI_PARAMS, id = "windowed_seasonal_ili", + params = params, id = "windowed_seasonal_ili", target_date_shift = 3L, sort_quantiles = TRUE ) %>% mutate(horizon = h) diff --git a/scripts/calibration/calibration_ws_backfill.R b/scripts/calibration/calibration_ws_backfill.R deleted file mode 100644 index 1593bf6a..00000000 --- a/scripts/calibration/calibration_ws_backfill.R +++ /dev/null @@ -1,126 +0,0 @@ -# Pseudo-hub history for calibration: replay flu prod's `windowed_seasonal` -# forecaster (scaled_pop_seasonal, seasonal_method = "window", trained on the -# nhsn + ILI+ + flusurv prod archive) over the NHSN seasons and write the result -# in the FluSight hub schema that calibrate_hub_forecasts() consumes. Together -# with scripts/calibration_ili_backfill.R this gives one forecaster's history -# from 2011 to today, so calibration can be studied without the submitted -# ensemble's changing composition in the way. -# -# The replay is the prod cell: nhsn_prod_archive from the flu_hosp_evaluation -# store, make_forecast_snapshot() + run_forecaster() with the grid row's spec -# values (copied from scripts/flu_hosp_prod.R; keep in sync), prod's seed and -# data substitutions. NHSN vintages begin 2024-11-19, so 2024-25 and 2025-26 are -# honest as-of replays; 2023-24 is replayed with as_of_policy = "cheating" -# (finalized data cut at the forecast date) and is marked as such in the output. -# -# Usage, from the repo root (~10 min cold, cached in cache/calibration/): -# distrobox enter rocker -- Rscript scripts/calibration_ws_backfill.R -# Then in R: ws <- ws_read_pseudo_hub() # forecasts; truth from get_nhsn_data_archive("flu") - -suppressPackageStartupMessages(source(here::here("R/load_all.R"))) - -WS_CACHE_DIR <- here::here("cache/calibration") -WS_FORECASTS_FILE <- file.path(WS_CACHE_DIR, "ws_pseudo_hub_forecasts.parquet") -WS_HORIZONS <- -1:3 -WS_ID <- "windowed_seasonal" -WS_STORE <- "flu_hosp_evaluation" -WS_SUBSTITUTIONS <- here::here("pipelines/flu_data_substitutions.csv") -# scripts/flu_hosp_prod.R: windowed_seasonal grid row, modeling params only. -WS_PARAMS <- list( - outcome = "value", - trainer = epipredict::quantile_reg(), - seasonal_method = "window", - pop_scaling = FALSE, - lags = list(c(0, 7)), - keys_to_ignore = list(list(c("source"), c("flusurv", "ILI+"))) -) - -#' Wednesday forecast dates with the generation-date delays prod's evaluation -#' mode uses, plus the vintage-free 2023-24 season under the cheating policy. -ws_schedule <- function(versions_end) { - delays <- c("2024-11-20" = "2024-11-21", "2024-12-25" = "2024-12-26", "2025-01-01" = "2025-01-02", "2025-12-24" = "2025-12-29") - through <- as.Date(versions_end) - through <- through - ((as.integer(format(through, "%u")) - 3) %% 7) - live <- tibble(forecast_date = seq.Date(as.Date("2024-11-20"), through, by = 7L), as_of_policy = "asof") %>% - mutate(generation_date = forecast_date) - hit <- match(as.character(live$forecast_date), names(delays)) - live$generation_date[!is.na(hit)] <- as.Date(delays[hit[!is.na(hit)]]) - cheat <- tibble( - forecast_date = seq.Date(as.Date("2023-10-11"), as.Date("2024-05-29"), by = 7L), - as_of_policy = "cheating" - ) %>% mutate(generation_date = forecast_date) - bind_rows(cheat, live) -} - -ws_geo_map <- function() { - gm <- get_population_data() %>% - distinct(state_id, .keep_all = TRUE) %>% - transmute(geo_value = state_id, location = state_code) - if (!"us" %in% gm$geo_value) gm <- bind_rows(gm, tibble(geo_value = "us", location = "US")) - gm -} - -ws_forecast_one <- function(archive, fd, gd, policy, geo_map) { - snapshot <- make_forecast_snapshot( - archive, forecast_date = fd, generation_date = gd, as_of_policy = policy, - substitutions = if (policy == "asof") WS_SUBSTITUTIONS else NULL - ) - purrr::map(WS_HORIZONS, function(h) { - set.seed(targets::tar_seed_create(paste(WS_ID, "nhsn", as.character(fd), h, sep = "/"))) - run_forecaster( - snapshot = snapshot, forecaster = scaled_pop_seasonal, aheads = h * 7L, - params = WS_PARAMS, id = WS_ID, target_date_shift = 3L, sort_quantiles = TRUE - ) %>% - mutate(horizon = h) - }) %>% - bind_rows() %>% - inner_join(geo_map, by = "geo_value") %>% - transmute( - reference_date = fd + 3L, - horizon, - target_end_date, - location, - level_index = hub_level_index(quantile), - level = quantile, - value, - as_of_policy = policy - ) -} - -ws_backfill <- function(refresh = FALSE) { - if (file.exists(WS_FORECASTS_FILE) && !refresh) { - cli::cli_alert_success("Pseudo-hub cache present at {.path {WS_FORECASTS_FILE}}; pass refresh = TRUE to rebuild.") - return(invisible(ws_read_pseudo_hub())) - } - withr::local_envvar(TAR_PROJECT = WS_STORE) - archive <- targets::tar_read(nhsn_prod_archive) - schedule <- ws_schedule(archive$versions_end) - geo_map <- ws_geo_map() - cli::cli_alert_info("Backfilling {nrow(schedule)} forecast dates x {length(WS_HORIZONS)} horizons of {WS_ID}") - failures <- list() - forecasts <- purrr::pmap(schedule, .progress = "windowed_seasonal backfill", function(forecast_date, as_of_policy, generation_date) { - tryCatch(ws_forecast_one(archive, forecast_date, generation_date, as_of_policy, geo_map), error = function(e) { - failures[[as.character(forecast_date)]] <<- conditionMessage(e) - NULL - }) - }) %>% - bind_rows() - if (length(failures) > 0) { - cli::cli_warn("{length(failures)} date{?s} failed: {.val {names(failures)}}") - print(failures) - } - dir.create(WS_CACHE_DIR, showWarnings = FALSE, recursive = TRUE) - nanoparquet::write_parquet(forecasts, WS_FORECASTS_FILE) - cli::cli_alert_success("{nrow(forecasts)} forecast rows over {n_distinct(forecasts$reference_date)} rounds -> {.path {WS_FORECASTS_FILE}}") - invisible(forecasts) -} - -#' Read the cached pseudo-hub forecasts in hub_read_forecasts() shape (plus `as_of_policy`). -ws_read_pseudo_hub <- function() { - nanoparquet::read_parquet(WS_FORECASTS_FILE) %>% - mutate(across(c(reference_date, target_end_date), as.Date), horizon = as.integer(horizon)) -} - -if (sys.nframe() == 0L) { - ws_backfill(refresh = "--refresh" %in% commandArgs(trailingOnly = TRUE)) -} diff --git a/scripts/calibration/calibration_ws_experiments.R b/scripts/calibration/calibration_ws_experiments.R deleted file mode 100644 index cbaef4cc..00000000 --- a/scripts/calibration/calibration_ws_experiments.R +++ /dev/null @@ -1,148 +0,0 @@ -# One forecaster across eras: does online calibration of `windowed_seasonal` -# behave the same on the ILI+ decade (2011-2023, percent scale, no revisions) -# as on the NHSN seasons (2023-2026, counts, real vintages), and how does that -# compare with calibrating the submitted ensemble? Prints markdown tables; -# results are recorded in notes/CALIBRATION.md. -# -# Inputs (all cached in cache/calibration/): -# scripts/calibration_ili_backfill.R -> windowed_seasonal on ILI+, 2010-2024 -# scripts/calibration_ws_backfill.R -> windowed_seasonal on NHSN, 2023-2026 -# hub_read_forecasts()/hub_read_truth() -> the submitted ensemble and hub truth -# -# Usage: distrobox enter rocker -- Rscript scripts/calibration_ws_experiments.R - -suppressPackageStartupMessages(source(here::here("R/load_all.R"))) -source(here::here("scripts/calibration_ili_backfill.R")) -source(here::here("scripts/calibration_ws_backfill.R")) - -ili <- ili_read_pseudo_hub() -ws <- ws_read_pseudo_hub() -hub_fc <- hub_read_forecasts() -hub_truth <- { - arch <- get_nhsn_data_archive("flu") - arch %>% - epix_as_of(arch$versions_end) %>% - mutate(geo_value = ifelse(geo_value == "usa", "us", geo_value)) %>% - left_join( - get_population_data() %>% select("state_id", location = "state_code"), - by = c("geo_value" = "state_id") - ) %>% - select(target_end_date = time_value, location, truth = value) %>% - filter(!is.na(truth)) %>% - arrange(location, target_end_date) -} -LIVE_SEASONS <- c("2024-2025", "2025-2026") -ERA_SPLIT <- as.Date("2023-08-01") # ILI+ history before, NHSN after - -# ILI+ seasons kept: drop 2010-11 (too little training data for h3), the two -# pandemic non-seasons, and 2023-24 (the NHSN era owns those rounds). -ili_rounds <- hub_label_seasons(unique(ili$forecasts$reference_date)) -ili_keep <- ili_rounds %>% filter(!season %in% c("2010-2011", "2020-2021", "2021-2022", "2023-2024")) -ili_fc <- ili$forecasts %>% filter(reference_date %in% ili_keep$round_date) - -# Locations present in every source, so eras are comparable per location. -common_loc <- Reduce(intersect, list(unique(ili_fc$location), unique(ws$location), unique(hub_fc$location))) -ili_fc <- ili_fc %>% filter(location %in% common_loc) -# The replay ran every Wednesday, summer included; keep only the hub's -# submission rounds so seasons, gaps and round counts match the ensemble's. -ws_fc <- ws %>% - filter(location %in% common_loc, reference_date %in% unique(hub_fc$reference_date)) %>% - select(-as_of_policy) -hub_fc <- hub_fc %>% filter(location %in% common_loc) - -combined_fc <- bind_rows(ili_fc, ws_fc) -combined_truth <- bind_rows( - ili$truth %>% filter(target_end_date < ERA_SPLIT), - hub_truth %>% filter(target_end_date >= ERA_SPLIT) -) %>% filter(location %in% common_loc) - -# Whitening across eras: divide each location's rounds by the 90th percentile of -# its in-season (Oct-May) truth for that era, so offsets and eta live in -# "fraction of a typical season peak" units in both eras. -in_season <- function(d) as.integer(format(d, "%m")) %in% c(10:12, 1:5) -era_scale <- function(truth, from, to) { - truth %>% - filter(target_end_date >= from, target_end_date < to, in_season(target_end_date)) %>% - group_by(location) %>% - summarize(scale = stats::quantile(truth, 0.9, names = FALSE), .groups = "drop") -} -scales <- bind_rows( - era_scale(ili$truth, as.Date("2011-08-01"), ERA_SPLIT) %>% mutate(from = as.Date("2000-01-01")), - era_scale(hub_truth, as.Date("2023-10-01"), as.Date("2024-06-01")) %>% mutate(from = ERA_SPLIT) -) %>% filter(location %in% common_loc, scale > 0) - -variants <- function(floor) list( - single = list(), - leaky = list(fast_decay = 0.1), - operating = list(slow_init = "burn_in_quantile", lr_slow = list(mult = 0.003), fast_decay = 0.1) -) -run_all <- function(fc, truth, burn_in, floor, ...) { - purrr::map(variants(floor), function(v) { - do.call(calibrate_hub_forecasts, c( - list(fc, truth, burn_in_seasons = burn_in, transform = "sqrt", lr_window = 20, - lr_args = list(mult = 0.03, floor = floor), progress = FALSE), - v, list(...) - )) - }) -} - -cli::cli_h1("A. submitted ensemble, NHSN only, burn-in 2023-24") -A <- run_all(hub_fc, hub_truth, "2023-2024", 1e-3) -cli::cli_h1("B. windowed_seasonal, NHSN only, burn-in 2023-24 (finalized-data replay)") -B <- run_all(ws_fc, hub_truth, "2023-2024", 1e-3) -cli::cli_h1("C. windowed_seasonal, ILI+ decade then NHSN, burn-in 2011-12, whitened") -C <- run_all(combined_fc, combined_truth, "2011-2012", 1e-4, scales = scales) - -season_table <- function(cals, seasons = LIVE_SEASONS) { - purrr::imap(cals, function(cal, nm) { - left_join( - hub_quantile_loss(cal, by = "season") %>% select(season, horizon, wis_pct = pct_improvement), - hub_coverage_summary(cal, by = "season") %>% select(season, horizon, cal_err_base = cal_error_base, cal_err_cal = cal_error_cal), - by = c("season", "horizon") - ) %>% mutate(variant = nm) - }) %>% - bind_rows() %>% - filter(season %in% seasons) -} -wide <- function(tbl, col, digits = 1) { - tbl %>% select(variant, season, horizon, value = all_of(col)) %>% - mutate(value = round(value, digits)) %>% - tidyr::pivot_wider(names_from = horizon, values_from = value, names_prefix = "h") %>% - arrange(season, variant) -} -report <- function(label, cals) { - tbl <- season_table(cals) - cat("\n\n### ", label, "\n\nWIS change % vs base (positive is better)\n\n", sep = "") - print(knitr::kable(wide(tbl, "wis_pct"))) - cat("\nCalibration error, calibrated (base in first row)\n\n") - base <- tbl %>% filter(variant == "single") %>% transmute(variant = "base", season, horizon, cal_err_cal = cal_err_base) - print(knitr::kable(wide(bind_rows(base, tbl), "cal_err_cal", 3))) -} -report("A. ensemble, NHSN only", A) -report("B. windowed_seasonal, NHSN only", B) -report("C. windowed_seasonal, ILI+ decade + NHSN, whitened", C) -cat("\n\n### C, the 2023-24 NHSN season (tracked live here; burn-in in A and B)\n\n") -print(knitr::kable(wide(season_table(C, seasons = "2023-2024"), "wis_pct"))) - -cat("\n\n### Base bias: fraction of truth above the base median, live seasons\n\n") -bias <- bind_rows( - A$single$forecasts %>% mutate(base = "ensemble"), - B$single$forecasts %>% mutate(base = "windowed_seasonal") -) %>% - filter(level == 0.5, !is.na(truth), !is.na(value_base), season %in% c("2023-2024", LIVE_SEASONS)) %>% - group_by(base, season, horizon) %>% - summarize(p = round(mean(truth > value_base), 2), .groups = "drop") %>% - tidyr::pivot_wider(names_from = horizon, values_from = p, names_prefix = "h") -print(knitr::kable(bias)) - -cat("\n\n### C, ILI+ seasons: WIS change % (single / operating)\n\n") -print(knitr::kable(wide(season_table(C, seasons = unique(ili_keep$season)) %>% filter(variant != "leaky"), "wis_pct"))) - -cat("\n\n### Base WIS per season and horizon (mean pinball x 2), ensemble vs windowed_seasonal\n\n") -bw <- bind_rows( - hub_quantile_loss(A$single, by = "season") %>% mutate(base = "ensemble"), - hub_quantile_loss(B$single, by = "season") %>% mutate(base = "windowed_seasonal") -) %>% filter(season %in% LIVE_SEASONS) %>% - transmute(base, season, horizon, wis = round(2 * loss_base, 1)) %>% - tidyr::pivot_wider(names_from = horizon, values_from = wis, names_prefix = "h") -print(knitr::kable(bw)) diff --git a/scripts/calibration/calibration_ws_replay.R b/scripts/calibration/calibration_ws_replay.R new file mode 100644 index 00000000..e27487f4 --- /dev/null +++ b/scripts/calibration/calibration_ws_replay.R @@ -0,0 +1,369 @@ +# Calibrate the clean evaluation replay of `windowed_seasonal` (no data +# substitutions, ROADMAP 1a) for flu and covid, and print base vs calibrated WIS +# and coverage per season and horizon as markdown tables. +# +# Forecasts come straight from the evaluation stores through the harness +# (ch_use() + ch_read_store()); truth and vintages from the same store's NHSN +# archive. Rounds are the hub's CMU-TimeSeries submission rounds, so the round +# axis matches the hub-based experiments in notes/calibration-ledger.md. The E12 +# configs use no burn-in season. +# +# With `burn_in`, warm-started configs also run with 2023-24 as a burn-in +# season: those rounds are backfilled weekly from 2023-10-11 to 2024-04-24 on +# the HHS-stitched archive (ch_inputs_with_burn_in(), ROADMAP 1b). Scores are +# reported over all locations, states only and US only. +# +# Usage: distrobox enter rocker -- Rscript scripts/calibration/calibration_ws_replay.R [workers] [burn_in] + +source(here::here("scripts/calibration/calibration_harness.R")) + +# Command-line arguments apply only when run as a script, not when sourced by a notebook. +args <- if (sys.nframe() == 0L) commandArgs(trailingOnly = TRUE) else character(0) +WORKERS <- if (length(args) >= 1) as.integer(args[[1]]) else 12L +BURN_IN <- length(args) >= 2 && args[[2]] == "burn_in" +BURN_IN_DATES <- c(from = "2023-10-11", through = "2024-04-24") +HUB_DIRS <- c(flu = "../FluSight-forecast-hub", covid = "../covid19-forecast-hub") +LIVE <- c("2024-2025", "2025-2026") + +# Seasons are August-July years, as in E10 (covid has no off-season gap). +season_of <- function(d) { + y <- as.integer(format(d, "%Y")) - (as.integer(format(d, "%m")) < 8) + paste0(y, "-", y + 1) +} + +# E11's constant rate (0.1 admissions per 100k per step, rate scale) and the +# covid prod config (REF-op without its warm start), both with no burn-in. With +# `burn_in`, also E05's sqrt single term cold and warm, REF-op with its warm +# start, and E11's constant rate warm-started. +ws_configs <- function(rate_scales, burn_in = BURN_IN) { + no_burn_in <- list(burn_in_seasons = character(0), slow_init = NULL) + warm <- list(burn_in_seasons = "2023-2024", slow_init = "burn_in_quantile") + sqrt_single <- list(ref = "paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3)) + cold <- list( + `E11 constant 0.1` = c(list(ref = "paper", scales = rate_scales, lr = 0.1), no_burn_in), + `REF-op cold` = c(list(ref = "op"), no_burn_in) + ) + if (!burn_in) { + return(cold) + } + c(cold, list( + `E05 single cold` = c(sqrt_single, no_burn_in), + `E05 warm + single` = c(sqrt_single, warm), + `REF-op warm` = c(list(ref = "op"), warm), + `E11 constant 0.1 warm` = c(list(ref = "paper", scales = rate_scales, lr = 0.1), warm) + )) +} + +#' The structure grid (E19, E20): two learning-rate anchors, each cold or +#' warm-started, each with or without the leak. Anchors: `adaptive 0.03` is +#' E01's sqrt adaptive rate (E18 step 4); `constant 0.018` is E16's constant +#' rate on sqrt per 100k. Each spec is `list(cfg, burn_in)` as the E18 steps +#' are, named `" | | "`. The no-leak cold and warm specs +#' are written as E16 and E18 wrote them, so their cached runs are reused. +ws_grid_configs <- function(rate_scales) { + anchors <- list( + `adaptive 0.03` = list( + ref = "paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3), + burn_in_seasons = character(0), slow_init = NULL, lr_window = 20, season_policy = "carry" + ), + `constant 0.018` = utils::modifyList(ws_configs(rate_scales, burn_in = FALSE)[["E11 constant 0.1"]], list(lr = 0.018, transform = "sqrt")) + ) + starts <- list(cold = NULL, warm = list(burn_in_seasons = "2023-2024", slow_init = "burn_in_quantile")) + leaks <- list(`no leak` = NULL, `leak 0.1` = list(fast_decay = 0.1)) + specs <- list() + for (a in names(anchors)) { + for (s in names(starts)) { + for (l in names(leaks)) { + cfg <- anchors[[a]] + if (!is.null(starts[[s]])) cfg <- utils::modifyList(cfg, starts[[s]]) + if (!is.null(leaks[[l]])) cfg <- utils::modifyList(cfg, leaks[[l]]) + specs[[paste(a, s, l, sep = " | ")]] <- list(cfg = cfg, burn_in = s == "warm") + } + } + } + specs +} + +#' Prod's two configs as grid-style specs: REF-op warm (flu prod) and REF-op +#' cold (covid prod). +ws_prod_specs <- function(rate_scales) { + list( + `REF-op warm` = list(cfg = ws_configs(rate_scales, burn_in = TRUE)[["REF-op warm"]], burn_in = TRUE), + `REF-op cold` = list(cfg = ws_configs(rate_scales, burn_in = FALSE)[["REF-op cold"]], burn_in = FALSE) + ) +} + +#' Run one spec (`list(cfg, burn_in)`) through the cache, with the burn-in +#' rounds prepended when the spec asks for them. `internals` keeps the +#' tracker's per-row state (E19). +ws_run_spec <- function(spec, wi, fc_burn_in, workers, internals = FALSE) { + fc <- if (spec$burn_in) fc_burn_in else wi$fc + do.call(cal_run_cached, c(list(fc, wi$truth), spec$cfg, list(learn = "exact", vintages = wi$vintages, workers = workers, internals = internals))) +} + +#' WIS reduction % and L1 coverage bias per config, season (live seasons and +#' "both") and horizon, states only, as a long tibble with `config`, `season`, +#' `h` ("h0".."h3"), `wis_pct`, `cal_err_base`, `cal_err_cal`. +ws_wis_by_season <- function(cals) { + purrr::imap(cals, function(cal, nm) { + fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) %>% filter(location != "US") + ws_scores(fc) %>% + select(season, horizon, which, wis, cal_err) %>% + tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err)) %>% + mutate(config = nm, wis_pct = 100 * (wis_base - wis_cal) / wis_base) + }) %>% + bind_rows() %>% + mutate( + config = factor(config, levels = names(cals)), + season = factor(season, levels = c(LIVE, "both")), + h = factor(paste0("h", horizon), levels = paste0("h", 0:3)) + ) +} + +#' Per-round tracker internals from an exact run, states only, h0–h3: the +#' learning rate and the median-level offsets relative to the base 90% interval +#' width (in the config's working units, so configs with and without `scales` +#' compare), plus the share of truths above the base median. Under exact +#' learning each round's rows come from that round's own re-run, so the state +#' is the one that played. Medians over states per (horizon, reference_date). +ws_internals <- function(cal, cfg, rate_scales) { + fc <- cal$forecasts %>% + filter(location != "US", !is_burn_in, horizon >= 0, !is.na(value_base)) + s <- if (is.null(cfg$scales)) fc %>% distinct(location) %>% mutate(scale = 1) else rate_scales %>% select(location, scale) + tf <- if (identical(cfg$transform, "sqrt")) sqrt else identity + fc %>% + left_join(s, by = "location") %>% + group_by(location, horizon, reference_date) %>% + summarize( + width_count = value_base[level == 0.95] - value_base[level == 0.05], + width_work = tf(value_base[level == 0.95] / scale[1]) - tf(value_base[level == 0.05] / scale[1]), + eta = mean(lr_level), + offset_mid = offset[level == 0.5], + offset_abs = mean(abs(offset)), + truth = truth[1], + base_mid = value_base[level == 0.5], + .groups = "drop" + ) %>% + mutate( + eta_rel = ifelse(eta == 0, NA_real_, eta) / width_work, + offset_rel = offset_mid / width_count, + offset_abs_rel = offset_abs / width_count + ) %>% + group_by(horizon, reference_date) %>% + summarize( + eta_rel = median(eta_rel, na.rm = TRUE), + offset_rel = median(offset_rel, na.rm = TRUE), + offset_abs_rel = median(offset_abs_rel, na.rm = TRUE), + above_base_median = mean(truth > base_mid, na.rm = TRUE), + .groups = "drop" + ) +} + +# 2023-24 burn-in rounds, backfilled on the HHS-stitched archive (cached). +ws_burn_in_forecasts <- function(disease) { + sched <- ch_schedule(through = BURN_IN_DATES[["through"]], from = as.Date(BURN_IN_DATES[["from"]])) + ch_to_hub(ch_backfill(ch_inputs_with_burn_in(), schedule = sched), disease) +} + +#' The replay's hub-round forecasts, truth and vintages, and per-100k rate +#' scales for one disease. Also used by the E14/E15 notebooks. +ws_inputs <- function(disease) { + ch_use(paste0(disease, "_windowed_seasonal")) + inp <- ch_inputs() + tv <- ch_hub_truth(inp) + hub_rounds <- suppressMessages(hub_read_forecasts( + hub_dir = here::here(HUB_DIRS[[disease]]), + target = c(flu = HUB_FLU_TARGET, covid = HUB_COVID_TARGET)[[disease]] + )) %>% distinct(reference_date) %>% pull() + fc <- ch_to_hub(ch_read_store(), disease) %>% filter(reference_date %in% hub_rounds) + cli::cli_alert_info("{disease}: {n_distinct(fc$reference_date)} rounds, {n_distinct(fc$location)} locations") + rate_scales <- get_population_data() %>% + distinct(state_code, .keep_all = TRUE) %>% + filter(state_code %in% unique(fc$location)) %>% + transmute(location = state_code, from = as.Date("2000-01-01"), scale = population / 1e5) + list(fc = fc, truth = tv$truth, vintages = tv$vintages, rate_scales = rate_scales) +} + +ws_run_disease <- function(disease) { + wi <- ws_inputs(disease) + fc <- wi$fc + tv <- wi[c("truth", "vintages")] + # Every backfilled 2023-24 round is kept, not only hub rounds (covid has none). + fc_burn_in <- if (BURN_IN) bind_rows(ws_burn_in_forecasts(disease), fc) + cals <- purrr::map(ws_configs(wi$rate_scales), function(cfg) { + t0 <- Sys.time() + fc_cfg <- if (length(cfg$burn_in_seasons) > 0) fc_burn_in else fc + cal <- do.call(cal_run_cached, c(list(fc_cfg, tv$truth), cfg, list(learn = "exact", vintages = tv$vintages, workers = WORKERS))) + cli::cli_alert_success("run took {format(round(Sys.time() - t0, 1))}") + cal$forecasts %>% mutate(season = season_of(reference_date)) + }) + list(cals = cals, truth = tv$truth) +} + +#' Base vs calibrated WIS (2 x mean pinball), 50%/90% interval coverage and L1 +#' coverage bias, per (season, horizon); season "both" pools the live seasons. +ws_scores <- function(fc) { + pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) + fc <- fc %>% filter(!is.na(truth), !is.na(value_base), season %in% LIVE) + fc <- bind_rows(fc, fc %>% mutate(season = "both")) + long <- fc %>% + tidyr::pivot_longer(c(value_base, value_cal), names_to = "which", values_to = "q") %>% + mutate(which = sub("value_", "", which)) + wis <- long %>% + group_by(season, horizon, which) %>% + summarize(wis = 2 * mean(pinball(truth, q, level)), .groups = "drop") + cov_int <- long %>% + filter(level %in% c(0.05, 0.25, 0.75, 0.95)) %>% + select(season, horizon, which, location, reference_date, truth, level, q) %>% + tidyr::pivot_wider(names_from = level, values_from = q, names_prefix = "q") %>% + group_by(season, horizon, which) %>% + summarize( + cov50 = mean(truth >= q0.25 & truth <= q0.75), + cov90 = mean(truth >= q0.05 & truth <= q0.95), + .groups = "drop" + ) + bias <- long %>% + group_by(season, horizon, which, level) %>% + summarize(gap = mean(truth <= q) - first(level), .groups = "drop") %>% + group_by(season, horizon, which) %>% + summarize(cal_err = mean(abs(gap)), .groups = "drop") + above <- fc %>% + filter(level == 0.5) %>% + group_by(season, horizon) %>% + summarize(above_median = mean(truth > value_base), .groups = "drop") + wis %>% + left_join(cov_int, by = c("season", "horizon", "which")) %>% + left_join(bias, by = c("season", "horizon", "which")) %>% + left_join(above, by = c("season", "horizon")) +} + +#' V-month per season: WIS reduction % (positive = calibrated WIS lower than base), L1 coverage bias and +#' the share of truth below the median, base vs calibrated, by calendar month of +#' the reference date. `share` is the month's share of the season's base WIS. +#' Groups by `config` too when present. Used by the E16/E17 notebooks. +ws_month_scores <- function(fc) { + pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) + by <- intersect(c("config", "season", "horizon"), names(fc)) + fc <- fc %>% + filter(!is.na(truth), !is.na(value_base), season %in% LIVE) %>% + mutate(month = factor(format(reference_date, "%b"), levels = month.abb[c(8:12, 1:7)])) + per_level <- fc %>% + group_by(across(all_of(c(by, "month", "level")))) %>% + summarize( + base = sum(pinball(truth, value_base, level)), cal = sum(pinball(truth, value_cal, level)), + gap_base = mean(truth <= value_base) - first(level), gap_cal = mean(truth <= value_cal) - first(level), + below_base = mean(truth < value_base), below_cal = mean(truth < value_cal), + n = n(), .groups = "drop" + ) + per_level %>% + group_by(across(all_of(c(by, "month")))) %>% + summarize( + wis_base = sum(base), wis_cal = sum(cal), + cal_err_base = mean(abs(gap_base)), cal_err_cal = mean(abs(gap_cal)), + below_med_base = below_base[level == 0.5], below_med_cal = below_cal[level == 0.5], + n = first(n), .groups = "drop" + ) %>% + group_by(across(all_of(by))) %>% + mutate(pct = 100 * (wis_base - wis_cal) / wis_base, share = 100 * wis_base / sum(wis_base)) %>% + ungroup() +} + +# The finalists' inputs for one disease: the submitted ensemble and the replay +# on the (round, location, horizon, level) rows both have, h0–h3. Cold runs see +# the live seasons only; warm runs also see the 2023-24 burn-in season (covid +# submissions start 2024-11-23, so covid has warm runs on the replay only). +fin_inputs <- function(disease) { + key <- c("reference_date", "location", "horizon", "level") + wi <- ws_inputs(disease) + # Right after ws_inputs(): both read the harness globals that ch_use() sets. + burn <- ws_burn_in_forecasts(disease) + hub <- suppressMessages(hub_read_forecasts( + hub_dir = here::here(HUB_DIRS[[disease]]), target = c(flu = HUB_FLU_TARGET, covid = HUB_COVID_TARGET)[[disease]] + )) %>% filter(horizon %in% 0:3) + ws <- bind_rows(burn, wi$fc) + common <- inner_join(distinct(hub, across(all_of(key))), distinct(ws, across(all_of(key))), by = key) + fcs <- list(ensemble = semi_join(hub, common, by = key), replay = semi_join(ws, common, by = key)) + warm <- fcs + if (!any(season_of(fcs$ensemble$reference_date) == "2023-2024")) { + warm <- list(replay = bind_rows(burn, fcs$replay)) + } + list(wi = wi, cold = purrr::map(fcs, \(fc) filter(fc, season_of(reference_date) %in% LIVE)), warm = warm) +} + +# The finalist configs: REF-op and sqrt constant 0.018 per 100k (E16), each cold and warm. +fin_candidates <- function(rate_scales) { + const <- ws_configs(rate_scales, burn_in = FALSE)[["E11 constant 0.1"]] + sqrt_018 <- utils::modifyList(const, list(lr = 0.018, transform = "sqrt")) + list( + `REF-op warm` = list(cfg = list(ref = "op"), warm = TRUE), + `REF-op cold` = list(cfg = list(ref = "op", burn_in_seasons = character(0), slow_init = NULL), warm = FALSE), + `sqrt 0.018` = list(cfg = sqrt_018, warm = FALSE), + `sqrt 0.018 warm` = list(cfg = utils::modifyList(sqrt_018, list(burn_in_seasons = "2023-2024", slow_init = "burn_in_quantile")), warm = TRUE), + # E20's best cell: the warm start plus the leak (same construction as ws_grid_configs()). + `sqrt 0.018 warm + leak` = list(cfg = utils::modifyList(sqrt_018, list(burn_in_seasons = "2023-2024", slow_init = "burn_in_quantile", fast_decay = 0.1)), warm = TRUE) + ) +} + +# Every finalist on each of `bases` that has its inputs, cached. Returns the +# inputs, the runs named ", ", their grid and the candidate names. +fin_runs <- function(disease, bases = c("ensemble", "replay"), workers = WORKERS) { + inp <- fin_inputs(disease) + cands <- fin_candidates(inp$wi$rate_scales) + grid <- tidyr::expand_grid(base = bases, cand = names(cands)) %>% + filter(!purrr::map_lgl(cands[cand], "warm") | base %in% names(inp$warm)) + cals <- purrr::pmap(grid, function(base, cand) { + cc <- cands[[cand]] + fc <- if (cc$warm) inp$warm[[base]] else inp$cold[[base]] + do.call(cal_run_cached, c(list(fc, inp$wi$truth), cc$cfg, list(learn = "exact", vintages = inp$wi$vintages, workers = workers))) + }) + list(inputs = inp, cals = setNames(cals, paste(grid$base, grid$cand, sep = ", ")), grid = grid, cands = names(cands)) +} + +# Location subsets scored separately: raw-count pooling makes US about half of +# the all-locations WIS. +WS_GEOS <- list(all = \(loc) rep(TRUE, length(loc)), states = \(loc) loc != "US", us = \(loc) loc == "US") + +ws_report <- function(disease, res) { + scores <- purrr::imap(WS_GEOS, function(keep, geos) { + purrr::imap(res$cals, function(fc, nm) ws_scores(fc %>% filter(keep(location))) %>% mutate(config = nm)) %>% + bind_rows() %>% + mutate(geos = geos) + }) %>% bind_rows() + base <- scores %>% filter(which == "base", config == names(res$cals)[1]) %>% mutate(config = "base") + tbl <- bind_rows(base, scores %>% filter(which == "cal")) %>% + group_by(geos, season, horizon) %>% + mutate(wis_pct = 100 * (wis[config == "base"] - wis) / wis[config == "base"]) %>% + ungroup() %>% + mutate( + config = factor(config, levels = c("base", names(res$cals))), season = factor(season, levels = c(LIVE, "both")), + geos = factor(geos, levels = names(WS_GEOS)) + ) + suffix <- if (BURN_IN) "_burn_in" else "" + readr::write_csv(tbl, file.path(CH_CACHE_DIR, glue::glue("ws_replay_scores{suffix}_{disease}.csv"))) + wide <- function(col, digits, g) { + tbl %>% + filter(geos == g) %>% + transmute(season, config, horizon, v = round(.data[[col]], digits)) %>% + tidyr::pivot_wider(names_from = horizon, values_from = v, names_prefix = "h") %>% + arrange(season, config) + } + for (g in names(WS_GEOS)) { + cat("\n\n## ", disease, ", ", g, " locations\n", sep = "") + for (spec in list( + c("wis", "WIS (2 x mean pinball)", 1), c("wis_pct", "WIS reduction % (positive = calibrated WIS lower than base)", 1), + c("cov50", "50% interval coverage", 3), c("cov90", "90% interval coverage", 3), + c("cal_err", "L1 coverage bias (lower is better)", 3) + )) { + cat("\n### ", spec[2], "\n\n", sep = "") + print(knitr::kable(wide(spec[1], as.integer(spec[3]), g))) + } + cat("\n### Fraction of truth above the base median\n\n") + print(knitr::kable(tbl %>% filter(config == "base", geos == g) %>% transmute(season, horizon, p = round(above_median, 2)) %>% + tidyr::pivot_wider(names_from = horizon, values_from = p, names_prefix = "h"))) + } + invisible(tbl) +} + +if (sys.nframe() == 0L) { + for (disease in c("flu", "covid")) ws_report(disease, ws_run_disease(disease)) +} diff --git a/scripts/calibration/render_experiments.R b/scripts/calibration/render_experiments.R new file mode 100644 index 00000000..7ccdc8a5 --- /dev/null +++ b/scripts/calibration/render_experiments.R @@ -0,0 +1,22 @@ +# Render calibration experiment notebooks into reports/calibration_experiments/ +# and rebuild its index.html (an overview plus links to every rendered notebook). +# +# Usage: Rscript scripts/calibration/render_experiments.R [learning_rate ...] +# With no arguments, renders every notebook; `--index` renders only the index. + +src_dir <- here::here("reports/writeups/calibration_experiments") +out_dir <- here::here("reports/calibration_experiments") +dir.create(out_dir, showWarnings = FALSE, recursive = TRUE) + +args <- commandArgs(trailingOnly = TRUE) +notebooks <- if (identical(args, "--index")) { + character(0) +} else if (length(args) > 0) { + file.path(src_dir, paste0(sub("[.]Rmd$", "", args), ".Rmd")) +} else { + setdiff(list.files(src_dir, pattern = "^[^_].*[.]Rmd$", full.names = TRUE), file.path(src_dir, "index.Rmd")) +} +for (nb in notebooks) { + rmarkdown::render(nb, output_dir = out_dir, envir = new.env(), quiet = TRUE) +} +rmarkdown::render(file.path(src_dir, "index.Rmd"), output_dir = out_dir, envir = new.env(), quiet = TRUE) diff --git a/scripts/one_offs/compare_ilinet_sources.py b/scripts/one_offs/compare_ilinet_sources.py new file mode 100644 index 00000000..9e957c28 --- /dev/null +++ b/scripts/one_offs/compare_ilinet_sources.py @@ -0,0 +1,356 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = ["duckdb", "typer"] +# /// +"""Compare versioned ILINet from epidata v4 `fluview` against v5 `fluview_ilinet`. + +Usage: + uv run scripts/one_offs/compare_ilinet_sources.py fetch # download into cache/ilinet/ + uv run scripts/one_offs/compare_ilinet_sources.py analyze # print markdown tables + +Needs DELPHI_EPIDATA_KEY in the environment (used for both APIs). Findings are in +notes/ilinet-sources.md. +""" + +import json +import os +import time +import urllib.parse +import urllib.request +from pathlib import Path + +import duckdb +import typer + +ROOT = Path(__file__).parents[2] +CACHE = ROOT / "cache" / "ilinet" +V4 = "https://api.delphi.cmu.edu/epidata/fluview/" +V5 = "https://delphi.cmu.edu/epidata/v5/archive/" +SIGNALS = ["wili", "ili", "num_ili", "num_patients", "num_providers"] +V5_GEO_TYPES = ["state", "nation"] +# v4 region codes for states + national. v4 `jfk` is NYC (v5 `nyc`) and v4 `ny` is all of NY +# (v5 `ny` with fill_method nyc_plus_ny_minus_nyc). v5 `ny_minus_nyc` has no v4 counterpart. +STATES = ( + "ak al ar az ca co ct dc de fl ga hi ia id il in ks ky la ma md me mi mn mo ms mt nc nd ne nh nj " + "nm nv ny nyc jfk oh ok or pa pr ri sc sd tn tx ut va vi vt wa wi wv wy" +).split() +V4_REGIONS = ["nat", *STATES] + +app = typer.Typer(add_completion=False) + + +def _get(url: str) -> bytes: + for attempt in range(4): + try: + with urllib.request.urlopen(url, timeout=600) as resp: + return resp.read() + except Exception as exc: # noqa: BLE001 + print(f" retry {attempt + 1} after {exc}") + time.sleep(10 * (attempt + 1)) + raise RuntimeError(f"failed: {url.split('?')[0]}") + + +@app.command() +def fetch() -> None: + """Download both sources into cache/ilinet/, skipping files already present.""" + key = os.environ["DELPHI_EPIDATA_KEY"] + CACHE.mkdir(parents=True, exist_ok=True) + for geo_type in V5_GEO_TYPES: + for signal in SIGNALS: + out = CACHE / f"v5_{geo_type}_{signal}.csv" + if out.exists(): + continue + qs = urllib.parse.urlencode( + {"source": "fluview_ilinet", "signal": signal, "geo_type": geo_type, "token": key} + ) + print(f"v5 {geo_type} {signal}") + out.write_bytes(_get(f"{V5}?{qs}")) + time.sleep(2) + for region in V4_REGIONS: + out = CACHE / f"v4_{region}.json" + if out.exists(): + continue + qs = urllib.parse.urlencode( + {"regions": region, "epiweeks": "199701-203053", "issues": "199701-203053", "api_key": key} + ) + print(f"v4 {region}") + body = json.loads(_get(f"{V4}?{qs}")) + if body["result"] != 1: + print(f" {region}: {body['message']}") + out.write_text(json.dumps(body.get("epidata", []))) + time.sleep(1) + + +def connect() -> duckdb.DuckDBPyConnection: + """Load both sources as long tables keyed by (geo, week_end, signal, version date).""" + con = duckdb.connect() + con.execute("SET TimeZone = 'UTC'") + con.execute(f""" + CREATE TABLE v5 AS + SELECT signal, + CAST(report_time AS TIMESTAMP) AS report_time, + CAST(report_time AS DATE) AS report_date, + CASE geo_value WHEN 'us' THEN 'nat' WHEN 'nyc' THEN 'jfk' ELSE geo_value END AS geo, + geo_type, + CAST(reference_time AS DATE) AS week_end, + value + FROM read_csv('{CACHE}/v5_*.csv', union_by_name = true) + WHERE age_group = 'all' + """) + unpivot = " UNION ALL ".join( + f"SELECT region AS geo, epiweek, issue, CAST(release_date AS DATE) AS raw_release_date, lag, " + f"'{s}' AS signal, CAST({s} AS DOUBLE) AS value FROM raw WHERE {s} IS NOT NULL" + for s in SIGNALS + ) + con.execute(f""" + CREATE TABLE raw AS SELECT * FROM read_json('{CACHE}/v4_*.json', format = 'array', + columns = {{region: 'VARCHAR', epiweek: 'INT', issue: 'INT', release_date: 'VARCHAR', lag: 'INT', + wili: 'DOUBLE', ili: 'DOUBLE', num_ili: 'DOUBLE', num_patients: 'DOUBLE', + num_providers: 'DOUBLE'}}); + -- Saturday ending MMWR week: week 1 starts on the Sunday on or before Jan 4. + CREATE MACRO ew_end(ew) AS CAST( + make_date(CAST(ew // 100 AS INT), 1, 4) + - to_days(CAST(dayofweek(make_date(CAST(ew // 100 AS INT), 1, 4)) AS INT)) + + to_days(CAST((ew % 100) * 7 - 1 AS INT)) AS DATE); + CREATE TABLE v4 AS + -- Some state rows lack release_date; those fall back to the Friday after the issue week. + SELECT u.*, ew_end(epiweek) AS week_end, ew_end(issue) AS issue_week_end, + coalesce(raw_release_date, ew_end(issue) + 6) AS release_date + FROM ({unpivot}) u; + """) + return con + + +def md(con: duckdb.DuckDBPyConnection, title: str, sql: str) -> None: + rel = con.sql(sql) + cols = rel.columns + rows = rel.fetchall() + print(f"\n### {title}\n") + print("| " + " | ".join(cols) + " |") + print("|" + "---|" * len(cols)) + for r in rows: + print("| " + " | ".join("" if v is None else (f"{v:.6g}" if isinstance(v, float) else str(v)) for v in r) + " |") + + +@app.command() +def analyze() -> None: + """Print the coverage, alignment, agreement, revision and finalized-value tables.""" + con = connect() + geo_class = "CASE WHEN geo = 'nat' THEN 'nat' ELSE 'state' END" + + # MMWR week sanity check: 202350 ends 2023-12-16. + assert str(con.sql("SELECT ew_end(202350)").fetchone()[0]) == "2023-12-16" + + md(con, "Coverage: v4 fluview (issue rows)", f""" + SELECT {geo_class} AS geo_class, signal, count(DISTINCT geo) AS n_geo, + min(week_end) AS first_week, max(week_end) AS last_week, + min(issue) AS first_issue, max(issue) AS last_issue, count(*) AS n_rows, + median(n) AS med_issues_per_geo_week + FROM (SELECT *, count(*) OVER (PARTITION BY geo, signal, epiweek) AS n FROM v4) + GROUP BY ALL ORDER BY ALL + """) + md(con, "Coverage: v5 fluview_ilinet (change rows)", f""" + SELECT {geo_class} AS geo_class, signal, count(DISTINCT geo) AS n_geo, + min(week_end) AS first_week, max(week_end) AS last_week, + min(report_date) AS first_report, max(report_date) AS last_report, count(*) AS n_rows, + median(n) AS med_versions_per_geo_week + FROM (SELECT *, count(*) OVER (PARTITION BY geo, signal, week_end) AS n FROM v5) + GROUP BY ALL ORDER BY ALL + """) + md(con, "Geos in one source only", """ + SELECT 'v4 only' AS side, string_agg(DISTINCT geo, ' ') AS geos FROM v4 WHERE geo NOT IN (SELECT geo FROM v5) + UNION ALL + SELECT 'v5 only', string_agg(DISTINCT geo, ' ') FROM v5 WHERE geo NOT IN (SELECT geo FROM v4) + """) + + # Version alignment: weekday of v5 report dates, and their relation to v4 release dates. + md(con, "v5 report_time weekday/hour distribution", """ + SELECT dayname(report_time) AS weekday, count(DISTINCT report_time) AS n_report_times, + min(report_date) AS first, max(report_date) AS last + FROM v5 GROUP BY ALL ORDER BY n_report_times DESC + """) + con.execute(""" + CREATE TABLE releases AS + SELECT issue, issue_week_end, min(release_date) AS release_date, max(release_date) AS max_release_date + FROM v4 WHERE geo = 'nat' GROUP BY ALL; + CREATE TABLE v5_reports AS + SELECT DISTINCT report_time, report_date FROM v5; + """) + md(con, "v4 release_date minus issue week end (days)", """ + SELECT release_date - issue_week_end AS days_after_week_end, count(*) AS n_issues, + min(issue) AS first_issue, max(issue) AS last_issue + FROM releases GROUP BY ALL ORDER BY n_issues DESC LIMIT 10 + """) + md(con, "v5 report dates matched to v4 release dates", """ + SELECT CASE WHEN r.release_date IS NOT NULL THEN 'equals a v4 release_date' + WHEN r2.release_date IS NOT NULL THEN 'within 6 days after a v4 release_date' + ELSE 'no nearby v4 release' END AS match, + count(*) AS n_report_times, min(v.report_date) AS first, max(v.report_date) AS last + FROM v5_reports v + LEFT JOIN releases r ON v.report_date = r.release_date + LEFT JOIN (SELECT DISTINCT release_date FROM releases) r2 + ON v.report_date > r2.release_date AND v.report_date <= r2.release_date + 6 AND r.release_date IS NULL + GROUP BY ALL ORDER BY ALL + """) + md(con, "v5 report times not on a v4 release date", """ + SELECT v.report_time, count(*) AS n_rows, min(week_end) AS min_week, max(week_end) AS max_week + FROM v5 v WHERE report_date NOT IN (SELECT release_date FROM releases) + GROUP BY ALL ORDER BY n_rows DESC LIMIT 15 + """) + + # v5 as of each v4 release date (v5 is change-only, so carry the last change forward). + con.execute(""" + CREATE TABLE paired AS + SELECT a.geo, a.signal, a.epiweek, a.week_end, a.issue, a.release_date, a.lag, + a.value AS v4_value, b.value AS v5_value, b.report_date AS v5_report_date + FROM v4 a ASOF LEFT JOIN v5 b + ON a.geo = b.geo AND a.signal = b.signal AND a.week_end = b.week_end + AND a.release_date >= b.report_date + """) + agree = f""" + SELECT {{grp}}, count(*) AS n, + round(100 * avg(CASE WHEN v5_value IS NULL THEN 1 ELSE 0 END), 2) AS pct_v5_missing, + round(100 * avg(CASE WHEN v4_value = v5_value THEN 1 ELSE 0 END), 2) AS pct_exact, + round(100 * avg(CASE WHEN abs(v4_value - v5_value) <= 5e-6 THEN 1 ELSE 0 END), 2) AS pct_within_5e6, + round(100 * avg(CASE WHEN abs(v4_value - v5_value) <= 1e-3 * greatest(abs(v4_value), 1) THEN 1 ELSE 0 END), 2) AS pct_within_0p1pct, + max(abs(v4_value - v5_value)) AS max_abs_diff + FROM paired {{where}} GROUP BY ALL ORDER BY ALL + """ + md(con, "Agreement at v4 (geo, week, issue), v5 as of the release date", agree.format( + grp=f"{geo_class} AS geo_class, signal", where="")) + md(con, "Agreement by issue season (ili, all geos)", agree.format( + grp="CASE WHEN issue % 100 >= 40 THEN issue // 100 ELSE issue // 100 - 1 END AS season_start", + where="WHERE signal = 'ili'")) + md(con, "Agreement on the lag-0 (first) issue", agree.format( + grp=f"{geo_class} AS geo_class, signal", where="WHERE lag = 0")) + md(con, "Agreement restricted to issues >= 2019w40 and v5 present", agree.format( + grp=f"{geo_class} AS geo_class, signal", where="WHERE issue >= 201940 AND v5_value IS NOT NULL")) + md(con, "Agreement excluding ny (v4 sums NY itself), the 2017w40-41 backfill issues, and rows v5 lacks", agree.format( + grp=f"{geo_class} AS geo_class, signal", where="WHERE geo != 'ny' AND issue > 201741 AND v5_value IS NOT NULL")) + md(con, "Largest mismatches (v5 present, diff > 5e-6)", """ + SELECT geo, signal, epiweek, issue, release_date, v5_report_date, v4_value, v5_value, + v4_value - v5_value AS diff + FROM paired WHERE v5_value IS NOT NULL AND abs(v4_value - v5_value) > 5e-6 + ORDER BY abs(v4_value - v5_value) / greatest(abs(v4_value), 1) DESC LIMIT 20 + """) + md(con, "Mismatch (diff > 5e-6, v5 present) counts by geo, top 15", """ + SELECT geo, count(*) AS n_mismatch, count(DISTINCT issue) AS n_issues, min(issue) AS first_issue, + max(issue) AS last_issue + FROM paired WHERE v5_value IS NOT NULL AND abs(v4_value - v5_value) > 5e-6 + GROUP BY ALL ORDER BY n_mismatch DESC LIMIT 15 + """) + # Shift test: does v5 as of release_date + k days agree better, i.e. is the version off by a week? + md(con, "Why v5 is missing at a v4 (geo, week, issue)", f""" + SELECT {geo_class} AS geo_class, + CASE WHEN p.release_date < f.first_report THEN 'v4 release before first v5 report of the week' + WHEN f.first_report IS NULL THEN 'week never in v5' + ELSE 'other' END AS reason, + count(*) AS n, min(p.issue) AS first_issue, max(p.issue) AS last_issue, + string_agg(DISTINCT p.geo, ' ') AS geos + FROM paired p LEFT JOIN (SELECT geo, signal, week_end, min(report_date) AS first_report FROM v5 GROUP BY ALL) f + USING (geo, signal, week_end) + WHERE p.v5_value IS NULL AND p.signal = 'ili' GROUP BY ALL ORDER BY ALL + """) + md(con, "Mismatches (ili, v5 present, diff > 5e-6) by geo", """ + SELECT geo, count(*) AS n_mismatch, count(DISTINCT issue) AS n_issues, min(issue) AS first_issue, + max(issue) AS last_issue, count(DISTINCT epiweek) AS n_weeks, min(epiweek) AS first_week, + max(epiweek) AS last_week, median(abs(v4_value - v5_value)) AS median_abs_diff + FROM paired WHERE signal = 'ili' AND v5_value IS NOT NULL AND abs(v4_value - v5_value) > 5e-6 + GROUP BY ALL ORDER BY n_mismatch DESC LIMIT 20 + """) + md(con, "Mismatches (ili, v5 present, diff > 5e-6, geo != ny) by issue, top 15", """ + SELECT issue, release_date, count(*) AS n_mismatch, count(DISTINCT geo) AS n_geo, + min(epiweek) AS first_week, max(epiweek) AS last_week, max(abs(v4_value - v5_value)) AS max_abs_diff + FROM paired WHERE signal = 'ili' AND geo != 'ny' AND v5_value IS NOT NULL AND abs(v4_value - v5_value) > 5e-6 + GROUP BY ALL ORDER BY n_mismatch DESC LIMIT 15 + """) + shift = " UNION ALL ".join( + f"""SELECT {k} AS k, abs(a.value - b.value) <= 5e-6 AS ok FROM + (SELECT *, release_date + {k} AS t FROM v4 WHERE signal = 'ili' AND issue >= 201940) a + ASOF LEFT JOIN v5 b ON a.geo = b.geo AND a.signal = b.signal AND a.week_end = b.week_end + AND a.t >= b.report_date""" + for k in [-7, -1, 0, 1, 6, 7] + ) + md(con, "Version shift test (ili, issues >= 2019w40): agreement when v5 is read k days after release", f""" + SELECT k, count(*) AS n, round(100 * avg(CASE WHEN ok THEN 1 ELSE 0 END), 2) AS pct_within_5e6 + FROM ({shift}) GROUP BY k ORDER BY k + """) + + # v5 change rows that v4 never shows as a value change on that release. + con.execute(""" + CREATE TABLE v4_changes AS + SELECT * FROM ( + SELECT *, lag(value) OVER (PARTITION BY geo, signal, epiweek ORDER BY issue) AS prev + FROM v4) WHERE prev IS NULL OR prev != value + """) + md(con, "Number of value changes per (geo, signal, week), issues >= 2019w40 vs v5 reports >= 2019-10-01", f""" + WITH a AS (SELECT {geo_class} AS g, signal, epiweek, count(*) AS n FROM v4_changes + WHERE issue >= 201940 GROUP BY ALL), + b AS (SELECT {geo_class} AS g, signal, week_end, count(*) AS n FROM v5 + WHERE report_date >= DATE '2019-10-01' GROUP BY ALL) + SELECT 'v4' AS src, g, signal, sum(n) AS n_changes, avg(n) AS mean_per_week FROM a GROUP BY ALL + UNION ALL SELECT 'v5', g, signal, sum(n), avg(n) FROM b GROUP BY ALL ORDER BY g, signal, src + """) + + # Finalized: latest version per (geo, signal, week). + con.execute(""" + CREATE TABLE fin AS + SELECT coalesce(a.geo, b.geo) AS geo, coalesce(a.signal, b.signal) AS signal, + coalesce(a.week_end, b.week_end) AS week_end, a.value AS v4_value, b.value AS v5_value + FROM (SELECT * FROM v4 QUALIFY row_number() OVER (PARTITION BY geo, signal, week_end ORDER BY issue DESC) = 1) a + FULL JOIN (SELECT * FROM v5 QUALIFY row_number() OVER (PARTITION BY geo, signal, week_end ORDER BY report_time DESC) = 1) b + USING (geo, signal, week_end) + """) + md(con, "Finalized (latest) values", f""" + SELECT {geo_class} AS geo_class, signal, count(*) AS n_geo_weeks, + sum(CASE WHEN v5_value IS NULL THEN 1 ELSE 0 END) AS v4_only, + sum(CASE WHEN v4_value IS NULL THEN 1 ELSE 0 END) AS v5_only, + round(100 * avg(CASE WHEN v4_value = v5_value THEN 1.0 WHEN v4_value IS NULL OR v5_value IS NULL THEN NULL ELSE 0 END), 2) AS pct_exact, + round(100 * avg(CASE WHEN abs(v4_value - v5_value) <= 5e-6 THEN 1.0 WHEN v4_value IS NULL OR v5_value IS NULL THEN NULL ELSE 0 END), 2) AS pct_within_5e6, + max(abs(v4_value - v5_value)) AS max_abs_diff + FROM fin GROUP BY ALL ORDER BY ALL + """) + md(con, "Finalized mismatches (diff > 5e-6), largest", """ + SELECT geo, signal, week_end, v4_value, v5_value, v4_value - v5_value AS diff FROM fin + WHERE abs(v4_value - v5_value) > 5e-6 + ORDER BY abs(v4_value - v5_value) / greatest(abs(v4_value), 1) DESC LIMIT 15 + """) + md(con, "Finalized coverage gaps by geo (weeks present in one source only)", f""" + SELECT geo, signal, sum(CASE WHEN v5_value IS NULL THEN 1 ELSE 0 END) AS v4_only, + min(CASE WHEN v5_value IS NULL THEN week_end END) AS v4_only_first, + max(CASE WHEN v5_value IS NULL THEN week_end END) AS v4_only_last, + sum(CASE WHEN v4_value IS NULL THEN 1 ELSE 0 END) AS v5_only, + min(CASE WHEN v4_value IS NULL THEN week_end END) AS v5_only_first, + max(CASE WHEN v4_value IS NULL THEN week_end END) AS v5_only_last + FROM fin WHERE signal = 'ili' GROUP BY ALL + HAVING v4_only + v5_only > 0 ORDER BY v4_only + v5_only DESC LIMIT 20 + """) + + md(con, "v4 state wili vs ili (v5 has no state wili)", """ + SELECT count(*) AS n, avg(CASE WHEN a.value = b.value THEN 1 ELSE 0 END) AS share_equal + FROM v4 a JOIN v4 b USING (geo, epiweek, issue) + WHERE a.signal = 'wili' AND b.signal = 'ili' AND a.geo != 'nat' + """) + md(con, "Issues (>= 2017w40) missing for some geos in v4", """ + WITH s AS (SELECT DISTINCT geo, issue FROM v4 WHERE signal = 'ili'), + i AS (SELECT DISTINCT issue FROM v4 WHERE issue >= 201740), + g AS (SELECT DISTINCT geo FROM s) + SELECT issue, count(*) AS n_geo_missing FROM g CROSS JOIN i ANTI JOIN s USING (geo, issue) + GROUP BY ALL ORDER BY issue + """) + + # Revision trajectories for spot checks: first 8 issues plus any issue where either source changes. + for geo, signal in [("ca", "ili"), ("ny", "ili"), ("tx", "ili"), ("nat", "wili")]: + for ew in [201952, 202250, 202350]: + md(con, f"Trajectory {geo} {signal} epiweek {ew}", f""" + SELECT issue, release_date, lag, v4_value, v5_value, v5_report_date, + round(v4_value - v5_value, 6) AS diff + FROM (SELECT *, lag(v4_value) OVER w AS p4, lag(v5_value) OVER w AS p5 FROM paired + WHERE geo = '{geo}' AND signal = '{signal}' AND epiweek = {ew} + WINDOW w AS (ORDER BY issue)) + WHERE lag <= 8 OR v4_value IS DISTINCT FROM p4 OR v5_value IS DISTINCT FROM p5 + ORDER BY issue + """) + +if __name__ == "__main__": + app() diff --git a/scripts/one_offs/hhs_2023_24_archive.R b/scripts/one_offs/hhs_2023_24_archive.R new file mode 100644 index 00000000..6eed4c65 --- /dev/null +++ b/scripts/one_offs/hhs_2023_24_archive.R @@ -0,0 +1,192 @@ +# Build a weekly, versioned 2023-24 hospital-admissions archive from Delphi's +# daily `hhs` source, for use as a burn-in season ahead of the NHSN-era replay, +# and validate its finalized values against NHSN's finalized 2023-24 values. +# +# Usage: Rscript scripts/one_offs/hhs_2023_24_archive.R [out_dir] [start] [versions_from] +# `start` (default 20230701) is the first admission day fetched; the 2023-24 +# burn-in uses 20200801 so a 2023-24 snapshot sees earlier seasons' history. +# Issues before `versions_from` (default 2023-07-01) collapse into one version +# at that date, which keeps the per-version loop small. +# Writes hhs_2023_24_weekly_archive.rds (epi_archive DT, both diseases), +# hhs_2023_24_weekly_archive.parquet and validation CSVs into out_dir. +# +# Decisions (see notes/ROADMAP.md item 1b): +# - Week: the hhs `time_value` is the admission day (healthdata.gov `date` - 1). +# NHSN's week ending Saturday S equals the sum over admission days S-7..S-1 +# (Sat..Fri, i.e. collection dates Sun..Sat); other alignments match far worse. +# Output time_value is the canonical Wednesday label, S - 3. +# - Versions: every daily issue is a version. A week's value as of v is the sum of +# its 7 daily values as of v; a week is only emitted once all 7 days exist. +# - US: the hhs `nation` signal, which equals the sum of all states incl. as/pr/vi. + +suppressPackageStartupMessages(source(here::here("R", "load_all.R"))) + +args <- commandArgs(trailingOnly = TRUE) +out_dir <- if (length(args) >= 1) args[[1]] else file.path(tempdir(), "hhs_2023_24") +dir.create(out_dir, recursive = TRUE, showWarnings = FALSE) +raw_cache <- file.path(out_dir, "hhs_daily_raw.rds") + +signals <- c(flu = "confirmed_admissions_influenza_1d", covid = "confirmed_admissions_covid_1d") +start <- if (length(args) >= 2) args[[2]] else "20230701" +versions_from <- as.Date(if (length(args) >= 3) args[[3]] else "2023-07-01") +time_range <- epidatr::epirange(as.integer(start), 20240531) + +# ---- Fetch daily data with all issues ---- +if (file.exists(raw_cache)) { + daily <- readRDS(raw_cache) +} else { + daily <- tidyr::expand_grid(disease = names(signals), geo_type = c("state", "nation")) %>% + purrr::pmap(\(disease, geo_type) { + epidatr::pub_covidcast( + "hhs", signals[[disease]], geo_type, "day", "*", + time_values = time_range, issues = "*" + ) %>% + transmute(disease = disease, geo_value, time_value, version = issue, value) + }) %>% + bind_rows() + saveRDS(daily, raw_cache) +} +# Flu has missing days before mid-2021 (reporting was optional); a week with one +# becomes NA, as in NHSN. From 2023-07 on there must be none. +stopifnot(!anyNA(daily$value[daily$time_value >= as.Date("2023-07-01")])) + +# Saturday week-ending for an hhs admission day: the Saturday after it. +week_ending_saturday <- function(admission_day) { + d <- admission_day + 1 + d + (6 - lubridate::wday(d, week_start = 7) + 1) %% 7 +} +stopifnot(week_ending_saturday(as.Date("2023-11-25")) == as.Date("2023-12-02")) # Sat -> next Sat +stopifnot(week_ending_saturday(as.Date("2023-12-01")) == as.Date("2023-12-02")) # Fri -> next day + +daily <- daily %>% mutate(week_end = week_ending_saturday(time_value)) + +# ---- Daily issues -> weekly versions ---- +versions <- sort(unique(pmax(daily$version, versions_from))) +weekly_long <- purrr::map(versions, \(v) { + daily %>% + filter(version <= v) %>% + group_by(disease, geo_value, time_value) %>% + slice_max(version, n = 1, with_ties = FALSE) %>% + group_by(disease, geo_value, week_end) %>% + summarize(value = sum(value), n_days = n(), .groups = "drop") %>% + filter(n_days == 7L) %>% + mutate(version = v) +}) %>% + bind_rows() + +# Weeks that were incomplete at some version, for the record. +incomplete_seen <- purrr::map(versions, \(v) { + daily %>% + filter(version <= v) %>% + distinct(disease, geo_value, week_end, time_value) %>% + count(disease, geo_value, week_end) %>% + filter(n < 7L) %>% + mutate(version = v) +}) %>% + bind_rows() + +# Canonical shape: Saturday season info, then the Wednesday label. +archive_dt <- weekly_long %>% + rename(time_value = week_end) %>% + select(disease, geo_value, time_value, version, value) %>% + add_season_info() %>% + mutate(time_value = time_value - 3L, source = "hhs") + +archives <- purrr::map(c(flu = "flu", covid = "covid"), \(d) { + archive_dt %>% + filter(disease == d) %>% + select(-disease) %>% + as_epi_archive(other_keys = "source", compactify = TRUE) +}) +saveRDS(purrr::map(archives, \(a) a$DT), file.path(out_dir, "hhs_2023_24_weekly_archive.rds")) +nanoparquet::write_parquet( + purrr::imap(archives, \(a, d) as_tibble(a$DT) %>% mutate(disease = d)) %>% bind_rows(), + file.path(out_dir, "hhs_2023_24_weekly_archive.parquet") +) + +# ---- Validation against NHSN ---- +nhsn_final <- purrr::map(c("flu", "covid"), \(d) { + get_nhsn_data_archive(d) %>% + epix_as_of_current() %>% + as_tibble() %>% + transmute(disease = d, geo_value = ifelse(geo_value == "usa", "us", geo_value), time_value = time_value - 3L, nhsn = value) +}) %>% + bind_rows() + +hhs_final <- purrr::imap(archives, \(a, d) epix_as_of_current(a) %>% as_tibble() %>% mutate(disease = d)) %>% + bind_rows() %>% + select(disease, geo_value, time_value, hhs = value) + +cmp <- inner_join(hhs_final, nhsn_final, by = c("disease", "geo_value", "time_value")) %>% + mutate(diff = hhs - nhsn, ratio = hhs / nhsn) +in_season <- \(x) filter(x, time_value >= as.Date("2023-10-04"), time_value <= as.Date("2024-04-24")) + +overall <- cmp %>% + in_season() %>% + group_by(disease) %>% + summarize( + geo_weeks = n(), + exact = mean(diff == 0), + sum_ratio = sum(hhs) / sum(nhsn), + median_ratio = median(ratio, na.rm = TRUE), + median_abs_diff = median(abs(diff)), + .groups = "drop" + ) +per_geo <- cmp %>% + in_season() %>% + group_by(disease, geo_value) %>% + summarize( + weeks = n(), + exact = mean(diff == 0), + sum_ratio = sum(hhs) / sum(nhsn), + median_ratio = median(ratio, na.rm = TRUE), + max_abs_diff = max(abs(diff)), + nhsn_total = sum(nhsn), + .groups = "drop" + ) +missing_geos <- anti_join( + nhsn_final %>% in_season() %>% distinct(disease, geo_value), + hhs_final %>% distinct(disease, geo_value), + by = c("disease", "geo_value") +) + +# ---- Revisions ---- +rev <- purrr::imap(archives, \(a, d) as_tibble(a$DT) %>% mutate(disease = d)) %>% + bind_rows() %>% + in_season() %>% + group_by(disease, geo_value, time_value) %>% + arrange(version, .by_group = TRUE) %>% + summarize( + n_versions = n(), + first_version = first(version), + first_value = first(value), + final_value = last(value), + .groups = "drop" + ) %>% + mutate( + lag_days = as.integer(first_version - (time_value + 3L)), + rel_revision = (final_value - first_value) / pmax(final_value, 1) + ) +revision_summary <- rev %>% + group_by(disease, us = geo_value == "us") %>% + summarize( + geo_weeks = n(), + median_versions = median(n_versions), + max_versions = max(n_versions), + median_lag_days = median(lag_days), + median_abs_rel_rev = median(abs(rel_revision)), + p90_abs_rel_rev = quantile(abs(rel_revision), 0.9), + .groups = "drop" + ) + +readr::write_csv(cmp, file.path(out_dir, "validation_weekly.csv")) +readr::write_csv(per_geo, file.path(out_dir, "validation_per_geo.csv")) +readr::write_csv(rev, file.path(out_dir, "revisions.csv")) + +print(knitr::kable(overall, digits = 3)) +print(knitr::kable(per_geo %>% arrange(disease, abs(log(sum_ratio)) * -1) %>% group_by(disease) %>% slice_head(n = 8), digits = 3)) +print(knitr::kable(revision_summary, digits = 3)) +cat("NHSN geos with no hhs data:\n") +print(missing_geos) +cat("Week-versions withheld as incomplete:", nrow(distinct(incomplete_seen, disease, geo_value, week_end, version)), "\n") +cat("Output:", out_dir, "\n") diff --git a/scripts/one_offs/sweep_submission_deviations.R b/scripts/one_offs/sweep_submission_deviations.R new file mode 100644 index 00000000..834464cc --- /dev/null +++ b/scripts/one_offs/sweep_submission_deviations.R @@ -0,0 +1,76 @@ +#!/usr/bin/env Rscript +# Sweep our hub submissions for forecasts far from the data, to catch pipeline +# mistakes. See notes/spoiled-submissions.md for what it has found. +# +# For every round, location and horizon it joins the submitted 2.5/50/97.5% +# quantiles with the NHSN value as published when the round was forecast +# (`asof_value`; `latest_value` is the latest week then reported) and the +# finalized value. It writes the joined table to +# cache/submission_sweep_{disease}.csv and prints two flags at h−1 and h0: +# +# - suspect: the median is off by more than 2x from BOTH the value known at +# the time and the finalized value, which a model rarely does on its own. +# - missed: the finalized value is outside the 95% interval and the median is +# off from it by more than 2x. This also catches data problems (bad first +# reports, data gaps), so read it alongside `asof_value`. +# +# Usage: Rscript scripts/one_offs/sweep_submission_deviations.R [flu|covid ...] + +suppressPackageStartupMessages(source(here::here("R/load_all.R"))) + +diseases <- commandArgs(trailingOnly = TRUE) +if (length(diseases) == 0) diseases <- c("flu", "covid") +hub_dirs <- c(flu = "../FluSight-forecast-hub", covid = "../covid19-forecast-hub") +targets <- c(flu = HUB_FLU_TARGET, covid = HUB_COVID_TARGET) + +sweep_one <- function(disease) { + archive <- get_nhsn_data_archive(disease) + vintages <- nhsn_read_vintages(disease, archive) + final <- nhsn_read_truth(disease, archive) %>% rename(final = truth) + fc <- suppressMessages(hub_read_forecasts(here::here(hub_dirs[[disease]]), target = targets[[disease]], drop_spoiled = FALSE)) %>% + filter(level %in% c(0.025, 0.5, 0.975), reference_date >= min(vintages$version) + 3) %>% + mutate(q = c(`0.025` = "lo", `0.5` = "med", `0.975` = "hi")[as.character(level)]) %>% + select(reference_date, horizon, target_end_date, location, q, value) %>% + tidyr::pivot_wider(names_from = q, values_from = value) + asof <- purrr::map(sort(unique(fc$reference_date)), function(rd) { + hub_vintage_snapshot(vintages, hub_round_asof(rd)) %>% + filter(target_end_date >= rd - 28) %>% + group_by(location) %>% + mutate(latest_week = max(target_end_date), latest_value = truth[target_end_date == max(target_end_date)]) %>% + ungroup() %>% + mutate(reference_date = rd) + }) %>% bind_rows() + fc %>% + left_join(select(asof, reference_date, location, target_end_date, asof_value = truth), + by = c("reference_date", "location", "target_end_date")) %>% + left_join(distinct(asof, reference_date, location, latest_week, latest_value), by = c("reference_date", "location")) %>% + left_join(final, by = c("location", "target_end_date")) %>% + mutate( + disease = disease, + known = coalesce(asof_value, latest_value), + lr_known = log(pmax(med, 0.5) / known), + lr_final = log(pmax(med, 0.5) / final) + ) +} + +show_flag <- function(df, title) { + cat("\n", title, "\n", sep = "") + df %>% + arrange(desc(abs(lr_final))) %>% + group_by(disease, reference_date, horizon) %>% + summarize( + n = n(), + `worst (median / known then / final)` = paste(head(sprintf("%s %.0f/%.0f/%.0f", location, med, known, final), 4), collapse = ", "), + .groups = "drop" + ) %>% + arrange(disease, reference_date, horizon) %>% + print(n = Inf, width = Inf) +} + +for (disease in diseases) { + res <- sweep_one(disease) + readr::write_csv(res, here::here("cache", paste0("submission_sweep_", disease, ".csv"))) + near <- res %>% filter(horizon %in% c(-1, 0), final >= 20, known >= 20) + show_flag(near %>% filter(abs(lr_known) > log(2), abs(lr_final) > log(2)), paste0("== ", disease, ": suspect")) + show_flag(near %>% filter(final < lo | final > hi, abs(lr_final) > log(2)), paste0("== ", disease, ": missed")) +} diff --git a/tests/testthat/test-calibrate.R b/tests/testthat/test-calibrate.R index 3c9f252c..2186af3d 100644 --- a/tests/testthat/test-calibrate.R +++ b/tests/testthat/test-calibrate.R @@ -87,5 +87,53 @@ test_that("metrics ignore rounds a location was not forecast at", { expect_true(anyNA(cal$forecasts$value_base)) expect_false(anyNA(hub_quantile_loss(cal)$loss_cal)) expect_false(anyNA(hub_coverage(cal)$coverage_cal)) - expect_false(anyNA(hub_rolling_tradeoff(cal, window = 5L)$qloss_cal)) +}) + +# Vintages where each week is first reported at half its value on the +# Wednesday after it ends and revised to the final value three weeks later. +make_synthetic_vintages <- function(truth) { + bind_rows( + truth %>% mutate(version = target_end_date + 4L, truth = 0.5 * truth), + truth %>% mutate(version = target_end_date + 25L) + ) +} + +test_that("vintage learning truth is the value as of the reveal round", { + hub <- make_synthetic_hub() + vintages <- make_synthetic_vintages(hub$truth) + learned <- function(settle_days) { + cal <- calibrate_hub_forecasts( + hub$forecasts, hub$truth, learn_truth = vintages, settle_days = settle_days, + burn_in_seasons = "2023-2024", progress = FALSE + ) + cal$forecasts %>% filter(level == 0.5, !is.na(truth_learned)) + } + # settle_days = 7 reveals each week at its first report; 28 after the revision. + # Burn-in weeks at the end of a season are revealed after the off-season gap, + # when they have already been revised, so only live rounds are checked. + first <- learned(7L) %>% filter(!is_burn_in) + expect_gt(nrow(first), 0) + expect_equal(first$truth_learned, 0.5 * first$truth) + settled <- learned(28L) + expect_equal(settled$truth_learned, settled$truth) +}) + +test_that("with no revisions, vintage and exact modes match learning from final truth", { + hub <- make_synthetic_hub() + vintages <- hub$truth %>% mutate(version = target_end_date + 4L) + args <- list( + burn_in_seasons = "2023-2024", transform = "sqrt", lr_window = 20, + lr_args = list(mult = 0.03, floor = 1e-3), slow_init = "burn_in_quantile", + lr_slow = list(mult = 0.003), fast_decay = 0.1 + ) + run <- function(f, ...) { + do.call(f, c(list(hub$forecasts, hub$truth, ...), args))$forecasts %>% + filter(!is_burn_in) %>% + arrange(location, horizon, round_index, level_index) + } + final <- run(calibrate_hub_forecasts, progress = FALSE) + cheap <- run(calibrate_hub_forecasts, learn_truth = vintages, progress = FALSE) + expect_equal(cheap$value_cal, final$value_cal) + exact <- run(calibrate_hub_forecasts_exact, vintages = vintages) + expect_equal(exact$value_cal, final$value_cal) }) diff --git a/uv.lock b/uv.lock index 6cfa491b..63bbcb8a 100644 --- a/uv.lock +++ b/uv.lock @@ -7,11 +7,15 @@ name = "exploration-tooling" version = "0.1.0" source = { virtual = "." } dependencies = [ + { name = "numpy" }, { name = "pandas" }, ] [package.metadata] -requires-dist = [{ name = "pandas", specifier = ">=2.3.3" }] +requires-dist = [ + { name = "numpy", specifier = ">=2.0" }, + { name = "pandas", specifier = ">=2.3.3" }, +] [[package]] name = "numpy"